Merge remote-tracking branch 'origin/pr-482'

# Conflicts:
#	.github/workflows/release.yml
This commit is contained in:
fawney19
2026-05-18 18:55:11 +08:00
14 changed files with 8649 additions and 600 deletions

View File

@@ -260,7 +260,8 @@ jobs:
root="package/${bundle}" root="package/${bundle}"
mkdir -p \ mkdir -p \
"${root}/bin" \ "${root}/bin" \
"${root}/frontend" "${root}/frontend" \
"${root}/scripts"
install -m 0755 "artifacts/aether-gateway-${platform}-${arch}/aether-gateway" "${root}/bin/aether-gateway" install -m 0755 "artifacts/aether-gateway-${platform}-${arch}/aether-gateway" "${root}/bin/aether-gateway"
cp -R artifacts/frontend-dist/. "${root}/frontend/" cp -R artifacts/frontend-dist/. "${root}/frontend/"
@@ -270,7 +271,9 @@ jobs:
install.sh > "${root}/install.sh" install.sh > "${root}/install.sh"
chmod 0755 "${root}/install.sh" chmod 0755 "${root}/install.sh"
install -m 0644 docker-compose.yml "${root}/docker-compose.yml" install -m 0644 docker-compose.yml "${root}/docker-compose.yml"
install -m 0644 docker-compose.sqlite.yml "${root}/docker-compose.sqlite.yml" install -m 0644 docker-compose.single-node.yml "${root}/docker-compose.single-node.yml"
install -m 0755 scripts/migrate-pg-compose-to-single-node.sh "${root}/scripts/migrate-pg-compose-to-single-node.sh"
install -m 0755 scripts/migrate-pg-to-single-node.sh "${root}/scripts/migrate-pg-to-single-node.sh"
install -m 0644 .env.example "${root}/.env.example" install -m 0644 .env.example "${root}/.env.example"
install -m 0755 generate_keys.sh "${root}/generate_keys.sh" install -m 0755 generate_keys.sh "${root}/generate_keys.sh"
install -m 0644 README.md "${root}/README.md" install -m 0644 README.md "${root}/README.md"

View File

@@ -48,14 +48,14 @@ cp .env.example .env
./generate_keys.sh ./generate_keys.sh
# 编辑 .env 设置 ADMIN_PASSWORD # 编辑 .env 设置 ADMIN_PASSWORD
# 3. 首次部署 / 更新 (从以下数据库、内存策略任选其一) # 3. 首次部署 / 更新 (从以下部署形态任选其一)
# Postgres + Redis (适用于企业或多人使用) # Postgres + Redis (适用于企业或多人使用)
docker compose pull && docker compose up -d docker compose pull && docker compose up -d
# 仅SQLite (适用于个人用户或朋友分享) # Single Node (适用于个人用户或朋友分享)
docker compose -f docker-compose.sqlite.yml pull && docker compose -f docker-compose.sqlite.yml up -d docker compose -f docker-compose.single-node.yml pull && docker compose -f docker-compose.single-node.yml up -d
``` ```
### 一键安装(可选部署方式 Linux: systemd; Mac: launchd ### 一键安装(默认 Single NodeLinux systemd / macOS launchd + SQLite
```bash ```bash
cd Aether && cd Aether cd Aether && cd Aether

View File

@@ -9,7 +9,10 @@ use clap::{Args as ClapArgs, Parser, Subcommand, ValueEnum};
use tracing::{debug, info, warn}; use tracing::{debug, info, warn};
use aether_crypto::warm_python_fernet_secret; use aether_crypto::warm_python_fernet_secret;
use aether_data::lifecycle::export::{export_database_jsonl, import_database_jsonl, ExportDomain}; use aether_data::lifecycle::export::{
copy_database_records, export_database_jsonl, import_database_jsonl, DataCopyOptions,
ExportDomain,
};
use aether_data::{DatabaseDriver, SqlDatabaseConfig, SqlPoolConfig, DEFAULT_SQLITE_DATABASE_URL}; use aether_data::{DatabaseDriver, SqlDatabaseConfig, SqlPoolConfig, DEFAULT_SQLITE_DATABASE_URL};
use aether_gateway::{ use aether_gateway::{
attach_static_frontend, build_router_with_state, set_gateway_frontdoor_app_port, AppState, attach_static_frontend, build_router_with_state, set_gateway_frontdoor_app_port, AppState,
@@ -579,6 +582,8 @@ enum DataCommand {
Export(DataExportArgs), Export(DataExportArgs),
/// Import database-neutral JSONL into the selected SQL database. /// Import database-neutral JSONL into the selected SQL database.
Import(DataImportArgs), Import(DataImportArgs),
/// Copy persistent SQL data directly between two databases without a JSONL file.
Copy(DataCopyArgs),
} }
#[derive(ClapArgs, Debug, Clone)] #[derive(ClapArgs, Debug, Clone)]
@@ -602,6 +607,27 @@ struct DataImportArgs {
input: PathBuf, input: PathBuf,
} }
#[derive(ClapArgs, Debug, Clone)]
struct DataCopyArgs {
#[arg(long, value_enum)]
source_driver: DatabaseDriverArg,
#[arg(long)]
source_url: String,
#[arg(long, value_enum)]
target_driver: DatabaseDriverArg,
#[arg(long)]
target_url: String,
#[arg(long, value_enum, value_delimiter = ',')]
domains: Vec<ExportDomainArg>,
#[arg(long)]
omit_request_body_details: bool,
}
impl GatewayLoggingArgs { impl GatewayLoggingArgs {
fn apply_to_runtime_config( fn apply_to_runtime_config(
&self, &self,
@@ -1251,6 +1277,7 @@ async fn run_data_command(command: &DataCommand) -> Result<(), Box<dyn std::erro
match command { match command {
DataCommand::Export(args) => run_data_export(args).await, DataCommand::Export(args) => run_data_export(args).await,
DataCommand::Import(args) => run_data_import(args).await, DataCommand::Import(args) => run_data_import(args).await,
DataCommand::Copy(args) => run_data_copy(args).await,
} }
} }
@@ -1267,11 +1294,11 @@ fn required_sql_database_config(
} }
fn requested_export_domains(args: &DataExportArgs) -> Vec<ExportDomain> { fn requested_export_domains(args: &DataExportArgs) -> Vec<ExportDomain> {
args.domains requested_domains(&args.domains)
.iter() }
.copied()
.map(Into::into) fn requested_domains(domains: &[ExportDomainArg]) -> Vec<ExportDomain> {
.collect::<Vec<_>>() domains.iter().copied().map(Into::into).collect::<Vec<_>>()
} }
fn current_unix_secs() -> Result<u64, std::time::SystemTimeError> { fn current_unix_secs() -> Result<u64, std::time::SystemTimeError> {
@@ -1324,6 +1351,61 @@ async fn run_data_import(args: &DataImportArgs) -> Result<(), Box<dyn std::error
Ok(()) Ok(())
} }
fn copy_database_config(
driver: DatabaseDriverArg,
url: &str,
label: &str,
) -> Result<SqlDatabaseConfig, Box<dyn std::error::Error>> {
let url = url.trim();
if url.is_empty() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
format!("{label} database URL must not be empty"),
)
.into());
}
let driver = DatabaseDriver::from(driver);
Ok(SqlDatabaseConfig::new(
driver,
url,
SqlPoolConfig {
require_ssl: false,
..SqlPoolConfig::default()
},
)?)
}
async fn run_data_copy(args: &DataCopyArgs) -> Result<(), Box<dyn std::error::Error>> {
let source = copy_database_config(args.source_driver, &args.source_url, "source")?;
let target = copy_database_config(args.target_driver, &args.target_url, "target")?;
let source_driver = source.driver;
let target_driver = target.driver;
let domains = requested_domains(&args.domains);
let created_at_unix_secs = current_unix_secs()?;
let imported = copy_database_records(
source,
target,
domains,
created_at_unix_secs,
DataCopyOptions {
omit_request_body_details: args.omit_request_body_details,
},
)
.await?;
info!(
source_driver = %source_driver,
target_driver = %target_driver,
imported,
"database copy complete"
);
println!(
"copied {} records from {} to {} without a JSONL file",
imported, source_driver, target_driver
);
Ok(())
}
async fn run_explicit_migrations(args: &Args) -> Result<(), Box<dyn std::error::Error>> { async fn run_explicit_migrations(args: &Args) -> Result<(), Box<dyn std::error::Error>> {
if args.data.effective_sql_database_config().is_none() { if args.data.effective_sql_database_config().is_none() {
return Err(std::io::Error::new( return Err(std::io::Error::new(

View File

@@ -1,5 +1,6 @@
use std::collections::{BTreeMap, BTreeSet}; use std::collections::{BTreeMap, BTreeSet};
use futures_util::TryStreamExt;
use serde_json::Value; use serde_json::Value;
use sqlx::{Column, Row, TypeInfo, ValueRef}; use sqlx::{Column, Row, TypeInfo, ValueRef};
@@ -130,6 +131,41 @@ pub struct ExportRow {
pub payload: Value, pub payload: Value,
} }
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct DataCopyOptions {
pub omit_request_body_details: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct SqliteCopyColumn {
name: String,
declared_type: String,
not_null: bool,
has_default: bool,
primary_key_position: i64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum SqliteCopyAffinity {
Integer,
Real,
Text,
Blob,
Numeric,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct SchemaCopyColumn {
sqlite: SqliteCopyColumn,
postgres: PostgresImportColumn,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct SchemaCopyTable {
table_name: String,
columns: Vec<SchemaCopyColumn>,
}
#[derive(Debug, Clone, PartialEq, Eq)] #[derive(Debug, Clone, PartialEq, Eq)]
struct PostgresImportColumn { struct PostgresImportColumn {
data_type: String, data_type: String,
@@ -141,6 +177,20 @@ struct PostgresImportColumn {
type PostgresImportColumns = BTreeMap<String, PostgresImportColumn>; type PostgresImportColumns = BTreeMap<String, PostgresImportColumn>;
type ImportColumnNames = BTreeSet<String>; type ImportColumnNames = BTreeSet<String>;
const USAGE_REQUEST_BODY_DETAIL_COLUMNS: &[&str] = &[
"request_body",
"response_body",
"provider_request_body",
"client_response_body",
"request_body_compressed",
"response_body_compressed",
"provider_request_body_compressed",
"client_response_body_compressed",
];
const REQUEST_BODY_DETAIL_TABLES: &[&str] = &["usage_body_blobs", "usage_http_audits"];
const LIFECYCLE_TABLES: &[&str] = &["_sqlx_migrations", "schema_backfills"];
pub fn encode_jsonl(records: &[DataExportRecord]) -> Result<String, DataLayerError> { pub fn encode_jsonl(records: &[DataExportRecord]) -> Result<String, DataLayerError> {
validate_export_records(records)?; validate_export_records(records)?;
@@ -344,6 +394,602 @@ pub async fn import_database_jsonl(
} }
} }
pub async fn copy_database_records(
source: SqlDatabaseConfig,
target: SqlDatabaseConfig,
domains: Vec<ExportDomain>,
created_at_unix_secs: u64,
options: DataCopyOptions,
) -> Result<usize, DataLayerError> {
if domains.is_empty()
&& source.driver == DatabaseDriver::Postgres
&& target.driver == DatabaseDriver::Sqlite
{
return copy_postgres_to_sqlite_from_target_schema(source, target, options).await;
}
let mut records =
decode_jsonl(&export_database_jsonl(source, domains, created_at_unix_secs).await?)?;
if options.omit_request_body_details {
omit_request_body_details_from_records(&mut records);
}
import_database_jsonl(target, &encode_jsonl(&records)?).await
}
fn omit_request_body_details_from_records(records: &mut [DataExportRecord]) {
for record in records {
let DataExportRecord::Row {
domain: ExportDomain::Usage,
payload,
..
} = record
else {
continue;
};
if let Some(object) = payload.as_object_mut() {
for column_name in USAGE_REQUEST_BODY_DETAIL_COLUMNS {
object.remove(*column_name);
}
}
}
}
async fn copy_postgres_to_sqlite_from_target_schema(
source: SqlDatabaseConfig,
mut target: SqlDatabaseConfig,
options: DataCopyOptions,
) -> Result<usize, DataLayerError> {
target.pool.min_connections = 1;
target.pool.max_connections = 1;
let postgres_pool =
crate::driver::postgres::PostgresPoolFactory::new(source.to_postgres_config()?)?
.connect_lazy()?;
let sqlite_pool = crate::driver::sqlite::SqlitePoolFactory::new(target)?.connect_lazy()?;
let source_tables = load_postgres_public_table_names(&postgres_pool).await?;
let target_tables = load_sqlite_copy_table_names(&sqlite_pool).await?;
ensure_no_nonempty_source_tables_outside_target_schema(
&postgres_pool,
&source_tables,
&target_tables,
options,
)
.await?;
let mut imported = 0usize;
sqlx::raw_sql("PRAGMA foreign_keys = OFF")
.execute(&sqlite_pool)
.await
.map_sql_err()?;
for table_name in target_tables {
if copy_table_is_lifecycle(&table_name)
|| copy_table_is_sqlite_internal(&table_name)
|| !source_tables.contains(&table_name)
|| (options.omit_request_body_details && copy_table_is_request_body_detail(&table_name))
{
continue;
}
let table_plan = build_postgres_sqlite_copy_table_plan(
&postgres_pool,
&sqlite_pool,
&table_name,
options,
)
.await?;
if table_plan.columns.is_empty() {
continue;
}
imported = imported.saturating_add(
copy_postgres_sqlite_table(&postgres_pool, &sqlite_pool, &table_plan).await?,
);
}
sqlx::raw_sql("PRAGMA foreign_keys = ON")
.execute(&sqlite_pool)
.await
.map_sql_err()?;
ensure_sqlite_foreign_key_check_passes(&sqlite_pool).await?;
Ok(imported)
}
async fn ensure_no_nonempty_source_tables_outside_target_schema(
postgres_pool: &crate::driver::postgres::PostgresPool,
source_tables: &BTreeSet<String>,
target_tables: &BTreeSet<String>,
options: DataCopyOptions,
) -> Result<(), DataLayerError> {
let mut missing = Vec::new();
for table_name in source_tables {
if copy_table_is_lifecycle(table_name)
|| (options.omit_request_body_details && copy_table_is_request_body_detail(table_name))
|| target_tables.contains(table_name)
{
continue;
}
if postgres_public_table_has_rows(postgres_pool, table_name).await? {
missing.push(table_name.clone());
}
}
if !missing.is_empty() {
return Err(DataLayerError::InvalidInput(format!(
"source Postgres has non-empty public tables that do not exist in the target SQLite schema: {}",
missing.join(", ")
)));
}
Ok(())
}
async fn build_postgres_sqlite_copy_table_plan(
postgres_pool: &crate::driver::postgres::PostgresPool,
sqlite_pool: &crate::driver::sqlite::SqlitePool,
table_name: &str,
options: DataCopyOptions,
) -> Result<SchemaCopyTable, DataLayerError> {
let sqlite_columns = load_sqlite_copy_columns(sqlite_pool, table_name).await?;
let postgres_columns =
load_postgres_import_columns(postgres_pool, &format!("public.{table_name}")).await?;
let source_has_rows = postgres_public_table_has_rows(postgres_pool, table_name).await?;
let mut columns = Vec::new();
for sqlite_column in sqlite_columns {
if options.omit_request_body_details
&& table_name == "usage"
&& USAGE_REQUEST_BODY_DETAIL_COLUMNS.contains(&sqlite_column.name.as_str())
{
continue;
}
if let Some(postgres_column) = postgres_columns.get(&sqlite_column.name) {
columns.push(SchemaCopyColumn {
sqlite: sqlite_column,
postgres: postgres_column.clone(),
});
continue;
}
if source_has_rows && sqlite_copy_column_is_required(&sqlite_column) {
return Err(DataLayerError::InvalidInput(format!(
"target SQLite table '{table_name}' has required column '{}' that does not exist in source Postgres",
sqlite_column.name
)));
}
}
if source_has_rows && columns.is_empty() {
return Err(DataLayerError::InvalidInput(format!(
"source Postgres table '{table_name}' has rows, but none of its columns exist in target SQLite"
)));
}
Ok(SchemaCopyTable {
table_name: table_name.to_string(),
columns,
})
}
async fn copy_postgres_sqlite_table(
postgres_pool: &crate::driver::postgres::PostgresPool,
sqlite_pool: &crate::driver::sqlite::SqlitePool,
table: &SchemaCopyTable,
) -> Result<usize, DataLayerError> {
let source_sql = postgres_schema_copy_select_sql(table)?;
let target_sql = sqlite_schema_copy_insert_sql(table)?;
let mut rows = sqlx::query(&source_sql).fetch(postgres_pool);
let mut imported = 0usize;
while let Some(row) = rows.try_next().await.map_sql_err()? {
let payload = row.try_get::<Value, _>("payload").map_sql_err()?;
let object = payload.as_object().ok_or_else(|| {
DataLayerError::UnexpectedValue(format!(
"postgres copy row for table '{}' did not produce a JSON object",
table.table_name
))
})?;
let mut query = sqlx::query(&target_sql);
for column in &table.columns {
let value = object.get(&column.sqlite.name).ok_or_else(|| {
DataLayerError::UnexpectedValue(format!(
"postgres copy row for table '{}' is missing column '{}'",
table.table_name, column.sqlite.name
))
})?;
query = bind_sqlite_copy_value(query, value, &column.sqlite)?;
}
query.execute(sqlite_pool).await.map_sql_err()?;
imported = imported.saturating_add(1);
}
Ok(imported)
}
fn postgres_schema_copy_select_sql(table: &SchemaCopyTable) -> Result<String, DataLayerError> {
let table_sql = format!(
"public.{}",
postgres_quote_identifier(table.table_name.as_str())?
);
let mut payload_parts = Vec::new();
for column in &table.columns {
if let Some(expr) = postgres_schema_copy_override_expr(column)? {
payload_parts.push(sql_string_literal(&column.sqlite.name));
payload_parts.push(expr);
}
}
let payload_sql = if payload_parts.is_empty() {
"to_jsonb(t)".to_string()
} else {
format!(
"to_jsonb(t) || jsonb_build_object({})",
payload_parts.join(", ")
)
};
let order_by = table
.columns
.iter()
.filter(|column| column.sqlite.primary_key_position > 0)
.map(|column| {
postgres_quote_identifier(&column.sqlite.name).map(|quoted| format!("t.{quoted} ASC"))
})
.collect::<Result<Vec<_>, _>>()?;
let order_sql = if order_by.is_empty() {
String::new()
} else {
format!(" ORDER BY {}", order_by.join(", "))
};
Ok(format!(
"SELECT {payload_sql} AS payload FROM {table_sql} AS t{order_sql}"
))
}
fn postgres_schema_copy_override_expr(
column: &SchemaCopyColumn,
) -> Result<Option<String>, DataLayerError> {
let column_sql = format!("t.{}", postgres_quote_identifier(&column.sqlite.name)?);
let affinity = sqlite_copy_affinity(&column.sqlite);
if affinity == SqliteCopyAffinity::Blob && is_postgres_bytea_column(&column.postgres) {
return Ok(Some(format!(
"CASE WHEN {column_sql} IS NULL THEN NULL ELSE encode({column_sql}, 'hex') END"
)));
}
if affinity == SqliteCopyAffinity::Integer && is_postgres_boolean_column(&column.postgres) {
return Ok(Some(format!(
"CASE WHEN {column_sql} IS NULL THEN NULL WHEN {column_sql} THEN 1 ELSE 0 END"
)));
}
if affinity == SqliteCopyAffinity::Integer
&& (is_postgres_timestamp_column(&column.postgres)
|| is_postgres_date_column(&column.postgres))
{
let timestamp_sql = if is_postgres_date_column(&column.postgres) {
format!("{column_sql}::timestamp")
} else {
column_sql.clone()
};
let multiplier = if sqlite_copy_column_stores_unix_millis(&column.sqlite.name) {
" * 1000"
} else {
""
};
return Ok(Some(format!(
"CASE WHEN {column_sql} IS NULL THEN NULL ELSE FLOOR(EXTRACT(EPOCH FROM {timestamp_sql}){multiplier})::bigint END"
)));
}
Ok(None)
}
fn sqlite_schema_copy_insert_sql(table: &SchemaCopyTable) -> Result<String, DataLayerError> {
let table_sql = sqlite_quote_identifier(&table.table_name)?;
let column_sql = table
.columns
.iter()
.map(|column| sqlite_quote_identifier(&column.sqlite.name))
.collect::<Result<Vec<_>, _>>()?
.join(", ");
let placeholder_sql = vec!["?"; table.columns.len()].join(", ");
Ok(format!(
"INSERT OR REPLACE INTO {table_sql} ({column_sql}) VALUES ({placeholder_sql})"
))
}
async fn load_postgres_public_table_names(
pool: &crate::driver::postgres::PostgresPool,
) -> Result<BTreeSet<String>, DataLayerError> {
let rows = sqlx::query(
r#"
SELECT table_name
FROM information_schema.tables
WHERE table_schema = 'public'
AND table_type = 'BASE TABLE'
ORDER BY table_name
"#,
)
.fetch_all(pool)
.await
.map_sql_err()?;
let mut tables = BTreeSet::new();
for row in rows {
tables.insert(row.try_get::<String, _>("table_name").map_sql_err()?);
}
Ok(tables)
}
async fn load_sqlite_copy_table_names(
pool: &crate::driver::sqlite::SqlitePool,
) -> Result<BTreeSet<String>, DataLayerError> {
let rows = sqlx::query(
r#"
SELECT name
FROM sqlite_schema
WHERE type = 'table'
AND name NOT LIKE 'sqlite_%'
ORDER BY name
"#,
)
.fetch_all(pool)
.await
.map_sql_err()?;
let mut tables = BTreeSet::new();
for row in rows {
let table_name = row.try_get::<String, _>("name").map_sql_err()?;
if !copy_table_is_lifecycle(&table_name) && !copy_table_is_sqlite_internal(&table_name) {
tables.insert(table_name);
}
}
Ok(tables)
}
async fn load_sqlite_copy_columns(
pool: &crate::driver::sqlite::SqlitePool,
table_name: &str,
) -> Result<Vec<SqliteCopyColumn>, DataLayerError> {
let table_sql = sqlite_quote_identifier(table_name)?;
let rows = sqlx::query(&format!("PRAGMA table_info({table_sql})"))
.fetch_all(pool)
.await
.map_sql_err()?;
let mut columns = Vec::new();
for row in rows {
columns.push(SqliteCopyColumn {
name: row.try_get::<String, _>("name").map_sql_err()?,
declared_type: row
.try_get::<Option<String>, _>("type")
.map_sql_err()?
.unwrap_or_default(),
not_null: row.try_get::<i64, _>("notnull").map_sql_err()? != 0,
has_default: row
.try_get::<Option<String>, _>("dflt_value")
.map_sql_err()?
.is_some(),
primary_key_position: row.try_get::<i64, _>("pk").map_sql_err()?,
});
}
if columns.is_empty() {
return Err(DataLayerError::UnexpectedValue(format!(
"target SQLite table '{table_name}' has no visible columns"
)));
}
Ok(columns)
}
async fn postgres_public_table_has_rows(
pool: &crate::driver::postgres::PostgresPool,
table_name: &str,
) -> Result<bool, DataLayerError> {
let table_sql = format!("public.{}", postgres_quote_identifier(table_name)?);
sqlx::query_scalar::<_, bool>(&format!(
"SELECT EXISTS (SELECT 1 FROM {table_sql} LIMIT 1)"
))
.fetch_one(pool)
.await
.map_sql_err()
}
async fn ensure_sqlite_foreign_key_check_passes(
pool: &crate::driver::sqlite::SqlitePool,
) -> Result<(), DataLayerError> {
let rows = sqlx::query("PRAGMA foreign_key_check")
.fetch_all(pool)
.await
.map_sql_err()?;
if rows.is_empty() {
return Ok(());
}
let mut violations = Vec::new();
for row in rows.iter().take(10) {
let table = row
.try_get::<Option<String>, _>("table")
.map_sql_err()?
.unwrap_or_else(|| "<unknown>".to_string());
let rowid = row.try_get::<Option<i64>, _>("rowid").map_sql_err()?;
let parent = row
.try_get::<Option<String>, _>("parent")
.map_sql_err()?
.unwrap_or_else(|| "<unknown>".to_string());
violations.push(format!("{table} rowid={rowid:?} parent={parent}"));
}
Err(DataLayerError::InvalidInput(format!(
"target SQLite foreign key check failed after copy: {}",
violations.join("; ")
)))
}
fn copy_table_is_lifecycle(table_name: &str) -> bool {
LIFECYCLE_TABLES.contains(&table_name)
}
fn copy_table_is_sqlite_internal(table_name: &str) -> bool {
table_name.starts_with("sqlite_")
}
fn copy_table_is_request_body_detail(table_name: &str) -> bool {
REQUEST_BODY_DETAIL_TABLES.contains(&table_name)
}
fn sqlite_copy_column_is_required(column: &SqliteCopyColumn) -> bool {
(column.not_null || column.primary_key_position > 0) && !column.has_default
}
fn sqlite_copy_column_stores_unix_millis(column_name: &str) -> bool {
column_name.ends_with("_unix_ms")
}
fn sqlite_copy_affinity(column: &SqliteCopyColumn) -> SqliteCopyAffinity {
let declared_type = column.declared_type.to_ascii_uppercase();
if declared_type.contains("INT") {
SqliteCopyAffinity::Integer
} else if declared_type.contains("CHAR")
|| declared_type.contains("CLOB")
|| declared_type.contains("TEXT")
{
SqliteCopyAffinity::Text
} else if declared_type.contains("BLOB") || declared_type.trim().is_empty() {
SqliteCopyAffinity::Blob
} else if declared_type.contains("REAL")
|| declared_type.contains("FLOA")
|| declared_type.contains("DOUB")
{
SqliteCopyAffinity::Real
} else {
SqliteCopyAffinity::Numeric
}
}
fn is_postgres_bytea_column(column: &PostgresImportColumn) -> bool {
column.data_type == "bytea" || column.udt_name == "bytea"
}
fn is_postgres_date_column(column: &PostgresImportColumn) -> bool {
column.data_type == "date" || column.udt_name == "date"
}
fn bind_sqlite_copy_value<'q>(
query: sqlx::query::Query<'q, sqlx::Sqlite, sqlx::sqlite::SqliteArguments<'q>>,
value: &'q Value,
column: &SqliteCopyColumn,
) -> Result<sqlx::query::Query<'q, sqlx::Sqlite, sqlx::sqlite::SqliteArguments<'q>>, DataLayerError>
{
Ok(match sqlite_copy_affinity(column) {
SqliteCopyAffinity::Integer => match value {
Value::Null => query.bind(Option::<i64>::None),
Value::Bool(value) => query.bind(i64::from(*value)),
Value::Number(number) => {
let value = number
.as_i64()
.or_else(|| number.as_u64().and_then(|value| i64::try_from(value).ok()))
.ok_or_else(|| {
DataLayerError::InvalidInput(format!(
"sqlite copy column '{}' expected integer, got {number}",
column.name
))
})?;
query.bind(value)
}
Value::String(value) => query.bind(value.parse::<i64>().map_err(|err| {
DataLayerError::InvalidInput(format!(
"sqlite copy column '{}' expected integer string: {err}",
column.name
))
})?),
Value::Array(_) | Value::Object(_) => {
return Err(DataLayerError::InvalidInput(format!(
"sqlite copy column '{}' expected integer-compatible value",
column.name
)));
}
},
SqliteCopyAffinity::Real => match value {
Value::Null => query.bind(Option::<f64>::None),
Value::Number(number) => query.bind(number.as_f64().ok_or_else(|| {
DataLayerError::InvalidInput(format!(
"sqlite copy column '{}' expected finite real value",
column.name
))
})?),
Value::String(value) => query.bind(value.parse::<f64>().map_err(|err| {
DataLayerError::InvalidInput(format!(
"sqlite copy column '{}' expected real string: {err}",
column.name
))
})?),
Value::Bool(value) => query.bind(if *value { 1.0 } else { 0.0 }),
Value::Array(_) | Value::Object(_) => {
return Err(DataLayerError::InvalidInput(format!(
"sqlite copy column '{}' expected real-compatible value",
column.name
)));
}
},
SqliteCopyAffinity::Blob => match value {
Value::Null => query.bind(Option::<Vec<u8>>::None),
Value::String(value) => query.bind(hex_decode(value, &column.name)?),
Value::Array(values) => {
let mut bytes = Vec::with_capacity(values.len());
for value in values {
let Some(byte) = value.as_u64().and_then(|value| u8::try_from(value).ok())
else {
return Err(DataLayerError::InvalidInput(format!(
"sqlite copy column '{}' contains non-byte array value",
column.name
)));
};
bytes.push(byte);
}
query.bind(bytes)
}
Value::Bool(_) | Value::Number(_) | Value::Object(_) => {
return Err(DataLayerError::InvalidInput(format!(
"sqlite copy column '{}' expected blob-compatible value",
column.name
)));
}
},
SqliteCopyAffinity::Text | SqliteCopyAffinity::Numeric => {
bind_sqlite_json_value(query, value)?
}
})
}
fn sql_string_literal(value: &str) -> String {
format!("'{}'", value.replace('\'', "''"))
}
fn hex_decode(value: &str, column_name: &str) -> Result<Vec<u8>, DataLayerError> {
let value = value.trim();
if !value.len().is_multiple_of(2) {
return Err(DataLayerError::InvalidInput(format!(
"sqlite copy column '{column_name}' has odd-length hex data"
)));
}
let mut bytes = Vec::with_capacity(value.len() / 2);
for index in (0..value.len()).step_by(2) {
let byte = u8::from_str_radix(&value[index..index + 2], 16).map_err(|err| {
DataLayerError::InvalidInput(format!(
"sqlite copy column '{column_name}' has invalid hex data at byte {}: {err}",
index / 2
))
})?;
bytes.push(byte);
}
Ok(bytes)
}
pub async fn export_sqlite_core_jsonl( pub async fn export_sqlite_core_jsonl(
pool: &crate::driver::sqlite::SqlitePool, pool: &crate::driver::sqlite::SqlitePool,
created_at_unix_secs: u64, created_at_unix_secs: u64,

View File

@@ -45,7 +45,15 @@ SELECT
m.provider_model_mappings AS model_provider_model_mappings, m.provider_model_mappings AS model_provider_model_mappings,
m.supports_streaming AS model_supports_streaming, m.supports_streaming AS model_supports_streaming,
m.is_active AS model_is_active, m.is_active AS model_is_active,
m.is_available AS model_is_available m.is_available AS model_is_available,
CASE
WHEN json_valid(p.config) THEN
CASE
WHEN json_type(p.config, '$.pool_advanced') IS NOT NULL THEN 1
ELSE 0
END
ELSE 0
END AS provider_pool_enabled
FROM providers p FROM providers p
INNER JOIN provider_endpoints pe ON pe.provider_id = p.id INNER JOIN provider_endpoints pe ON pe.provider_id = p.id
INNER JOIN provider_api_keys pak ON pak.provider_id = p.id INNER JOIN provider_api_keys pak ON pak.provider_id = p.id
@@ -67,52 +75,139 @@ pub struct SqliteMinimalCandidateSelectionReadRepository {
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
struct CandidateSelectionRow { struct CandidateSelectionRow {
row: StoredMinimalCandidateSelectionRow, row: StoredMinimalCandidateSelectionRow,
provider_pool_enabled: bool,
key_auth_config: Option<String>, key_auth_config: Option<String>,
key_last_used_at_unix_secs: Option<u64>, key_last_used_at_unix_secs: Option<u64>,
} }
#[derive(Debug, Clone, Copy)]
enum SelectedRowsOrder {
WithGlobalModel,
WithoutGlobalModel,
}
#[derive(Debug, Clone, Copy)]
enum SelectedRowsFilter<'a> {
None,
GlobalModel(&'a str),
RequestedModel(&'a str),
}
#[derive(Debug, Clone, Copy)]
struct SqlPage {
limit: i64,
offset: i64,
}
impl SqliteMinimalCandidateSelectionReadRepository { impl SqliteMinimalCandidateSelectionReadRepository {
pub fn new(pool: SqlitePool) -> Self { pub fn new(pool: SqlitePool) -> Self {
Self { pool } Self { pool }
} }
async fn load_rows_for_api_format(
&self,
api_format: &str,
) -> Result<Vec<CandidateSelectionRow>, DataLayerError> {
let canonical_api_format = normalize_api_format(api_format);
let storage_aliases = api_format_aliases(&canonical_api_format);
let match_aliases = sql_match_aliases(&storage_aliases);
let mut builder = QueryBuilder::<Sqlite>::new(CANDIDATE_SELECTION_COLUMNS);
builder.push(" AND LOWER(pe.api_format) IN (");
{
let mut separated = builder.separated(", ");
for alias in &match_aliases {
separated.push_bind(alias);
}
}
builder.push(")");
let rows = builder.build().fetch_all(&self.pool).await.map_sql_err()?;
let mut items = rows
.iter()
.map(map_candidate_selection_row)
.collect::<Result<Vec<_>, _>>()?;
items.retain(|item| {
api_format_matches(&item.row.endpoint_api_format, &canonical_api_format)
&& item.row.key_supports_api_format(&canonical_api_format)
&& key_auth_channel_matches(item, &canonical_api_format)
});
Ok(items)
}
async fn selected_rows_for_api_format( async fn selected_rows_for_api_format(
&self, &self,
api_format: &str, api_format: &str,
) -> Result<Vec<StoredMinimalCandidateSelectionRow>, DataLayerError> { ) -> Result<Vec<StoredMinimalCandidateSelectionRow>, DataLayerError> {
let rows = self.load_rows_for_api_format(api_format).await?; self.load_selected_rows_for_api_format(
Ok(sort_rows(select_pool_rows(rows), true)) api_format,
SelectedRowsFilter::None,
SelectedRowsOrder::WithGlobalModel,
None,
)
.await
}
async fn load_selected_rows_for_api_format(
&self,
api_format: &str,
filter: SelectedRowsFilter<'_>,
order: SelectedRowsOrder,
page: Option<SqlPage>,
) -> Result<Vec<StoredMinimalCandidateSelectionRow>, DataLayerError> {
let canonical_api_format = normalize_api_format(api_format);
let storage_aliases = api_format_aliases(&canonical_api_format);
let match_aliases = sql_match_aliases(&storage_aliases);
let mut rows = Vec::new();
for storage_api_format in storage_aliases {
let mut builder = QueryBuilder::<Sqlite>::new("WITH candidate_rows AS (");
builder.push(CANDIDATE_SELECTION_COLUMNS);
push_candidate_sql_filters(&mut builder, &storage_api_format, &match_aliases);
match filter {
SelectedRowsFilter::None => {}
SelectedRowsFilter::GlobalModel(global_model_name) => {
builder.push(" AND gm.name = ");
builder.push_bind(global_model_name);
}
SelectedRowsFilter::RequestedModel(requested_model_name) => {
push_requested_model_sql_filter(
&mut builder,
requested_model_name,
&match_aliases,
);
}
}
builder.push(
r#"
),
pool_rows AS (
SELECT candidate.*
FROM candidate_rows candidate
WHERE candidate.provider_pool_enabled = 1
AND NOT EXISTS (
SELECT 1
FROM candidate_rows other
WHERE other.provider_pool_enabled = 1
AND other.provider_id = candidate.provider_id
AND other.endpoint_id = candidate.endpoint_id
AND other.model_id = candidate.model_id
AND (
other.key_internal_priority < candidate.key_internal_priority
OR (
other.key_internal_priority = candidate.key_internal_priority
AND other.key_id < candidate.key_id
)
)
)
),
selected_rows AS (
SELECT * FROM candidate_rows WHERE provider_pool_enabled = 0
UNION ALL
SELECT * FROM pool_rows
)
SELECT * FROM selected_rows
"#,
);
push_selected_rows_order(&mut builder, order);
if let Some(page) = page {
builder.push(" LIMIT ");
builder.push_bind(page.limit);
builder.push(" OFFSET ");
builder.push_bind(page.offset);
}
let query_rows = builder.build().fetch_all(&self.pool).await.map_sql_err()?;
let mut items = query_rows
.iter()
.map(map_candidate_selection_row)
.collect::<Result<Vec<_>, _>>()?;
items.retain(|item| {
api_format_matches(&item.row.endpoint_api_format, &canonical_api_format)
&& item.row.key_supports_api_format(&canonical_api_format)
&& key_auth_channel_matches(item, &canonical_api_format)
});
rows.extend(items.into_iter().map(|item| item.row));
}
let rows = match filter {
SelectedRowsFilter::RequestedModel(requested_model_name) => rows
.into_iter()
.filter(|row| {
row_matches_requested_model(row, requested_model_name, &canonical_api_format)
})
.collect(),
_ => rows,
};
Ok(dedupe_candidate_selection_rows(rows))
} }
} }
@@ -130,14 +225,13 @@ impl MinimalCandidateSelectionReadRepository for SqliteMinimalCandidateSelection
api_format: &str, api_format: &str,
global_model_name: &str, global_model_name: &str,
) -> Result<Vec<StoredMinimalCandidateSelectionRow>, DataLayerError> { ) -> Result<Vec<StoredMinimalCandidateSelectionRow>, DataLayerError> {
Ok(sort_rows( self.load_selected_rows_for_api_format(
self.selected_rows_for_api_format(api_format) api_format,
.await? SelectedRowsFilter::GlobalModel(global_model_name),
.into_iter() SelectedRowsOrder::WithoutGlobalModel,
.filter(|row| row.global_model_name == global_model_name) None,
.collect(), )
false, .await
))
} }
async fn list_for_exact_api_format_and_requested_model( async fn list_for_exact_api_format_and_requested_model(
@@ -145,13 +239,11 @@ impl MinimalCandidateSelectionReadRepository for SqliteMinimalCandidateSelection
api_format: &str, api_format: &str,
requested_model_name: &str, requested_model_name: &str,
) -> Result<Vec<StoredMinimalCandidateSelectionRow>, DataLayerError> { ) -> Result<Vec<StoredMinimalCandidateSelectionRow>, DataLayerError> {
self.list_for_exact_api_format_and_requested_model_page( self.load_selected_rows_for_api_format(
&StoredRequestedModelCandidateRowsQuery { api_format,
api_format: api_format.to_string(), SelectedRowsFilter::RequestedModel(requested_model_name),
requested_model_name: requested_model_name.to_string(), SelectedRowsOrder::WithGlobalModel,
offset: 0, None,
limit: u32::MAX,
},
) )
.await .await
} }
@@ -160,42 +252,74 @@ impl MinimalCandidateSelectionReadRepository for SqliteMinimalCandidateSelection
&self, &self,
query: &StoredRequestedModelCandidateRowsQuery, query: &StoredRequestedModelCandidateRowsQuery,
) -> Result<Vec<StoredMinimalCandidateSelectionRow>, DataLayerError> { ) -> Result<Vec<StoredMinimalCandidateSelectionRow>, DataLayerError> {
let rows = self self.load_selected_rows_for_api_format(
.selected_rows_for_api_format(&query.api_format) &query.api_format,
.await? SelectedRowsFilter::RequestedModel(&query.requested_model_name),
.into_iter() SelectedRowsOrder::WithGlobalModel,
.filter(|row| { Some(SqlPage {
row_matches_requested_model(row, &query.requested_model_name, &query.api_format) limit: i64::from(query.limit.max(1)),
}) offset: i64::from(query.offset),
.collect::<Vec<_>>(); }),
Ok(sort_rows(rows, true) )
.into_iter() .await
.skip(query.offset as usize)
.take(query.limit as usize)
.collect())
} }
async fn list_pool_key_rows_for_group( async fn list_pool_key_rows_for_group(
&self, &self,
query: &StoredPoolKeyCandidateRowsQuery, query: &StoredPoolKeyCandidateRowsQuery,
) -> Result<Vec<StoredMinimalCandidateSelectionRow>, DataLayerError> { ) -> Result<Vec<StoredMinimalCandidateSelectionRow>, DataLayerError> {
let rows = self let canonical_api_format = normalize_api_format(&query.api_format);
.load_rows_for_api_format(&query.api_format) let storage_aliases = api_format_aliases(&canonical_api_format);
.await? let match_aliases = sql_match_aliases(&storage_aliases);
.into_iter() let mut rows = Vec::<CandidateSelectionRow>::new();
.filter(|row| { let page_in_sql = !matches!(query.order, StoredPoolKeyCandidateOrder::LoadBalance { .. });
row.row.provider_id == query.provider_id
&& row.row.endpoint_id == query.endpoint_id for storage_api_format in storage_aliases {
&& row.row.model_id == query.model_id let mut builder = QueryBuilder::<Sqlite>::new(CANDIDATE_SELECTION_COLUMNS);
}) push_candidate_sql_filters(&mut builder, &storage_api_format, &match_aliases);
.collect::<Vec<_>>(); builder.push(" AND p.id = ");
let mut rows = sort_pool_key_rows(rows, &query.order); builder.push_bind(&query.provider_id);
Ok(rows builder.push(" AND pe.id = ");
.drain(..) builder.push_bind(&query.endpoint_id);
.skip(query.offset as usize) builder.push(" AND m.id = ");
.take(query.limit as usize) builder.push_bind(&query.model_id);
.map(|item| item.row) if page_in_sql {
.collect()) push_pool_key_order(&mut builder, &query.order);
builder.push(" LIMIT ");
builder.push_bind(i64::from(query.limit.max(1)));
builder.push(" OFFSET ");
builder.push_bind(i64::from(query.offset));
} else {
builder.push(" ORDER BY pak.id ASC");
}
let query_rows = builder.build().fetch_all(&self.pool).await.map_sql_err()?;
let mut items = query_rows
.iter()
.map(map_candidate_selection_row)
.collect::<Result<Vec<_>, _>>()?;
items.retain(|item| {
api_format_matches(&item.row.endpoint_api_format, &canonical_api_format)
&& item.row.key_supports_api_format(&canonical_api_format)
&& key_auth_channel_matches(item, &canonical_api_format)
});
rows.extend(items);
}
if page_in_sql {
Ok(dedupe_candidate_selection_rows(
rows.into_iter().map(|item| item.row).collect(),
))
} else {
Ok(dedupe_candidate_selection_rows(
sort_pool_key_rows(rows, &query.order)
.into_iter()
.skip(query.offset as usize)
.take(query.limit as usize)
.map(|item| item.row)
.collect(),
))
}
} }
async fn list_pool_key_rows_for_group_key_ids( async fn list_pool_key_rows_for_group_key_ids(
@@ -211,75 +335,330 @@ impl MinimalCandidateSelectionReadRepository for SqliteMinimalCandidateSelection
.enumerate() .enumerate()
.map(|(index, key_id)| (key_id.as_str(), index)) .map(|(index, key_id)| (key_id.as_str(), index))
.collect::<BTreeMap<_, _>>(); .collect::<BTreeMap<_, _>>();
let mut rows = self let canonical_api_format = normalize_api_format(&query.api_format);
.load_rows_for_api_format(&query.api_format) let storage_aliases = api_format_aliases(&canonical_api_format);
.await? let match_aliases = sql_match_aliases(&storage_aliases);
.into_iter() let mut rows = Vec::new();
.filter(|row| {
row.row.provider_id == query.provider_id for storage_api_format in storage_aliases {
&& row.row.endpoint_id == query.endpoint_id let mut builder = QueryBuilder::<Sqlite>::new(CANDIDATE_SELECTION_COLUMNS);
&& row.row.model_id == query.model_id push_candidate_sql_filters(&mut builder, &storage_api_format, &match_aliases);
&& key_order.contains_key(row.row.key_id.as_str()) builder.push(" AND p.id = ");
}) builder.push_bind(&query.provider_id);
.map(|item| item.row) builder.push(" AND pe.id = ");
.collect::<Vec<_>>(); builder.push_bind(&query.endpoint_id);
builder.push(" AND m.id = ");
builder.push_bind(&query.model_id);
builder.push(" AND pak.id IN (");
{
let mut separated = builder.separated(", ");
for key_id in &query.key_ids {
separated.push_bind(key_id);
}
}
builder.push(")");
builder.push(" ORDER BY CASE pak.id");
for (index, key_id) in query.key_ids.iter().enumerate() {
builder.push(" WHEN ");
builder.push_bind(key_id);
builder.push(" THEN ");
builder.push_bind(i64::try_from(index).map_err(|_| {
DataLayerError::UnexpectedValue("key id order index overflowed".to_string())
})?);
}
builder.push(" ELSE ");
builder.push_bind(i64::try_from(query.key_ids.len()).map_err(|_| {
DataLayerError::UnexpectedValue("key id order length overflowed".to_string())
})?);
builder.push(" END ASC, pak.id ASC");
let query_rows = builder.build().fetch_all(&self.pool).await.map_sql_err()?;
let mut items = query_rows
.iter()
.map(map_candidate_selection_row)
.collect::<Result<Vec<_>, _>>()?;
items.retain(|item| {
api_format_matches(&item.row.endpoint_api_format, &canonical_api_format)
&& item.row.key_supports_api_format(&canonical_api_format)
&& key_auth_channel_matches(item, &canonical_api_format)
});
rows.extend(items.into_iter().map(|item| item.row));
}
let mut rows = dedupe_candidate_selection_rows(rows);
rows.sort_by(|left, right| { rows.sort_by(|left, right| {
key_order key_order
.get(left.key_id.as_str()) .get(left.key_id.as_str())
.cmp(&key_order.get(right.key_id.as_str())) .cmp(&key_order.get(right.key_id.as_str()))
.then(left.key_id.cmp(&right.key_id)) .then(left.key_id.cmp(&right.key_id))
}); });
Ok(dedupe_candidate_selection_rows(rows)) Ok(rows)
} }
} }
fn select_pool_rows(rows: Vec<CandidateSelectionRow>) -> Vec<StoredMinimalCandidateSelectionRow> { fn push_candidate_sql_filters(
let mut selected = Vec::new(); builder: &mut QueryBuilder<'_, Sqlite>,
let mut pool_rows = storage_api_format: &str,
BTreeMap::<(String, String, String), StoredMinimalCandidateSelectionRow>::new(); match_aliases: &[String],
for item in rows { ) {
if !item.provider_pool_enabled { builder.push(" AND LOWER(COALESCE(pe.api_format, '')) = ");
selected.push(item.row); builder.push_bind(storage_api_format.trim().to_ascii_lowercase());
continue; push_key_api_format_sql_filter(builder, match_aliases);
} push_key_auth_channel_sql_filter(builder, storage_api_format);
let key = (
item.row.provider_id.clone(),
item.row.endpoint_id.clone(),
item.row.model_id.clone(),
);
match pool_rows.get(&key) {
Some(existing)
if (existing.key_internal_priority, existing.key_id.as_str())
<= (item.row.key_internal_priority, item.row.key_id.as_str()) => {}
_ => {
pool_rows.insert(key, item.row);
}
}
}
selected.extend(pool_rows.into_values());
dedupe_candidate_selection_rows(selected)
} }
fn sort_rows( fn push_key_api_format_sql_filter(
mut rows: Vec<StoredMinimalCandidateSelectionRow>, builder: &mut QueryBuilder<'_, Sqlite>,
include_global_model: bool, match_aliases: &[String],
) -> Vec<StoredMinimalCandidateSelectionRow> { ) {
rows.sort_by(|left, right| { builder.push(
if include_global_model { r#"
let ordering = left.global_model_name.cmp(&right.global_model_name); AND (
if !ordering.is_eq() { pak.api_formats IS NULL
return ordering; OR TRIM(pak.api_formats) = ''
} OR CASE
WHEN json_valid(pak.api_formats) THEN
(
(
json_type(pak.api_formats) = 'array'
AND EXISTS (
SELECT 1
FROM json_each(pak.api_formats) AS fmt
WHERE LOWER(TRIM(CAST(fmt.value AS TEXT))) IN (
"#,
);
push_bind_list(builder, match_aliases);
builder.push(
r#"
)
)
)
OR (
json_type(pak.api_formats) = 'text'
AND LOWER(TRIM(CAST(json_extract(pak.api_formats, '$') AS TEXT))) IN (
"#,
);
push_bind_list(builder, match_aliases);
builder.push(
r#"
)
)
OR (
json_type(pak.api_formats) = 'text'
AND EXISTS (
SELECT 1
FROM json_each(
CASE
WHEN json_valid(CAST(json_extract(pak.api_formats, '$') AS TEXT))
THEN CAST(json_extract(pak.api_formats, '$') AS TEXT)
ELSE '[]'
END
) AS fmt
WHERE LOWER(TRIM(CAST(fmt.value AS TEXT))) IN (
"#,
);
push_bind_list(builder, match_aliases);
builder.push(
r#"
)
)
)
)
ELSE 0
END
OR LOWER(TRIM(pak.api_formats)) IN (
"#,
);
push_bind_list(builder, match_aliases);
builder.push(
r#"
)
)
"#,
);
}
fn push_key_auth_channel_sql_filter(
builder: &mut QueryBuilder<'_, Sqlite>,
storage_api_format: &str,
) {
let api_format = normalize_api_format(storage_api_format);
builder.push(
r#"
AND (
(
LOWER(TRIM(p.provider_type)) = 'codex'
AND LOWER(TRIM(pak.auth_type)) = 'oauth'
AND "#,
);
builder.push_bind(api_format.clone());
builder.push(
r#" IN ('openai:responses', 'openai:responses:compact', 'openai:image')
)
OR (
LOWER(TRIM(p.provider_type)) = 'chatgpt_web'
AND LOWER(TRIM(pak.auth_type)) IN ('oauth', 'bearer')
AND "#,
);
builder.push_bind(api_format.clone());
builder.push(
r#" = 'openai:image'
)
OR (
LOWER(TRIM(p.provider_type)) = 'claude_code'
AND LOWER(TRIM(pak.auth_type)) = 'oauth'
AND "#,
);
builder.push_bind(api_format.clone());
builder.push(
r#" = 'claude:messages'
)
OR (
LOWER(TRIM(p.provider_type)) = 'kiro'
AND "#,
);
builder.push_bind(api_format.clone());
builder.push(
r#" = 'claude:messages'
AND (
LOWER(TRIM(pak.auth_type)) = 'oauth'
OR (
LOWER(TRIM(pak.auth_type)) = 'bearer'
AND pak.auth_config IS NOT NULL
AND TRIM(pak.auth_config) <> ''
)
)
)
OR (
LOWER(TRIM(p.provider_type)) IN ('gemini_cli', 'antigravity')
AND LOWER(TRIM(pak.auth_type)) = 'oauth'
AND "#,
);
builder.push_bind(api_format.clone());
builder.push(
r#" = 'gemini:generate_content'
)
OR (
LOWER(TRIM(p.provider_type)) = 'vertex_ai'
AND (
(
LOWER(TRIM(pak.auth_type)) = 'api_key'
AND "#,
);
builder.push_bind(api_format.clone());
builder.push(
r#" = 'gemini:generate_content'
)
OR (
LOWER(TRIM(pak.auth_type)) IN ('service_account', 'vertex_ai')
AND "#,
);
builder.push_bind(api_format.clone());
builder.push(
r#" IN ('claude:messages', 'gemini:generate_content')
)
)
)
OR (
LOWER(TRIM(p.provider_type)) NOT IN (
'chatgpt_web',
'claude_code',
'codex',
'gemini_cli',
'vertex_ai',
'antigravity',
'kiro'
)
AND LOWER(TRIM(pak.auth_type)) <> 'oauth'
)
)
"#,
);
}
fn push_requested_model_sql_filter(
builder: &mut QueryBuilder<'_, Sqlite>,
requested_model_name: &str,
_match_aliases: &[String],
) {
builder.push(
r#"
AND (
gm.name = "#,
);
builder.push_bind(requested_model_name.to_string());
builder.push(
r#"
OR m.provider_model_name = "#,
);
builder.push_bind(requested_model_name.to_string());
builder.push(
r#"
OR (
m.provider_model_mappings IS NOT NULL
AND m.provider_model_mappings LIKE "#,
);
builder.push_bind(format!(
"%{}%",
requested_model_name
.replace('\\', "\\\\")
.replace('%', "\\%")
.replace('_', "\\_")
));
builder.push(
r#"
ESCAPE '\'
)
)
"#,
);
}
fn push_selected_rows_order(builder: &mut QueryBuilder<'_, Sqlite>, order: SelectedRowsOrder) {
builder.push(" ORDER BY ");
if matches!(order, SelectedRowsOrder::WithGlobalModel) {
builder.push("global_model_name ASC, ");
}
builder.push(
"provider_priority ASC, key_internal_priority ASC, provider_id ASC, endpoint_id ASC, key_id ASC, model_id ASC",
);
}
fn push_pool_key_order(
builder: &mut QueryBuilder<'_, Sqlite>,
order: &StoredPoolKeyCandidateOrder,
) {
match order {
StoredPoolKeyCandidateOrder::InternalPriority => {
builder.push(" ORDER BY pak.internal_priority ASC, pak.id ASC");
} }
left.provider_priority StoredPoolKeyCandidateOrder::Lru => {
.cmp(&right.provider_priority) builder.push(
.then(left.key_internal_priority.cmp(&right.key_internal_priority)) " ORDER BY pak.last_used_at IS NOT NULL ASC, pak.last_used_at ASC, pak.internal_priority ASC, pak.id ASC",
.then(left.provider_id.cmp(&right.provider_id)) );
.then(left.endpoint_id.cmp(&right.endpoint_id)) }
.then(left.key_id.cmp(&right.key_id)) StoredPoolKeyCandidateOrder::CacheAffinity => {
.then(left.model_id.cmp(&right.model_id)) builder.push(
}); " ORDER BY pak.last_used_at IS NULL ASC, pak.last_used_at DESC, pak.internal_priority ASC, pak.id ASC",
rows );
}
StoredPoolKeyCandidateOrder::SingleAccount => {
builder.push(
" ORDER BY pak.internal_priority ASC, pak.last_used_at IS NULL ASC, pak.last_used_at DESC, pak.id ASC",
);
}
StoredPoolKeyCandidateOrder::LoadBalance { seed } => {
let _ = seed;
builder.push(" ORDER BY pak.id ASC");
}
}
}
fn push_bind_list(builder: &mut QueryBuilder<'_, Sqlite>, values: &[String]) {
let mut separated = builder.separated(", ");
for value in values {
separated.push_bind(value.clone());
}
} }
fn sort_pool_key_rows( fn sort_pool_key_rows(
@@ -473,9 +852,8 @@ fn dedupe_candidate_selection_rows(
} }
fn map_candidate_selection_row(row: &SqliteRow) -> Result<CandidateSelectionRow, DataLayerError> { fn map_candidate_selection_row(row: &SqliteRow) -> Result<CandidateSelectionRow, DataLayerError> {
let provider_config = parse_json(row.try_get("provider_config").ok().flatten())?; let _provider_config = parse_json(row.try_get("provider_config").ok().flatten())?;
let global_model_config = parse_json(row.try_get("global_model_config").ok().flatten())?; let global_model_config = parse_json(row.try_get("global_model_config").ok().flatten())?;
let provider_pool_enabled = json_object_field_present(&provider_config, "pool_advanced");
let global_model_mappings = global_model_config let global_model_mappings = global_model_config
.as_ref() .as_ref()
.and_then(|value| value.get("model_mappings").cloned()); .and_then(|value| value.get("model_mappings").cloned());
@@ -528,7 +906,6 @@ fn map_candidate_selection_row(row: &SqliteRow) -> Result<CandidateSelectionRow,
model_is_active: row.try_get("model_is_active").map_sql_err()?, model_is_active: row.try_get("model_is_active").map_sql_err()?,
model_is_available: row.try_get("model_is_available").map_sql_err()?, model_is_available: row.try_get("model_is_available").map_sql_err()?,
}, },
provider_pool_enabled,
key_auth_config: row.try_get("key_auth_config").map_sql_err()?, key_auth_config: row.try_get("key_auth_config").map_sql_err()?,
key_last_used_at_unix_secs: row key_last_used_at_unix_secs: row
.try_get::<Option<i64>, _>("key_last_used_at_unix_secs") .try_get::<Option<i64>, _>("key_last_used_at_unix_secs")
@@ -550,13 +927,6 @@ fn parse_json(value: Option<String>) -> Result<Option<serde_json::Value>, DataLa
.transpose() .transpose()
} }
fn json_object_field_present(value: &Option<serde_json::Value>, field: &str) -> bool {
value
.as_ref()
.and_then(|value| value.get(field))
.is_some_and(|value| !value.is_null())
}
fn json_bool(value: &serde_json::Value) -> Option<bool> { fn json_bool(value: &serde_json::Value) -> Option<bool> {
value.as_bool().or_else(|| { value.as_bool().or_else(|| {
value value

View File

@@ -1,16 +1,279 @@
use async_trait::async_trait; use async_trait::async_trait;
use sqlx::{sqlite::SqliteRow, Row}; use sqlx::{sqlite::SqliteRow, QueryBuilder, Row, Sqlite};
use super::{ use super::{
InMemoryProviderCatalogReadRepository, ProviderCatalogKeyListQuery, ProviderCatalogKeyListOrder, ProviderCatalogKeyListQuery, ProviderCatalogReadRepository,
ProviderCatalogReadRepository, ProviderCatalogWriteRepository, StoredProviderCatalogEndpoint, ProviderCatalogWriteRepository, StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
StoredProviderCatalogKey, StoredProviderCatalogKeyPage, StoredProviderCatalogKeyStats, StoredProviderCatalogKeyPage, StoredProviderCatalogKeyStats, StoredProviderCatalogProvider,
StoredProviderCatalogProvider,
}; };
use crate::driver::sqlite::{sqlite_optional_real, SqlitePool}; use crate::driver::sqlite::{sqlite_optional_real, SqlitePool};
use crate::error::SqlResultExt; use crate::error::SqlResultExt;
use crate::DataLayerError; use crate::DataLayerError;
const LIST_PROVIDERS_BY_IDS_PREFIX: &str = r#"
SELECT
id,
name,
description,
website,
provider_type,
billing_type,
CAST(monthly_quota_usd AS REAL) AS monthly_quota_usd,
CAST(monthly_used_usd AS REAL) AS monthly_used_usd,
quota_reset_day,
quota_last_reset_at AS quota_last_reset_at_unix_secs,
quota_expires_at AS quota_expires_at_unix_secs,
provider_priority,
is_active,
keep_priority_on_conversion,
enable_format_conversion,
concurrent_limit,
max_retries,
proxy,
request_timeout,
stream_first_byte_timeout,
config,
created_at AS created_at_unix_ms,
updated_at AS updated_at_unix_secs
FROM providers
WHERE id IN (
"#;
const LIST_ENDPOINTS_BY_IDS_PREFIX: &str = r#"
SELECT
id,
provider_id,
api_format,
api_family,
endpoint_kind,
is_active,
health_score,
base_url,
header_rules,
body_rules,
max_retries,
custom_path,
config,
format_acceptance_config,
proxy,
created_at AS created_at_unix_ms,
updated_at AS updated_at_unix_secs
FROM provider_endpoints
WHERE id IN (
"#;
const LIST_ENDPOINTS_BY_PROVIDER_IDS_PREFIX: &str = r#"
SELECT
id,
provider_id,
api_format,
api_family,
endpoint_kind,
is_active,
health_score,
base_url,
header_rules,
body_rules,
max_retries,
custom_path,
config,
format_acceptance_config,
proxy,
created_at AS created_at_unix_ms,
updated_at AS updated_at_unix_secs
FROM provider_endpoints
WHERE provider_id IN (
"#;
const LIST_KEYS_BY_IDS_PREFIX: &str = r#"
SELECT
id,
provider_id,
name,
auth_type,
capabilities,
is_active,
api_formats,
auth_type_by_format,
allow_auth_channel_mismatch_formats,
COALESCE(api_key, encrypted_key) AS api_key,
auth_config,
note,
internal_priority,
rate_multipliers,
global_priority_by_format,
allowed_models,
expires_at AS expires_at_unix_secs,
cache_ttl_minutes,
max_probe_interval_minutes,
proxy,
fingerprint,
rpm_limit,
concurrent_limit,
learned_rpm_limit,
concurrent_429_count,
rpm_429_count,
last_429_at AS last_429_at_unix_secs,
last_429_type,
adjustment_history,
utilization_samples,
last_probe_increase_at AS last_probe_increase_at_unix_secs,
last_rpm_peak,
request_count,
total_tokens,
CAST(total_cost_usd AS REAL) AS total_cost_usd,
success_count,
error_count,
total_response_time_ms,
last_used_at AS last_used_at_unix_secs,
auto_fetch_models,
last_models_fetch_at AS last_models_fetch_at_unix_secs,
last_models_fetch_error,
locked_models,
model_include_patterns,
model_exclude_patterns,
upstream_metadata,
oauth_invalid_at AS oauth_invalid_at_unix_secs,
oauth_invalid_reason,
status_snapshot,
created_at AS created_at_unix_ms,
updated_at AS updated_at_unix_secs,
health_by_format,
circuit_breaker_by_format
FROM provider_api_keys
WHERE id IN (
"#;
const LIST_KEYS_BY_PROVIDER_IDS_PREFIX: &str = r#"
SELECT
id,
provider_id,
name,
auth_type,
capabilities,
is_active,
api_formats,
auth_type_by_format,
allow_auth_channel_mismatch_formats,
COALESCE(api_key, encrypted_key) AS api_key,
auth_config,
note,
internal_priority,
rate_multipliers,
global_priority_by_format,
allowed_models,
expires_at AS expires_at_unix_secs,
cache_ttl_minutes,
max_probe_interval_minutes,
proxy,
fingerprint,
rpm_limit,
concurrent_limit,
learned_rpm_limit,
concurrent_429_count,
rpm_429_count,
last_429_at AS last_429_at_unix_secs,
last_429_type,
adjustment_history,
utilization_samples,
last_probe_increase_at AS last_probe_increase_at_unix_secs,
last_rpm_peak,
request_count,
total_tokens,
CAST(total_cost_usd AS REAL) AS total_cost_usd,
success_count,
error_count,
total_response_time_ms,
last_used_at AS last_used_at_unix_secs,
auto_fetch_models,
last_models_fetch_at AS last_models_fetch_at_unix_secs,
last_models_fetch_error,
locked_models,
model_include_patterns,
model_exclude_patterns,
upstream_metadata,
oauth_invalid_at AS oauth_invalid_at_unix_secs,
oauth_invalid_reason,
status_snapshot,
created_at AS created_at_unix_ms,
updated_at AS updated_at_unix_secs,
health_by_format,
circuit_breaker_by_format
FROM provider_api_keys
WHERE provider_id IN (
"#;
const LIST_KEY_SUMMARIES_BY_PROVIDER_IDS_PREFIX: &str = r#"
SELECT
id,
provider_id,
COALESCE(NULLIF(name, ''), id) AS name,
COALESCE(NULLIF(auth_type, ''), 'summary') AS auth_type,
NULL AS capabilities,
is_active,
api_formats,
NULL AS auth_type_by_format,
NULL AS allow_auth_channel_mismatch_formats,
'summary' AS api_key,
CASE
WHEN auth_config IS NULL THEN NULL
ELSE '{}'
END AS auth_config,
NULL AS note,
NULL AS internal_priority,
NULL AS rate_multipliers,
NULL AS global_priority_by_format,
NULL AS allowed_models,
NULL AS expires_at_unix_secs,
NULL AS cache_ttl_minutes,
NULL AS max_probe_interval_minutes,
NULL AS proxy,
NULL AS fingerprint,
NULL AS rpm_limit,
NULL AS concurrent_limit,
NULL AS learned_rpm_limit,
NULL AS concurrent_429_count,
NULL AS rpm_429_count,
NULL AS last_429_at_unix_secs,
NULL AS last_429_type,
NULL AS adjustment_history,
NULL AS utilization_samples,
NULL AS last_probe_increase_at_unix_secs,
NULL AS last_rpm_peak,
NULL AS request_count,
0 AS total_tokens,
0.0 AS total_cost_usd,
NULL AS success_count,
NULL AS error_count,
NULL AS total_response_time_ms,
NULL AS last_used_at_unix_secs,
FALSE AS auto_fetch_models,
NULL AS last_models_fetch_at_unix_secs,
NULL AS last_models_fetch_error,
NULL AS locked_models,
NULL AS model_include_patterns,
NULL AS model_exclude_patterns,
NULL AS upstream_metadata,
NULL AS oauth_invalid_at_unix_secs,
NULL AS oauth_invalid_reason,
NULL AS status_snapshot,
NULL AS created_at_unix_ms,
NULL AS updated_at_unix_secs,
health_by_format,
NULL AS circuit_breaker_by_format
FROM provider_api_keys
WHERE provider_id IN (
"#;
const LIST_KEY_STATS_BY_PROVIDER_IDS_PREFIX: &str = r#"
SELECT
provider_id,
COUNT(*) AS total_keys,
SUM(CASE WHEN is_active THEN 1 ELSE 0 END) AS active_keys
FROM provider_api_keys
WHERE provider_id IN (
"#;
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct SqliteProviderCatalogReadRepository { pub struct SqliteProviderCatalogReadRepository {
pool: SqlitePool, pool: SqlitePool,
@@ -21,92 +284,340 @@ impl SqliteProviderCatalogReadRepository {
Self { pool } Self { pool }
} }
async fn load_memory(&self) -> Result<InMemoryProviderCatalogReadRepository, DataLayerError> { pub async fn list_providers_by_ids(
Ok(InMemoryProviderCatalogReadRepository::seed( &self,
self.load_providers().await?, provider_ids: &[String],
self.load_endpoints().await?, ) -> Result<Vec<StoredProviderCatalogProvider>, DataLayerError> {
self.load_keys().await?, if provider_ids.is_empty() {
)) return Ok(Vec::new());
} }
async fn load_providers(&self) -> Result<Vec<StoredProviderCatalogProvider>, DataLayerError> { let rows = build_list_query(
let rows = sqlx::query( LIST_PROVIDERS_BY_IDS_PREFIX,
r#" provider_ids,
SELECT " ORDER BY name ASC",
id, name, description, website, provider_type, billing_type,
CAST(monthly_quota_usd AS REAL) AS monthly_quota_usd,
CAST(monthly_used_usd AS REAL) AS monthly_used_usd,
quota_reset_day,
quota_last_reset_at AS quota_last_reset_at_unix_secs,
quota_expires_at AS quota_expires_at_unix_secs,
provider_priority, is_active, keep_priority_on_conversion,
enable_format_conversion, concurrent_limit, max_retries, proxy,
request_timeout, stream_first_byte_timeout, config,
created_at AS created_at_unix_ms,
updated_at AS updated_at_unix_secs
FROM providers
"#,
) )
.build()
.fetch_all(&self.pool) .fetch_all(&self.pool)
.await .await
.map_sql_err()?; .map_sql_err()?;
rows.iter().map(map_provider_row).collect() rows.iter().map(map_provider_row).collect()
} }
async fn load_endpoints(&self) -> Result<Vec<StoredProviderCatalogEndpoint>, DataLayerError> { pub async fn list_providers(
&self,
active_only: bool,
) -> Result<Vec<StoredProviderCatalogProvider>, DataLayerError> {
let rows = sqlx::query( let rows = sqlx::query(
r#" r#"
SELECT SELECT
id, provider_id, api_format, api_family, endpoint_kind, is_active, id,
health_score, base_url, header_rules, body_rules, max_retries, name,
custom_path, config, format_acceptance_config, proxy, description,
website,
provider_type,
billing_type,
CAST(monthly_quota_usd AS REAL) AS monthly_quota_usd,
CAST(monthly_used_usd AS REAL) AS monthly_used_usd,
quota_reset_day,
quota_last_reset_at AS quota_last_reset_at_unix_secs,
quota_expires_at AS quota_expires_at_unix_secs,
provider_priority,
is_active,
keep_priority_on_conversion,
enable_format_conversion,
concurrent_limit,
max_retries,
proxy,
request_timeout,
stream_first_byte_timeout,
config,
created_at AS created_at_unix_ms, created_at AS created_at_unix_ms,
updated_at AS updated_at_unix_secs updated_at AS updated_at_unix_secs
FROM provider_endpoints FROM providers
WHERE api_format IS NOT NULL WHERE (? = FALSE OR is_active = TRUE)
ORDER BY provider_priority ASC, name ASC
"#, "#,
) )
.bind(active_only)
.fetch_all(&self.pool)
.await
.map_sql_err()?;
rows.iter().map(map_provider_row).collect()
}
pub async fn list_endpoints_by_ids(
&self,
endpoint_ids: &[String],
) -> Result<Vec<StoredProviderCatalogEndpoint>, DataLayerError> {
if endpoint_ids.is_empty() {
return Ok(Vec::new());
}
let rows = build_list_query(
LIST_ENDPOINTS_BY_IDS_PREFIX,
endpoint_ids,
" ORDER BY api_format ASC, id ASC",
)
.build()
.fetch_all(&self.pool) .fetch_all(&self.pool)
.await .await
.map_sql_err()?; .map_sql_err()?;
rows.iter().map(map_endpoint_row).collect() rows.iter().map(map_endpoint_row).collect()
} }
async fn load_keys(&self) -> Result<Vec<StoredProviderCatalogKey>, DataLayerError> { pub async fn list_endpoints_by_provider_ids(
let rows = sqlx::query( &self,
r#" provider_ids: &[String],
SELECT ) -> Result<Vec<StoredProviderCatalogEndpoint>, DataLayerError> {
id, provider_id, name, auth_type, capabilities, is_active, api_formats, if provider_ids.is_empty() {
auth_type_by_format, allow_auth_channel_mismatch_formats, return Ok(Vec::new());
COALESCE(api_key, encrypted_key) AS api_key, }
auth_config, note, internal_priority, rate_multipliers,
global_priority_by_format, allowed_models, let rows = build_list_query(
expires_at AS expires_at_unix_secs, LIST_ENDPOINTS_BY_PROVIDER_IDS_PREFIX,
cache_ttl_minutes, max_probe_interval_minutes, proxy, fingerprint, provider_ids,
rpm_limit, concurrent_limit, learned_rpm_limit, concurrent_429_count, " ORDER BY provider_id ASC, api_format ASC, id ASC",
rpm_429_count, last_429_at AS last_429_at_unix_secs, last_429_type,
adjustment_history, utilization_samples,
last_probe_increase_at AS last_probe_increase_at_unix_secs,
last_rpm_peak, request_count, total_tokens, total_cost_usd,
success_count, error_count, total_response_time_ms,
last_used_at AS last_used_at_unix_secs, auto_fetch_models,
last_models_fetch_at AS last_models_fetch_at_unix_secs,
last_models_fetch_error, locked_models, model_include_patterns,
model_exclude_patterns, upstream_metadata,
oauth_invalid_at AS oauth_invalid_at_unix_secs,
oauth_invalid_reason, status_snapshot,
created_at AS created_at_unix_ms,
updated_at AS updated_at_unix_secs,
health_by_format, circuit_breaker_by_format
FROM provider_api_keys
"#,
) )
.build()
.fetch_all(&self.pool)
.await
.map_sql_err()?;
rows.iter().map(map_endpoint_row).collect()
}
pub async fn list_keys_by_ids(
&self,
key_ids: &[String],
) -> Result<Vec<StoredProviderCatalogKey>, DataLayerError> {
if key_ids.is_empty() {
return Ok(Vec::new());
}
let rows = build_list_query(
LIST_KEYS_BY_IDS_PREFIX,
key_ids,
" ORDER BY name ASC, id ASC",
)
.build()
.fetch_all(&self.pool) .fetch_all(&self.pool)
.await .await
.map_sql_err()?; .map_sql_err()?;
rows.iter().map(map_key_row).collect() rows.iter().map(map_key_row).collect()
} }
pub async fn list_keys_by_provider_ids(
&self,
provider_ids: &[String],
) -> Result<Vec<StoredProviderCatalogKey>, DataLayerError> {
if provider_ids.is_empty() {
return Ok(Vec::new());
}
let rows = build_list_query(
LIST_KEYS_BY_PROVIDER_IDS_PREFIX,
provider_ids,
" ORDER BY provider_id ASC, name ASC, id ASC",
)
.build()
.fetch_all(&self.pool)
.await
.map_sql_err()?;
rows.iter().map(map_key_row).collect()
}
pub async fn list_key_summaries_by_provider_ids(
&self,
provider_ids: &[String],
) -> Result<Vec<StoredProviderCatalogKey>, DataLayerError> {
if provider_ids.is_empty() {
return Ok(Vec::new());
}
let rows = build_list_query(
LIST_KEY_SUMMARIES_BY_PROVIDER_IDS_PREFIX,
provider_ids,
" ORDER BY provider_id ASC, id ASC",
)
.build()
.fetch_all(&self.pool)
.await
.map_sql_err()?;
rows.iter().map(map_key_row).collect()
}
pub async fn list_keys_page(
&self,
query: &ProviderCatalogKeyListQuery,
) -> Result<StoredProviderCatalogKeyPage, DataLayerError> {
if query.provider_id.trim().is_empty() {
return Err(DataLayerError::InvalidInput(
"provider catalog provider_id is empty".to_string(),
));
}
let offset = i64::try_from(query.offset).map_err(|_| {
DataLayerError::InvalidInput(format!(
"invalid provider catalog key offset: {}",
query.offset
))
})?;
let limit = i64::try_from(query.limit).map_err(|_| {
DataLayerError::InvalidInput(format!(
"invalid provider catalog key limit: {}",
query.limit
))
})?;
let search_pattern = query
.search
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
.map(|value| format!("%{}%", value.to_ascii_lowercase()));
let order_by = match query.order {
ProviderCatalogKeyListOrder::Name => "internal_priority ASC, name ASC, id ASC",
ProviderCatalogKeyListOrder::CreatedAt => {
"internal_priority ASC, COALESCE(created_at, 0) ASC, id ASC"
}
ProviderCatalogKeyListOrder::CreatedAtAsc => {
"created_at IS NULL ASC, created_at ASC, name ASC, id ASC"
}
ProviderCatalogKeyListOrder::CreatedAtDesc => {
"created_at IS NULL ASC, created_at DESC, name ASC, id ASC"
}
ProviderCatalogKeyListOrder::LastUsedAtAsc => {
"last_used_at IS NULL ASC, last_used_at ASC, name ASC, id ASC"
}
ProviderCatalogKeyListOrder::LastUsedAtDesc => {
"last_used_at IS NULL ASC, last_used_at DESC, name ASC, id ASC"
}
};
let count_row = sqlx::query(
r#"
SELECT COUNT(*) AS total
FROM provider_api_keys
WHERE provider_id = ?
AND (? IS NULL OR LOWER(name) LIKE ? OR LOWER(id) LIKE ?)
AND (? IS NULL OR is_active = ?)
"#,
)
.bind(&query.provider_id)
.bind(search_pattern.as_deref())
.bind(search_pattern.as_deref())
.bind(search_pattern.as_deref())
.bind(query.is_active)
.bind(query.is_active)
.fetch_one(&self.pool)
.await
.map_sql_err()?;
let total = count_row.try_get::<i64, _>("total").map_sql_err()?.max(0) as usize;
let sql = format!(
r#"
SELECT
id,
provider_id,
name,
auth_type,
capabilities,
is_active,
api_formats,
auth_type_by_format,
allow_auth_channel_mismatch_formats,
COALESCE(api_key, encrypted_key) AS api_key,
auth_config,
note,
internal_priority,
rate_multipliers,
global_priority_by_format,
allowed_models,
expires_at AS expires_at_unix_secs,
cache_ttl_minutes,
max_probe_interval_minutes,
proxy,
fingerprint,
rpm_limit,
concurrent_limit,
learned_rpm_limit,
concurrent_429_count,
rpm_429_count,
last_429_at AS last_429_at_unix_secs,
last_429_type,
adjustment_history,
utilization_samples,
last_probe_increase_at AS last_probe_increase_at_unix_secs,
last_rpm_peak,
request_count,
total_tokens,
CAST(total_cost_usd AS REAL) AS total_cost_usd,
success_count,
error_count,
total_response_time_ms,
last_used_at AS last_used_at_unix_secs,
auto_fetch_models,
last_models_fetch_at AS last_models_fetch_at_unix_secs,
last_models_fetch_error,
locked_models,
model_include_patterns,
model_exclude_patterns,
upstream_metadata,
oauth_invalid_at AS oauth_invalid_at_unix_secs,
oauth_invalid_reason,
status_snapshot,
created_at AS created_at_unix_ms,
updated_at AS updated_at_unix_secs,
health_by_format,
circuit_breaker_by_format
FROM provider_api_keys
WHERE provider_id = ?
AND (? IS NULL OR LOWER(name) LIKE ? OR LOWER(id) LIKE ?)
AND (? IS NULL OR is_active = ?)
ORDER BY {order_by}
LIMIT ?
OFFSET ?
"#,
);
let rows = sqlx::query(&sql)
.bind(&query.provider_id)
.bind(search_pattern.as_deref())
.bind(search_pattern.as_deref())
.bind(search_pattern.as_deref())
.bind(query.is_active)
.bind(query.is_active)
.bind(limit)
.bind(offset)
.fetch_all(&self.pool)
.await
.map_sql_err()?;
let items = rows
.iter()
.map(map_key_row)
.collect::<Result<Vec<_>, _>>()?;
Ok(StoredProviderCatalogKeyPage { items, total })
}
pub async fn list_key_stats_by_provider_ids(
&self,
provider_ids: &[String],
) -> Result<Vec<StoredProviderCatalogKeyStats>, DataLayerError> {
if provider_ids.is_empty() {
return Ok(Vec::new());
}
let rows = build_list_query(
LIST_KEY_STATS_BY_PROVIDER_IDS_PREFIX,
provider_ids,
"\nGROUP BY provider_id\nORDER BY provider_id ASC",
)
.build()
.fetch_all(&self.pool)
.await
.map_sql_err()?;
rows.iter().map(map_key_stats_row).collect()
}
pub async fn create_provider( pub async fn create_provider(
&self, &self,
provider: &StoredProviderCatalogProvider, provider: &StoredProviderCatalogProvider,
@@ -870,81 +1381,63 @@ impl ProviderCatalogReadRepository for SqliteProviderCatalogReadRepository {
&self, &self,
active_only: bool, active_only: bool,
) -> Result<Vec<StoredProviderCatalogProvider>, DataLayerError> { ) -> Result<Vec<StoredProviderCatalogProvider>, DataLayerError> {
self.load_memory().await?.list_providers(active_only).await Self::list_providers(self, active_only).await
} }
async fn list_providers_by_ids( async fn list_providers_by_ids(
&self, &self,
provider_ids: &[String], provider_ids: &[String],
) -> Result<Vec<StoredProviderCatalogProvider>, DataLayerError> { ) -> Result<Vec<StoredProviderCatalogProvider>, DataLayerError> {
self.load_memory() Self::list_providers_by_ids(self, provider_ids).await
.await?
.list_providers_by_ids(provider_ids)
.await
} }
async fn list_endpoints_by_ids( async fn list_endpoints_by_ids(
&self, &self,
endpoint_ids: &[String], endpoint_ids: &[String],
) -> Result<Vec<StoredProviderCatalogEndpoint>, DataLayerError> { ) -> Result<Vec<StoredProviderCatalogEndpoint>, DataLayerError> {
self.load_memory() Self::list_endpoints_by_ids(self, endpoint_ids).await
.await?
.list_endpoints_by_ids(endpoint_ids)
.await
} }
async fn list_endpoints_by_provider_ids( async fn list_endpoints_by_provider_ids(
&self, &self,
provider_ids: &[String], provider_ids: &[String],
) -> Result<Vec<StoredProviderCatalogEndpoint>, DataLayerError> { ) -> Result<Vec<StoredProviderCatalogEndpoint>, DataLayerError> {
self.load_memory() Self::list_endpoints_by_provider_ids(self, provider_ids).await
.await?
.list_endpoints_by_provider_ids(provider_ids)
.await
} }
async fn list_keys_by_ids( async fn list_keys_by_ids(
&self, &self,
key_ids: &[String], key_ids: &[String],
) -> Result<Vec<StoredProviderCatalogKey>, DataLayerError> { ) -> Result<Vec<StoredProviderCatalogKey>, DataLayerError> {
self.load_memory().await?.list_keys_by_ids(key_ids).await Self::list_keys_by_ids(self, key_ids).await
} }
async fn list_keys_by_provider_ids( async fn list_keys_by_provider_ids(
&self, &self,
provider_ids: &[String], provider_ids: &[String],
) -> Result<Vec<StoredProviderCatalogKey>, DataLayerError> { ) -> Result<Vec<StoredProviderCatalogKey>, DataLayerError> {
self.load_memory() Self::list_keys_by_provider_ids(self, provider_ids).await
.await?
.list_keys_by_provider_ids(provider_ids)
.await
} }
async fn list_key_summaries_by_provider_ids( async fn list_key_summaries_by_provider_ids(
&self, &self,
provider_ids: &[String], provider_ids: &[String],
) -> Result<Vec<StoredProviderCatalogKey>, DataLayerError> { ) -> Result<Vec<StoredProviderCatalogKey>, DataLayerError> {
self.load_memory() Self::list_key_summaries_by_provider_ids(self, provider_ids).await
.await?
.list_key_summaries_by_provider_ids(provider_ids)
.await
} }
async fn list_keys_page( async fn list_keys_page(
&self, &self,
query: &ProviderCatalogKeyListQuery, query: &ProviderCatalogKeyListQuery,
) -> Result<StoredProviderCatalogKeyPage, DataLayerError> { ) -> Result<StoredProviderCatalogKeyPage, DataLayerError> {
self.load_memory().await?.list_keys_page(query).await Self::list_keys_page(self, query).await
} }
async fn list_key_stats_by_provider_ids( async fn list_key_stats_by_provider_ids(
&self, &self,
provider_ids: &[String], provider_ids: &[String],
) -> Result<Vec<StoredProviderCatalogKeyStats>, DataLayerError> { ) -> Result<Vec<StoredProviderCatalogKeyStats>, DataLayerError> {
self.load_memory() Self::list_key_stats_by_provider_ids(self, provider_ids).await
.await?
.list_key_stats_by_provider_ids(provider_ids)
.await
} }
} }
@@ -1063,6 +1556,21 @@ impl ProviderCatalogWriteRepository for SqliteProviderCatalogReadRepository {
} }
} }
fn build_list_query<'a>(
prefix: &'static str,
ids: &'a [String],
suffix: &'static str,
) -> QueryBuilder<'a, Sqlite> {
let mut builder = QueryBuilder::<Sqlite>::new(prefix);
let mut separated = builder.separated(", ");
for id in ids {
separated.push_bind(id);
}
separated.push_unseparated(")");
builder.push(suffix);
builder
}
fn current_unix_secs() -> u64 { fn current_unix_secs() -> u64 {
chrono::Utc::now().timestamp().max(0) as u64 chrono::Utc::now().timestamp().max(0) as u64
} }
@@ -1353,6 +1861,14 @@ fn map_endpoint_row(row: &SqliteRow) -> Result<StoredProviderCatalogEndpoint, Da
) )
} }
fn map_key_stats_row(row: &SqliteRow) -> Result<StoredProviderCatalogKeyStats, DataLayerError> {
StoredProviderCatalogKeyStats::new(
row.try_get("provider_id").map_sql_err()?,
row.try_get("total_keys").map_sql_err()?,
row.try_get("active_keys").map_sql_err()?,
)
}
fn map_key_row(row: &SqliteRow) -> Result<StoredProviderCatalogKey, DataLayerError> { fn map_key_row(row: &SqliteRow) -> Result<StoredProviderCatalogKey, DataLayerError> {
let total_cost_usd = sqlite_optional_real(row, "total_cost_usd")?.unwrap_or(0.0); let total_cost_usd = sqlite_optional_real(row, "total_cost_usd")?.unwrap_or(0.0);
if !total_cost_usd.is_finite() { if !total_cost_usd.is_finite() {
@@ -1549,8 +2065,8 @@ mod tests {
use super::SqliteProviderCatalogReadRepository; use super::SqliteProviderCatalogReadRepository;
use crate::lifecycle::migrate::run_sqlite_migrations; use crate::lifecycle::migrate::run_sqlite_migrations;
use crate::repository::provider_catalog::{ use crate::repository::provider_catalog::{
ProviderCatalogKeyListOrder, ProviderCatalogKeyListQuery, ProviderCatalogReadRepository, ProviderCatalogKeyListOrder, ProviderCatalogKeyListQuery, StoredProviderCatalogEndpoint,
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider, StoredProviderCatalogKey, StoredProviderCatalogProvider,
}; };
use serde_json::json; use serde_json::json;

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -3,7 +3,7 @@ services:
image: ${APP_IMAGE:-ghcr.io/fawney19/aether:latest} image: ${APP_IMAGE:-ghcr.io/fawney19/aether:latest}
container_name: aether-app container_name: aether-app
env_file: env_file:
- .env - ${AETHER_ENV_FILE:-.env}
environment: environment:
TZ: Asia/Shanghai TZ: Asia/Shanghai
AETHER_DATABASE_DRIVER: sqlite AETHER_DATABASE_DRIVER: sqlite

View File

@@ -0,0 +1,238 @@
# Postgres to Aether Single Node Migration
Chinese version: [pg-to-single-node-migration.zh-CN.md](pg-to-single-node-migration.zh-CN.md)
This runbook migrates an existing Docker Compose Postgres deployment to Aether
single-node. In this repository, **single-node** means the default SQLite installer mode:
`install.sh --mode single-node`, a system service backed by SQLite. The Docker Compose
single-node template is `docker-compose.single-node.yml`, exposed through `--mode compose-single-node`.
The migration script is:
```bash
scripts/migrate-pg-to-single-node.sh
```
If the target should stay on Docker Compose instead of becoming a system
service, use the image-based Compose migration script:
```bash
scripts/migrate-pg-compose-to-single-node.sh
```
Both migration scripts pull/install the target single-node version before
downtime, stop only the source `app`, copy Postgres records directly into a
temporary SQLite DB without writing a JSONL file, replace the target
`aether.db`, and start single-node.
You can also use the installer as the unified entrypoint and let `--mode`
select the migration target:
```bash
# In interactive mode, first choose the target deployment mode:
# 1) Docker Compose standard deployment (Postgres + Redis)
# 2) Docker Compose single-node deployment (SQLite)
# 3) System service single-node deployment (SQLite)
# After choosing 2 or 3, choose the data initialization mode:
# 1) Fresh initialization (do not migrate existing data)
# 2) Migrate from an existing Docker Compose PG database
install.sh
# Migrate into a new single-node Docker Compose directory.
install.sh \
--mode compose-single-node \
--migrate-from-compose /root/Aether/docker-compose.yml \
--compose-dir /opt/aether-single \
--replace-existing
# Migrate into the system service + SQLite layout.
sudo install.sh \
--mode single-node \
--migrate-from-compose /root/Aether/docker-compose.yml \
--replace-existing
```
Interactive mode first asks for the target deployment shape. If the target is
`compose-single-node` or `single-node`, the installer then asks for the data
initialization mode: fresh initialization, or migration from an existing Docker
Compose PG database. If you choose migration, it tries to detect the source PG
Compose file from `docker compose ls`, then verifies that the Compose config
contains the default `app` and `postgres` services. If exactly one match is
found, it is used as the default prompt value. If detection is ambiguous or
fails, the installer stops; rerun it with `--migrate-from-compose` to specify
the source compose path.
The installer only normalizes the entrypoint: `compose-single-node` delegates to
`scripts/migrate-pg-compose-to-single-node.sh`, while `single-node` delegates to
`scripts/migrate-pg-to-single-node.sh`.
## What It Does
The script keeps the production cutover window short:
1. Reads the source Compose `.env`.
2. Builds a single-node env file that preserves `JWT_SECRET_KEY`, `ENCRYPTION_KEY` or
`AETHER_GATEWAY_DATA_ENCRYPTION_KEY`, admin settings, port, and app config.
3. Installs the single-node release with `install.sh --mode single-node --skip-start`.
4. Preflights SQLite migrations with the installed single-node binary.
5. Pulls the target single-node image, confirms its `copy` command is available,
and verifies that its Docker image ID matches the currently running source
`app` image ID.
6. Uses the target SQLite schema as the migration plan: same-name source
Postgres tables and columns are copied into the temporary SQLite DB.
7. Applies the compressed body and HTTP body detail policy. The default is full,
and you can opt into an omit mode for large artifacts.
8. Checks that the work directory and target SQLite directory have enough free
disk space for the temporary and final SQLite files.
9. Stops only the source `app` service, leaving Postgres and Redis running.
10. Copies source Postgres records directly into a temporary SQLite database
without generating JSONL files.
11. Replaces the target SQLite DB, including SQLite `-wal`/`-shm` sidecar files
when present, and starts the single-node service.
The image check compares Docker image IDs, not just tag strings. If both source
and target say `latest` but resolve to different image IDs, migration stops.
Upgrade the source PG Compose `app` to the target single-node version first,
verify it is healthy, then run the migration. The scripts also check that the
target image supports direct copy and the request-body omit flag; using a new
script with an old image stops before cutover to avoid missing data.
## Production Cutover
Before production cutover, take a normal server backup or snapshot. Then run:
```bash
sudo scripts/migrate-pg-to-single-node.sh \
--source-compose /root/Aether/docker-compose.yml \
--replace-existing
```
For Docker Compose single-node cutover instead of a system service:
```bash
scripts/migrate-pg-compose-to-single-node.sh \
--source-compose /root/Aether/docker-compose.yml \
--replace-existing
```
The source Postgres compose directory and target single-node compose directory
can be different. For example:
```bash
install.sh \
--mode compose-single-node \
--migrate-from-compose /root/Aether/docker-compose.yml \
--compose-dir /opt/aether-single \
--replace-existing
```
Equivalently, call the lower-level script and pass each target path explicitly:
```bash
scripts/migrate-pg-compose-to-single-node.sh \
--source-compose /root/Aether/docker-compose.yml \
--target-compose /opt/aether-single/docker-compose.single-node.yml \
--target-env /opt/aether-single/.env.single-node \
--target-db /opt/aether-single/data/aether.db \
--replace-existing
```
During cutover, the script stops and removes only the source `app` container to
free the fixed `aether-app` container name. Postgres, Redis, and their volumes
remain in place for rollback.
Defaults:
| Setting | Default |
| --- | --- |
| Source Compose | `docker-compose.yml` |
| Single Node install root | `/opt/aether` |
| Single Node config dir | `/etc/aether` |
| Target SQLite DB | `/opt/aether/data/aether.db` |
| Source app service | `app` |
| Source Postgres service | `postgres` |
| Single Node service | `aether-gateway` |
The script writes migration artifacts under `./data/pg-to-single-node-<timestamp>` next
to the source Compose file unless `--work-dir` is provided.
## Rollback
The script leaves the original Postgres and Redis volumes in place. If cutover
finishes but you need to roll back:
```bash
sudo systemctl stop aether-gateway
cd /root/Aether
docker compose -f docker-compose.yml up -d app
```
For the Compose single-node script, rollback is the same idea: start the app
again from the original Postgres compose file.
If the migration fails before cutover completes, the script attempts to restart
the source `app` service automatically. Pass `--keep-source-stopped-on-error` if
you want to inspect the stopped source deployment manually instead.
## Data Coverage Guard
The migration does not maintain a separate business-domain table list. The
target single-node image first builds a temporary SQLite database with its
normal migrations, then `aether-gateway copy` reads that SQLite schema and copies
matching public Postgres tables and columns.
If the source Postgres database has a non-empty public table that does not exist
in the target SQLite schema, the copy stops instead of silently dropping it. It
ignores lifecycle metadata tables such as `_sqlx_migrations` and
`schema_backfills`. Extra source columns that are absent from the target schema
are not copied.
## Request Body Detail Policy
The production migration migrates all migratable data by default. The only
optional exclusion is request body detail data.
When you choose to skip request bodies, the migration does not copy
`usage_body_blobs`, `usage_http_audits`, or legacy `usage` request body columns
such as `request_body`, `provider_request_body`, `response_body`,
`client_response_body`, and `*_body_compressed`.
Interactive installation lets you choose:
```text
1) Full migration: migrate all migratable data, including request body details
2) Skip request bodies: migrate all other data; skip only request body large fields and HTTP body detail tables; source PG is unchanged
```
For non-interactive full runs:
```bash
scripts/migrate-pg-to-single-node.sh \
--request-body-mode full
```
For non-interactive omit runs:
```bash
scripts/migrate-pg-to-single-node.sh \
--request-body-mode omit
```
`omit` only skips writing those large artifacts and detail tables into the
target SQLite database. It does not delete or clear the source Postgres data.
## Notes
- Single Node requires root or sudo because it writes `/opt/aether`, `/etc/aether`, and
the system service definition.
- The script does not decrypt or re-encrypt provider keys. It preserves the
original encryption key and moves encrypted data as-is.
- Existing target SQLite databases, including `-wal`/`-shm` sidecars, are not
replaced unless `--replace-existing` is provided.
- Disk space checks use `pg_database_size(current_database()) * 2 + 1 GiB` as the
conservative estimate for one SQLite copy. If the work directory and target DB
directory are on the same filesystem, the script requires enough space for both
the temporary and final SQLite files. With `--request-body-mode omit`, the
estimate subtracts `usage_body_blobs` and `usage_http_audits` relation sizes.
- For non-standard source Compose files, set `--app-service` and
`--postgres-service` to match the service names.

View File

@@ -0,0 +1,221 @@
# Postgres 到 Aether Single Node 迁移
英文版:[pg-to-single-node-migration.md](pg-to-single-node-migration.md)
本文档用于把现有 Docker Compose Postgres 部署迁移到 Aether
single-node。当前版本里**single-node** 指默认 SQLite 安装模式:
`install.sh --mode single-node`,也就是系统服务加 SQLite。Docker Compose
单机模板是 `docker-compose.single-node.yml`,安装脚本入口是
`--mode compose-single-node`
迁移脚本:
```bash
scripts/migrate-pg-to-single-node.sh
```
如果目标形态仍然要保持 Docker Compose而不是系统服务使用镜像版迁移脚本
```bash
scripts/migrate-pg-compose-to-single-node.sh
```
两种迁移脚本都会先拉取/安装目标 single-node 版本,再停止源 `app`,把 Postgres
记录直接写入临时 SQLite DB不落 JSONL 中间文件;复制成功后替换目标
`aether.db`,最后启动 single-node。
也可以直接用安装脚本作为统一入口,由 `--mode` 选择迁移目标:
```bash
# 交互式执行时,先选择目标部署模式:
# 1) Docker Compose 标准部署Postgres + Redis
# 2) Docker Compose 单节点部署SQLite
# 3) 系统服务单节点部署SQLite
# 选择 2 或 3 后,再选择数据初始化方式:
# 1) 全新初始化(不迁移现有数据)
# 2) 从现有 Docker Compose PG 数据库迁移
install.sh
# 迁移到新的 single-node Docker Compose 目录
install.sh \
--mode compose-single-node \
--migrate-from-compose /root/Aether/docker-compose.yml \
--compose-dir /opt/aether-single \
--replace-existing
# 迁移到系统服务 + SQLite
sudo install.sh \
--mode single-node \
--migrate-from-compose /root/Aether/docker-compose.yml \
--replace-existing
```
交互模式会先选择目标部署形态。如果目标是 `compose-single-node`
`single-node`,安装脚本会再询问数据初始化方式:全新初始化,或从现有 Docker
Compose PG 数据库迁移。选择迁移后,脚本会通过 `docker compose ls` 自动探测源
PG Compose 文件,并确认该 Compose 配置里存在默认的 `app``postgres` 服务;
如果能唯一识别,会作为默认值带入提示。探测不到或存在多个候选时会直接中止;
此时请用 `--migrate-from-compose` 显式指定源 compose 路径。
安装脚本只是统一参数入口:`compose-single-node` 会委托给
`scripts/migrate-pg-compose-to-single-node.sh``single-node` 会委托给
`scripts/migrate-pg-to-single-node.sh`
## 迁移内容
脚本会尽量缩短生产停机窗口:
1. 读取源 Compose 目录下的 `.env`
2. 生成 single-node 环境文件,保留 `JWT_SECRET_KEY``ENCRYPTION_KEY`
`AETHER_GATEWAY_DATA_ENCRYPTION_KEY`、管理员配置、端口和应用配置。
3. 执行 `install.sh --mode single-node --skip-start`,提前安装 single-node
release但不启动服务。
4. 使用已安装的 single-node 二进制预检 SQLite schema migration。
5. 拉取目标 single-node 镜像,确认其 `copy` 命令可用,并检查源 `app`
当前运行镜像 ID 与目标镜像 ID 一致。
6. 以目标 SQLite schema 作为迁移计划:把源 Postgres 中同名表、同名字段
复制到临时 SQLite DB。
7. 检查请求体明细迁移策略;默认全部迁移,也可以选择只跳过请求体明细。
8. 检查 work-dir 和目标 SQLite 目录是否有足够空间容纳临时库和正式库。
9. 只停止源 Compose 的 `app` 服务,保留 Postgres 和 Redis 运行,方便回滚。
10. 从源 Postgres 直接复制记录到临时 SQLite 数据库,不生成 JSONL 中间文件。
11. 复制完成后替换目标 SQLite DB包括 SQLite `-wal``-shm` 边车文件,
然后启动 single-node 系统服务。
镜像一致性检查比较的是 Docker 镜像 ID不只是 tag 字符串。即使源和目标都写着
`latest`,只要实际镜像 ID 不同,迁移也会中止。请先把源 PG Compose 的 `app`
升级到目标 single-node 相同版本,确认运行正常后再迁移。迁移脚本也会检查目标镜像
是否支持直接 copy 和请求体跳过开关;如果只是换了脚本但镜像还是旧版本,脚本会
直接中止,避免漏迁。
## 生产切换
切换前先做一次常规服务器备份或快照。确认后执行:
```bash
sudo scripts/migrate-pg-to-single-node.sh \
--source-compose /root/Aether/docker-compose.yml \
--replace-existing
```
如果要迁移到 Docker Compose single-node而不是系统服务
```bash
scripts/migrate-pg-compose-to-single-node.sh \
--source-compose /root/Aether/docker-compose.yml \
--replace-existing
```
源 Postgres Compose 目录和目标 single-node Compose 目录可以不一样。例如:
```bash
install.sh \
--mode compose-single-node \
--migrate-from-compose /root/Aether/docker-compose.yml \
--compose-dir /opt/aether-single \
--replace-existing
```
等价地,也可以直接调底层脚本并显式传入每个目标路径:
```bash
scripts/migrate-pg-compose-to-single-node.sh \
--source-compose /root/Aether/docker-compose.yml \
--target-compose /opt/aether-single/docker-compose.single-node.yml \
--target-env /opt/aether-single/.env.single-node \
--target-db /opt/aether-single/data/aether.db \
--replace-existing
```
切换时脚本只会停止并移除源 `app` 容器,用来释放固定的 `aether-app`
容器名Postgres、Redis 和它们的 volume 都会保留,方便回滚。
默认路径和服务名:
| 配置项 | 默认值 |
| --- | --- |
| 源 Compose 文件 | `docker-compose.yml` |
| single-node 安装目录 | `/opt/aether` |
| single-node 配置目录 | `/etc/aether` |
| 目标 SQLite DB | `/opt/aether/data/aether.db` |
| 源 app 服务 | `app` |
| 源 Postgres 服务 | `postgres` |
| single-node 服务 | `aether-gateway` |
除非显式传入 `--work-dir`,脚本会把迁移产物写到源 Compose 文件旁边的
`./data/pg-to-single-node-<timestamp>`
## 回滚
脚本会保留原 Postgres 和 Redis volume。迁移已经完成但需要回滚时
```bash
sudo systemctl stop aether-gateway
cd /root/Aether
docker compose -f docker-compose.yml up -d app
```
对于 Compose single-node 脚本,回滚思路相同:重新用原 Postgres compose 文件
拉起 `app`
如果迁移在切换完成前失败,脚本默认会尝试自动拉起源 `app` 服务。需要失败后
保持源应用停止以便人工排查时,增加:
```bash
--keep-source-stopped-on-error
```
## 数据覆盖保护
迁移不再维护一份额外的业务表清单。目标 single-node 镜像会先用正常
migrations 建出临时 SQLite 数据库,然后 `aether-gateway copy` 读取这个
SQLite schema把源 Postgres 里同名表、同名字段复制过去。
如果源 Postgres 里存在非空 public 表,但目标 SQLite schema 中没有同名表,
copy 会直接中止,不会静默丢弃。生命周期元数据表 `_sqlx_migrations`
`schema_backfills` 会被忽略。源表中存在但目标 SQLite 不存在的额外字段不会复制。
## 请求体明细策略
single-node SQLite 生产迁移默认迁移所有可迁移数据,唯一可选的跳过项是请求体明细。
选择“不迁移请求体”时,不会迁移 `usage_body_blobs``usage_http_audits`,也不会迁移 `usage`
表里的 `request_body` / `provider_request_body` / `response_body` /
`client_response_body` / `*_body_compressed` 等请求体大字段。
交互安装时可以选择:
```text
1) 全部迁移:迁移所有可迁移数据,包括请求体明细
2) 不迁移请求体:迁移其他所有数据;仅跳过请求体大字段和 HTTP 请求体明细,源 PG 不清除
```
非交互执行时,全部迁移可以显式指定:
```bash
scripts/migrate-pg-to-single-node.sh \
--request-body-mode full
```
不迁移请求体可以显式指定:
```bash
scripts/migrate-pg-to-single-node.sh \
--request-body-mode omit
```
`omit` 只是不把这些大字段和明细表写进目标 SQLite不会删除或清空源 Postgres。
## 注意事项
- single-node 安装需要 root 或 sudo 权限,因为会写入 `/opt/aether`
`/etc/aether` 和系统服务定义。
- 脚本不会解密或重新加密供应商密钥;它会沿用源环境的加密密钥,并原样迁移已加密数据。
- 已存在的目标 SQLite DB包括 `-wal``-shm` 边车文件,只有在传入
`--replace-existing` 时才会被替换。
- 空间检查会用 `pg_database_size(current_database()) * 2 + 1 GiB` 作为单份
SQLite 的保守估算。如果 work-dir 和目标 DB 目录在同一个文件系统,会要求同时
容纳临时 SQLite 和正式 SQLite。选择 `--request-body-mode omit` 时,
估算会扣除 `usage_body_blobs``usage_http_audits` 的表空间。
- 非标准 Compose 服务名需要通过 `--app-service``--postgres-service`
明确指定。

View File

@@ -49,6 +49,19 @@ ADMIN_PASSWORD_SOURCE=""
UI_LANG="${AETHER_LANG:-${AETHER_LANGUAGE:-auto}}" UI_LANG="${AETHER_LANG:-${AETHER_LANGUAGE:-auto}}"
RELEASE_KEEP="${AETHER_RELEASE_KEEP:-3}" RELEASE_KEEP="${AETHER_RELEASE_KEEP:-3}"
RELEASE_ARCHIVE_URL="${AETHER_RELEASE_ARCHIVE_URL:-${AETHER_DOWNLOAD_URL:-}}" RELEASE_ARCHIVE_URL="${AETHER_RELEASE_ARCHIVE_URL:-${AETHER_DOWNLOAD_URL:-}}"
MIGRATE_FROM_COMPOSE=""
MIGRATE_TARGET_COMPOSE=""
MIGRATE_TARGET_ENV=""
MIGRATE_TARGET_DB=""
MIGRATE_WORK_DIR=""
MIGRATE_APP_SERVICE=""
MIGRATE_POSTGRES_SERVICE=""
MIGRATE_SINGLE_NODE_SERVICE=""
MIGRATE_REPLACE_EXISTING="false"
MIGRATE_REPLACE_TARGET_COMPOSE="false"
MIGRATE_KEEP_SOURCE_STOPPED_ON_ERROR="false"
MIGRATE_INTERACTIVE="false"
MIGRATE_REQUEST_BODY_MODE=""
usage() { usage() {
cat <<'EOF' cat <<'EOF'
@@ -57,10 +70,10 @@ Usage: install.sh [options]
Install Aether Gateway. Install Aether Gateway.
Options: Options:
--mode MODE Deployment mode: compose, compose-sqlite, or single --mode MODE Deployment mode: compose, compose-single-node, or single-node
compose: Docker Compose app + Postgres + Redis compose: Docker Compose app + Postgres + Redis
compose-sqlite: Docker Compose app + SQLite compose-single-node: Docker Compose single-node app
single: system service with SQLite single-node: single-node system service
Linux services use systemd; macOS services use launchd Linux services use systemd; macOS services use launchd
--channel CHANNEL Release channel to resolve when --version is omitted: stable, latest, rc, or beta --channel CHANNEL Release channel to resolve when --version is omitted: stable, latest, rc, or beta
stable/latest resolves the latest stable tag (default) stable/latest resolves the latest stable tag (default)
@@ -79,8 +92,31 @@ Options:
--lang LANG Installer language: zh or en --lang LANG Installer language: zh or en
--skip-start Install files, but do not start Docker Compose or restart the service --skip-start Install files, but do not start Docker Compose or restart the service
--keep-releases N Keep the latest N releases, prune older ones (default: 3, 0=disable) --keep-releases N Keep the latest N releases, prune older ones (default: 3, 0=disable)
--migrate-from-compose PATH
Migrate an existing Postgres Compose deployment into the selected single-node mode
--target-compose PATH
Migration target compose file for --mode compose-single-node
--target-env PATH Migration target env file for --mode compose-single-node
--target-db PATH Migration target SQLite DB path
--work-dir PATH Migration working directory
--app-service NAME Source compose app service for migration
--postgres-service NAME
Source compose Postgres service for migration
--single-node-service NAME
Target compose service for --mode compose-single-node migration
--replace-existing Allow replacing an existing target SQLite DB during migration
--replace-target-compose
Overwrite target compose file from the single-node template during migration
--request-body-mode MODE
Request/response body detail handling during migration: full/1 or omit/2
--keep-source-stopped-on-error
Do not auto-restart source app if migration fails after stopping it
-h, --help Show this help -h, --help Show this help
Migration examples:
install.sh --mode compose-single-node --migrate-from-compose /root/Aether/docker-compose.yml --compose-dir /opt/aether-single --replace-existing
sudo install.sh --mode single-node --migrate-from-compose /root/Aether/docker-compose.yml --replace-existing
Environment overrides: Environment overrides:
AETHER_REPO, AETHER_SOURCE_REF, AETHER_INSTALL_MODE, AETHER_CHANNEL, AETHER_VERSION AETHER_REPO, AETHER_SOURCE_REF, AETHER_INSTALL_MODE, AETHER_CHANNEL, AETHER_VERSION
AETHER_LANG or AETHER_LANGUAGE AETHER_LANG or AETHER_LANGUAGE
@@ -158,7 +194,7 @@ select_language() {
请选择安装语言 / Choose installer language: 请选择安装语言 / Choose installer language:
1) 中文 1) 中文
2) 英语 / English 2) English
请输入选项 / Enter choice [1]: 请输入选项 / Enter choice [1]:
EOF EOF
@@ -264,6 +300,63 @@ parse_args() {
RELEASE_KEEP="$2" RELEASE_KEEP="$2"
shift 2 shift 2
;; ;;
--migrate-from-compose)
[[ $# -ge 2 ]] || die "--migrate-from-compose requires a path"
MIGRATE_FROM_COMPOSE="$2"
shift 2
;;
--target-compose)
[[ $# -ge 2 ]] || die "--target-compose requires a path"
MIGRATE_TARGET_COMPOSE="$2"
shift 2
;;
--target-env)
[[ $# -ge 2 ]] || die "--target-env requires a path"
MIGRATE_TARGET_ENV="$2"
shift 2
;;
--target-db)
[[ $# -ge 2 ]] || die "--target-db requires a path"
MIGRATE_TARGET_DB="$2"
shift 2
;;
--work-dir)
[[ $# -ge 2 ]] || die "--work-dir requires a path"
MIGRATE_WORK_DIR="$2"
shift 2
;;
--app-service)
[[ $# -ge 2 ]] || die "--app-service requires a service name"
MIGRATE_APP_SERVICE="$2"
shift 2
;;
--postgres-service)
[[ $# -ge 2 ]] || die "--postgres-service requires a service name"
MIGRATE_POSTGRES_SERVICE="$2"
shift 2
;;
--single-node-service)
[[ $# -ge 2 ]] || die "--single-node-service requires a service name"
MIGRATE_SINGLE_NODE_SERVICE="$2"
shift 2
;;
--replace-existing)
MIGRATE_REPLACE_EXISTING="true"
shift
;;
--replace-target-compose)
MIGRATE_REPLACE_TARGET_COMPOSE="true"
shift
;;
--request-body-mode)
[[ $# -ge 2 ]] || die "--request-body-mode requires a value"
MIGRATE_REQUEST_BODY_MODE="$2"
shift 2
;;
--keep-source-stopped-on-error)
MIGRATE_KEEP_SOURCE_STOPPED_ON_ERROR="true"
shift
;;
-h|--help) -h|--help)
usage usage
exit 0 exit 0
@@ -441,25 +534,25 @@ select_mode() {
MODE="compose" MODE="compose"
return return
;; ;;
compose-sqlite|sqlite-compose|compose-solo|solo|docker-solo|docker-solo-compose) compose-single-node|docker-single-node|docker-single-node-compose)
MODE="compose-sqlite" MODE="compose-single-node"
return return
;; ;;
single|service|systemd|launchd|sqlite) single-node|service|systemd|launchd|sqlite)
MODE="single" MODE="single-node"
return return
;; ;;
cluster|multi|multi-node) cluster|multi|multi-node)
if ui_is_zh; then if ui_is_zh; then
die "集群部署模式暂未开放;请先选择 compose、compose-sqlite 或 single" die "集群部署模式暂未开放;请先选择 compose、compose-single-node 或 single-node"
else else
die "cluster deployment mode is temporarily disabled; choose compose, compose-sqlite, or single" die "cluster deployment mode is temporarily disabled; choose compose, compose-single-node, or single-node"
fi fi
;; ;;
auto|"") auto|"")
;; ;;
*) *)
die "unsupported install mode: ${MODE}; expected compose, compose-sqlite, or single" die "unsupported install mode: ${MODE}; expected compose, compose-single-node, or single-node"
;; ;;
esac esac
@@ -468,34 +561,34 @@ select_mode() {
cat >/dev/tty <<EOF cat >/dev/tty <<EOF
请选择 Aether 部署模式: 请选择 Aether 部署模式:
1) Docker Compose 应用: Postgres + Redis 1) Docker Compose 标准部署(Postgres + Redis
3) Docker Compose 应用: 仅SQLite 2) Docker Compose 单节点部署(SQLite
4) 系统服务: 仅SQLite 3) 系统服务单节点部署(SQLite
请输入选项 [4]: 请输入选项 [3]:
EOF EOF
else else
cat >/dev/tty <<EOF cat >/dev/tty <<EOF
Choose Aether deployment mode: Choose Aether deployment mode:
1) Docker Compose app: Postgres + Redis 1) Docker Compose standard deployment (Postgres + Redis)
3) Docker Compose app: SQLite only 2) Docker Compose single-node deployment (SQLite)
4) System service: SQLite only 3) System service single-node deployment (SQLite)
Enter choice [4]: Enter choice [3]:
EOF EOF
fi fi
local choice local choice
IFS= read -r choice </dev/tty || choice="" IFS= read -r choice </dev/tty || choice=""
case "${choice:-4}" in case "${choice:-3}" in
1) 1)
MODE="compose" MODE="compose"
;; ;;
3) 2)
MODE="compose-sqlite" MODE="compose-single-node"
;; ;;
4) 3)
MODE="single" MODE="single-node"
;; ;;
*) *)
if ui_is_zh; then if ui_is_zh; then
@@ -505,8 +598,45 @@ EOF
fi fi
;; ;;
esac esac
if [[ -z "${MIGRATE_FROM_COMPOSE}" && "${MODE}" != "compose" ]]; then
if ui_is_zh; then
cat >/dev/tty <<'EOF'
请选择数据初始化方式:
1) 全新初始化(不迁移现有数据)
2) 从现有 Docker Compose PG 数据库迁移
请输入选项 [1]:
EOF
else
cat >/dev/tty <<'EOF'
Choose data initialization mode:
1) Fresh initialization (do not migrate existing data)
2) Migrate from an existing Docker Compose PG database
Enter choice [1]:
EOF
fi
local init_choice
IFS= read -r init_choice </dev/tty || init_choice=""
case "${init_choice:-1}" in
1)
;;
2)
MIGRATE_INTERACTIVE="true"
;;
*)
if ui_is_zh; then
die "无效的数据初始化方式选项: ${init_choice}"
else
die "invalid data initialization choice: ${init_choice}"
fi
;;
esac
fi
else else
MODE="single" MODE="single-node"
fi fi
} }
@@ -857,6 +987,434 @@ start_compose_deployment() {
fi fi
} }
migration_options_requested() {
[[ -n "${MIGRATE_TARGET_COMPOSE}" ]] && return 0
[[ -n "${MIGRATE_TARGET_ENV}" ]] && return 0
[[ -n "${MIGRATE_TARGET_DB}" ]] && return 0
[[ -n "${MIGRATE_WORK_DIR}" ]] && return 0
[[ -n "${MIGRATE_APP_SERVICE}" ]] && return 0
[[ -n "${MIGRATE_POSTGRES_SERVICE}" ]] && return 0
[[ -n "${MIGRATE_SINGLE_NODE_SERVICE}" ]] && return 0
[[ "${MIGRATE_REPLACE_EXISTING}" == "true" ]] && return 0
[[ "${MIGRATE_REPLACE_TARGET_COMPOSE}" == "true" ]] && return 0
[[ -n "${MIGRATE_REQUEST_BODY_MODE}" ]] && return 0
[[ "${MIGRATE_KEEP_SOURCE_STOPPED_ON_ERROR}" == "true" ]] && return 0
return 1
}
normalize_migration_request_body_mode() {
case "${MIGRATE_REQUEST_BODY_MODE}" in
""|1|full|all|include)
MIGRATE_REQUEST_BODY_MODE="full"
;;
2|omit|skip)
MIGRATE_REQUEST_BODY_MODE="omit"
;;
*)
die "--request-body-mode must be full/1 or omit/2"
;;
esac
}
prompt_with_default() {
local prompt="$1"
local default_value="$2"
local value
if [[ -n "${default_value}" ]]; then
printf '%s [%s]: ' "${prompt}" "${default_value}" >/dev/tty
else
printf '%s: ' "${prompt}" >/dev/tty
fi
IFS= read -r value </dev/tty || value=""
if [[ -z "${value}" ]]; then
printf '%s\n' "${default_value}"
else
printf '%s\n' "${value}"
fi
}
prompt_yes_no() {
local prompt="$1"
local default_value="$2"
local suffix choice
case "${default_value}" in
yes)
if ui_is_zh; then
suffix="[y/n默认 y]"
else
suffix="[y/n, default y]"
fi
;;
*)
default_value="no"
if ui_is_zh; then
suffix="[y/n默认 n]"
else
suffix="[y/n, default n]"
fi
;;
esac
while true; do
printf '%s %s: ' "${prompt}" "${suffix}" >/dev/tty
IFS= read -r choice </dev/tty || choice=""
choice="$(printf '%s' "${choice}" | tr '[:upper:]' '[:lower:]')"
case "${choice:-${default_value}}" in
y|yes)
return 0
;;
n|no)
return 1
;;
*)
if ui_is_zh; then
echo "请输入 y 或 n。" >/dev/tty
else
echo "Enter y or n." >/dev/tty
fi
;;
esac
done
}
docker_compose_ls_config_files() {
local output
output="$(docker compose ls --format json 2>/dev/null || true)"
if [[ -n "${output}" && "${output}" == *ConfigFiles* ]]; then
printf '%s' "${output}" |
tr '{' '\n' |
sed -n 's/.*"ConfigFiles"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p'
return
fi
docker compose ls 2>/dev/null | awk 'NR > 1 && NF > 0 { print $NF }'
}
compose_file_has_source_services() {
local compose_file="$1"
local app_service="${MIGRATE_APP_SERVICE:-app}"
local postgres_service="${MIGRATE_POSTGRES_SERVICE:-postgres}"
local services
services="$(docker compose -f "${compose_file}" config --services 2>/dev/null || true)"
[[ -n "${services}" ]] || return 1
printf '%s\n' "${services}" | grep -Fxq "${app_service}" || return 1
printf '%s\n' "${services}" | grep -Fxq "${postgres_service}" || return 1
}
append_unique_candidate() {
local candidate="$1"
shift
local existing
for existing in "$@"; do
[[ "${existing}" != "${candidate}" ]] || return 1
done
printf '%s\n' "${candidate}"
}
detect_source_compose_from_docker_compose_ls() {
local config_files compose_file candidate
local -a candidates=()
command -v docker >/dev/null 2>&1 || return 0
docker compose version >/dev/null 2>&1 || return 0
while IFS= read -r config_files || [[ -n "${config_files}" ]]; do
config_files="$(trim_whitespace "${config_files}")"
[[ -n "${config_files}" ]] || continue
# The migration scripts currently accept one source compose file. If the
# source project was launched with multiple compose files, ask explicitly.
[[ "${config_files}" != *,* ]] || continue
compose_file="${config_files}"
[[ -f "${compose_file}" ]] || continue
compose_file="$(absolute_path "${compose_file}")"
compose_file_has_source_services "${compose_file}" || continue
candidate="$(append_unique_candidate "${compose_file}" "${candidates[@]}" || true)"
[[ -z "${candidate}" ]] || candidates+=("${candidate}")
done < <(docker_compose_ls_config_files)
case "${#candidates[@]}" in
0)
return 0
;;
1)
printf '%s\n' "${candidates[0]}"
;;
*)
if interactive_tty_available; then
if ui_is_zh; then
echo "从 docker compose ls 找到多个可能的源 Compose无法安全自动选择" >/dev/tty
else
echo "docker compose ls found multiple possible source Compose files and cannot choose safely:" >/dev/tty
fi
printf ' %s\n' "${candidates[@]}" >/dev/tty
fi
return 0
;;
esac
}
collect_interactive_migration_options() {
local detected_source
local prompt
local source_compose_abs
local source_compose_dir
[[ "${MIGRATE_INTERACTIVE}" == "true" ]] || return
interactive_tty_available || die "interactive migration selection requires a terminal"
if ui_is_zh; then
cat >/dev/tty <<'EOF'
迁移会先做预检并拉取/安装目标 single-node再在切换窗口停止源 app。
源 Postgres 和 Redis 会保留,便于回滚。
EOF
else
cat >/dev/tty <<'EOF'
Migration will preflight and pull/install the target single-node release first.
During cutover it stops only the source app. Source Postgres and Redis remain for rollback.
EOF
fi
if [[ -z "${MIGRATE_FROM_COMPOSE}" ]]; then
detected_source="$(detect_source_compose_from_docker_compose_ls || true)"
if [[ -z "${detected_source}" ]]; then
if ui_is_zh; then
die "未能通过 docker compose ls 唯一识别源 PG Compose请使用 --migrate-from-compose 显式指定"
else
die "could not uniquely detect source PG Compose from docker compose ls; pass --migrate-from-compose explicitly"
fi
fi
if ui_is_zh; then
printf '已通过 docker compose ls 探测到源 Compose: %s\n' "${detected_source}" >/dev/tty
else
printf 'Detected source Compose from docker compose ls: %s\n' "${detected_source}" >/dev/tty
fi
if ui_is_zh; then
prompt="确认使用该源 Compose 进行迁移"
else
prompt="Use this source Compose for migration"
fi
if prompt_yes_no "${prompt}" "yes"; then
MIGRATE_FROM_COMPOSE="${detected_source}"
else
if ui_is_zh; then
die "已取消迁移;如需指定其他源 Compose请使用 --migrate-from-compose"
else
die "migration cancelled; pass --migrate-from-compose to use another source Compose"
fi
fi
fi
[[ -n "${MIGRATE_FROM_COMPOSE}" ]] || die "--migrate-from-compose cannot be empty"
source_compose_abs="$(absolute_path "${MIGRATE_FROM_COMPOSE}")"
source_compose_dir="$(dirname "${source_compose_abs}")"
if [[ "${MODE}" == "compose-single-node" ]]; then
if [[ "${COMPOSE_DIR_EXPLICIT}" != "true" ]]; then
COMPOSE_DIR="${source_compose_dir}-single-node"
fi
if ui_is_zh; then
printf '已自动选择目标 single-node Compose 目录: %s\n' "${COMPOSE_DIR}" >/dev/tty
prompt="确认使用该目标目录"
else
printf 'Selected target single-node Compose directory: %s\n' "${COMPOSE_DIR}" >/dev/tty
prompt="Use this target directory"
fi
if ! prompt_yes_no "${prompt}" "yes"; then
if ui_is_zh; then
die "已取消迁移;如需指定其他目标目录,请使用 --compose-dir"
else
die "migration cancelled; pass --compose-dir to use another target directory"
fi
fi
fi
if [[ "${MIGRATE_REPLACE_EXISTING}" != "true" ]]; then
if ui_is_zh; then
prompt="如果目标 SQLite 已存在,是否允许备份后替换"
else
prompt="If the target SQLite DB already exists, allow backup and replacement"
fi
if prompt_yes_no "${prompt}" "no"; then
MIGRATE_REPLACE_EXISTING="true"
fi
fi
if [[ -z "${MIGRATE_REQUEST_BODY_MODE}" ]]; then
if ui_is_zh; then
cat >/dev/tty <<'EOF'
请求体明细迁移策略:
1) 全部迁移:迁移所有可迁移数据,包括请求体明细
2) 不迁移请求体:迁移其他所有数据;仅跳过请求体大字段和 HTTP 请求体明细,源 PG 不清除
请输入选项 [1]:
EOF
else
cat >/dev/tty <<'EOF'
Request/response body detail migration mode:
1) Full migration: migrate all migratable data, including request body details
2) Skip request bodies: migrate all other data; skip only request body large fields and HTTP body detail tables; source PG is unchanged
Enter choice [1]:
EOF
fi
local body_choice
IFS= read -r body_choice </dev/tty || body_choice=""
MIGRATE_REQUEST_BODY_MODE="${body_choice:-1}"
fi
normalize_migration_request_body_mode
}
install_migration_project_file() {
local source_path="$1"
local mode="$2"
local target_path
ensure_tmp_root
target_path="${TMP_ROOT}/$(basename "${source_path}")"
install_project_file "${source_path}" "${target_path}" "${mode}"
printf '%s\n' "${target_path}"
}
run_compose_single_node_migration() {
local migration_script
local target_template
local source_compose_abs
local source_compose_dir
local compose_dir_abs
local target_compose
local target_compose_abs
local target_compose_dir
local target_env
local target_env_abs
local table
local -a migrate_args
source_compose_abs="$(absolute_path "${MIGRATE_FROM_COMPOSE}")"
[[ -f "${source_compose_abs}" ]] || die "source compose file not found: ${MIGRATE_FROM_COMPOSE}"
source_compose_dir="$(dirname "${source_compose_abs}")"
compose_dir_abs="$(absolute_path_maybe_missing "${COMPOSE_DIR}")"
if [[ -n "${MIGRATE_TARGET_COMPOSE}" ]]; then
target_compose="${MIGRATE_TARGET_COMPOSE}"
elif [[ "${compose_dir_abs}" == "${source_compose_dir}" ]]; then
target_compose="${compose_dir_abs}/docker-compose.single-node.yml"
else
target_compose="${compose_dir_abs}/docker-compose.yml"
fi
target_compose_abs="$(absolute_path_maybe_missing "${target_compose}")"
target_compose_dir="$(dirname "${target_compose_abs}")"
if [[ -n "${MIGRATE_TARGET_ENV}" ]]; then
target_env="${MIGRATE_TARGET_ENV}"
elif [[ "$(basename "${target_compose_abs}")" == "docker-compose.yml" ]]; then
target_env="${target_compose_dir}/.env"
else
target_env="${target_compose_dir}/.env.single-node"
fi
target_env_abs="$(absolute_path_maybe_missing "${target_env}")"
[[ "${target_compose_abs}" != "${source_compose_abs}" ]] || die "target compose would overwrite the source compose file; pass --target-compose or --compose-dir"
[[ "${target_env_abs}" != "${source_compose_dir}/.env" ]] || die "target env would overwrite the source .env; pass --target-env or --compose-dir"
migration_script="$(install_migration_project_file "scripts/migrate-pg-compose-to-single-node.sh" "0755")"
target_template="$(install_migration_project_file "docker-compose.single-node.yml" "0644")"
migrate_args=(
"${migration_script}"
--source-compose "${source_compose_abs}"
--target-compose "${target_compose_abs}"
--target-template "${target_template}"
--target-env "${target_env_abs}"
--app-image "$(compose_image)"
)
[[ -z "${MIGRATE_TARGET_DB}" ]] || migrate_args+=(--target-db "${MIGRATE_TARGET_DB}")
[[ -z "${MIGRATE_WORK_DIR}" ]] || migrate_args+=(--work-dir "${MIGRATE_WORK_DIR}")
[[ -z "${MIGRATE_APP_SERVICE}" ]] || migrate_args+=(--app-service "${MIGRATE_APP_SERVICE}")
[[ -z "${MIGRATE_POSTGRES_SERVICE}" ]] || migrate_args+=(--postgres-service "${MIGRATE_POSTGRES_SERVICE}")
[[ -z "${MIGRATE_SINGLE_NODE_SERVICE}" ]] || migrate_args+=(--single-node-service "${MIGRATE_SINGLE_NODE_SERVICE}")
[[ "${MIGRATE_REPLACE_EXISTING}" != "true" ]] || migrate_args+=(--replace-existing)
[[ "${MIGRATE_REPLACE_TARGET_COMPOSE}" != "true" ]] || migrate_args+=(--replace-target-compose)
[[ -z "${MIGRATE_REQUEST_BODY_MODE}" ]] || migrate_args+=(--request-body-mode "${MIGRATE_REQUEST_BODY_MODE}")
[[ "${MIGRATE_KEEP_SOURCE_STOPPED_ON_ERROR}" != "true" ]] || migrate_args+=(--keep-source-stopped-on-error)
bash "${migrate_args[@]}"
}
run_single_node_service_migration() {
local migration_script
local installer
local table
local -a migrate_args
[[ -z "${MIGRATE_TARGET_COMPOSE}" ]] || die "--target-compose is only valid with --mode compose-single-node"
[[ -z "${MIGRATE_TARGET_ENV}" ]] || die "--target-env is only valid with --mode compose-single-node"
[[ -z "${MIGRATE_SINGLE_NODE_SERVICE}" ]] || die "--single-node-service is only valid with --mode compose-single-node"
[[ "${MIGRATE_REPLACE_TARGET_COMPOSE}" != "true" ]] || die "--replace-target-compose is only valid with --mode compose-single-node"
migration_script="$(install_migration_project_file "scripts/migrate-pg-to-single-node.sh" "0755")"
installer="$(install_migration_project_file "install.sh" "0755")"
migrate_args=(
"${migration_script}"
--source-compose "${MIGRATE_FROM_COMPOSE}"
--installer "${installer}"
--install-root "${INSTALL_ROOT}"
--config-dir "${CONFIG_DIR}"
--service-name "${SERVICE_NAME}"
--service-user "${SERVICE_USER}"
--service-group "${SERVICE_GROUP}"
--app-image "$(compose_image)"
--install-channel "${CHANNEL}"
--install-repo "${REPO}"
--install-source-ref "${SOURCE_REF}"
)
[[ -z "${VERSION}" ]] || migrate_args+=(--install-version "${VERSION}")
[[ -z "${ARCHIVE_PATH}" ]] || migrate_args+=(--install-archive "${ARCHIVE_PATH}")
[[ -z "${RELEASE_ARCHIVE_URL}" ]] || migrate_args+=(--install-download-url "${RELEASE_ARCHIVE_URL}")
[[ -z "${MIGRATE_TARGET_DB}" ]] || migrate_args+=(--target-db "${MIGRATE_TARGET_DB}")
[[ -z "${MIGRATE_WORK_DIR}" ]] || migrate_args+=(--work-dir "${MIGRATE_WORK_DIR}")
[[ -z "${MIGRATE_APP_SERVICE}" ]] || migrate_args+=(--app-service "${MIGRATE_APP_SERVICE}")
[[ -z "${MIGRATE_POSTGRES_SERVICE}" ]] || migrate_args+=(--postgres-service "${MIGRATE_POSTGRES_SERVICE}")
[[ "${MIGRATE_REPLACE_EXISTING}" != "true" ]] || migrate_args+=(--replace-existing)
[[ -z "${MIGRATE_REQUEST_BODY_MODE}" ]] || migrate_args+=(--request-body-mode "${MIGRATE_REQUEST_BODY_MODE}")
[[ "${MIGRATE_KEEP_SOURCE_STOPPED_ON_ERROR}" != "true" ]] || migrate_args+=(--keep-source-stopped-on-error)
bash "${migrate_args[@]}"
}
run_migration_from_compose() {
if [[ -n "${MIGRATE_REQUEST_BODY_MODE}" ]]; then
normalize_migration_request_body_mode
fi
case "${MODE}" in
compose-single-node)
run_compose_single_node_migration
;;
single-node)
run_single_node_service_migration
;;
compose)
die "--migrate-from-compose target mode must be compose-single-node or single-node"
;;
*)
die "unsupported migration target mode: ${MODE}"
;;
esac
}
resolve_version() { resolve_version() {
if [[ -n "${VERSION}" ]]; then if [[ -n "${VERSION}" ]]; then
echo "${VERSION}" echo "${VERSION}"
@@ -899,6 +1457,36 @@ current_script_dir() {
fi fi
} }
ensure_tmp_root() {
if [[ -z "${TMP_ROOT}" ]]; then
TMP_ROOT="$(mktemp -d)"
fi
}
absolute_path() {
local path="$1"
local dir
local base
if [[ "${path}" == /* ]]; then
printf '%s\n' "${path}"
return
fi
dir="$(dirname "${path}")"
base="$(basename "${path}")"
printf '%s/%s\n' "$(cd "${dir}" && pwd -P)" "${base}"
}
absolute_path_maybe_missing() {
local path="$1"
if [[ "${path}" == /* ]]; then
printf '%s\n' "${path}"
else
printf '%s/%s\n' "$(pwd -P)" "${path}"
fi
}
local_bundle_dir() { local_bundle_dir() {
local dir local dir
dir="$(current_script_dir)" dir="$(current_script_dir)"
@@ -1092,6 +1680,8 @@ AETHER_GATEWAY_AUTO_PREPARE_DATABASE=true
AETHER_RUNTIME_BACKEND=memory AETHER_RUNTIME_BACKEND=memory
API_KEY_PREFIX=sk API_KEY_PREFIX=sk
AETHER_DATABASE_DRIVER=sqlite
AETHER_DATABASE_URL=sqlite://${INSTALL_ROOT}/data/aether.db
DATABASE_URL=sqlite://${INSTALL_ROOT}/data/aether.db DATABASE_URL=sqlite://${INSTALL_ROOT}/data/aether.db
JWT_SECRET_KEY=${jwt_key} JWT_SECRET_KEY=${jwt_key}
@@ -1192,7 +1782,7 @@ generate_compose_env() {
replace_or_append_env "${output}" "AETHER_GATEWAY_AUTO_PREPARE_DATABASE" "true" replace_or_append_env "${output}" "AETHER_GATEWAY_AUTO_PREPARE_DATABASE" "true"
} }
generate_compose_sqlite_env() { generate_compose_single_node_env() {
local output="$1" local output="$1"
local jwt_key encryption_key local jwt_key encryption_key
prompt_admin_password prompt_admin_password
@@ -1218,6 +1808,8 @@ AETHER_GATEWAY_AUTO_PREPARE_DATABASE=true
AETHER_RUNTIME_BACKEND=memory AETHER_RUNTIME_BACKEND=memory
API_KEY_PREFIX=sk API_KEY_PREFIX=sk
AETHER_DATABASE_DRIVER=sqlite
AETHER_DATABASE_URL=sqlite:///app/data/aether.db
DATABASE_URL=sqlite:///app/data/aether.db DATABASE_URL=sqlite:///app/data/aether.db
JWT_SECRET_KEY=${JWT_SECRET_KEY:-${jwt_key}} JWT_SECRET_KEY=${JWT_SECRET_KEY:-${jwt_key}}
@@ -1374,9 +1966,9 @@ ensure_env_matches_requested_mode() {
topology="${topology:-single-node}" topology="${topology:-single-node}"
if [[ "${mode}" == "cluster" ]]; then if [[ "${mode}" == "cluster" ]]; then
[[ "${topology}" == "multi-node" ]] || die "existing env ${file} is ${topology}; set AETHER_GATEWAY_DEPLOYMENT_TOPOLOGY=multi-node or use --mode single" [[ "${topology}" == "multi-node" ]] || die "existing env ${file} is ${topology}; set AETHER_GATEWAY_DEPLOYMENT_TOPOLOGY=multi-node or use --mode single-node"
cluster_env_has_required_backends "${file}" || die "existing multi-node env ${file} must define DATABASE_URL and REDIS_URL" cluster_env_has_required_backends "${file}" || die "existing multi-node env ${file} must define DATABASE_URL and REDIS_URL"
elif [[ "${mode}" == "single" && "${topology}" == "multi-node" ]]; then elif [[ "${mode}" == "single-node" && "${topology}" == "multi-node" ]]; then
die "existing env ${file} is multi-node; cluster mode is temporarily disabled, edit the env file" die "existing env ${file} is multi-node; cluster mode is temporarily disabled, edit the env file"
fi fi
} }
@@ -1580,7 +2172,7 @@ EOF
exit 1 exit 1
fi fi
else else
info "generating first-install SQLite env file" info "generating first-install single-node env file"
generate_first_install_env "${GENERATED_ENV}" generate_first_install_env "${GENERATED_ENV}"
fi fi
echo "${GENERATED_ENV}" echo "${GENERATED_ENV}"
@@ -1624,14 +2216,14 @@ EOF
compose_next_steps compose_next_steps
} }
install_compose_sqlite_mode() { install_compose_single_node_mode() {
resolve_compose_dir resolve_compose_dir
info "preparing Docker Compose SQLite deployment in ${COMPOSE_DIR}" info "preparing Docker Compose single-node deployment in ${COMPOSE_DIR}"
ensure_directory "${COMPOSE_DIR}" ensure_directory "${COMPOSE_DIR}"
ensure_directory "${COMPOSE_DIR}/logs" ensure_directory "${COMPOSE_DIR}/logs"
ensure_directory "${COMPOSE_DIR}/data" ensure_directory "${COMPOSE_DIR}/data"
install_project_file "docker-compose.sqlite.yml" "${COMPOSE_DIR}/docker-compose.yml" "0644" install_project_file "docker-compose.single-node.yml" "${COMPOSE_DIR}/docker-compose.yml" "0644"
install_project_file ".env.example" "${COMPOSE_DIR}/.env.example" "0644" install_project_file ".env.example" "${COMPOSE_DIR}/.env.example" "0644"
install_generate_keys_script "${COMPOSE_DIR}/generate_keys.sh" install_generate_keys_script "${COMPOSE_DIR}/generate_keys.sh"
@@ -1639,13 +2231,13 @@ install_compose_sqlite_mode() {
warn "keeping existing ${COMPOSE_DIR}/.env" warn "keeping existing ${COMPOSE_DIR}/.env"
else else
info "generating ${COMPOSE_DIR}/.env" info "generating ${COMPOSE_DIR}/.env"
generate_compose_sqlite_env "${COMPOSE_DIR}/.env" generate_compose_single_node_env "${COMPOSE_DIR}/.env"
chmod 0600 "${COMPOSE_DIR}/.env" chmod 0600 "${COMPOSE_DIR}/.env"
fi fi
cat <<EOF cat <<EOF
Docker Compose SQLite files are ready: Docker Compose single-node files are ready:
${COMPOSE_DIR}/docker-compose.yml ${COMPOSE_DIR}/docker-compose.yml
${COMPOSE_DIR}/.env ${COMPOSE_DIR}/.env
${COMPOSE_DIR}/.env.example ${COMPOSE_DIR}/.env.example
@@ -2077,11 +2669,20 @@ main() {
apply_platform_defaults apply_platform_defaults
select_version select_version
select_mode select_mode
collect_interactive_migration_options
if [[ -n "${MIGRATE_FROM_COMPOSE}" ]]; then
run_migration_from_compose
return
fi
if migration_options_requested; then
die "migration options require --migrate-from-compose"
fi
if [[ "${MODE}" == "compose" ]]; then if [[ "${MODE}" == "compose" ]]; then
install_compose_mode install_compose_mode
elif [[ "${MODE}" == "compose-sqlite" ]]; then elif [[ "${MODE}" == "compose-single-node" ]]; then
install_compose_sqlite_mode install_compose_single_node_mode
else else
require_root require_root
require_service_manager require_service_manager

View File

@@ -0,0 +1,873 @@
#!/usr/bin/env bash
set -euo pipefail
SOURCE_COMPOSE="docker-compose.yml"
TARGET_COMPOSE=""
TARGET_COMPOSE_TEMPLATE="./docker-compose.single-node.yml"
TARGET_ENV=""
TARGET_DB=""
WORK_DIR=""
APP_SERVICE="app"
POSTGRES_SERVICE="postgres"
SINGLE_NODE_SERVICE="app"
APP_IMAGE=""
REPLACE_EXISTING="false"
REPLACE_TARGET_COMPOSE="false"
DRY_RUN="false"
KEEP_SOURCE_STOPPED_ON_ERROR="false"
DISK_SPACE_MULTIPLIER="${AETHER_MIGRATION_DISK_SPACE_MULTIPLIER:-2}"
DISK_SPACE_MIN_FREE_BYTES="${AETHER_MIGRATION_MIN_FREE_BYTES:-1073741824}"
REQUEST_BODY_MODE="${AETHER_MIGRATION_REQUEST_BODY_MODE:-full}"
APP_STOPPED="false"
CUTOVER_COMPLETE="false"
SOURCE_COMPOSE_ABS=""
SOURCE_COMPOSE_DIR=""
SOURCE_ENV=""
SOURCE_NETWORK=""
TARGET_COMPOSE_ABS=""
TARGET_COMPOSE_DIR=""
TARGET_ENV_ABS=""
DB_USER=""
DB_NAME=""
DB_PASSWORD=""
NOW=""
usage() {
cat <<'EOF'
Usage: scripts/migrate-pg-compose-to-single-node.sh [options]
Migrate an existing Docker Compose Postgres deployment to Docker Compose single-node.
This script pulls the single-node app image before downtime, stops only the source app,
then copies Postgres data directly into a temporary SQLite DB without writing a JSONL
intermediate file. After the copy succeeds, it starts the single-node Compose app.
Options:
--source-compose PATH Source Postgres docker compose file (default: docker-compose.yml)
--target-compose PATH Target single-node compose file (default: SOURCE_DIR/docker-compose.single-node.yml)
--target-template PATH Template copied when target compose is missing (default: ./docker-compose.single-node.yml)
--target-env PATH Target env file (default: TARGET_COMPOSE_DIR/.env.single-node)
--target-db PATH Final SQLite DB path (default: TARGET_COMPOSE_DIR/data/aether.db)
--work-dir PATH Working directory (default: SOURCE_DIR/data/pg-compose-to-single-node-<timestamp>)
--app-service NAME Source compose app service (default: app)
--postgres-service NAME Source compose Postgres service (default: postgres)
--single-node-service NAME Target single-node compose service (default: app)
--app-image IMAGE Override APP_IMAGE for the target single-node compose env
--replace-existing Allow replacing an existing target SQLite database
--replace-target-compose Overwrite target compose file from the template
--dry-run Pull/preflight/direct-copy without stopping or switching
--request-body-mode MODE Request/response body detail handling: full/1 or omit/2
full: migrate all migratable data, including request body details
omit: migrate all other data; skip only request body large fields and HTTP body detail tables; source PG is unchanged
--keep-source-stopped-on-error
Do not auto-restart source app if migration fails after stopping it
-h, --help Show this help
Default cutover behavior:
1. Copy/prepare docker-compose.single-node.yml and .env.single-node.
2. Pull the target single-node app image while the source app is still running.
3. Verify the running source app image ID matches the target single-node image ID.
4. Preflight SQLite migrations in a temporary DB with the target image.
5. Re-check migration coverage and available disk space.
6. Stop/remove only the source app container; keep Postgres/Redis running.
7. Copy records directly into SQLite, replace TARGET_DB, and start single-node.
EOF
}
log() {
printf '>>> %s\n' "$*"
}
warn() {
printf 'WARN: %s\n' "$*" >&2
}
die() {
printf 'ERROR: %s\n' "$*" >&2
exit 1
}
trim() {
local value="$1"
value="${value#"${value%%[![:space:]]*}"}"
value="${value%"${value##*[![:space:]]}"}"
printf '%s' "$value"
}
strip_optional_quotes() {
local value="$1"
if [[ "${#value}" -ge 2 ]]; then
if [[ "${value:0:1}" == "\"" && "${value: -1}" == "\"" ]]; then
printf '%s' "${value:1:${#value}-2}"
return
fi
if [[ "${value:0:1}" == "'" && "${value: -1}" == "'" ]]; then
printf '%s' "${value:1:${#value}-2}"
return
fi
fi
printf '%s' "$value"
}
absolute_path() {
local path="$1"
local dir
local base
if [[ "$path" == /* ]]; then
printf '%s\n' "$path"
return
fi
dir="$(dirname "$path")"
base="$(basename "$path")"
printf '%s/%s\n' "$(cd "$dir" && pwd -P)" "$base"
}
absolute_path_maybe_missing() {
local path="$1"
if [[ "$path" == /* ]]; then
printf '%s\n' "$path"
else
printf '%s/%s\n' "$(pwd -P)" "$path"
fi
}
parse_args() {
while [[ $# -gt 0 ]]; do
case "$1" in
--source-compose)
[[ $# -ge 2 ]] || die "--source-compose requires a value"
SOURCE_COMPOSE="$2"
shift 2
;;
--target-compose)
[[ $# -ge 2 ]] || die "--target-compose requires a value"
TARGET_COMPOSE="$2"
shift 2
;;
--target-template)
[[ $# -ge 2 ]] || die "--target-template requires a value"
TARGET_COMPOSE_TEMPLATE="$2"
shift 2
;;
--target-env)
[[ $# -ge 2 ]] || die "--target-env requires a value"
TARGET_ENV="$2"
shift 2
;;
--target-db)
[[ $# -ge 2 ]] || die "--target-db requires a value"
TARGET_DB="$2"
shift 2
;;
--work-dir)
[[ $# -ge 2 ]] || die "--work-dir requires a value"
WORK_DIR="$2"
shift 2
;;
--app-service)
[[ $# -ge 2 ]] || die "--app-service requires a value"
APP_SERVICE="$2"
shift 2
;;
--postgres-service)
[[ $# -ge 2 ]] || die "--postgres-service requires a value"
POSTGRES_SERVICE="$2"
shift 2
;;
--single-node-service)
[[ $# -ge 2 ]] || die "--single-node-service requires a value"
SINGLE_NODE_SERVICE="$2"
shift 2
;;
--app-image)
[[ $# -ge 2 ]] || die "--app-image requires a value"
APP_IMAGE="$2"
shift 2
;;
--replace-existing)
REPLACE_EXISTING="true"
shift
;;
--replace-target-compose)
REPLACE_TARGET_COMPOSE="true"
shift
;;
--dry-run)
DRY_RUN="true"
shift
;;
--request-body-mode)
[[ $# -ge 2 ]] || die "--request-body-mode requires a value"
REQUEST_BODY_MODE="$2"
shift 2
;;
--keep-source-stopped-on-error)
KEEP_SOURCE_STOPPED_ON_ERROR="true"
shift
;;
-h|--help)
usage
exit 0
;;
*)
die "unknown argument: $1"
;;
esac
done
}
require_command() {
command -v "$1" >/dev/null 2>&1 || die "required command not found: $1"
}
normalize_request_body_mode() {
case "$REQUEST_BODY_MODE" in
""|1|full|all|include)
REQUEST_BODY_MODE="full"
;;
2|omit|skip)
REQUEST_BODY_MODE="omit"
;;
*)
die "--request-body-mode must be full/1 or omit/2"
;;
esac
}
env_file_get() {
local file="$1"
local wanted="$2"
local line key value
local found=""
while IFS= read -r line || [[ -n "$line" ]]; do
line="${line%$'\r'}"
line="$(trim "$line")"
[[ -z "$line" || "${line:0:1}" == "#" ]] && continue
[[ "$line" == export\ * ]] && line="${line#export }"
key="$(trim "${line%%=*}")"
[[ "$key" == "$wanted" ]] || continue
value="${line#*=}"
found="$(strip_optional_quotes "$(trim "$value")")"
done < "$file"
printf '%s' "$found"
}
validate_env_line_for_copy() {
local line="$1"
local line_no="$2"
[[ "$line" == *'${'* ]] && die "source env line ${line_no} uses variable expansion; write a concrete value before migration"
[[ "$line" == *'$('* ]] && die "source env line ${line_no} uses command substitution; write a concrete value before migration"
[[ "$line" == *'`'* ]] && die "source env line ${line_no} uses command substitution; write a concrete value before migration"
return 0
}
should_skip_single_node_env_key() {
case "$1" in
APP_IMAGE|LOCAL_APP_IMAGE|APP_PORT|DB_HOST|DB_PORT|DB_USER|DB_NAME|DB_PASSWORD|POSTGRES_*|MYSQL_*|REDIS_HOST|REDIS_PORT|REDIS_PASSWORD|REDIS_URL|AETHER_GATEWAY_DATA_REDIS_URL|AETHER_GATEWAY_DATA_REDIS_KEY_PREFIX|DATABASE_URL|AETHER_DATABASE_URL|AETHER_DATABASE_DRIVER|AETHER_GATEWAY_DATA_POSTGRES_URL|AETHER_RUNTIME_BACKEND|AETHER_RUNTIME_REDIS_URL|AETHER_RUNTIME_REDIS_KEY_PREFIX|AETHER_GATEWAY_DEPLOYMENT_TOPOLOGY|AETHER_GATEWAY_NODE_ROLE|AETHER_GATEWAY_STATIC_DIR|AETHER_LOG_DIR|AETHER_GATEWAY_AUTO_PREPARE_DATABASE)
return 0
;;
*)
return 1
;;
esac
}
write_single_node_env() {
local output="$1"
local line raw_line key
local line_no=0
local app_port app_image jwt_key encryption_key
app_port="$(env_file_get "$SOURCE_ENV" "APP_PORT")"
app_port="${app_port:-8084}"
app_image="${APP_IMAGE:-$(env_file_get "$SOURCE_ENV" "APP_IMAGE")}"
app_image="${app_image:-ghcr.io/fawney19/aether:latest}"
jwt_key="$(env_file_get "$SOURCE_ENV" "JWT_SECRET_KEY")"
encryption_key="$(env_file_get "$SOURCE_ENV" "ENCRYPTION_KEY")"
: > "$output"
{
printf '# Generated by scripts/migrate-pg-compose-to-single-node.sh from %s\n' "$SOURCE_ENV"
printf '# single-node means Docker Compose app + SQLite for this migration target.\n\n'
} >> "$output"
while IFS= read -r raw_line || [[ -n "$raw_line" ]]; do
line_no=$((line_no + 1))
line="${raw_line%$'\r'}"
line="$(trim "$line")"
[[ -z "$line" || "${line:0:1}" == "#" ]] && continue
[[ "$line" == export\ * ]] && die "source env line ${line_no} uses export; write KEY=VALUE before migration"
[[ "$line" =~ ^[A-Za-z_][A-Za-z0-9_]*= ]] || die "source env line ${line_no} must be KEY=VALUE"
validate_env_line_for_copy "$line" "$line_no"
key="${line%%=*}"
if should_skip_single_node_env_key "$key"; then
continue
fi
printf '%s\n' "$line" >> "$output"
done < "$SOURCE_ENV"
{
printf '\n# Single-node Compose runtime overrides\n'
printf 'APP_IMAGE=%s\n' "$app_image"
printf 'APP_PORT=%s\n' "$app_port"
printf 'AETHER_GATEWAY_STATIC_DIR=/srv/frontend\n'
printf 'AETHER_LOG_DESTINATION=both\n'
printf 'AETHER_LOG_FORMAT=pretty\n'
printf 'AETHER_LOG_DIR=/app/logs\n'
printf 'AETHER_DATABASE_DRIVER=sqlite\n'
printf 'AETHER_DATABASE_URL=sqlite://./data/aether.db\n'
printf 'DATABASE_URL=sqlite://./data/aether.db\n'
printf 'AETHER_RUNTIME_BACKEND=memory\n'
printf 'AETHER_GATEWAY_DEPLOYMENT_TOPOLOGY=single-node\n'
printf 'AETHER_GATEWAY_NODE_ROLE=all\n'
printf 'AETHER_GATEWAY_AUTO_PREPARE_DATABASE=true\n'
printf 'JWT_SECRET_KEY=%s\n' "$jwt_key"
printf 'ENCRYPTION_KEY=%s\n' "$encryption_key"
} >> "$output"
}
source_compose() {
docker compose -f "$SOURCE_COMPOSE_ABS" "$@"
}
target_compose() {
AETHER_ENV_FILE="$TARGET_ENV_ABS" docker compose --env-file "$TARGET_ENV_ABS" -f "$TARGET_COMPOSE_ABS" "$@"
}
target_run_app() {
local database_url="$1"
shift
target_compose run --rm --no-deps \
-v "${WORK_DIR}:/migration" \
-e AETHER_LOG_DESTINATION=stdout \
-e AETHER_DATABASE_DRIVER=sqlite \
-e "AETHER_DATABASE_URL=${database_url}" \
-e "DATABASE_URL=${database_url}" \
"$SINGLE_NODE_SERVICE" "$@"
}
run_psql_stdin() {
local sql_file="$1"
source_compose exec -T \
-e "PGPASSWORD=${DB_PASSWORD}" \
"$POSTGRES_SERVICE" \
psql -h 127.0.0.1 -U "$DB_USER" -d "$DB_NAME" -v ON_ERROR_STOP=1 -At -f - < "$sql_file"
}
source_database_size_bytes() {
local sql_file
local result
sql_file="${WORK_DIR}/source-database-size.sql"
if [[ "$REQUEST_BODY_MODE" == "omit" ]]; then
cat > "$sql_file" <<'SQL'
SELECT GREATEST(
pg_database_size(current_database())
- COALESCE(pg_total_relation_size(to_regclass('public.usage_body_blobs')), 0)
- COALESCE(pg_total_relation_size(to_regclass('public.usage_http_audits')), 0),
0
);
SQL
else
printf 'SELECT pg_database_size(current_database());\n' > "$sql_file"
fi
result="$(run_psql_stdin "$sql_file" | tr -d '[:space:]')"
[[ "$result" =~ ^[0-9]+$ ]] || die "could not determine source Postgres database size"
printf '%s\n' "$result"
}
file_size_bytes() {
local path="$1"
if [[ ! -e "$path" ]]; then
printf '0\n'
return
fi
stat -c '%s' "$path" 2>/dev/null || stat -f '%z' "$path" 2>/dev/null || die "could not stat file: ${path}"
}
target_sqlite_size_bytes() {
local total=0
local suffix
local size
for suffix in "" "-wal" "-shm"; do
size="$(file_size_bytes "${TARGET_DB}${suffix}")"
total=$((total + size))
done
printf '%s\n' "$total"
}
available_bytes_for_path() {
local path="$1"
df -Pk "$path" | awk 'NR == 2 { printf "%.0f\n", $4 * 1024 }'
}
filesystem_key_for_path() {
local path="$1"
df -Pk "$path" | awk 'NR == 2 { print $1 }'
}
format_bytes() {
local bytes="$1"
awk -v bytes="$bytes" 'BEGIN {
if (bytes >= 1073741824) {
printf "%.1f GiB", bytes / 1073741824
} else {
printf "%.1f MiB", bytes / 1048576
}
}'
}
assert_available_space() {
local path="$1"
local required_bytes="$2"
local label="$3"
local available_bytes
available_bytes="$(available_bytes_for_path "$path")"
[[ "$available_bytes" =~ ^[0-9]+$ ]] || die "could not determine free disk space for ${path}"
log "${label} free space: $(format_bytes "$available_bytes"); required: $(format_bytes "$required_bytes")"
if (( available_bytes < required_bytes )); then
die "${label} does not have enough free disk space; required $(format_bytes "$required_bytes"), available $(format_bytes "$available_bytes")"
fi
}
check_disk_space() {
local source_bytes
local estimated_db_bytes
local backup_bytes=0
local target_dir
local work_fs
local target_fs
local required_bytes
[[ "$DISK_SPACE_MULTIPLIER" =~ ^[1-9][0-9]*$ ]] || die "AETHER_MIGRATION_DISK_SPACE_MULTIPLIER must be a positive integer"
[[ "$DISK_SPACE_MIN_FREE_BYTES" =~ ^[0-9]+$ ]] || die "AETHER_MIGRATION_MIN_FREE_BYTES must be a non-negative integer"
source_bytes="$(source_database_size_bytes)"
estimated_db_bytes=$((source_bytes * DISK_SPACE_MULTIPLIER + DISK_SPACE_MIN_FREE_BYTES))
target_dir="$(dirname "$TARGET_DB")"
mkdir -p "$target_dir"
if [[ "$REPLACE_EXISTING" == "true" ]]; then
backup_bytes="$(target_sqlite_size_bytes)"
fi
log "source Postgres size used for disk estimate: $(format_bytes "$source_bytes")"
if [[ "$DRY_RUN" == "true" ]]; then
assert_available_space "$WORK_DIR" "$estimated_db_bytes" "work dir"
return
fi
work_fs="$(filesystem_key_for_path "$WORK_DIR")"
target_fs="$(filesystem_key_for_path "$target_dir")"
if [[ "$work_fs" == "$target_fs" ]]; then
required_bytes=$((estimated_db_bytes * 2 + backup_bytes))
assert_available_space "$WORK_DIR" "$required_bytes" "work/target filesystem"
else
assert_available_space "$WORK_DIR" "$((estimated_db_bytes + backup_bytes))" "work dir"
assert_available_space "$target_dir" "$estimated_db_bytes" "target DB dir"
fi
}
check_request_body_artifacts() {
local sql_file
local result_file
sql_file="${WORK_DIR}/check-request-body-artifacts.sql"
result_file="${WORK_DIR}/request-body-artifacts.txt"
cat > "$sql_file" <<'SQL'
CREATE TEMP TABLE aether_request_body_artifacts (
artifact text PRIMARY KEY
) ON COMMIT PRESERVE ROWS;
DO $$
DECLARE
candidate record;
has_rows boolean;
BEGIN
IF EXISTS (
SELECT 1
FROM information_schema.tables
WHERE table_schema = 'public'
AND table_name = 'usage_body_blobs'
) THEN
EXECUTE 'SELECT EXISTS (SELECT 1 FROM public.usage_body_blobs LIMIT 1)'
INTO has_rows;
IF has_rows THEN
INSERT INTO aether_request_body_artifacts(artifact)
VALUES ('usage_body_blobs')
ON CONFLICT DO NOTHING;
END IF;
END IF;
IF EXISTS (
SELECT 1
FROM information_schema.tables
WHERE table_schema = 'public'
AND table_name = 'usage_http_audits'
) THEN
EXECUTE 'SELECT EXISTS (SELECT 1 FROM public.usage_http_audits LIMIT 1)'
INTO has_rows;
IF has_rows THEN
INSERT INTO aether_request_body_artifacts(artifact)
VALUES ('usage_http_audits')
ON CONFLICT DO NOTHING;
END IF;
END IF;
IF EXISTS (
SELECT 1
FROM information_schema.tables
WHERE table_schema = 'public'
AND table_name = 'usage'
) THEN
FOR candidate IN
SELECT unnest(ARRAY[
'request_body',
'response_body',
'provider_request_body',
'client_response_body',
'request_body_compressed',
'response_body_compressed',
'provider_request_body_compressed',
'client_response_body_compressed'
]) AS column_name
LOOP
IF EXISTS (
SELECT 1
FROM information_schema.columns
WHERE table_schema = 'public'
AND table_name = 'usage'
AND column_name = candidate.column_name
) THEN
EXECUTE format('SELECT EXISTS (SELECT 1 FROM public.usage WHERE %I IS NOT NULL LIMIT 1)', candidate.column_name)
INTO has_rows;
IF has_rows THEN
INSERT INTO aether_request_body_artifacts(artifact)
VALUES ('usage.' || candidate.column_name)
ON CONFLICT DO NOTHING;
END IF;
END IF;
END LOOP;
END IF;
END
$$;
SELECT artifact FROM aether_request_body_artifacts ORDER BY artifact;
SQL
run_psql_stdin "$sql_file" > "$result_file"
if [[ ! -s "$result_file" ]]; then
return
fi
if [[ "$REQUEST_BODY_MODE" == "omit" ]]; then
warn "source has request/response body details that will not be copied into single-node SQLite"
cat "$result_file" >&2
return
fi
warn "source has request/response body details that will be copied into single-node SQLite"
cat "$result_file" >&2
}
source_service_is_running() {
source_compose ps --services --status running | grep -Fxq "$1"
}
source_app_container_id() {
source_compose ps -q "$APP_SERVICE"
}
docker_image_id() {
local image="$1"
docker image inspect -f '{{.Id}}' "$image" 2>/dev/null || true
}
assert_source_and_target_images_match() {
local target_image="$1"
local source_container_id
local source_image_ref
local source_image_id
local target_image_id
source_container_id="$(source_app_container_id)"
[[ -n "$source_container_id" ]] || die "could not resolve running source app container for service: ${APP_SERVICE}"
source_image_ref="$(docker inspect -f '{{.Config.Image}}' "$source_container_id")"
source_image_id="$(docker inspect -f '{{.Image}}' "$source_container_id")"
target_image_id="$(docker_image_id "$target_image")"
[[ -n "$target_image_id" ]] || die "target single-node image is not available locally after pull: ${target_image}"
log "source app image: ${source_image_ref} (${source_image_id})"
log "target single-node image: ${target_image} (${target_image_id})"
if [[ "$source_image_id" != "$target_image_id" ]]; then
die "source app image and target single-node image are different; upgrade the source PG Compose app to ${target_image} before migration"
fi
}
assert_target_copy_command_available() {
local target_image="$1"
local help_output
help_output="$(docker run --rm --entrypoint aether-gateway "$target_image" copy --help 2>&1 || true)"
if [[ "$help_output" != *"--source-driver"* || "$help_output" != *"--target-driver"* || "$help_output" != *"--omit-request-body-details"* ]]; then
die "target single-node image does not support direct PG-to-SQLite copy; use a matching Aether release image"
fi
}
resolve_source_network() {
local container_id
local network
container_id="$(source_compose ps -q "$POSTGRES_SERVICE")"
[[ -n "$container_id" ]] || die "could not resolve container id for source Postgres service: ${POSTGRES_SERVICE}"
SOURCE_NETWORK=""
while IFS= read -r network; do
[[ -n "$network" ]] || continue
SOURCE_NETWORK="$network"
break
done < <(docker inspect -f '{{range $name, $_ := .NetworkSettings.Networks}}{{println $name}}{{end}}' "$container_id")
[[ -n "$SOURCE_NETWORK" ]] || die "could not resolve Docker network for source Postgres service: ${POSTGRES_SERVICE}"
}
prepare_target_compose() {
local template_abs
TARGET_COMPOSE="${TARGET_COMPOSE:-${SOURCE_COMPOSE_DIR}/docker-compose.single-node.yml}"
TARGET_COMPOSE_ABS="$(absolute_path_maybe_missing "$TARGET_COMPOSE")"
TARGET_COMPOSE_DIR="$(dirname "$TARGET_COMPOSE_ABS")"
template_abs="$(absolute_path "$TARGET_COMPOSE_TEMPLATE")"
[[ -f "$template_abs" ]] || die "target compose template not found: ${TARGET_COMPOSE_TEMPLATE}"
mkdir -p "$TARGET_COMPOSE_DIR"
if [[ ! -f "$TARGET_COMPOSE_ABS" || "$REPLACE_TARGET_COMPOSE" == "true" ]]; then
log "installing target single-node compose file at ${TARGET_COMPOSE_ABS}"
install -m 0644 "$template_abs" "$TARGET_COMPOSE_ABS"
else
log "keeping existing target compose file ${TARGET_COMPOSE_ABS}"
fi
TARGET_ENV="${TARGET_ENV:-${TARGET_COMPOSE_DIR}/.env.single-node}"
TARGET_ENV_ABS="$(absolute_path_maybe_missing "$TARGET_ENV")"
}
target_sqlite_file_exists() {
[[ -e "$TARGET_DB" || -e "${TARGET_DB}-wal" || -e "${TARGET_DB}-shm" ]]
}
finalize_target_db() {
local temp_db="$1"
local target_dir
local backup_path
local suffix
target_dir="$(dirname "$TARGET_DB")"
mkdir -p "$target_dir"
if target_sqlite_file_exists; then
[[ "$REPLACE_EXISTING" == "true" ]] || die "target DB already exists: ${TARGET_DB}; pass --replace-existing to replace it"
for suffix in "" "-wal" "-shm"; do
if [[ -e "${TARGET_DB}${suffix}" ]]; then
backup_path="${WORK_DIR}/$(basename "$TARGET_DB")${suffix}.backup.${NOW}"
log "backing up existing target SQLite file to ${backup_path}"
cp -p "${TARGET_DB}${suffix}" "$backup_path"
fi
done
fi
log "installing migrated SQLite DB at ${TARGET_DB}"
install -m 0640 "$temp_db" "$TARGET_DB"
for suffix in "-wal" "-shm"; do
if [[ -e "${temp_db}${suffix}" ]]; then
install -m 0640 "${temp_db}${suffix}" "${TARGET_DB}${suffix}"
else
rm -f "${TARGET_DB}${suffix}"
fi
done
}
cleanup_on_exit() {
local status=$?
if [[ "$status" -eq 0 ]]; then
return
fi
warn "migration failed with exit status ${status}"
if [[ "$APP_STOPPED" == "true" && "$CUTOVER_COMPLETE" != "true" && "$KEEP_SOURCE_STOPPED_ON_ERROR" != "true" ]]; then
warn "attempting to restart source compose app because cutover did not complete"
source_compose up -d "$APP_SERVICE" || warn "source app restart failed; check ${SOURCE_COMPOSE_ABS}"
fi
}
preflight() {
require_command docker
require_command awk
require_command df
docker compose version >/dev/null
SOURCE_COMPOSE_ABS="$(absolute_path "$SOURCE_COMPOSE")"
[[ -f "$SOURCE_COMPOSE_ABS" ]] || die "source compose file not found: ${SOURCE_COMPOSE}"
SOURCE_COMPOSE_DIR="$(dirname "$SOURCE_COMPOSE_ABS")"
SOURCE_ENV="${SOURCE_COMPOSE_DIR}/.env"
[[ -f "$SOURCE_ENV" ]] || die "source env file not found: ${SOURCE_ENV}"
NOW="$(date +%Y%m%d%H%M%S)"
if [[ -z "$WORK_DIR" ]]; then
WORK_DIR="${SOURCE_COMPOSE_DIR}/data/pg-compose-to-single-node-${NOW}"
fi
WORK_DIR="$(absolute_path_maybe_missing "$WORK_DIR")"
mkdir -p "$WORK_DIR"
prepare_target_compose
TARGET_DB="${TARGET_DB:-${TARGET_COMPOSE_DIR}/data/aether.db}"
TARGET_DB="$(absolute_path_maybe_missing "$TARGET_DB")"
DB_USER="$(env_file_get "$SOURCE_ENV" "DB_USER")"
DB_USER="${DB_USER:-postgres}"
DB_NAME="$(env_file_get "$SOURCE_ENV" "DB_NAME")"
DB_NAME="${DB_NAME:-aether}"
DB_PASSWORD="$(env_file_get "$SOURCE_ENV" "DB_PASSWORD")"
DB_PASSWORD="${DB_PASSWORD:-aether}"
[[ -n "$(env_file_get "$SOURCE_ENV" "JWT_SECRET_KEY")" ]] || die "source env must define JWT_SECRET_KEY"
if [[ -z "$(env_file_get "$SOURCE_ENV" "ENCRYPTION_KEY")" && -z "$(env_file_get "$SOURCE_ENV" "AETHER_GATEWAY_DATA_ENCRYPTION_KEY")" ]]; then
die "source env must define ENCRYPTION_KEY or AETHER_GATEWAY_DATA_ENCRYPTION_KEY"
fi
write_single_node_env "$TARGET_ENV_ABS"
chmod 0600 "$TARGET_ENV_ABS"
log "source compose: ${SOURCE_COMPOSE_ABS}"
log "source env: ${SOURCE_ENV}"
log "target compose: ${TARGET_COMPOSE_ABS}"
log "target env: ${TARGET_ENV_ABS}"
log "target SQLite DB: ${TARGET_DB}"
log "work dir: ${WORK_DIR}"
source_service_is_running "$POSTGRES_SERVICE" || die "source Postgres service is not running: ${POSTGRES_SERVICE}"
resolve_source_network
log "source Docker network: ${SOURCE_NETWORK}"
}
source_postgres_url() {
printf 'postgresql://%s:%s@%s:5432/%s' "$DB_USER" "$DB_PASSWORD" "$POSTGRES_SERVICE" "$DB_NAME"
}
copy_source_to_sqlite() {
local target_temp_db="$1"
local target_url
local image
local -a copy_args
image="$(env_file_get "$TARGET_ENV_ABS" "APP_IMAGE")"
[[ -n "$image" ]] || die "target env must define APP_IMAGE"
target_url="sqlite:///migration/$(basename "$target_temp_db")"
rm -f "$target_temp_db" "${target_temp_db}-wal" "${target_temp_db}-shm"
target_run_app "$target_url" --migrate
copy_args=(
copy
--source-driver postgres
--source-url "$(source_postgres_url)"
--target-driver sqlite
--target-url "$target_url"
)
if [[ "$REQUEST_BODY_MODE" == "omit" ]]; then
copy_args+=(--omit-request-body-details)
fi
docker run --rm \
--network "$SOURCE_NETWORK" \
-v "${WORK_DIR}:/migration" \
--env-file "$TARGET_ENV_ABS" \
-e AETHER_LOG_DESTINATION=stdout \
"$image" \
"${copy_args[@]}"
}
main() {
local preflight_db
local dry_run_db
local target_temp_db
local target_image
parse_args "$@"
normalize_request_body_mode
trap cleanup_on_exit EXIT
preflight
preflight_db="${WORK_DIR}/single-node-preflight.db"
dry_run_db="${WORK_DIR}/dry-run-target-aether.db"
target_temp_db="${WORK_DIR}/target-aether.db"
if target_sqlite_file_exists && [[ "$REPLACE_EXISTING" != "true" && "$DRY_RUN" != "true" ]]; then
die "target DB already exists: ${TARGET_DB}; pass --replace-existing to replace it"
fi
log "pulling target single-node image before downtime"
target_compose pull "$SINGLE_NODE_SERVICE"
target_image="$(env_file_get "$TARGET_ENV_ABS" "APP_IMAGE")"
[[ -n "$target_image" ]] || die "target env must define APP_IMAGE"
log "checking target image copy command is available"
assert_target_copy_command_available "$target_image"
log "checking source app image matches target single-node image"
assert_source_and_target_images_match "$target_image"
log "preflighting target SQLite schema migration"
rm -f "$preflight_db" "${preflight_db}-wal" "${preflight_db}-shm"
target_run_app "sqlite:///migration/$(basename "$preflight_db")" --migrate
log "checking request body detail policy"
check_request_body_artifacts
log "checking available disk space before copy"
check_disk_space
if [[ "$DRY_RUN" == "true" ]]; then
warn "dry-run copy happens while the source app may still be writing; use only for rehearsal"
log "copying source Postgres tables directly into dry-run SQLite target"
copy_source_to_sqlite "$dry_run_db"
log "dry run complete; temporary SQLite DB is ${dry_run_db}"
return
fi
log "stopping source app service; Postgres and Redis stay running"
source_compose stop "$APP_SERVICE"
APP_STOPPED="true"
log "checking request body detail policy again after the app has stopped"
check_request_body_artifacts
log "copying source Postgres tables directly into temporary SQLite DB"
copy_source_to_sqlite "$target_temp_db"
finalize_target_db "$target_temp_db"
log "removing stopped source app container to free the app container name"
source_compose rm -f "$APP_SERVICE"
log "starting single-node compose app"
target_compose up -d "$SINGLE_NODE_SERVICE"
CUTOVER_COMPLETE="true"
log "migration complete"
log "source Postgres/Redis volumes were left in place for rollback"
}
main "$@"

File diff suppressed because it is too large Load Diff