i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-01 17:00:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add postgres to single-node migration

Browse Source
This commit is contained in:
HsungKayphoon
2026-05-16 15:22:42 +08:00
parent 75b7319465
commit 4b12ec8913
13 changed files with 5386 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.github/workflows/release.yml vendored
View File

@@ -258,13 +258,17 @@ jobs:
root="package/${bundle}"
mkdir -p \
"${root}/bin" \
"${root}/frontend"
"${root}/frontend" \
"${root}/scripts"
install -m 0755 "artifacts/aether-gateway-${platform}-${arch}/aether-gateway" "${root}/bin/aether-gateway"
cp -R artifacts/frontend-dist/. "${root}/frontend/"
sed "s/^VERSION=\"\${AETHER_VERSION:-}\"/VERSION=\"\${AETHER_VERSION:-${VERSION}}\"/" install.sh > "${root}/install.sh"
chmod 0755 "${root}/install.sh"
install -m 0644 docker-compose.yml "${root}/docker-compose.yml"
install -m 0644 docker-compose.single-node.yml "${root}/docker-compose.single-node.yml"
install -m 0755 scripts/migrate-pg-compose-to-single-node.sh "${root}/scripts/migrate-pg-compose-to-single-node.sh"
install -m 0755 scripts/migrate-pg-to-single-node.sh "${root}/scripts/migrate-pg-to-single-node.sh"
install -m 0644 .env.example "${root}/.env.example"
install -m 0755 generate_keys.sh "${root}/generate_keys.sh"
install -m 0644 README.md "${root}/README.md"

8
README.md
View File

@@ -48,14 +48,14 @@ cp .env.example .env
./generate_keys.sh
# 编辑 .env 设置 ADMIN_PASSWORD
# 3. 首次部署 / 更新 (从以下数据库、内存策略任选其一)
# 3. 首次部署 / 更新 (从以下部署形态任选其一)
# Postgres + Redis (适用于企业或多人使用)
docker compose pull && docker compose up -d
# 仅SQLite (适用于个人用户或朋友分享)
docker compose -f docker-compose.sqlite.yml pull && docker compose -f docker-compose.sqlite.yml up -d
# Single Node (适用于个人用户或朋友分享)
docker compose -f docker-compose.single-node.yml pull && docker compose -f docker-compose.single-node.yml up -d
```
### 一键安装（可选部署方式 Linux: systemd; Mac: launchd）
### 一键安装（默认 Single Node：Linux systemd / macOS launchd + SQLite）
```bash
cd Aether && cd Aether

98
apps/aether-gateway/src/main.rs
View File

@@ -9,7 +9,10 @@ use clap::{Args as ClapArgs, Parser, Subcommand, ValueEnum};
use tracing::{debug, info, warn};
use aether_crypto::warm_python_fernet_secret;
use aether_data::lifecycle::export::{export_database_jsonl, import_database_jsonl, ExportDomain};
use aether_data::lifecycle::export::{
copy_database_records, export_database_jsonl, import_database_jsonl, DataCopyOptions,
ExportDomain,
};
use aether_data::{DatabaseDriver, SqlDatabaseConfig, SqlPoolConfig, DEFAULT_SQLITE_DATABASE_URL};
use aether_gateway::{
attach_static_frontend, build_router_with_state, set_gateway_frontdoor_app_port, AppState,
@@ -77,6 +80,8 @@ enum ExportDomainArg {
Wallets,
Usage,
Billing,
Stats,
Auxiliary,
}
impl From<ExportDomainArg> for ExportDomain {
@@ -93,6 +98,8 @@ impl From<ExportDomainArg> for ExportDomain {
ExportDomainArg::Wallets => ExportDomain::Wallets,
ExportDomainArg::Usage => ExportDomain::Usage,
ExportDomainArg::Billing => ExportDomain::Billing,
ExportDomainArg::Stats => ExportDomain::Stats,
ExportDomainArg::Auxiliary => ExportDomain::Auxiliary,
}
}
}
@@ -579,6 +586,8 @@ enum DataCommand {
Export(DataExportArgs),
/// Import database-neutral JSONL into the selected SQL database.
Import(DataImportArgs),
/// Copy persistent SQL data directly between two databases without a JSONL file.
Copy(DataCopyArgs),
}
#[derive(ClapArgs, Debug, Clone)]
@@ -602,6 +611,27 @@ struct DataImportArgs {
input: PathBuf,
}
#[derive(ClapArgs, Debug, Clone)]
struct DataCopyArgs {
#[arg(long, value_enum)]
source_driver: DatabaseDriverArg,
#[arg(long)]
source_url: String,
#[arg(long, value_enum)]
target_driver: DatabaseDriverArg,
#[arg(long)]
target_url: String,
#[arg(long, value_enum, value_delimiter = ',')]
domains: Vec<ExportDomainArg>,
#[arg(long)]
omit_request_body_details: bool,
}
impl GatewayLoggingArgs {
fn apply_to_runtime_config(
&self,
@@ -1251,6 +1281,7 @@ async fn run_data_command(command: &DataCommand) -> Result<(), Box<dyn std::erro
match command {
DataCommand::Export(args) => run_data_export(args).await,
DataCommand::Import(args) => run_data_import(args).await,
DataCommand::Copy(args) => run_data_copy(args).await,
}
}
@@ -1267,11 +1298,11 @@ fn required_sql_database_config(
}
fn requested_export_domains(args: &DataExportArgs) -> Vec<ExportDomain> {
args.domains
.iter()
.copied()
.map(Into::into)
.collect::<Vec<_>>()
requested_domains(&args.domains)
}
fn requested_domains(domains: &[ExportDomainArg]) -> Vec<ExportDomain> {
domains.iter().copied().map(Into::into).collect::<Vec<_>>()
}
fn current_unix_secs() -> Result<u64, std::time::SystemTimeError> {
@@ -1324,6 +1355,61 @@ async fn run_data_import(args: &DataImportArgs) -> Result<(), Box<dyn std::error
Ok(())
}
fn copy_database_config(
driver: DatabaseDriverArg,
url: &str,
label: &str,
) -> Result<SqlDatabaseConfig, Box<dyn std::error::Error>> {
let url = url.trim();
if url.is_empty() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
format!("{label} database URL must not be empty"),
)
.into());
}
let driver = DatabaseDriver::from(driver);
Ok(SqlDatabaseConfig::new(
driver,
url,
SqlPoolConfig {
require_ssl: false,
..SqlPoolConfig::default()
},
)?)
}
async fn run_data_copy(args: &DataCopyArgs) -> Result<(), Box<dyn std::error::Error>> {
let source = copy_database_config(args.source_driver, &args.source_url, "source")?;
let target = copy_database_config(args.target_driver, &args.target_url, "target")?;
let source_driver = source.driver;
let target_driver = target.driver;
let domains = requested_domains(&args.domains);
let created_at_unix_secs = current_unix_secs()?;
let imported = copy_database_records(
source,
target,
domains,
created_at_unix_secs,
DataCopyOptions {
omit_request_body_details: args.omit_request_body_details,
},
)
.await?;
info!(
source_driver = %source_driver,
target_driver = %target_driver,
imported,
"database copy complete"
);
println!(
"copied {} records from {} to {} without a JSONL file",
imported, source_driver, target_driver
);
Ok(())
}
async fn run_explicit_migrations(args: &Args) -> Result<(), Box<dyn std::error::Error>> {
if args.data.effective_sql_database_config().is_none() {
return Err(std::io::Error::new(

34
crates/aether-data/migrations/sqlite/20260513000000_add_usage_body_detail_tables.sql Normal file
View File

@@ -0,0 +1,34 @@
CREATE TABLE IF NOT EXISTS usage_body_blobs (
body_ref TEXT PRIMARY KEY NOT NULL,
request_id TEXT NOT NULL,
body_field TEXT NOT NULL,
payload_gzip BLOB NOT NULL,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
UNIQUE (request_id, body_field),
FOREIGN KEY (request_id) REFERENCES "usage"(request_id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS usage_body_blobs_request_id_idx
ON usage_body_blobs (request_id);
CREATE TABLE IF NOT EXISTS usage_http_audits (
request_id TEXT PRIMARY KEY NOT NULL,
request_headers TEXT,
provider_request_headers TEXT,
response_headers TEXT,
client_response_headers TEXT,
request_body_ref TEXT,
provider_request_body_ref TEXT,
response_body_ref TEXT,
client_response_body_ref TEXT,
request_body_state TEXT,
provider_request_body_state TEXT,
response_body_state TEXT,
client_response_body_state TEXT,
body_capture_mode TEXT NOT NULL DEFAULT 'none',
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (request_id) REFERENCES "usage"(request_id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS usage_http_audits_updated_at_idx
ON usage_http_audits (updated_at);

570
crates/aether-data/migrations/sqlite/20260514000000_expand_single_node_migration_tables.sql Normal file
View File

@@ -0,0 +1,570 @@
CREATE TABLE IF NOT EXISTS api_key_provider_mappings (
id TEXT PRIMARY KEY NOT NULL,
api_key_id TEXT NOT NULL,
provider_id TEXT NOT NULL,
priority_adjustment INTEGER NOT NULL DEFAULT 0,
weight_multiplier REAL NOT NULL DEFAULT 1,
is_enabled INTEGER NOT NULL DEFAULT 1,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
UNIQUE (api_key_id, provider_id)
);
CREATE INDEX IF NOT EXISTS api_key_provider_mappings_api_key_id_idx
ON api_key_provider_mappings (api_key_id);
CREATE INDEX IF NOT EXISTS api_key_provider_mappings_provider_id_idx
ON api_key_provider_mappings (provider_id);
CREATE INDEX IF NOT EXISTS idx_apikey_provider_enabled
ON api_key_provider_mappings (api_key_id, is_enabled);
CREATE TABLE IF NOT EXISTS provider_usage_tracking (
id TEXT PRIMARY KEY NOT NULL,
provider_id TEXT NOT NULL,
window_start INTEGER NOT NULL,
window_end INTEGER NOT NULL,
total_requests INTEGER NOT NULL DEFAULT 0,
successful_requests INTEGER NOT NULL DEFAULT 0,
failed_requests INTEGER NOT NULL DEFAULT 0,
avg_response_time_ms REAL NOT NULL DEFAULT 0,
total_response_time_ms REAL NOT NULL DEFAULT 0,
total_cost_usd REAL NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE INDEX IF NOT EXISTS provider_usage_tracking_provider_id_idx
ON provider_usage_tracking (provider_id);
CREATE INDEX IF NOT EXISTS provider_usage_tracking_window_start_idx
ON provider_usage_tracking (window_start);
CREATE INDEX IF NOT EXISTS idx_provider_window
ON provider_usage_tracking (provider_id, window_start);
CREATE INDEX IF NOT EXISTS idx_window_time
ON provider_usage_tracking (window_start, window_end);
CREATE TABLE IF NOT EXISTS usage_routing_snapshots (
request_id TEXT PRIMARY KEY NOT NULL,
candidate_id TEXT,
candidate_index INTEGER,
key_name TEXT,
planner_kind TEXT,
route_family TEXT,
route_kind TEXT,
execution_path TEXT,
local_execution_runtime_miss_reason TEXT,
selected_provider_id TEXT,
selected_endpoint_id TEXT,
selected_provider_api_key_id TEXT,
has_format_conversion INTEGER,
created_at INTEGER NOT NULL DEFAULT 0,
updated_at INTEGER NOT NULL DEFAULT 0,
FOREIGN KEY (request_id) REFERENCES "usage"(request_id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS ix_usage_routing_snapshots_route_family_kind
ON usage_routing_snapshots (route_family, route_kind);
CREATE INDEX IF NOT EXISTS ix_usage_routing_snapshots_candidate_id
ON usage_routing_snapshots (candidate_id);
CREATE TABLE IF NOT EXISTS stats_summary (
id TEXT PRIMARY KEY NOT NULL,
cutoff_date INTEGER NOT NULL,
all_time_requests INTEGER NOT NULL DEFAULT 0,
all_time_success_requests INTEGER NOT NULL DEFAULT 0,
all_time_error_requests INTEGER NOT NULL DEFAULT 0,
all_time_input_tokens INTEGER NOT NULL DEFAULT 0,
all_time_output_tokens INTEGER NOT NULL DEFAULT 0,
all_time_cache_creation_tokens INTEGER NOT NULL DEFAULT 0,
all_time_cache_read_tokens INTEGER NOT NULL DEFAULT 0,
all_time_cost REAL NOT NULL DEFAULT 0,
all_time_actual_cost REAL NOT NULL DEFAULT 0,
total_users INTEGER NOT NULL DEFAULT 0,
active_users INTEGER NOT NULL DEFAULT 0,
total_api_keys INTEGER NOT NULL DEFAULT 0,
active_api_keys INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS user_model_usage_counts (
id TEXT PRIMARY KEY NOT NULL,
user_id TEXT NOT NULL,
model TEXT NOT NULL,
usage_count INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
UNIQUE (user_id, model)
);
CREATE INDEX IF NOT EXISTS idx_user_model_usage_user
ON user_model_usage_counts (user_id);
CREATE INDEX IF NOT EXISTS idx_user_model_usage_model
ON user_model_usage_counts (model);
ALTER TABLE stats_hourly
ADD COLUMN response_time_sum_ms REAL NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly
ADD COLUMN response_time_samples INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly
ADD COLUMN cache_hit_total_requests INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly
ADD COLUMN cache_hit_requests INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly
ADD COLUMN completed_total_requests INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly
ADD COLUMN completed_cache_hit_requests INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly
ADD COLUMN completed_input_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly
ADD COLUMN completed_cache_creation_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly
ADD COLUMN completed_cache_read_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly
ADD COLUMN completed_total_input_context INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly
ADD COLUMN completed_cache_creation_cost REAL NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly
ADD COLUMN completed_cache_read_cost REAL NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly
ADD COLUMN settled_total_cost REAL NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly
ADD COLUMN settled_total_requests INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly
ADD COLUMN settled_input_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly
ADD COLUMN settled_output_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly
ADD COLUMN settled_cache_creation_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly
ADD COLUMN settled_cache_read_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly
ADD COLUMN settled_first_finalized_at_unix_secs INTEGER;
ALTER TABLE stats_hourly
ADD COLUMN settled_last_finalized_at_unix_secs INTEGER;
ALTER TABLE stats_hourly_user
ADD COLUMN cache_creation_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly_user
ADD COLUMN cache_read_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly_user
ADD COLUMN actual_total_cost REAL NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly_user
ADD COLUMN response_time_sum_ms REAL NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly_user
ADD COLUMN response_time_samples INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly_user
ADD COLUMN settled_total_cost REAL NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly_user
ADD COLUMN settled_total_requests INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly_user
ADD COLUMN settled_input_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly_user
ADD COLUMN settled_output_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly_user
ADD COLUMN settled_cache_creation_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly_user
ADD COLUMN settled_cache_read_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly_user
ADD COLUMN settled_first_finalized_at_unix_secs INTEGER;
ALTER TABLE stats_hourly_user
ADD COLUMN settled_last_finalized_at_unix_secs INTEGER;
ALTER TABLE stats_hourly_user_model
ADD COLUMN response_time_sum_ms REAL NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly_user_model
ADD COLUMN response_time_samples INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly_model
ADD COLUMN response_time_sum_ms REAL NOT NULL DEFAULT 0;
ALTER TABLE stats_hourly_model
ADD COLUMN response_time_samples INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_daily
ADD COLUMN effective_input_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_daily
ADD COLUMN total_input_context INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_daily
ADD COLUMN response_time_sum_ms REAL NOT NULL DEFAULT 0;
ALTER TABLE stats_daily
ADD COLUMN response_time_samples INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_daily
ADD COLUMN cache_creation_ephemeral_5m_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_daily
ADD COLUMN cache_creation_ephemeral_1h_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_daily
ADD COLUMN cache_hit_total_requests INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_daily
ADD COLUMN cache_hit_requests INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_daily
ADD COLUMN completed_total_requests INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_daily
ADD COLUMN completed_cache_hit_requests INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_daily
ADD COLUMN completed_input_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_daily
ADD COLUMN completed_cache_creation_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_daily
ADD COLUMN completed_cache_read_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_daily
ADD COLUMN completed_total_input_context INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_daily
ADD COLUMN completed_cache_creation_cost REAL NOT NULL DEFAULT 0;
ALTER TABLE stats_daily
ADD COLUMN completed_cache_read_cost REAL NOT NULL DEFAULT 0;
ALTER TABLE stats_daily
ADD COLUMN settled_total_cost REAL NOT NULL DEFAULT 0;
ALTER TABLE stats_daily
ADD COLUMN settled_total_requests INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_daily
ADD COLUMN settled_input_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_daily
ADD COLUMN settled_output_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_daily
ADD COLUMN settled_cache_creation_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_daily
ADD COLUMN settled_cache_read_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_daily
ADD COLUMN settled_first_finalized_at_unix_secs INTEGER;
ALTER TABLE stats_daily
ADD COLUMN settled_last_finalized_at_unix_secs INTEGER;
ALTER TABLE stats_daily_model
ADD COLUMN cache_creation_ephemeral_5m_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_daily_model
ADD COLUMN cache_creation_ephemeral_1h_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_daily_model
ADD COLUMN response_time_sum_ms REAL NOT NULL DEFAULT 0;
ALTER TABLE stats_daily_model
ADD COLUMN response_time_samples INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_user_daily
ADD COLUMN actual_total_cost REAL NOT NULL DEFAULT 0;
ALTER TABLE stats_user_daily
ADD COLUMN response_time_sum_ms REAL NOT NULL DEFAULT 0;
ALTER TABLE stats_user_daily
ADD COLUMN response_time_samples INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_user_daily
ADD COLUMN effective_input_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_user_daily
ADD COLUMN total_input_context INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_user_daily
ADD COLUMN cache_creation_cost REAL NOT NULL DEFAULT 0;
ALTER TABLE stats_user_daily
ADD COLUMN cache_read_cost REAL NOT NULL DEFAULT 0;
ALTER TABLE stats_user_daily
ADD COLUMN cache_creation_ephemeral_5m_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_user_daily
ADD COLUMN cache_creation_ephemeral_1h_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_user_daily
ADD COLUMN settled_total_cost REAL NOT NULL DEFAULT 0;
ALTER TABLE stats_user_daily
ADD COLUMN settled_total_requests INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_user_daily
ADD COLUMN settled_input_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_user_daily
ADD COLUMN settled_output_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_user_daily
ADD COLUMN settled_cache_creation_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_user_daily
ADD COLUMN settled_cache_read_tokens INTEGER NOT NULL DEFAULT 0;
ALTER TABLE stats_user_daily
ADD COLUMN settled_first_finalized_at_unix_secs INTEGER;
ALTER TABLE stats_user_daily
ADD COLUMN settled_last_finalized_at_unix_secs INTEGER;
CREATE TABLE IF NOT EXISTS stats_user_summary (
id TEXT PRIMARY KEY NOT NULL,
user_id TEXT NOT NULL,
username TEXT,
cutoff_date INTEGER NOT NULL,
all_time_requests INTEGER NOT NULL DEFAULT 0,
all_time_success_requests INTEGER NOT NULL DEFAULT 0,
all_time_error_requests INTEGER NOT NULL DEFAULT 0,
all_time_input_tokens INTEGER NOT NULL DEFAULT 0,
all_time_output_tokens INTEGER NOT NULL DEFAULT 0,
all_time_cache_creation_tokens INTEGER NOT NULL DEFAULT 0,
all_time_cache_read_tokens INTEGER NOT NULL DEFAULT 0,
all_time_cost REAL NOT NULL DEFAULT 0,
all_time_actual_cost REAL NOT NULL DEFAULT 0,
active_days INTEGER NOT NULL DEFAULT 0,
first_active_date INTEGER,
last_active_date INTEGER,
created_at INTEGER NOT NULL DEFAULT 0,
updated_at INTEGER NOT NULL DEFAULT 0,
UNIQUE (user_id)
);
CREATE INDEX IF NOT EXISTS idx_stats_user_summary_cutoff_date
ON stats_user_summary (cutoff_date);
CREATE TABLE IF NOT EXISTS stats_user_daily_model (
id TEXT PRIMARY KEY NOT NULL,
user_id TEXT NOT NULL,
username TEXT,
date INTEGER NOT NULL,
model TEXT NOT NULL,
total_requests INTEGER NOT NULL DEFAULT 0,
success_requests INTEGER NOT NULL DEFAULT 0,
input_tokens INTEGER NOT NULL DEFAULT 0,
effective_input_tokens INTEGER NOT NULL DEFAULT 0,
output_tokens INTEGER NOT NULL DEFAULT 0,
total_tokens INTEGER NOT NULL DEFAULT 0,
total_input_context INTEGER NOT NULL DEFAULT 0,
cache_creation_tokens INTEGER NOT NULL DEFAULT 0,
cache_creation_ephemeral_5m_tokens INTEGER NOT NULL DEFAULT 0,
cache_creation_ephemeral_1h_tokens INTEGER NOT NULL DEFAULT 0,
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
total_cost REAL NOT NULL DEFAULT 0,
actual_total_cost REAL NOT NULL DEFAULT 0,
response_time_sum_ms REAL NOT NULL DEFAULT 0,
response_time_samples INTEGER NOT NULL DEFAULT 0,
successful_response_time_sum_ms REAL NOT NULL DEFAULT 0,
successful_response_time_samples INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL DEFAULT 0,
updated_at INTEGER NOT NULL DEFAULT 0,
UNIQUE (user_id, date, model)
);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_model_date
ON stats_user_daily_model (date);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_model_user_id
ON stats_user_daily_model (user_id);
CREATE TABLE IF NOT EXISTS stats_user_daily_provider (
id TEXT PRIMARY KEY NOT NULL,
user_id TEXT NOT NULL,
username TEXT,
date INTEGER NOT NULL,
provider_name TEXT NOT NULL,
total_requests INTEGER NOT NULL DEFAULT 0,
success_requests INTEGER NOT NULL DEFAULT 0,
input_tokens INTEGER NOT NULL DEFAULT 0,
effective_input_tokens INTEGER NOT NULL DEFAULT 0,
output_tokens INTEGER NOT NULL DEFAULT 0,
total_tokens INTEGER NOT NULL DEFAULT 0,
total_input_context INTEGER NOT NULL DEFAULT 0,
cache_creation_tokens INTEGER NOT NULL DEFAULT 0,
cache_creation_ephemeral_5m_tokens INTEGER NOT NULL DEFAULT 0,
cache_creation_ephemeral_1h_tokens INTEGER NOT NULL DEFAULT 0,
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
total_cost REAL NOT NULL DEFAULT 0,
actual_total_cost REAL NOT NULL DEFAULT 0,
response_time_sum_ms REAL NOT NULL DEFAULT 0,
response_time_samples INTEGER NOT NULL DEFAULT 0,
successful_response_time_sum_ms REAL NOT NULL DEFAULT 0,
successful_response_time_samples INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL DEFAULT 0,
updated_at INTEGER NOT NULL DEFAULT 0,
UNIQUE (user_id, date, provider_name)
);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_provider_date
ON stats_user_daily_provider (date);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_provider_user_id
ON stats_user_daily_provider (user_id);
CREATE TABLE IF NOT EXISTS stats_user_daily_api_format (
id TEXT PRIMARY KEY NOT NULL,
user_id TEXT NOT NULL,
username TEXT,
date INTEGER NOT NULL,
api_format TEXT NOT NULL,
total_requests INTEGER NOT NULL DEFAULT 0,
success_requests INTEGER NOT NULL DEFAULT 0,
input_tokens INTEGER NOT NULL DEFAULT 0,
effective_input_tokens INTEGER NOT NULL DEFAULT 0,
output_tokens INTEGER NOT NULL DEFAULT 0,
total_tokens INTEGER NOT NULL DEFAULT 0,
total_input_context INTEGER NOT NULL DEFAULT 0,
cache_creation_tokens INTEGER NOT NULL DEFAULT 0,
cache_creation_ephemeral_5m_tokens INTEGER NOT NULL DEFAULT 0,
cache_creation_ephemeral_1h_tokens INTEGER NOT NULL DEFAULT 0,
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
total_cost REAL NOT NULL DEFAULT 0,
actual_total_cost REAL NOT NULL DEFAULT 0,
response_time_sum_ms REAL NOT NULL DEFAULT 0,
response_time_samples INTEGER NOT NULL DEFAULT 0,
successful_response_time_sum_ms REAL NOT NULL DEFAULT 0,
successful_response_time_samples INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL DEFAULT 0,
updated_at INTEGER NOT NULL DEFAULT 0,
UNIQUE (user_id, date, api_format)
);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_api_format_date
ON stats_user_daily_api_format (date);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_api_format_user_id
ON stats_user_daily_api_format (user_id);
CREATE TABLE IF NOT EXISTS stats_daily_model_provider (
id TEXT PRIMARY KEY NOT NULL,
date INTEGER NOT NULL,
model TEXT NOT NULL,
provider_name TEXT NOT NULL,
total_requests INTEGER NOT NULL DEFAULT 0,
total_tokens INTEGER NOT NULL DEFAULT 0,
total_cost REAL NOT NULL DEFAULT 0,
response_time_sum_ms REAL NOT NULL DEFAULT 0,
response_time_samples INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL DEFAULT 0,
updated_at INTEGER NOT NULL DEFAULT 0,
UNIQUE (date, model, provider_name)
);
CREATE INDEX IF NOT EXISTS idx_stats_daily_model_provider_date
ON stats_daily_model_provider (date);
CREATE INDEX IF NOT EXISTS idx_stats_daily_model_provider_date_model_provider
ON stats_daily_model_provider (date, model, provider_name);
CREATE TABLE IF NOT EXISTS stats_user_daily_model_provider (
id TEXT PRIMARY KEY NOT NULL,
user_id TEXT NOT NULL,
username TEXT,
date INTEGER NOT NULL,
model TEXT NOT NULL,
provider_name TEXT NOT NULL,
total_requests INTEGER NOT NULL DEFAULT 0,
total_tokens INTEGER NOT NULL DEFAULT 0,
total_cost REAL NOT NULL DEFAULT 0,
response_time_sum_ms REAL NOT NULL DEFAULT 0,
response_time_samples INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL DEFAULT 0,
updated_at INTEGER NOT NULL DEFAULT 0,
UNIQUE (user_id, date, model, provider_name)
);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_model_provider_date
ON stats_user_daily_model_provider (date);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_model_provider_user_date
ON stats_user_daily_model_provider (user_id, date);
CREATE TABLE IF NOT EXISTS stats_daily_cost_savings (
id TEXT PRIMARY KEY NOT NULL,
date INTEGER NOT NULL,
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
cache_read_cost REAL NOT NULL DEFAULT 0,
cache_creation_cost REAL NOT NULL DEFAULT 0,
estimated_full_cost REAL NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL DEFAULT 0,
updated_at INTEGER NOT NULL DEFAULT 0,
UNIQUE (date)
);
CREATE INDEX IF NOT EXISTS idx_stats_daily_cost_savings_date
ON stats_daily_cost_savings (date);
CREATE TABLE IF NOT EXISTS stats_daily_cost_savings_provider (
id TEXT PRIMARY KEY NOT NULL,
date INTEGER NOT NULL,
provider_name TEXT NOT NULL,
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
cache_read_cost REAL NOT NULL DEFAULT 0,
cache_creation_cost REAL NOT NULL DEFAULT 0,
estimated_full_cost REAL NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL DEFAULT 0,
updated_at INTEGER NOT NULL DEFAULT 0,
UNIQUE (date, provider_name)
);
CREATE INDEX IF NOT EXISTS idx_stats_daily_cost_savings_provider_date
ON stats_daily_cost_savings_provider (date);
CREATE INDEX IF NOT EXISTS idx_stats_daily_cost_savings_provider_date_provider
ON stats_daily_cost_savings_provider (date, provider_name);
CREATE TABLE IF NOT EXISTS stats_daily_cost_savings_model (
id TEXT PRIMARY KEY NOT NULL,
date INTEGER NOT NULL,
model TEXT NOT NULL,
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
cache_read_cost REAL NOT NULL DEFAULT 0,
cache_creation_cost REAL NOT NULL DEFAULT 0,
estimated_full_cost REAL NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL DEFAULT 0,
updated_at INTEGER NOT NULL DEFAULT 0,
UNIQUE (date, model)
);
CREATE INDEX IF NOT EXISTS idx_stats_daily_cost_savings_model_date
ON stats_daily_cost_savings_model (date);
CREATE INDEX IF NOT EXISTS idx_stats_daily_cost_savings_model_date_model
ON stats_daily_cost_savings_model (date, model);
CREATE TABLE IF NOT EXISTS stats_daily_cost_savings_model_provider (
id TEXT PRIMARY KEY NOT NULL,
date INTEGER NOT NULL,
model TEXT NOT NULL,
provider_name TEXT NOT NULL,
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
cache_read_cost REAL NOT NULL DEFAULT 0,
cache_creation_cost REAL NOT NULL DEFAULT 0,
estimated_full_cost REAL NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL DEFAULT 0,
updated_at INTEGER NOT NULL DEFAULT 0,
UNIQUE (date, model, provider_name)
);
CREATE INDEX IF NOT EXISTS idx_stats_daily_cost_savings_model_provider_date
ON stats_daily_cost_savings_model_provider (date);
CREATE INDEX IF NOT EXISTS idx_stats_daily_cost_savings_model_provider_date_dims
ON stats_daily_cost_savings_model_provider (date, model, provider_name);
CREATE TABLE IF NOT EXISTS stats_user_daily_cost_savings (
id TEXT PRIMARY KEY NOT NULL,
user_id TEXT NOT NULL,
username TEXT,
date INTEGER NOT NULL,
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
cache_read_cost REAL NOT NULL DEFAULT 0,
cache_creation_cost REAL NOT NULL DEFAULT 0,
estimated_full_cost REAL NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL DEFAULT 0,
updated_at INTEGER NOT NULL DEFAULT 0,
UNIQUE (user_id, date)
);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_cost_savings_date
ON stats_user_daily_cost_savings (date);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_cost_savings_user_date
ON stats_user_daily_cost_savings (user_id, date);
CREATE TABLE IF NOT EXISTS stats_user_daily_cost_savings_provider (
id TEXT PRIMARY KEY NOT NULL,
user_id TEXT NOT NULL,
username TEXT,
date INTEGER NOT NULL,
provider_name TEXT NOT NULL,
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
cache_read_cost REAL NOT NULL DEFAULT 0,
cache_creation_cost REAL NOT NULL DEFAULT 0,
estimated_full_cost REAL NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL DEFAULT 0,
updated_at INTEGER NOT NULL DEFAULT 0,
UNIQUE (user_id, date, provider_name)
);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_cost_savings_provider_date
ON stats_user_daily_cost_savings_provider (date);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_cost_savings_provider_user_date
ON stats_user_daily_cost_savings_provider (user_id, date);
CREATE TABLE IF NOT EXISTS stats_user_daily_cost_savings_model (
id TEXT PRIMARY KEY NOT NULL,
user_id TEXT NOT NULL,
username TEXT,
date INTEGER NOT NULL,
model TEXT NOT NULL,
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
cache_read_cost REAL NOT NULL DEFAULT 0,
cache_creation_cost REAL NOT NULL DEFAULT 0,
estimated_full_cost REAL NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL DEFAULT 0,
updated_at INTEGER NOT NULL DEFAULT 0,
UNIQUE (user_id, date, model)
);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_cost_savings_model_date
ON stats_user_daily_cost_savings_model (date);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_cost_savings_model_user_date
ON stats_user_daily_cost_savings_model (user_id, date);
CREATE TABLE IF NOT EXISTS stats_user_daily_cost_savings_model_provider (
id TEXT PRIMARY KEY NOT NULL,
user_id TEXT NOT NULL,
username TEXT,
date INTEGER NOT NULL,
model TEXT NOT NULL,
provider_name TEXT NOT NULL,
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
cache_read_cost REAL NOT NULL DEFAULT 0,
cache_creation_cost REAL NOT NULL DEFAULT 0,
estimated_full_cost REAL NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL DEFAULT 0,
updated_at INTEGER NOT NULL DEFAULT 0,
UNIQUE (user_id, date, model, provider_name)
);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_cost_savings_model_provider_date
ON stats_user_daily_cost_savings_model_provider (date);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_cost_savings_model_provider_user_date
ON stats_user_daily_cost_savings_model_provider (user_id, date);

1357
crates/aether-data/src/lifecycle/export.rs
View File

File diff suppressed because it is too large Load Diff

2
crates/aether-data/src/lifecycle/migrate/tests.rs
View File

@@ -603,6 +603,8 @@ fn mysql_and_sqlite_migrations_include_enabled_incrementals() {
20260512000000,
20260512090000,
20260512110000,
20260513000000,
20260514000000,
]
);
}

2
docker-compose.sqlite.yml → docker-compose.single-node.yml
View File

@@ -3,7 +3,7 @@ services:
image: ${APP_IMAGE:-ghcr.io/fawney19/aether:latest}
container_name: aether-app
env_file:
- .env
- ${AETHER_ENV_FILE:-.env}
environment:
TZ: Asia/Shanghai
AETHER_DATABASE_DRIVER: sqlite

246
docs/operations/pg-to-single-node-migration.md Normal file
View File

@@ -0,0 +1,246 @@
# Postgres to Aether Single Node Migration
Chinese version: [pg-to-single-node-migration.zh-CN.md](pg-to-single-node-migration.zh-CN.md)
This runbook migrates an existing Docker Compose Postgres deployment to Aether
single-node. In this repository, **single-node** means the default SQLite installer mode:
`install.sh --mode single-node`, a system service backed by SQLite. The Docker Compose
single-node template is `docker-compose.single-node.yml`, exposed through `--mode compose-single-node`.
The migration script is:
```bash
scripts/migrate-pg-to-single-node.sh
```
If the target should stay on Docker Compose instead of becoming a system
service, use the image-based Compose migration script:
```bash
scripts/migrate-pg-compose-to-single-node.sh
```
Both migration scripts pull/install the target single-node version before
downtime, stop only the source `app`, copy Postgres records directly into a
temporary SQLite DB without writing a JSONL file, replace the target
`aether.db`, and start single-node.
You can also use the installer as the unified entrypoint and let `--mode`
select the migration target:
```bash
# In interactive mode, first choose the target deployment mode:
# 1) Docker Compose standard deployment (Postgres + Redis)
# 2) Docker Compose single-node deployment (SQLite)
# 3) System service single-node deployment (SQLite)
# After choosing 2 or 3, choose the data initialization mode:
# 1) Fresh initialization (do not migrate existing data)
# 2) Migrate from an existing Docker Compose PG database
install.sh
# Migrate into a new single-node Docker Compose directory.
install.sh \
--mode compose-single-node \
--migrate-from-compose /root/Aether/docker-compose.yml \
--compose-dir /opt/aether-single \
--replace-existing
# Migrate into the system service + SQLite layout.
sudo install.sh \
--mode single-node \
--migrate-from-compose /root/Aether/docker-compose.yml \
--replace-existing
```
Interactive mode first asks for the target deployment shape. If the target is
`compose-single-node` or `single-node`, the installer then asks for the data
initialization mode: fresh initialization, or migration from an existing Docker
Compose PG database. If you choose migration, it tries to detect the source PG
Compose file from `docker compose ls`, then verifies that the Compose config
contains the default `app` and `postgres` services. If exactly one match is
found, it is used as the default prompt value. If detection is ambiguous or
fails, the installer stops; rerun it with `--migrate-from-compose` to specify
the source compose path.
The installer only normalizes the entrypoint: `compose-single-node` delegates to
`scripts/migrate-pg-compose-to-single-node.sh`, while `single-node` delegates to
`scripts/migrate-pg-to-single-node.sh`.
## What It Does
The script keeps the production cutover window short:
1. Reads the source Compose `.env`.
2. Builds a single-node env file that preserves `JWT_SECRET_KEY`, `ENCRYPTION_KEY` or
`AETHER_GATEWAY_DATA_ENCRYPTION_KEY`, admin settings, port, and app config.
3. Installs the single-node release with `install.sh --mode single-node --skip-start`.
4. Preflights SQLite migrations with the installed single-node binary.
5. Pulls the target single-node image, confirms its `copy` command supports the
current migration domains, and verifies that its Docker image ID matches the
currently running source `app` image ID.
6. Checks for non-empty Postgres tables not covered by the migration domains.
7. Applies the compressed body and HTTP body detail policy. The default is full,
and you can opt into an omit mode for large artifacts.
8. Checks that the work directory and target SQLite directory have enough free
disk space for the temporary and final SQLite files.
9. Stops only the source `app` service, leaving Postgres and Redis running.
10. Copies source Postgres records directly into a temporary SQLite database
without generating JSONL files.
11. Replaces the target SQLite DB, including SQLite `-wal`/`-shm` sidecar files
when present, and starts the single-node service.
The image check compares Docker image IDs, not just tag strings. If both source
and target say `latest` but resolve to different image IDs, migration stops.
Upgrade the source PG Compose `app` to the target single-node version first,
verify it is healthy, then run the migration. The scripts also check that the
target image supports `stats`, `auxiliary`, and the request-body omit flag; using
a new script with an old image stops before cutover to avoid missing data.
## Production Cutover
Before production cutover, take a normal server backup or snapshot. Then run:
```bash
sudo scripts/migrate-pg-to-single-node.sh \
--source-compose /root/Aether/docker-compose.yml \
--replace-existing
```
For Docker Compose single-node cutover instead of a system service:
```bash
scripts/migrate-pg-compose-to-single-node.sh \
--source-compose /root/Aether/docker-compose.yml \
--replace-existing
```
The source Postgres compose directory and target single-node compose directory
can be different. For example:
```bash
install.sh \
--mode compose-single-node \
--migrate-from-compose /root/Aether/docker-compose.yml \
--compose-dir /opt/aether-single \
--replace-existing
```
Equivalently, call the lower-level script and pass each target path explicitly:
```bash
scripts/migrate-pg-compose-to-single-node.sh \
--source-compose /root/Aether/docker-compose.yml \
--target-compose /opt/aether-single/docker-compose.single-node.yml \
--target-env /opt/aether-single/.env.single-node \
--target-db /opt/aether-single/data/aether.db \
--replace-existing
```
During cutover, the script stops and removes only the source `app` container to
free the fixed `aether-app` container name. Postgres, Redis, and their volumes
remain in place for rollback.
Defaults:
| Setting | Default |
| --- | --- |
| Source Compose | `docker-compose.yml` |
| Single Node install root | `/opt/aether` |
| Single Node config dir | `/etc/aether` |
| Target SQLite DB | `/opt/aether/data/aether.db` |
| Source app service | `app` |
| Source Postgres service | `postgres` |
| Single Node service | `aether-gateway` |
The script writes migration artifacts under `./data/pg-to-single-node-<timestamp>` next
to the source Compose file unless `--work-dir` is provided.
## Rollback
The script leaves the original Postgres and Redis volumes in place. If cutover
finishes but you need to roll back:
```bash
sudo systemctl stop aether-gateway
cd /root/Aether
docker compose -f docker-compose.yml up -d app
```
For the Compose single-node script, rollback is the same idea: start the app
again from the original Postgres compose file.
If the migration fails before cutover completes, the script attempts to restart
the source `app` service automatically. Pass `--keep-source-stopped-on-error` if
you want to inspect the stopped source deployment manually instead.
## Data Coverage Guard
The current migration covers these persistent domains: users, API keys,
providers, provider keys, endpoints, models, global models, auth modules, OAuth
links, user groups, proxy nodes, system configs, wallets, usage, and billing
data.
Before stopping the app, and again after the source app has stopped, the script
checks the source Postgres database for non-empty tables outside that migration
coverage. It does not run source Postgres migrations or backfills during the
cutover. It ignores lifecycle metadata tables such as `_sqlx_migrations` and
`schema_backfills`. Any other non-empty uncovered table blocks the migration.
## Request Body Detail Policy
The production migration migrates all migratable data by default. The only
optional exclusion is request body detail data.
When you choose to skip request bodies, the migration does not copy
`usage_body_blobs`, `usage_http_audits`, or legacy `usage` request body columns
such as `request_body`, `provider_request_body`, `response_body`,
`client_response_body`, and `*_body_compressed`.
Interactive installation lets you choose:
```text
1) Full migration: migrate all migratable data, including request body details
2) Skip request bodies: migrate all other data; skip only request body large fields and HTTP body detail tables; source PG is unchanged
```
For non-interactive full runs:
```bash
scripts/migrate-pg-to-single-node.sh \
--request-body-mode full
```
For non-interactive omit runs:
```bash
scripts/migrate-pg-to-single-node.sh \
--request-body-mode omit
```
`omit` only skips writing those large artifacts and detail tables into the
target SQLite database. It does not delete or clear the source Postgres data.
If a table is intentionally excluded, allow it explicitly:
```bash
scripts/migrate-pg-to-single-node.sh \
--allow-non-exported-table legacy_custom_table
```
Use that only after confirming the table is not required in the single-node target.
## Notes
- Single Node requires root or sudo because it writes `/opt/aether`, `/etc/aether`, and
the system service definition.
- The script does not decrypt or re-encrypt provider keys. It preserves the
original encryption key and moves encrypted data as-is.
- Existing target SQLite databases, including `-wal`/`-shm` sidecars, are not
replaced unless `--replace-existing` is provided.
- Disk space checks use `pg_database_size(current_database()) * 2 + 1 GiB` as the
conservative estimate for one SQLite copy. If the work directory and target DB
directory are on the same filesystem, the script requires enough space for both
the temporary and final SQLite files. With `--request-body-mode omit`, the
estimate subtracts `usage_body_blobs` and `usage_http_audits` relation sizes.
- For non-standard source Compose files, set `--app-service` and
`--postgres-service` to match the service names.

230
docs/operations/pg-to-single-node-migration.zh-CN.md Normal file
View File

@@ -0,0 +1,230 @@
# Postgres 到 Aether Single Node 迁移
英文版：[pg-to-single-node-migration.md](pg-to-single-node-migration.md)
本文档用于把现有 Docker Compose Postgres 部署迁移到 Aether
single-node。当前版本里，**single-node** 指默认 SQLite 安装模式：
`install.sh --mode single-node`，也就是系统服务加 SQLite。Docker Compose
单机模板是 `docker-compose.single-node.yml`，安装脚本入口是
`--mode compose-single-node`。
迁移脚本：
```bash
scripts/migrate-pg-to-single-node.sh
```
如果目标形态仍然要保持 Docker Compose，而不是系统服务，使用镜像版迁移脚本：
```bash
scripts/migrate-pg-compose-to-single-node.sh
```
两种迁移脚本都会先拉取/安装目标 single-node 版本，再停止源 `app`，把 Postgres
记录直接写入临时 SQLite DB，不落 JSONL 中间文件；复制成功后替换目标
`aether.db`，最后启动 single-node。
也可以直接用安装脚本作为统一入口，由 `--mode` 选择迁移目标：
```bash
# 交互式执行时，先选择目标部署模式：
# 1) Docker Compose 标准部署（Postgres + Redis）
# 2) Docker Compose 单节点部署（SQLite）
# 3) 系统服务单节点部署（SQLite）
# 选择 2 或 3 后，再选择数据初始化方式：
# 1) 全新初始化（不迁移现有数据）
# 2) 从现有 Docker Compose PG 数据库迁移
install.sh
# 迁移到新的 single-node Docker Compose 目录
install.sh \
--mode compose-single-node \
--migrate-from-compose /root/Aether/docker-compose.yml \
--compose-dir /opt/aether-single \
--replace-existing
# 迁移到系统服务 + SQLite
sudo install.sh \
--mode single-node \
--migrate-from-compose /root/Aether/docker-compose.yml \
--replace-existing
```
交互模式会先选择目标部署形态。如果目标是 `compose-single-node` 或
`single-node`，安装脚本会再询问数据初始化方式：全新初始化，或从现有 Docker
Compose PG 数据库迁移。选择迁移后，脚本会通过 `docker compose ls` 自动探测源
PG Compose 文件，并确认该 Compose 配置里存在默认的 `app` 和 `postgres` 服务；
如果能唯一识别，会作为默认值带入提示。探测不到或存在多个候选时会直接中止；
此时请用 `--migrate-from-compose` 显式指定源 compose 路径。
安装脚本只是统一参数入口：`compose-single-node` 会委托给
`scripts/migrate-pg-compose-to-single-node.sh`，`single-node` 会委托给
`scripts/migrate-pg-to-single-node.sh`。
## 迁移内容
脚本会尽量缩短生产停机窗口：
1. 读取源 Compose 目录下的 `.env`。
2. 生成 single-node 环境文件，保留 `JWT_SECRET_KEY`、`ENCRYPTION_KEY` 或
`AETHER_GATEWAY_DATA_ENCRYPTION_KEY`、管理员配置、端口和应用配置。
3. 执行 `install.sh --mode single-node --skip-start`，提前安装 single-node
release，但不启动服务。
4. 使用已安装的 single-node 二进制预检 SQLite schema migration。
5. 拉取目标 single-node 镜像，确认其 `copy` 命令支持当前迁移域，并检查源
`app` 当前运行镜像 ID 与目标镜像 ID 一致。
6. 检查 Postgres 里是否存在当前迁移域没有覆盖、但又非空的表。
7. 检查请求体明细迁移策略；默认全部迁移，也可以选择只跳过请求体明细。
8. 检查 work-dir 和目标 SQLite 目录是否有足够空间容纳临时库和正式库。
9. 只停止源 Compose 的 `app` 服务，保留 Postgres 和 Redis 运行，方便回滚。
10. 从源 Postgres 直接复制记录到临时 SQLite 数据库，不生成 JSONL 中间文件。
11. 复制完成后替换目标 SQLite DB，包括 SQLite `-wal`、`-shm` 边车文件，
然后启动 single-node 系统服务。
镜像一致性检查比较的是 Docker 镜像 ID，不只是 tag 字符串。即使源和目标都写着
`latest`，只要实际镜像 ID 不同，迁移也会中止。请先把源 PG Compose 的 `app`
升级到目标 single-node 相同版本，确认运行正常后再迁移。迁移脚本也会检查目标镜像
是否支持 `stats`、`auxiliary` 和请求体跳过开关；如果只是换了脚本但镜像还是旧版本，
脚本会直接中止，避免漏迁。
## 生产切换
切换前先做一次常规服务器备份或快照。确认后执行：
```bash
sudo scripts/migrate-pg-to-single-node.sh \
--source-compose /root/Aether/docker-compose.yml \
--replace-existing
```
如果要迁移到 Docker Compose single-node，而不是系统服务：
```bash
scripts/migrate-pg-compose-to-single-node.sh \
--source-compose /root/Aether/docker-compose.yml \
--replace-existing
```
源 Postgres Compose 目录和目标 single-node Compose 目录可以不一样。例如：
```bash
install.sh \
--mode compose-single-node \
--migrate-from-compose /root/Aether/docker-compose.yml \
--compose-dir /opt/aether-single \
--replace-existing
```
等价地，也可以直接调底层脚本并显式传入每个目标路径：
```bash
scripts/migrate-pg-compose-to-single-node.sh \
--source-compose /root/Aether/docker-compose.yml \
--target-compose /opt/aether-single/docker-compose.single-node.yml \
--target-env /opt/aether-single/.env.single-node \
--target-db /opt/aether-single/data/aether.db \
--replace-existing
```
切换时脚本只会停止并移除源 `app` 容器，用来释放固定的 `aether-app`
容器名；Postgres、Redis 和它们的 volume 都会保留，方便回滚。
默认路径和服务名：
| 配置项 | 默认值 |
| --- | --- |
| 源 Compose 文件 | `docker-compose.yml` |
| single-node 安装目录 | `/opt/aether` |
| single-node 配置目录 | `/etc/aether` |
| 目标 SQLite DB | `/opt/aether/data/aether.db` |
| 源 app 服务 | `app` |
| 源 Postgres 服务 | `postgres` |
| single-node 服务 | `aether-gateway` |
除非显式传入 `--work-dir`，脚本会把迁移产物写到源 Compose 文件旁边的
`./data/pg-to-single-node-<timestamp>`。
## 回滚
脚本会保留原 Postgres 和 Redis volume。迁移已经完成但需要回滚时：
```bash
sudo systemctl stop aether-gateway
cd /root/Aether
docker compose -f docker-compose.yml up -d app
```
对于 Compose single-node 脚本，回滚思路相同：重新用原 Postgres compose 文件
拉起 `app`。
如果迁移在切换完成前失败，脚本默认会尝试自动拉起源 `app` 服务。需要失败后
保持源应用停止以便人工排查时，增加：
```bash
--keep-source-stopped-on-error
```
## 数据覆盖保护
当前迁移覆盖的持久化域包括：用户、API Key、供应商、供应商 Key、
端点、模型、全局模型、认证模块、OAuth 关联、用户组、代理节点、系统配置、
钱包、用量和计费数据。
脚本会在停机前，以及源 `app` 停止之后，再检查一次源 Postgres：如果发现
当前迁移域没有覆盖的非空表，会直接中止迁移，避免漏迁。切换期间不会对源
Postgres 执行 migrations 或 backfills。生命周期元数据表 `_sqlx_migrations`
和 `schema_backfills` 会被忽略。
## 请求体明细策略
single-node SQLite 生产迁移默认迁移所有可迁移数据，唯一可选的跳过项是请求体明细。
选择“不迁移请求体”时，不会迁移 `usage_body_blobs`、`usage_http_audits`，也不会迁移 `usage`
表里的 `request_body` / `provider_request_body` / `response_body` /
`client_response_body` / `*_body_compressed` 等请求体大字段。
交互安装时可以选择：
```text
1) 全部迁移：迁移所有可迁移数据，包括请求体明细
2) 不迁移请求体：迁移其他所有数据；仅跳过请求体大字段和 HTTP 请求体明细，源 PG 不清除
```
非交互执行时，全部迁移可以显式指定：
```bash
scripts/migrate-pg-to-single-node.sh \
--request-body-mode full
```
不迁移请求体可以显式指定：
```bash
scripts/migrate-pg-to-single-node.sh \
--request-body-mode omit
```
`omit` 只是不把这些大字段和明细表写进目标 SQLite，不会删除或清空源 Postgres。
如果某个表确认不需要迁移，可以显式允许：
```bash
scripts/migrate-pg-to-single-node.sh \
--allow-non-exported-table legacy_custom_table
```
只有在确认该表对 single-node 目标库不重要时才这样做。
## 注意事项
- single-node 安装需要 root 或 sudo 权限，因为会写入 `/opt/aether`、
`/etc/aether` 和系统服务定义。
- 脚本不会解密或重新加密供应商密钥；它会沿用源环境的加密密钥，并原样迁移已加密数据。
- 已存在的目标 SQLite DB，包括 `-wal`、`-shm` 边车文件，只有在传入
`--replace-existing` 时才会被替换。
- 空间检查会用 `pg_database_size(current_database()) * 2 + 1 GiB` 作为单份
SQLite 的保守估算。如果 work-dir 和目标 DB 目录在同一个文件系统，会要求同时
容纳临时 SQLite 和正式 SQLite。选择 `--request-body-mode omit` 时，
估算会扣除 `usage_body_blobs` 和 `usage_http_audits` 的表空间。
- 非标准 Compose 服务名需要通过 `--app-service` 和 `--postgres-service`
明确指定。

688
install.sh
View File

@@ -49,6 +49,20 @@ ADMIN_PASSWORD_SOURCE=""
UI_LANG="${AETHER_LANG:-${AETHER_LANGUAGE:-auto}}"
RELEASE_KEEP="${AETHER_RELEASE_KEEP:-3}"
RELEASE_ARCHIVE_URL="${AETHER_RELEASE_ARCHIVE_URL:-${AETHER_DOWNLOAD_URL:-}}"
MIGRATE_FROM_COMPOSE=""
MIGRATE_TARGET_COMPOSE=""
MIGRATE_TARGET_ENV=""
MIGRATE_TARGET_DB=""
MIGRATE_WORK_DIR=""
MIGRATE_APP_SERVICE=""
MIGRATE_POSTGRES_SERVICE=""
MIGRATE_SINGLE_NODE_SERVICE=""
MIGRATE_REPLACE_EXISTING="false"
MIGRATE_REPLACE_TARGET_COMPOSE="false"
MIGRATE_KEEP_SOURCE_STOPPED_ON_ERROR="false"
MIGRATE_INTERACTIVE="false"
MIGRATE_REQUEST_BODY_MODE=""
MIGRATE_ALLOW_NON_EXPORTED_TABLES=()
usage() {
cat <<'EOF'
@@ -57,10 +71,10 @@ Usage: install.sh [options]
Install Aether Gateway.
Options:
--mode MODE Deployment mode: compose, compose-sqlite, or single
--mode MODE Deployment mode: compose, compose-single-node, or single-node
compose: Docker Compose app + Postgres + Redis
compose-sqlite: Docker Compose app + SQLite
single: system service with SQLite
compose-single-node: Docker Compose single-node app
single-node: single-node system service
Linux services use systemd; macOS services use launchd
--channel CHANNEL Release channel to resolve when --version is omitted: stable, latest, rc, or beta
stable/latest resolves the latest stable tag (default)
@@ -79,8 +93,33 @@ Options:
--lang LANG Installer language: zh or en
--skip-start Install files, but do not start Docker Compose or restart the service
--keep-releases N Keep the latest N releases, prune older ones (default: 3, 0=disable)
--migrate-from-compose PATH
Migrate an existing Postgres Compose deployment into the selected single-node mode
--target-compose PATH
Migration target compose file for --mode compose-single-node
--target-env PATH Migration target env file for --mode compose-single-node
--target-db PATH Migration target SQLite DB path
--work-dir PATH Migration working directory
--app-service NAME Source compose app service for migration
--postgres-service NAME
Source compose Postgres service for migration
--single-node-service NAME
Target compose service for --mode compose-single-node migration
--allow-non-exported-table TABLE
Allow one source table outside export coverage to be non-empty
--replace-existing Allow replacing an existing target SQLite DB during migration
--replace-target-compose
Overwrite target compose file from the single-node template during migration
--request-body-mode MODE
Request/response body detail handling during migration: full/1 or omit/2
--keep-source-stopped-on-error
Do not auto-restart source app if migration fails after stopping it
-h, --help Show this help
Migration examples:
install.sh --mode compose-single-node --migrate-from-compose /root/Aether/docker-compose.yml --compose-dir /opt/aether-single --replace-existing
sudo install.sh --mode single-node --migrate-from-compose /root/Aether/docker-compose.yml --replace-existing
Environment overrides:
AETHER_REPO, AETHER_SOURCE_REF, AETHER_INSTALL_MODE, AETHER_CHANNEL, AETHER_VERSION
AETHER_LANG or AETHER_LANGUAGE
@@ -158,7 +197,7 @@ select_language() {
请选择安装语言 / Choose installer language:
1) 中文
2) 英语 / English
2) English
请输入选项 / Enter choice [1]:
EOF
@@ -264,6 +303,68 @@ parse_args() {
RELEASE_KEEP="$2"
shift 2
;;
--migrate-from-compose)
[[ $# -ge 2 ]] || die "--migrate-from-compose requires a path"
MIGRATE_FROM_COMPOSE="$2"
shift 2
;;
--target-compose)
[[ $# -ge 2 ]] || die "--target-compose requires a path"
MIGRATE_TARGET_COMPOSE="$2"
shift 2
;;
--target-env)
[[ $# -ge 2 ]] || die "--target-env requires a path"
MIGRATE_TARGET_ENV="$2"
shift 2
;;
--target-db)
[[ $# -ge 2 ]] || die "--target-db requires a path"
MIGRATE_TARGET_DB="$2"
shift 2
;;
--work-dir)
[[ $# -ge 2 ]] || die "--work-dir requires a path"
MIGRATE_WORK_DIR="$2"
shift 2
;;
--app-service)
[[ $# -ge 2 ]] || die "--app-service requires a service name"
MIGRATE_APP_SERVICE="$2"
shift 2
;;
--postgres-service)
[[ $# -ge 2 ]] || die "--postgres-service requires a service name"
MIGRATE_POSTGRES_SERVICE="$2"
shift 2
;;
--single-node-service)
[[ $# -ge 2 ]] || die "--single-node-service requires a service name"
MIGRATE_SINGLE_NODE_SERVICE="$2"
shift 2
;;
--allow-non-exported-table)
[[ $# -ge 2 ]] || die "--allow-non-exported-table requires a table name"
MIGRATE_ALLOW_NON_EXPORTED_TABLES+=("$2")
shift 2
;;
--replace-existing)
MIGRATE_REPLACE_EXISTING="true"
shift
;;
--replace-target-compose)
MIGRATE_REPLACE_TARGET_COMPOSE="true"
shift
;;
--request-body-mode)
[[ $# -ge 2 ]] || die "--request-body-mode requires a value"
MIGRATE_REQUEST_BODY_MODE="$2"
shift 2
;;
--keep-source-stopped-on-error)
MIGRATE_KEEP_SOURCE_STOPPED_ON_ERROR="true"
shift
;;
-h|--help)
usage
exit 0
@@ -441,25 +542,25 @@ select_mode() {
MODE="compose"
return
;;
compose-sqlite|sqlite-compose|compose-solo|solo|docker-solo|docker-solo-compose)
MODE="compose-sqlite"
compose-single-node|docker-single-node|docker-single-node-compose)
MODE="compose-single-node"
return
;;
single|service|systemd|launchd|sqlite)
MODE="single"
single-node|service|systemd|launchd|sqlite)
MODE="single-node"
return
;;
cluster|multi|multi-node)
if ui_is_zh; then
die "集群部署模式暂未开放；请先选择 compose、compose-sqlite 或 single"
die "集群部署模式暂未开放；请先选择 compose、compose-single-node 或 single-node"
else
die "cluster deployment mode is temporarily disabled; choose compose, compose-sqlite, or single"
die "cluster deployment mode is temporarily disabled; choose compose, compose-single-node, or single-node"
fi
;;
auto|"")
;;
*)
die "unsupported install mode: ${MODE}; expected compose, compose-sqlite, or single"
die "unsupported install mode: ${MODE}; expected compose, compose-single-node, or single-node"
;;
esac
@@ -468,34 +569,34 @@ select_mode() {
cat >/dev/tty <<EOF
请选择 Aether 部署模式:
1) Docker Compose 应用: Postgres + Redis
3) Docker Compose 应用: 仅SQLite
4) 系统服务: 仅SQLite
1) Docker Compose 标准部署（Postgres + Redis）
2) Docker Compose 单节点部署（SQLite）
3) 系统服务单节点部署（SQLite）
请输入选项 [4]:
请输入选项 [3]:
EOF
else
cat >/dev/tty <<EOF
Choose Aether deployment mode:
1) Docker Compose app: Postgres + Redis
3) Docker Compose app: SQLite only
4) System service: SQLite only
1) Docker Compose standard deployment (Postgres + Redis)
2) Docker Compose single-node deployment (SQLite)
3) System service single-node deployment (SQLite)
Enter choice [4]:
Enter choice [3]:
EOF
fi
local choice
IFS= read -r choice </dev/tty || choice=""
case "${choice:-4}" in
case "${choice:-3}" in
1)
MODE="compose"
;;
3)
MODE="compose-sqlite"
2)
MODE="compose-single-node"
;;
4)
MODE="single"
3)
MODE="single-node"
;;
*)
if ui_is_zh; then
@@ -505,8 +606,45 @@ EOF
fi
;;
esac
if [[ -z "${MIGRATE_FROM_COMPOSE}" && "${MODE}" != "compose" ]]; then
if ui_is_zh; then
cat >/dev/tty <<'EOF'
请选择数据初始化方式:
1) 全新初始化（不迁移现有数据）
2) 从现有 Docker Compose PG 数据库迁移
请输入选项 [1]:
EOF
else
cat >/dev/tty <<'EOF'
Choose data initialization mode:
1) Fresh initialization (do not migrate existing data)
2) Migrate from an existing Docker Compose PG database
Enter choice [1]:
EOF
fi
local init_choice
IFS= read -r init_choice </dev/tty || init_choice=""
case "${init_choice:-1}" in
1)
;;
2)
MIGRATE_INTERACTIVE="true"
;;
*)
if ui_is_zh; then
die "无效的数据初始化方式选项: ${init_choice}"
else
die "invalid data initialization choice: ${init_choice}"
fi
;;
esac
fi
else
MODE="single"
MODE="single-node"
fi
}
@@ -857,6 +995,441 @@ start_compose_deployment() {
fi
}
migration_options_requested() {
[[ -n "${MIGRATE_TARGET_COMPOSE}" ]] && return 0
[[ -n "${MIGRATE_TARGET_ENV}" ]] && return 0
[[ -n "${MIGRATE_TARGET_DB}" ]] && return 0
[[ -n "${MIGRATE_WORK_DIR}" ]] && return 0
[[ -n "${MIGRATE_APP_SERVICE}" ]] && return 0
[[ -n "${MIGRATE_POSTGRES_SERVICE}" ]] && return 0
[[ -n "${MIGRATE_SINGLE_NODE_SERVICE}" ]] && return 0
[[ "${MIGRATE_REPLACE_EXISTING}" == "true" ]] && return 0
[[ "${MIGRATE_REPLACE_TARGET_COMPOSE}" == "true" ]] && return 0
[[ -n "${MIGRATE_REQUEST_BODY_MODE}" ]] && return 0
[[ "${MIGRATE_KEEP_SOURCE_STOPPED_ON_ERROR}" == "true" ]] && return 0
[[ "${#MIGRATE_ALLOW_NON_EXPORTED_TABLES[@]}" -gt 0 ]] && return 0
return 1
}
normalize_migration_request_body_mode() {
case "${MIGRATE_REQUEST_BODY_MODE}" in
""|1|full|all|include)
MIGRATE_REQUEST_BODY_MODE="full"
;;
2|omit|skip)
MIGRATE_REQUEST_BODY_MODE="omit"
;;
*)
die "--request-body-mode must be full/1 or omit/2"
;;
esac
}
prompt_with_default() {
local prompt="$1"
local default_value="$2"
local value
if [[ -n "${default_value}" ]]; then
printf '%s [%s]: ' "${prompt}" "${default_value}" >/dev/tty
else
printf '%s: ' "${prompt}" >/dev/tty
fi
IFS= read -r value </dev/tty || value=""
if [[ -z "${value}" ]]; then
printf '%s\n' "${default_value}"
else
printf '%s\n' "${value}"
fi
}
prompt_yes_no() {
local prompt="$1"
local default_value="$2"
local suffix choice
case "${default_value}" in
yes)
if ui_is_zh; then
suffix="[y/n，默认 y]"
else
suffix="[y/n, default y]"
fi
;;
*)
default_value="no"
if ui_is_zh; then
suffix="[y/n，默认 n]"
else
suffix="[y/n, default n]"
fi
;;
esac
while true; do
printf '%s %s: ' "${prompt}" "${suffix}" >/dev/tty
IFS= read -r choice </dev/tty || choice=""
choice="$(printf '%s' "${choice}" | tr '[:upper:]' '[:lower:]')"
case "${choice:-${default_value}}" in
y|yes)
return 0
;;
n|no)
return 1
;;
*)
if ui_is_zh; then
echo "请输入 y 或 n。" >/dev/tty
else
echo "Enter y or n." >/dev/tty
fi
;;
esac
done
}
docker_compose_ls_config_files() {
local output
output="$(docker compose ls --format json 2>/dev/null || true)"
if [[ -n "${output}" && "${output}" == *ConfigFiles* ]]; then
printf '%s' "${output}" |
tr '{' '\n' |
sed -n 's/.*"ConfigFiles"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p'
return
fi
docker compose ls 2>/dev/null | awk 'NR > 1 && NF > 0 { print $NF }'
}
compose_file_has_source_services() {
local compose_file="$1"
local app_service="${MIGRATE_APP_SERVICE:-app}"
local postgres_service="${MIGRATE_POSTGRES_SERVICE:-postgres}"
local services
services="$(docker compose -f "${compose_file}" config --services 2>/dev/null || true)"
[[ -n "${services}" ]] || return 1
printf '%s\n' "${services}" | grep -Fxq "${app_service}" || return 1
printf '%s\n' "${services}" | grep -Fxq "${postgres_service}" || return 1
}
append_unique_candidate() {
local candidate="$1"
shift
local existing
for existing in "$@"; do
[[ "${existing}" != "${candidate}" ]] || return 1
done
printf '%s\n' "${candidate}"
}
detect_source_compose_from_docker_compose_ls() {
local config_files compose_file candidate
local -a candidates=()
command -v docker >/dev/null 2>&1 || return 0
docker compose version >/dev/null 2>&1 || return 0
while IFS= read -r config_files || [[ -n "${config_files}" ]]; do
config_files="$(trim_whitespace "${config_files}")"
[[ -n "${config_files}" ]] || continue
# The migration scripts currently accept one source compose file. If the
# source project was launched with multiple compose files, ask explicitly.
[[ "${config_files}" != *,* ]] || continue
compose_file="${config_files}"
[[ -f "${compose_file}" ]] || continue
compose_file="$(absolute_path "${compose_file}")"
compose_file_has_source_services "${compose_file}" || continue
candidate="$(append_unique_candidate "${compose_file}" "${candidates[@]}" || true)"
[[ -z "${candidate}" ]] || candidates+=("${candidate}")
done < <(docker_compose_ls_config_files)
case "${#candidates[@]}" in
0)
return 0
;;
1)
printf '%s\n' "${candidates[0]}"
;;
*)
if interactive_tty_available; then
if ui_is_zh; then
echo "从 docker compose ls 找到多个可能的源 Compose，无法安全自动选择：" >/dev/tty
else
echo "docker compose ls found multiple possible source Compose files and cannot choose safely:" >/dev/tty
fi
printf ' %s\n' "${candidates[@]}" >/dev/tty
fi
return 0
;;
esac
}
collect_interactive_migration_options() {
local detected_source
local prompt
local source_compose_abs
local source_compose_dir
[[ "${MIGRATE_INTERACTIVE}" == "true" ]] || return
interactive_tty_available || die "interactive migration selection requires a terminal"
if ui_is_zh; then
cat >/dev/tty <<'EOF'
迁移会先做预检并拉取/安装目标 single-node，再在切换窗口停止源 app。
源 Postgres 和 Redis 会保留，便于回滚。
EOF
else
cat >/dev/tty <<'EOF'
Migration will preflight and pull/install the target single-node release first.
During cutover it stops only the source app. Source Postgres and Redis remain for rollback.
EOF
fi
if [[ -z "${MIGRATE_FROM_COMPOSE}" ]]; then
detected_source="$(detect_source_compose_from_docker_compose_ls || true)"
if [[ -z "${detected_source}" ]]; then
if ui_is_zh; then
die "未能通过 docker compose ls 唯一识别源 PG Compose；请使用 --migrate-from-compose 显式指定"
else
die "could not uniquely detect source PG Compose from docker compose ls; pass --migrate-from-compose explicitly"
fi
fi
if ui_is_zh; then
printf '已通过 docker compose ls 探测到源 Compose: %s\n' "${detected_source}" >/dev/tty
else
printf 'Detected source Compose from docker compose ls: %s\n' "${detected_source}" >/dev/tty
fi
if ui_is_zh; then
prompt="确认使用该源 Compose 进行迁移"
else
prompt="Use this source Compose for migration"
fi
if prompt_yes_no "${prompt}" "yes"; then
MIGRATE_FROM_COMPOSE="${detected_source}"
else
if ui_is_zh; then
die "已取消迁移；如需指定其他源 Compose，请使用 --migrate-from-compose"
else
die "migration cancelled; pass --migrate-from-compose to use another source Compose"
fi
fi
fi
[[ -n "${MIGRATE_FROM_COMPOSE}" ]] || die "--migrate-from-compose cannot be empty"
source_compose_abs="$(absolute_path "${MIGRATE_FROM_COMPOSE}")"
source_compose_dir="$(dirname "${source_compose_abs}")"
if [[ "${MODE}" == "compose-single-node" ]]; then
if [[ "${COMPOSE_DIR_EXPLICIT}" != "true" ]]; then
COMPOSE_DIR="${source_compose_dir}-single-node"
fi
if ui_is_zh; then
printf '已自动选择目标 single-node Compose 目录: %s\n' "${COMPOSE_DIR}" >/dev/tty
prompt="确认使用该目标目录"
else
printf 'Selected target single-node Compose directory: %s\n' "${COMPOSE_DIR}" >/dev/tty
prompt="Use this target directory"
fi
if ! prompt_yes_no "${prompt}" "yes"; then
if ui_is_zh; then
die "已取消迁移；如需指定其他目标目录，请使用 --compose-dir"
else
die "migration cancelled; pass --compose-dir to use another target directory"
fi
fi
fi
if [[ "${MIGRATE_REPLACE_EXISTING}" != "true" ]]; then
if ui_is_zh; then
prompt="如果目标 SQLite 已存在，是否允许备份后替换"
else
prompt="If the target SQLite DB already exists, allow backup and replacement"
fi
if prompt_yes_no "${prompt}" "no"; then
MIGRATE_REPLACE_EXISTING="true"
fi
fi
if [[ -z "${MIGRATE_REQUEST_BODY_MODE}" ]]; then
if ui_is_zh; then
cat >/dev/tty <<'EOF'
请求体明细迁移策略:
1) 全部迁移：迁移所有可迁移数据，包括请求体明细
2) 不迁移请求体：迁移其他所有数据；仅跳过请求体大字段和 HTTP 请求体明细，源 PG 不清除
请输入选项 [1]:
EOF
else
cat >/dev/tty <<'EOF'
Request/response body detail migration mode:
1) Full migration: migrate all migratable data, including request body details
2) Skip request bodies: migrate all other data; skip only request body large fields and HTTP body detail tables; source PG is unchanged
Enter choice [1]:
EOF
fi
local body_choice
IFS= read -r body_choice </dev/tty || body_choice=""
MIGRATE_REQUEST_BODY_MODE="${body_choice:-1}"
fi
normalize_migration_request_body_mode
}
install_migration_project_file() {
local source_path="$1"
local mode="$2"
local target_path
ensure_tmp_root
target_path="${TMP_ROOT}/$(basename "${source_path}")"
install_project_file "${source_path}" "${target_path}" "${mode}"
printf '%s\n' "${target_path}"
}
run_compose_single_node_migration() {
local migration_script
local target_template
local source_compose_abs
local source_compose_dir
local compose_dir_abs
local target_compose
local target_compose_abs
local target_compose_dir
local target_env
local target_env_abs
local table
local -a migrate_args
source_compose_abs="$(absolute_path "${MIGRATE_FROM_COMPOSE}")"
[[ -f "${source_compose_abs}" ]] || die "source compose file not found: ${MIGRATE_FROM_COMPOSE}"
source_compose_dir="$(dirname "${source_compose_abs}")"
compose_dir_abs="$(absolute_path_maybe_missing "${COMPOSE_DIR}")"
if [[ -n "${MIGRATE_TARGET_COMPOSE}" ]]; then
target_compose="${MIGRATE_TARGET_COMPOSE}"
elif [[ "${compose_dir_abs}" == "${source_compose_dir}" ]]; then
target_compose="${compose_dir_abs}/docker-compose.single-node.yml"
else
target_compose="${compose_dir_abs}/docker-compose.yml"
fi
target_compose_abs="$(absolute_path_maybe_missing "${target_compose}")"
target_compose_dir="$(dirname "${target_compose_abs}")"
if [[ -n "${MIGRATE_TARGET_ENV}" ]]; then
target_env="${MIGRATE_TARGET_ENV}"
elif [[ "$(basename "${target_compose_abs}")" == "docker-compose.yml" ]]; then
target_env="${target_compose_dir}/.env"
else
target_env="${target_compose_dir}/.env.single-node"
fi
target_env_abs="$(absolute_path_maybe_missing "${target_env}")"
[[ "${target_compose_abs}" != "${source_compose_abs}" ]] || die "target compose would overwrite the source compose file; pass --target-compose or --compose-dir"
[[ "${target_env_abs}" != "${source_compose_dir}/.env" ]] || die "target env would overwrite the source .env; pass --target-env or --compose-dir"
migration_script="$(install_migration_project_file "scripts/migrate-pg-compose-to-single-node.sh" "0755")"
target_template="$(install_migration_project_file "docker-compose.single-node.yml" "0644")"
migrate_args=(
"${migration_script}"
--source-compose "${source_compose_abs}"
--target-compose "${target_compose_abs}"
--target-template "${target_template}"
--target-env "${target_env_abs}"
--app-image "$(compose_image)"
)
[[ -z "${MIGRATE_TARGET_DB}" ]] || migrate_args+=(--target-db "${MIGRATE_TARGET_DB}")
[[ -z "${MIGRATE_WORK_DIR}" ]] || migrate_args+=(--work-dir "${MIGRATE_WORK_DIR}")
[[ -z "${MIGRATE_APP_SERVICE}" ]] || migrate_args+=(--app-service "${MIGRATE_APP_SERVICE}")
[[ -z "${MIGRATE_POSTGRES_SERVICE}" ]] || migrate_args+=(--postgres-service "${MIGRATE_POSTGRES_SERVICE}")
[[ -z "${MIGRATE_SINGLE_NODE_SERVICE}" ]] || migrate_args+=(--single-node-service "${MIGRATE_SINGLE_NODE_SERVICE}")
for table in "${MIGRATE_ALLOW_NON_EXPORTED_TABLES[@]}"; do
migrate_args+=(--allow-non-exported-table "${table}")
done
[[ "${MIGRATE_REPLACE_EXISTING}" != "true" ]] || migrate_args+=(--replace-existing)
[[ "${MIGRATE_REPLACE_TARGET_COMPOSE}" != "true" ]] || migrate_args+=(--replace-target-compose)
[[ -z "${MIGRATE_REQUEST_BODY_MODE}" ]] || migrate_args+=(--request-body-mode "${MIGRATE_REQUEST_BODY_MODE}")
[[ "${MIGRATE_KEEP_SOURCE_STOPPED_ON_ERROR}" != "true" ]] || migrate_args+=(--keep-source-stopped-on-error)
bash "${migrate_args[@]}"
}
run_single_node_service_migration() {
local migration_script
local installer
local table
local -a migrate_args
[[ -z "${MIGRATE_TARGET_COMPOSE}" ]] || die "--target-compose is only valid with --mode compose-single-node"
[[ -z "${MIGRATE_TARGET_ENV}" ]] || die "--target-env is only valid with --mode compose-single-node"
[[ -z "${MIGRATE_SINGLE_NODE_SERVICE}" ]] || die "--single-node-service is only valid with --mode compose-single-node"
[[ "${MIGRATE_REPLACE_TARGET_COMPOSE}" != "true" ]] || die "--replace-target-compose is only valid with --mode compose-single-node"
migration_script="$(install_migration_project_file "scripts/migrate-pg-to-single-node.sh" "0755")"
installer="$(install_migration_project_file "install.sh" "0755")"
migrate_args=(
"${migration_script}"
--source-compose "${MIGRATE_FROM_COMPOSE}"
--installer "${installer}"
--install-root "${INSTALL_ROOT}"
--config-dir "${CONFIG_DIR}"
--service-name "${SERVICE_NAME}"
--service-user "${SERVICE_USER}"
--service-group "${SERVICE_GROUP}"
--app-image "$(compose_image)"
--install-channel "${CHANNEL}"
--install-repo "${REPO}"
--install-source-ref "${SOURCE_REF}"
)
[[ -z "${VERSION}" ]] || migrate_args+=(--install-version "${VERSION}")
[[ -z "${ARCHIVE_PATH}" ]] || migrate_args+=(--install-archive "${ARCHIVE_PATH}")
[[ -z "${RELEASE_ARCHIVE_URL}" ]] || migrate_args+=(--install-download-url "${RELEASE_ARCHIVE_URL}")
[[ -z "${MIGRATE_TARGET_DB}" ]] || migrate_args+=(--target-db "${MIGRATE_TARGET_DB}")
[[ -z "${MIGRATE_WORK_DIR}" ]] || migrate_args+=(--work-dir "${MIGRATE_WORK_DIR}")
[[ -z "${MIGRATE_APP_SERVICE}" ]] || migrate_args+=(--app-service "${MIGRATE_APP_SERVICE}")
[[ -z "${MIGRATE_POSTGRES_SERVICE}" ]] || migrate_args+=(--postgres-service "${MIGRATE_POSTGRES_SERVICE}")
for table in "${MIGRATE_ALLOW_NON_EXPORTED_TABLES[@]}"; do
migrate_args+=(--allow-non-exported-table "${table}")
done
[[ "${MIGRATE_REPLACE_EXISTING}" != "true" ]] || migrate_args+=(--replace-existing)
[[ -z "${MIGRATE_REQUEST_BODY_MODE}" ]] || migrate_args+=(--request-body-mode "${MIGRATE_REQUEST_BODY_MODE}")
[[ "${MIGRATE_KEEP_SOURCE_STOPPED_ON_ERROR}" != "true" ]] || migrate_args+=(--keep-source-stopped-on-error)
bash "${migrate_args[@]}"
}
run_migration_from_compose() {
if [[ -n "${MIGRATE_REQUEST_BODY_MODE}" ]]; then
normalize_migration_request_body_mode
fi
case "${MODE}" in
compose-single-node)
run_compose_single_node_migration
;;
single-node)
run_single_node_service_migration
;;
compose)
die "--migrate-from-compose target mode must be compose-single-node or single-node"
;;
*)
die "unsupported migration target mode: ${MODE}"
;;
esac
}
resolve_version() {
if [[ -n "${VERSION}" ]]; then
echo "${VERSION}"
@@ -899,6 +1472,36 @@ current_script_dir() {
fi
}
ensure_tmp_root() {
if [[ -z "${TMP_ROOT}" ]]; then
TMP_ROOT="$(mktemp -d)"
fi
}
absolute_path() {
local path="$1"
local dir
local base
if [[ "${path}" == /* ]]; then
printf '%s\n' "${path}"
return
fi
dir="$(dirname "${path}")"
base="$(basename "${path}")"
printf '%s/%s\n' "$(cd "${dir}" && pwd -P)" "${base}"
}
absolute_path_maybe_missing() {
local path="$1"
if [[ "${path}" == /* ]]; then
printf '%s\n' "${path}"
else
printf '%s/%s\n' "$(pwd -P)" "${path}"
fi
}
local_bundle_dir() {
local dir
dir="$(current_script_dir)"
@@ -1092,6 +1695,8 @@ AETHER_GATEWAY_AUTO_PREPARE_DATABASE=true
AETHER_RUNTIME_BACKEND=memory
API_KEY_PREFIX=sk
AETHER_DATABASE_DRIVER=sqlite
AETHER_DATABASE_URL=sqlite://${INSTALL_ROOT}/data/aether.db
DATABASE_URL=sqlite://${INSTALL_ROOT}/data/aether.db
JWT_SECRET_KEY=${jwt_key}
@@ -1192,7 +1797,7 @@ generate_compose_env() {
replace_or_append_env "${output}" "AETHER_GATEWAY_AUTO_PREPARE_DATABASE" "true"
}
generate_compose_sqlite_env() {
generate_compose_single_node_env() {
local output="$1"
local jwt_key encryption_key
prompt_admin_password
@@ -1218,6 +1823,8 @@ AETHER_GATEWAY_AUTO_PREPARE_DATABASE=true
AETHER_RUNTIME_BACKEND=memory
API_KEY_PREFIX=sk
AETHER_DATABASE_DRIVER=sqlite
AETHER_DATABASE_URL=sqlite:///app/data/aether.db
DATABASE_URL=sqlite:///app/data/aether.db
JWT_SECRET_KEY=${JWT_SECRET_KEY:-${jwt_key}}
@@ -1374,9 +1981,9 @@ ensure_env_matches_requested_mode() {
topology="${topology:-single-node}"
if [[ "${mode}" == "cluster" ]]; then
[[ "${topology}" == "multi-node" ]] || die "existing env ${file} is ${topology}; set AETHER_GATEWAY_DEPLOYMENT_TOPOLOGY=multi-node or use --mode single"
[[ "${topology}" == "multi-node" ]] || die "existing env ${file} is ${topology}; set AETHER_GATEWAY_DEPLOYMENT_TOPOLOGY=multi-node or use --mode single-node"
cluster_env_has_required_backends "${file}" || die "existing multi-node env ${file} must define DATABASE_URL and REDIS_URL"
elif [[ "${mode}" == "single" && "${topology}" == "multi-node" ]]; then
elif [[ "${mode}" == "single-node" && "${topology}" == "multi-node" ]]; then
die "existing env ${file} is multi-node; cluster mode is temporarily disabled, edit the env file"
fi
}
@@ -1580,7 +2187,7 @@ EOF
exit 1
fi
else
info "generating first-install SQLite env file"
info "generating first-install single-node env file"
generate_first_install_env "${GENERATED_ENV}"
fi
echo "${GENERATED_ENV}"
@@ -1624,14 +2231,14 @@ EOF
compose_next_steps
}
install_compose_sqlite_mode() {
install_compose_single_node_mode() {
resolve_compose_dir
info "preparing Docker Compose SQLite deployment in ${COMPOSE_DIR}"
info "preparing Docker Compose single-node deployment in ${COMPOSE_DIR}"
ensure_directory "${COMPOSE_DIR}"
ensure_directory "${COMPOSE_DIR}/logs"
ensure_directory "${COMPOSE_DIR}/data"
install_project_file "docker-compose.sqlite.yml" "${COMPOSE_DIR}/docker-compose.yml" "0644"
install_project_file "docker-compose.single-node.yml" "${COMPOSE_DIR}/docker-compose.yml" "0644"
install_project_file ".env.example" "${COMPOSE_DIR}/.env.example" "0644"
install_generate_keys_script "${COMPOSE_DIR}/generate_keys.sh"
@@ -1639,13 +2246,13 @@ install_compose_sqlite_mode() {
warn "keeping existing ${COMPOSE_DIR}/.env"
else
info "generating ${COMPOSE_DIR}/.env"
generate_compose_sqlite_env "${COMPOSE_DIR}/.env"
generate_compose_single_node_env "${COMPOSE_DIR}/.env"
chmod 0600 "${COMPOSE_DIR}/.env"
fi
cat <<EOF
Docker Compose SQLite files are ready:
Docker Compose single-node files are ready:
${COMPOSE_DIR}/docker-compose.yml
${COMPOSE_DIR}/.env
${COMPOSE_DIR}/.env.example
@@ -2077,11 +2684,20 @@ main() {
apply_platform_defaults
select_version
select_mode
collect_interactive_migration_options
if [[ -n "${MIGRATE_FROM_COMPOSE}" ]]; then
run_migration_from_compose
return
fi
if migration_options_requested; then
die "migration options require --migrate-from-compose"
fi
if [[ "${MODE}" == "compose" ]]; then
install_compose_mode
elif [[ "${MODE}" == "compose-sqlite" ]]; then
install_compose_sqlite_mode
elif [[ "${MODE}" == "compose-single-node" ]]; then
install_compose_single_node_mode
else
require_root
require_service_manager

1028
scripts/migrate-pg-compose-to-single-node.sh Executable file
View File

File diff suppressed because it is too large Load Diff

1185
scripts/migrate-pg-to-single-node.sh Executable file
View File

File diff suppressed because it is too large Load Diff