mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 01:10:23 +08:00
Merge remote-tracking branch 'origin/pr-482'
# Conflicts: # .github/workflows/release.yml
This commit is contained in:
7
.github/workflows/release.yml
vendored
7
.github/workflows/release.yml
vendored
@@ -260,7 +260,8 @@ jobs:
|
|||||||
root="package/${bundle}"
|
root="package/${bundle}"
|
||||||
mkdir -p \
|
mkdir -p \
|
||||||
"${root}/bin" \
|
"${root}/bin" \
|
||||||
"${root}/frontend"
|
"${root}/frontend" \
|
||||||
|
"${root}/scripts"
|
||||||
|
|
||||||
install -m 0755 "artifacts/aether-gateway-${platform}-${arch}/aether-gateway" "${root}/bin/aether-gateway"
|
install -m 0755 "artifacts/aether-gateway-${platform}-${arch}/aether-gateway" "${root}/bin/aether-gateway"
|
||||||
cp -R artifacts/frontend-dist/. "${root}/frontend/"
|
cp -R artifacts/frontend-dist/. "${root}/frontend/"
|
||||||
@@ -270,7 +271,9 @@ jobs:
|
|||||||
install.sh > "${root}/install.sh"
|
install.sh > "${root}/install.sh"
|
||||||
chmod 0755 "${root}/install.sh"
|
chmod 0755 "${root}/install.sh"
|
||||||
install -m 0644 docker-compose.yml "${root}/docker-compose.yml"
|
install -m 0644 docker-compose.yml "${root}/docker-compose.yml"
|
||||||
install -m 0644 docker-compose.sqlite.yml "${root}/docker-compose.sqlite.yml"
|
install -m 0644 docker-compose.single-node.yml "${root}/docker-compose.single-node.yml"
|
||||||
|
install -m 0755 scripts/migrate-pg-compose-to-single-node.sh "${root}/scripts/migrate-pg-compose-to-single-node.sh"
|
||||||
|
install -m 0755 scripts/migrate-pg-to-single-node.sh "${root}/scripts/migrate-pg-to-single-node.sh"
|
||||||
install -m 0644 .env.example "${root}/.env.example"
|
install -m 0644 .env.example "${root}/.env.example"
|
||||||
install -m 0755 generate_keys.sh "${root}/generate_keys.sh"
|
install -m 0755 generate_keys.sh "${root}/generate_keys.sh"
|
||||||
install -m 0644 README.md "${root}/README.md"
|
install -m 0644 README.md "${root}/README.md"
|
||||||
|
|||||||
@@ -48,14 +48,14 @@ cp .env.example .env
|
|||||||
./generate_keys.sh
|
./generate_keys.sh
|
||||||
# 编辑 .env 设置 ADMIN_PASSWORD
|
# 编辑 .env 设置 ADMIN_PASSWORD
|
||||||
|
|
||||||
# 3. 首次部署 / 更新 (从以下数据库、内存策略任选其一)
|
# 3. 首次部署 / 更新 (从以下部署形态任选其一)
|
||||||
# Postgres + Redis (适用于企业或多人使用)
|
# Postgres + Redis (适用于企业或多人使用)
|
||||||
docker compose pull && docker compose up -d
|
docker compose pull && docker compose up -d
|
||||||
# 仅SQLite (适用于个人用户或朋友分享)
|
# Single Node (适用于个人用户或朋友分享)
|
||||||
docker compose -f docker-compose.sqlite.yml pull && docker compose -f docker-compose.sqlite.yml up -d
|
docker compose -f docker-compose.single-node.yml pull && docker compose -f docker-compose.single-node.yml up -d
|
||||||
```
|
```
|
||||||
|
|
||||||
### 一键安装(可选部署方式 Linux: systemd; Mac: launchd)
|
### 一键安装(默认 Single Node:Linux systemd / macOS launchd + SQLite)
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd Aether && cd Aether
|
cd Aether && cd Aether
|
||||||
|
|||||||
@@ -9,7 +9,10 @@ use clap::{Args as ClapArgs, Parser, Subcommand, ValueEnum};
|
|||||||
use tracing::{debug, info, warn};
|
use tracing::{debug, info, warn};
|
||||||
|
|
||||||
use aether_crypto::warm_python_fernet_secret;
|
use aether_crypto::warm_python_fernet_secret;
|
||||||
use aether_data::lifecycle::export::{export_database_jsonl, import_database_jsonl, ExportDomain};
|
use aether_data::lifecycle::export::{
|
||||||
|
copy_database_records, export_database_jsonl, import_database_jsonl, DataCopyOptions,
|
||||||
|
ExportDomain,
|
||||||
|
};
|
||||||
use aether_data::{DatabaseDriver, SqlDatabaseConfig, SqlPoolConfig, DEFAULT_SQLITE_DATABASE_URL};
|
use aether_data::{DatabaseDriver, SqlDatabaseConfig, SqlPoolConfig, DEFAULT_SQLITE_DATABASE_URL};
|
||||||
use aether_gateway::{
|
use aether_gateway::{
|
||||||
attach_static_frontend, build_router_with_state, set_gateway_frontdoor_app_port, AppState,
|
attach_static_frontend, build_router_with_state, set_gateway_frontdoor_app_port, AppState,
|
||||||
@@ -579,6 +582,8 @@ enum DataCommand {
|
|||||||
Export(DataExportArgs),
|
Export(DataExportArgs),
|
||||||
/// Import database-neutral JSONL into the selected SQL database.
|
/// Import database-neutral JSONL into the selected SQL database.
|
||||||
Import(DataImportArgs),
|
Import(DataImportArgs),
|
||||||
|
/// Copy persistent SQL data directly between two databases without a JSONL file.
|
||||||
|
Copy(DataCopyArgs),
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(ClapArgs, Debug, Clone)]
|
#[derive(ClapArgs, Debug, Clone)]
|
||||||
@@ -602,6 +607,27 @@ struct DataImportArgs {
|
|||||||
input: PathBuf,
|
input: PathBuf,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(ClapArgs, Debug, Clone)]
|
||||||
|
struct DataCopyArgs {
|
||||||
|
#[arg(long, value_enum)]
|
||||||
|
source_driver: DatabaseDriverArg,
|
||||||
|
|
||||||
|
#[arg(long)]
|
||||||
|
source_url: String,
|
||||||
|
|
||||||
|
#[arg(long, value_enum)]
|
||||||
|
target_driver: DatabaseDriverArg,
|
||||||
|
|
||||||
|
#[arg(long)]
|
||||||
|
target_url: String,
|
||||||
|
|
||||||
|
#[arg(long, value_enum, value_delimiter = ',')]
|
||||||
|
domains: Vec<ExportDomainArg>,
|
||||||
|
|
||||||
|
#[arg(long)]
|
||||||
|
omit_request_body_details: bool,
|
||||||
|
}
|
||||||
|
|
||||||
impl GatewayLoggingArgs {
|
impl GatewayLoggingArgs {
|
||||||
fn apply_to_runtime_config(
|
fn apply_to_runtime_config(
|
||||||
&self,
|
&self,
|
||||||
@@ -1251,6 +1277,7 @@ async fn run_data_command(command: &DataCommand) -> Result<(), Box<dyn std::erro
|
|||||||
match command {
|
match command {
|
||||||
DataCommand::Export(args) => run_data_export(args).await,
|
DataCommand::Export(args) => run_data_export(args).await,
|
||||||
DataCommand::Import(args) => run_data_import(args).await,
|
DataCommand::Import(args) => run_data_import(args).await,
|
||||||
|
DataCommand::Copy(args) => run_data_copy(args).await,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1267,11 +1294,11 @@ fn required_sql_database_config(
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn requested_export_domains(args: &DataExportArgs) -> Vec<ExportDomain> {
|
fn requested_export_domains(args: &DataExportArgs) -> Vec<ExportDomain> {
|
||||||
args.domains
|
requested_domains(&args.domains)
|
||||||
.iter()
|
}
|
||||||
.copied()
|
|
||||||
.map(Into::into)
|
fn requested_domains(domains: &[ExportDomainArg]) -> Vec<ExportDomain> {
|
||||||
.collect::<Vec<_>>()
|
domains.iter().copied().map(Into::into).collect::<Vec<_>>()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn current_unix_secs() -> Result<u64, std::time::SystemTimeError> {
|
fn current_unix_secs() -> Result<u64, std::time::SystemTimeError> {
|
||||||
@@ -1324,6 +1351,61 @@ async fn run_data_import(args: &DataImportArgs) -> Result<(), Box<dyn std::error
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn copy_database_config(
|
||||||
|
driver: DatabaseDriverArg,
|
||||||
|
url: &str,
|
||||||
|
label: &str,
|
||||||
|
) -> Result<SqlDatabaseConfig, Box<dyn std::error::Error>> {
|
||||||
|
let url = url.trim();
|
||||||
|
if url.is_empty() {
|
||||||
|
return Err(std::io::Error::new(
|
||||||
|
std::io::ErrorKind::InvalidInput,
|
||||||
|
format!("{label} database URL must not be empty"),
|
||||||
|
)
|
||||||
|
.into());
|
||||||
|
}
|
||||||
|
let driver = DatabaseDriver::from(driver);
|
||||||
|
Ok(SqlDatabaseConfig::new(
|
||||||
|
driver,
|
||||||
|
url,
|
||||||
|
SqlPoolConfig {
|
||||||
|
require_ssl: false,
|
||||||
|
..SqlPoolConfig::default()
|
||||||
|
},
|
||||||
|
)?)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn run_data_copy(args: &DataCopyArgs) -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
let source = copy_database_config(args.source_driver, &args.source_url, "source")?;
|
||||||
|
let target = copy_database_config(args.target_driver, &args.target_url, "target")?;
|
||||||
|
let source_driver = source.driver;
|
||||||
|
let target_driver = target.driver;
|
||||||
|
let domains = requested_domains(&args.domains);
|
||||||
|
let created_at_unix_secs = current_unix_secs()?;
|
||||||
|
let imported = copy_database_records(
|
||||||
|
source,
|
||||||
|
target,
|
||||||
|
domains,
|
||||||
|
created_at_unix_secs,
|
||||||
|
DataCopyOptions {
|
||||||
|
omit_request_body_details: args.omit_request_body_details,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
info!(
|
||||||
|
source_driver = %source_driver,
|
||||||
|
target_driver = %target_driver,
|
||||||
|
imported,
|
||||||
|
"database copy complete"
|
||||||
|
);
|
||||||
|
println!(
|
||||||
|
"copied {} records from {} to {} without a JSONL file",
|
||||||
|
imported, source_driver, target_driver
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
async fn run_explicit_migrations(args: &Args) -> Result<(), Box<dyn std::error::Error>> {
|
async fn run_explicit_migrations(args: &Args) -> Result<(), Box<dyn std::error::Error>> {
|
||||||
if args.data.effective_sql_database_config().is_none() {
|
if args.data.effective_sql_database_config().is_none() {
|
||||||
return Err(std::io::Error::new(
|
return Err(std::io::Error::new(
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
use std::collections::{BTreeMap, BTreeSet};
|
use std::collections::{BTreeMap, BTreeSet};
|
||||||
|
|
||||||
|
use futures_util::TryStreamExt;
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use sqlx::{Column, Row, TypeInfo, ValueRef};
|
use sqlx::{Column, Row, TypeInfo, ValueRef};
|
||||||
|
|
||||||
@@ -130,6 +131,41 @@ pub struct ExportRow {
|
|||||||
pub payload: Value,
|
pub payload: Value,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||||
|
pub struct DataCopyOptions {
|
||||||
|
pub omit_request_body_details: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
struct SqliteCopyColumn {
|
||||||
|
name: String,
|
||||||
|
declared_type: String,
|
||||||
|
not_null: bool,
|
||||||
|
has_default: bool,
|
||||||
|
primary_key_position: i64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
enum SqliteCopyAffinity {
|
||||||
|
Integer,
|
||||||
|
Real,
|
||||||
|
Text,
|
||||||
|
Blob,
|
||||||
|
Numeric,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
struct SchemaCopyColumn {
|
||||||
|
sqlite: SqliteCopyColumn,
|
||||||
|
postgres: PostgresImportColumn,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
struct SchemaCopyTable {
|
||||||
|
table_name: String,
|
||||||
|
columns: Vec<SchemaCopyColumn>,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
struct PostgresImportColumn {
|
struct PostgresImportColumn {
|
||||||
data_type: String,
|
data_type: String,
|
||||||
@@ -141,6 +177,20 @@ struct PostgresImportColumn {
|
|||||||
type PostgresImportColumns = BTreeMap<String, PostgresImportColumn>;
|
type PostgresImportColumns = BTreeMap<String, PostgresImportColumn>;
|
||||||
type ImportColumnNames = BTreeSet<String>;
|
type ImportColumnNames = BTreeSet<String>;
|
||||||
|
|
||||||
|
const USAGE_REQUEST_BODY_DETAIL_COLUMNS: &[&str] = &[
|
||||||
|
"request_body",
|
||||||
|
"response_body",
|
||||||
|
"provider_request_body",
|
||||||
|
"client_response_body",
|
||||||
|
"request_body_compressed",
|
||||||
|
"response_body_compressed",
|
||||||
|
"provider_request_body_compressed",
|
||||||
|
"client_response_body_compressed",
|
||||||
|
];
|
||||||
|
|
||||||
|
const REQUEST_BODY_DETAIL_TABLES: &[&str] = &["usage_body_blobs", "usage_http_audits"];
|
||||||
|
const LIFECYCLE_TABLES: &[&str] = &["_sqlx_migrations", "schema_backfills"];
|
||||||
|
|
||||||
pub fn encode_jsonl(records: &[DataExportRecord]) -> Result<String, DataLayerError> {
|
pub fn encode_jsonl(records: &[DataExportRecord]) -> Result<String, DataLayerError> {
|
||||||
validate_export_records(records)?;
|
validate_export_records(records)?;
|
||||||
|
|
||||||
@@ -344,6 +394,602 @@ pub async fn import_database_jsonl(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn copy_database_records(
|
||||||
|
source: SqlDatabaseConfig,
|
||||||
|
target: SqlDatabaseConfig,
|
||||||
|
domains: Vec<ExportDomain>,
|
||||||
|
created_at_unix_secs: u64,
|
||||||
|
options: DataCopyOptions,
|
||||||
|
) -> Result<usize, DataLayerError> {
|
||||||
|
if domains.is_empty()
|
||||||
|
&& source.driver == DatabaseDriver::Postgres
|
||||||
|
&& target.driver == DatabaseDriver::Sqlite
|
||||||
|
{
|
||||||
|
return copy_postgres_to_sqlite_from_target_schema(source, target, options).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut records =
|
||||||
|
decode_jsonl(&export_database_jsonl(source, domains, created_at_unix_secs).await?)?;
|
||||||
|
if options.omit_request_body_details {
|
||||||
|
omit_request_body_details_from_records(&mut records);
|
||||||
|
}
|
||||||
|
import_database_jsonl(target, &encode_jsonl(&records)?).await
|
||||||
|
}
|
||||||
|
|
||||||
|
fn omit_request_body_details_from_records(records: &mut [DataExportRecord]) {
|
||||||
|
for record in records {
|
||||||
|
let DataExportRecord::Row {
|
||||||
|
domain: ExportDomain::Usage,
|
||||||
|
payload,
|
||||||
|
..
|
||||||
|
} = record
|
||||||
|
else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
if let Some(object) = payload.as_object_mut() {
|
||||||
|
for column_name in USAGE_REQUEST_BODY_DETAIL_COLUMNS {
|
||||||
|
object.remove(*column_name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn copy_postgres_to_sqlite_from_target_schema(
|
||||||
|
source: SqlDatabaseConfig,
|
||||||
|
mut target: SqlDatabaseConfig,
|
||||||
|
options: DataCopyOptions,
|
||||||
|
) -> Result<usize, DataLayerError> {
|
||||||
|
target.pool.min_connections = 1;
|
||||||
|
target.pool.max_connections = 1;
|
||||||
|
|
||||||
|
let postgres_pool =
|
||||||
|
crate::driver::postgres::PostgresPoolFactory::new(source.to_postgres_config()?)?
|
||||||
|
.connect_lazy()?;
|
||||||
|
let sqlite_pool = crate::driver::sqlite::SqlitePoolFactory::new(target)?.connect_lazy()?;
|
||||||
|
|
||||||
|
let source_tables = load_postgres_public_table_names(&postgres_pool).await?;
|
||||||
|
let target_tables = load_sqlite_copy_table_names(&sqlite_pool).await?;
|
||||||
|
|
||||||
|
ensure_no_nonempty_source_tables_outside_target_schema(
|
||||||
|
&postgres_pool,
|
||||||
|
&source_tables,
|
||||||
|
&target_tables,
|
||||||
|
options,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let mut imported = 0usize;
|
||||||
|
sqlx::raw_sql("PRAGMA foreign_keys = OFF")
|
||||||
|
.execute(&sqlite_pool)
|
||||||
|
.await
|
||||||
|
.map_sql_err()?;
|
||||||
|
|
||||||
|
for table_name in target_tables {
|
||||||
|
if copy_table_is_lifecycle(&table_name)
|
||||||
|
|| copy_table_is_sqlite_internal(&table_name)
|
||||||
|
|| !source_tables.contains(&table_name)
|
||||||
|
|| (options.omit_request_body_details && copy_table_is_request_body_detail(&table_name))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let table_plan = build_postgres_sqlite_copy_table_plan(
|
||||||
|
&postgres_pool,
|
||||||
|
&sqlite_pool,
|
||||||
|
&table_name,
|
||||||
|
options,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
if table_plan.columns.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
imported = imported.saturating_add(
|
||||||
|
copy_postgres_sqlite_table(&postgres_pool, &sqlite_pool, &table_plan).await?,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
sqlx::raw_sql("PRAGMA foreign_keys = ON")
|
||||||
|
.execute(&sqlite_pool)
|
||||||
|
.await
|
||||||
|
.map_sql_err()?;
|
||||||
|
ensure_sqlite_foreign_key_check_passes(&sqlite_pool).await?;
|
||||||
|
Ok(imported)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn ensure_no_nonempty_source_tables_outside_target_schema(
|
||||||
|
postgres_pool: &crate::driver::postgres::PostgresPool,
|
||||||
|
source_tables: &BTreeSet<String>,
|
||||||
|
target_tables: &BTreeSet<String>,
|
||||||
|
options: DataCopyOptions,
|
||||||
|
) -> Result<(), DataLayerError> {
|
||||||
|
let mut missing = Vec::new();
|
||||||
|
for table_name in source_tables {
|
||||||
|
if copy_table_is_lifecycle(table_name)
|
||||||
|
|| (options.omit_request_body_details && copy_table_is_request_body_detail(table_name))
|
||||||
|
|| target_tables.contains(table_name)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if postgres_public_table_has_rows(postgres_pool, table_name).await? {
|
||||||
|
missing.push(table_name.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !missing.is_empty() {
|
||||||
|
return Err(DataLayerError::InvalidInput(format!(
|
||||||
|
"source Postgres has non-empty public tables that do not exist in the target SQLite schema: {}",
|
||||||
|
missing.join(", ")
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn build_postgres_sqlite_copy_table_plan(
|
||||||
|
postgres_pool: &crate::driver::postgres::PostgresPool,
|
||||||
|
sqlite_pool: &crate::driver::sqlite::SqlitePool,
|
||||||
|
table_name: &str,
|
||||||
|
options: DataCopyOptions,
|
||||||
|
) -> Result<SchemaCopyTable, DataLayerError> {
|
||||||
|
let sqlite_columns = load_sqlite_copy_columns(sqlite_pool, table_name).await?;
|
||||||
|
let postgres_columns =
|
||||||
|
load_postgres_import_columns(postgres_pool, &format!("public.{table_name}")).await?;
|
||||||
|
let source_has_rows = postgres_public_table_has_rows(postgres_pool, table_name).await?;
|
||||||
|
let mut columns = Vec::new();
|
||||||
|
|
||||||
|
for sqlite_column in sqlite_columns {
|
||||||
|
if options.omit_request_body_details
|
||||||
|
&& table_name == "usage"
|
||||||
|
&& USAGE_REQUEST_BODY_DETAIL_COLUMNS.contains(&sqlite_column.name.as_str())
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(postgres_column) = postgres_columns.get(&sqlite_column.name) {
|
||||||
|
columns.push(SchemaCopyColumn {
|
||||||
|
sqlite: sqlite_column,
|
||||||
|
postgres: postgres_column.clone(),
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if source_has_rows && sqlite_copy_column_is_required(&sqlite_column) {
|
||||||
|
return Err(DataLayerError::InvalidInput(format!(
|
||||||
|
"target SQLite table '{table_name}' has required column '{}' that does not exist in source Postgres",
|
||||||
|
sqlite_column.name
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if source_has_rows && columns.is_empty() {
|
||||||
|
return Err(DataLayerError::InvalidInput(format!(
|
||||||
|
"source Postgres table '{table_name}' has rows, but none of its columns exist in target SQLite"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(SchemaCopyTable {
|
||||||
|
table_name: table_name.to_string(),
|
||||||
|
columns,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn copy_postgres_sqlite_table(
|
||||||
|
postgres_pool: &crate::driver::postgres::PostgresPool,
|
||||||
|
sqlite_pool: &crate::driver::sqlite::SqlitePool,
|
||||||
|
table: &SchemaCopyTable,
|
||||||
|
) -> Result<usize, DataLayerError> {
|
||||||
|
let source_sql = postgres_schema_copy_select_sql(table)?;
|
||||||
|
let target_sql = sqlite_schema_copy_insert_sql(table)?;
|
||||||
|
let mut rows = sqlx::query(&source_sql).fetch(postgres_pool);
|
||||||
|
let mut imported = 0usize;
|
||||||
|
|
||||||
|
while let Some(row) = rows.try_next().await.map_sql_err()? {
|
||||||
|
let payload = row.try_get::<Value, _>("payload").map_sql_err()?;
|
||||||
|
let object = payload.as_object().ok_or_else(|| {
|
||||||
|
DataLayerError::UnexpectedValue(format!(
|
||||||
|
"postgres copy row for table '{}' did not produce a JSON object",
|
||||||
|
table.table_name
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
let mut query = sqlx::query(&target_sql);
|
||||||
|
for column in &table.columns {
|
||||||
|
let value = object.get(&column.sqlite.name).ok_or_else(|| {
|
||||||
|
DataLayerError::UnexpectedValue(format!(
|
||||||
|
"postgres copy row for table '{}' is missing column '{}'",
|
||||||
|
table.table_name, column.sqlite.name
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
query = bind_sqlite_copy_value(query, value, &column.sqlite)?;
|
||||||
|
}
|
||||||
|
query.execute(sqlite_pool).await.map_sql_err()?;
|
||||||
|
imported = imported.saturating_add(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(imported)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn postgres_schema_copy_select_sql(table: &SchemaCopyTable) -> Result<String, DataLayerError> {
|
||||||
|
let table_sql = format!(
|
||||||
|
"public.{}",
|
||||||
|
postgres_quote_identifier(table.table_name.as_str())?
|
||||||
|
);
|
||||||
|
let mut payload_parts = Vec::new();
|
||||||
|
for column in &table.columns {
|
||||||
|
if let Some(expr) = postgres_schema_copy_override_expr(column)? {
|
||||||
|
payload_parts.push(sql_string_literal(&column.sqlite.name));
|
||||||
|
payload_parts.push(expr);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let payload_sql = if payload_parts.is_empty() {
|
||||||
|
"to_jsonb(t)".to_string()
|
||||||
|
} else {
|
||||||
|
format!(
|
||||||
|
"to_jsonb(t) || jsonb_build_object({})",
|
||||||
|
payload_parts.join(", ")
|
||||||
|
)
|
||||||
|
};
|
||||||
|
|
||||||
|
let order_by = table
|
||||||
|
.columns
|
||||||
|
.iter()
|
||||||
|
.filter(|column| column.sqlite.primary_key_position > 0)
|
||||||
|
.map(|column| {
|
||||||
|
postgres_quote_identifier(&column.sqlite.name).map(|quoted| format!("t.{quoted} ASC"))
|
||||||
|
})
|
||||||
|
.collect::<Result<Vec<_>, _>>()?;
|
||||||
|
let order_sql = if order_by.is_empty() {
|
||||||
|
String::new()
|
||||||
|
} else {
|
||||||
|
format!(" ORDER BY {}", order_by.join(", "))
|
||||||
|
};
|
||||||
|
|
||||||
|
Ok(format!(
|
||||||
|
"SELECT {payload_sql} AS payload FROM {table_sql} AS t{order_sql}"
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn postgres_schema_copy_override_expr(
|
||||||
|
column: &SchemaCopyColumn,
|
||||||
|
) -> Result<Option<String>, DataLayerError> {
|
||||||
|
let column_sql = format!("t.{}", postgres_quote_identifier(&column.sqlite.name)?);
|
||||||
|
let affinity = sqlite_copy_affinity(&column.sqlite);
|
||||||
|
|
||||||
|
if affinity == SqliteCopyAffinity::Blob && is_postgres_bytea_column(&column.postgres) {
|
||||||
|
return Ok(Some(format!(
|
||||||
|
"CASE WHEN {column_sql} IS NULL THEN NULL ELSE encode({column_sql}, 'hex') END"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
if affinity == SqliteCopyAffinity::Integer && is_postgres_boolean_column(&column.postgres) {
|
||||||
|
return Ok(Some(format!(
|
||||||
|
"CASE WHEN {column_sql} IS NULL THEN NULL WHEN {column_sql} THEN 1 ELSE 0 END"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
if affinity == SqliteCopyAffinity::Integer
|
||||||
|
&& (is_postgres_timestamp_column(&column.postgres)
|
||||||
|
|| is_postgres_date_column(&column.postgres))
|
||||||
|
{
|
||||||
|
let timestamp_sql = if is_postgres_date_column(&column.postgres) {
|
||||||
|
format!("{column_sql}::timestamp")
|
||||||
|
} else {
|
||||||
|
column_sql.clone()
|
||||||
|
};
|
||||||
|
let multiplier = if sqlite_copy_column_stores_unix_millis(&column.sqlite.name) {
|
||||||
|
" * 1000"
|
||||||
|
} else {
|
||||||
|
""
|
||||||
|
};
|
||||||
|
return Ok(Some(format!(
|
||||||
|
"CASE WHEN {column_sql} IS NULL THEN NULL ELSE FLOOR(EXTRACT(EPOCH FROM {timestamp_sql}){multiplier})::bigint END"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(None)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sqlite_schema_copy_insert_sql(table: &SchemaCopyTable) -> Result<String, DataLayerError> {
|
||||||
|
let table_sql = sqlite_quote_identifier(&table.table_name)?;
|
||||||
|
let column_sql = table
|
||||||
|
.columns
|
||||||
|
.iter()
|
||||||
|
.map(|column| sqlite_quote_identifier(&column.sqlite.name))
|
||||||
|
.collect::<Result<Vec<_>, _>>()?
|
||||||
|
.join(", ");
|
||||||
|
let placeholder_sql = vec!["?"; table.columns.len()].join(", ");
|
||||||
|
Ok(format!(
|
||||||
|
"INSERT OR REPLACE INTO {table_sql} ({column_sql}) VALUES ({placeholder_sql})"
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn load_postgres_public_table_names(
|
||||||
|
pool: &crate::driver::postgres::PostgresPool,
|
||||||
|
) -> Result<BTreeSet<String>, DataLayerError> {
|
||||||
|
let rows = sqlx::query(
|
||||||
|
r#"
|
||||||
|
SELECT table_name
|
||||||
|
FROM information_schema.tables
|
||||||
|
WHERE table_schema = 'public'
|
||||||
|
AND table_type = 'BASE TABLE'
|
||||||
|
ORDER BY table_name
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
.map_sql_err()?;
|
||||||
|
|
||||||
|
let mut tables = BTreeSet::new();
|
||||||
|
for row in rows {
|
||||||
|
tables.insert(row.try_get::<String, _>("table_name").map_sql_err()?);
|
||||||
|
}
|
||||||
|
Ok(tables)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn load_sqlite_copy_table_names(
|
||||||
|
pool: &crate::driver::sqlite::SqlitePool,
|
||||||
|
) -> Result<BTreeSet<String>, DataLayerError> {
|
||||||
|
let rows = sqlx::query(
|
||||||
|
r#"
|
||||||
|
SELECT name
|
||||||
|
FROM sqlite_schema
|
||||||
|
WHERE type = 'table'
|
||||||
|
AND name NOT LIKE 'sqlite_%'
|
||||||
|
ORDER BY name
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
.map_sql_err()?;
|
||||||
|
|
||||||
|
let mut tables = BTreeSet::new();
|
||||||
|
for row in rows {
|
||||||
|
let table_name = row.try_get::<String, _>("name").map_sql_err()?;
|
||||||
|
if !copy_table_is_lifecycle(&table_name) && !copy_table_is_sqlite_internal(&table_name) {
|
||||||
|
tables.insert(table_name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(tables)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn load_sqlite_copy_columns(
|
||||||
|
pool: &crate::driver::sqlite::SqlitePool,
|
||||||
|
table_name: &str,
|
||||||
|
) -> Result<Vec<SqliteCopyColumn>, DataLayerError> {
|
||||||
|
let table_sql = sqlite_quote_identifier(table_name)?;
|
||||||
|
let rows = sqlx::query(&format!("PRAGMA table_info({table_sql})"))
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
.map_sql_err()?;
|
||||||
|
|
||||||
|
let mut columns = Vec::new();
|
||||||
|
for row in rows {
|
||||||
|
columns.push(SqliteCopyColumn {
|
||||||
|
name: row.try_get::<String, _>("name").map_sql_err()?,
|
||||||
|
declared_type: row
|
||||||
|
.try_get::<Option<String>, _>("type")
|
||||||
|
.map_sql_err()?
|
||||||
|
.unwrap_or_default(),
|
||||||
|
not_null: row.try_get::<i64, _>("notnull").map_sql_err()? != 0,
|
||||||
|
has_default: row
|
||||||
|
.try_get::<Option<String>, _>("dflt_value")
|
||||||
|
.map_sql_err()?
|
||||||
|
.is_some(),
|
||||||
|
primary_key_position: row.try_get::<i64, _>("pk").map_sql_err()?,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if columns.is_empty() {
|
||||||
|
return Err(DataLayerError::UnexpectedValue(format!(
|
||||||
|
"target SQLite table '{table_name}' has no visible columns"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(columns)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn postgres_public_table_has_rows(
|
||||||
|
pool: &crate::driver::postgres::PostgresPool,
|
||||||
|
table_name: &str,
|
||||||
|
) -> Result<bool, DataLayerError> {
|
||||||
|
let table_sql = format!("public.{}", postgres_quote_identifier(table_name)?);
|
||||||
|
sqlx::query_scalar::<_, bool>(&format!(
|
||||||
|
"SELECT EXISTS (SELECT 1 FROM {table_sql} LIMIT 1)"
|
||||||
|
))
|
||||||
|
.fetch_one(pool)
|
||||||
|
.await
|
||||||
|
.map_sql_err()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn ensure_sqlite_foreign_key_check_passes(
|
||||||
|
pool: &crate::driver::sqlite::SqlitePool,
|
||||||
|
) -> Result<(), DataLayerError> {
|
||||||
|
let rows = sqlx::query("PRAGMA foreign_key_check")
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
.map_sql_err()?;
|
||||||
|
if rows.is_empty() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut violations = Vec::new();
|
||||||
|
for row in rows.iter().take(10) {
|
||||||
|
let table = row
|
||||||
|
.try_get::<Option<String>, _>("table")
|
||||||
|
.map_sql_err()?
|
||||||
|
.unwrap_or_else(|| "<unknown>".to_string());
|
||||||
|
let rowid = row.try_get::<Option<i64>, _>("rowid").map_sql_err()?;
|
||||||
|
let parent = row
|
||||||
|
.try_get::<Option<String>, _>("parent")
|
||||||
|
.map_sql_err()?
|
||||||
|
.unwrap_or_else(|| "<unknown>".to_string());
|
||||||
|
violations.push(format!("{table} rowid={rowid:?} parent={parent}"));
|
||||||
|
}
|
||||||
|
Err(DataLayerError::InvalidInput(format!(
|
||||||
|
"target SQLite foreign key check failed after copy: {}",
|
||||||
|
violations.join("; ")
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn copy_table_is_lifecycle(table_name: &str) -> bool {
|
||||||
|
LIFECYCLE_TABLES.contains(&table_name)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn copy_table_is_sqlite_internal(table_name: &str) -> bool {
|
||||||
|
table_name.starts_with("sqlite_")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn copy_table_is_request_body_detail(table_name: &str) -> bool {
|
||||||
|
REQUEST_BODY_DETAIL_TABLES.contains(&table_name)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sqlite_copy_column_is_required(column: &SqliteCopyColumn) -> bool {
|
||||||
|
(column.not_null || column.primary_key_position > 0) && !column.has_default
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sqlite_copy_column_stores_unix_millis(column_name: &str) -> bool {
|
||||||
|
column_name.ends_with("_unix_ms")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sqlite_copy_affinity(column: &SqliteCopyColumn) -> SqliteCopyAffinity {
|
||||||
|
let declared_type = column.declared_type.to_ascii_uppercase();
|
||||||
|
if declared_type.contains("INT") {
|
||||||
|
SqliteCopyAffinity::Integer
|
||||||
|
} else if declared_type.contains("CHAR")
|
||||||
|
|| declared_type.contains("CLOB")
|
||||||
|
|| declared_type.contains("TEXT")
|
||||||
|
{
|
||||||
|
SqliteCopyAffinity::Text
|
||||||
|
} else if declared_type.contains("BLOB") || declared_type.trim().is_empty() {
|
||||||
|
SqliteCopyAffinity::Blob
|
||||||
|
} else if declared_type.contains("REAL")
|
||||||
|
|| declared_type.contains("FLOA")
|
||||||
|
|| declared_type.contains("DOUB")
|
||||||
|
{
|
||||||
|
SqliteCopyAffinity::Real
|
||||||
|
} else {
|
||||||
|
SqliteCopyAffinity::Numeric
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_postgres_bytea_column(column: &PostgresImportColumn) -> bool {
|
||||||
|
column.data_type == "bytea" || column.udt_name == "bytea"
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_postgres_date_column(column: &PostgresImportColumn) -> bool {
|
||||||
|
column.data_type == "date" || column.udt_name == "date"
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bind_sqlite_copy_value<'q>(
|
||||||
|
query: sqlx::query::Query<'q, sqlx::Sqlite, sqlx::sqlite::SqliteArguments<'q>>,
|
||||||
|
value: &'q Value,
|
||||||
|
column: &SqliteCopyColumn,
|
||||||
|
) -> Result<sqlx::query::Query<'q, sqlx::Sqlite, sqlx::sqlite::SqliteArguments<'q>>, DataLayerError>
|
||||||
|
{
|
||||||
|
Ok(match sqlite_copy_affinity(column) {
|
||||||
|
SqliteCopyAffinity::Integer => match value {
|
||||||
|
Value::Null => query.bind(Option::<i64>::None),
|
||||||
|
Value::Bool(value) => query.bind(i64::from(*value)),
|
||||||
|
Value::Number(number) => {
|
||||||
|
let value = number
|
||||||
|
.as_i64()
|
||||||
|
.or_else(|| number.as_u64().and_then(|value| i64::try_from(value).ok()))
|
||||||
|
.ok_or_else(|| {
|
||||||
|
DataLayerError::InvalidInput(format!(
|
||||||
|
"sqlite copy column '{}' expected integer, got {number}",
|
||||||
|
column.name
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
query.bind(value)
|
||||||
|
}
|
||||||
|
Value::String(value) => query.bind(value.parse::<i64>().map_err(|err| {
|
||||||
|
DataLayerError::InvalidInput(format!(
|
||||||
|
"sqlite copy column '{}' expected integer string: {err}",
|
||||||
|
column.name
|
||||||
|
))
|
||||||
|
})?),
|
||||||
|
Value::Array(_) | Value::Object(_) => {
|
||||||
|
return Err(DataLayerError::InvalidInput(format!(
|
||||||
|
"sqlite copy column '{}' expected integer-compatible value",
|
||||||
|
column.name
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
},
|
||||||
|
SqliteCopyAffinity::Real => match value {
|
||||||
|
Value::Null => query.bind(Option::<f64>::None),
|
||||||
|
Value::Number(number) => query.bind(number.as_f64().ok_or_else(|| {
|
||||||
|
DataLayerError::InvalidInput(format!(
|
||||||
|
"sqlite copy column '{}' expected finite real value",
|
||||||
|
column.name
|
||||||
|
))
|
||||||
|
})?),
|
||||||
|
Value::String(value) => query.bind(value.parse::<f64>().map_err(|err| {
|
||||||
|
DataLayerError::InvalidInput(format!(
|
||||||
|
"sqlite copy column '{}' expected real string: {err}",
|
||||||
|
column.name
|
||||||
|
))
|
||||||
|
})?),
|
||||||
|
Value::Bool(value) => query.bind(if *value { 1.0 } else { 0.0 }),
|
||||||
|
Value::Array(_) | Value::Object(_) => {
|
||||||
|
return Err(DataLayerError::InvalidInput(format!(
|
||||||
|
"sqlite copy column '{}' expected real-compatible value",
|
||||||
|
column.name
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
},
|
||||||
|
SqliteCopyAffinity::Blob => match value {
|
||||||
|
Value::Null => query.bind(Option::<Vec<u8>>::None),
|
||||||
|
Value::String(value) => query.bind(hex_decode(value, &column.name)?),
|
||||||
|
Value::Array(values) => {
|
||||||
|
let mut bytes = Vec::with_capacity(values.len());
|
||||||
|
for value in values {
|
||||||
|
let Some(byte) = value.as_u64().and_then(|value| u8::try_from(value).ok())
|
||||||
|
else {
|
||||||
|
return Err(DataLayerError::InvalidInput(format!(
|
||||||
|
"sqlite copy column '{}' contains non-byte array value",
|
||||||
|
column.name
|
||||||
|
)));
|
||||||
|
};
|
||||||
|
bytes.push(byte);
|
||||||
|
}
|
||||||
|
query.bind(bytes)
|
||||||
|
}
|
||||||
|
Value::Bool(_) | Value::Number(_) | Value::Object(_) => {
|
||||||
|
return Err(DataLayerError::InvalidInput(format!(
|
||||||
|
"sqlite copy column '{}' expected blob-compatible value",
|
||||||
|
column.name
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
},
|
||||||
|
SqliteCopyAffinity::Text | SqliteCopyAffinity::Numeric => {
|
||||||
|
bind_sqlite_json_value(query, value)?
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sql_string_literal(value: &str) -> String {
|
||||||
|
format!("'{}'", value.replace('\'', "''"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hex_decode(value: &str, column_name: &str) -> Result<Vec<u8>, DataLayerError> {
|
||||||
|
let value = value.trim();
|
||||||
|
if !value.len().is_multiple_of(2) {
|
||||||
|
return Err(DataLayerError::InvalidInput(format!(
|
||||||
|
"sqlite copy column '{column_name}' has odd-length hex data"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut bytes = Vec::with_capacity(value.len() / 2);
|
||||||
|
for index in (0..value.len()).step_by(2) {
|
||||||
|
let byte = u8::from_str_radix(&value[index..index + 2], 16).map_err(|err| {
|
||||||
|
DataLayerError::InvalidInput(format!(
|
||||||
|
"sqlite copy column '{column_name}' has invalid hex data at byte {}: {err}",
|
||||||
|
index / 2
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
bytes.push(byte);
|
||||||
|
}
|
||||||
|
Ok(bytes)
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn export_sqlite_core_jsonl(
|
pub async fn export_sqlite_core_jsonl(
|
||||||
pool: &crate::driver::sqlite::SqlitePool,
|
pool: &crate::driver::sqlite::SqlitePool,
|
||||||
created_at_unix_secs: u64,
|
created_at_unix_secs: u64,
|
||||||
|
|||||||
@@ -45,7 +45,15 @@ SELECT
|
|||||||
m.provider_model_mappings AS model_provider_model_mappings,
|
m.provider_model_mappings AS model_provider_model_mappings,
|
||||||
m.supports_streaming AS model_supports_streaming,
|
m.supports_streaming AS model_supports_streaming,
|
||||||
m.is_active AS model_is_active,
|
m.is_active AS model_is_active,
|
||||||
m.is_available AS model_is_available
|
m.is_available AS model_is_available,
|
||||||
|
CASE
|
||||||
|
WHEN json_valid(p.config) THEN
|
||||||
|
CASE
|
||||||
|
WHEN json_type(p.config, '$.pool_advanced') IS NOT NULL THEN 1
|
||||||
|
ELSE 0
|
||||||
|
END
|
||||||
|
ELSE 0
|
||||||
|
END AS provider_pool_enabled
|
||||||
FROM providers p
|
FROM providers p
|
||||||
INNER JOIN provider_endpoints pe ON pe.provider_id = p.id
|
INNER JOIN provider_endpoints pe ON pe.provider_id = p.id
|
||||||
INNER JOIN provider_api_keys pak ON pak.provider_id = p.id
|
INNER JOIN provider_api_keys pak ON pak.provider_id = p.id
|
||||||
@@ -67,52 +75,139 @@ pub struct SqliteMinimalCandidateSelectionReadRepository {
|
|||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
struct CandidateSelectionRow {
|
struct CandidateSelectionRow {
|
||||||
row: StoredMinimalCandidateSelectionRow,
|
row: StoredMinimalCandidateSelectionRow,
|
||||||
provider_pool_enabled: bool,
|
|
||||||
key_auth_config: Option<String>,
|
key_auth_config: Option<String>,
|
||||||
key_last_used_at_unix_secs: Option<u64>,
|
key_last_used_at_unix_secs: Option<u64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy)]
|
||||||
|
enum SelectedRowsOrder {
|
||||||
|
WithGlobalModel,
|
||||||
|
WithoutGlobalModel,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy)]
|
||||||
|
enum SelectedRowsFilter<'a> {
|
||||||
|
None,
|
||||||
|
GlobalModel(&'a str),
|
||||||
|
RequestedModel(&'a str),
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy)]
|
||||||
|
struct SqlPage {
|
||||||
|
limit: i64,
|
||||||
|
offset: i64,
|
||||||
|
}
|
||||||
|
|
||||||
impl SqliteMinimalCandidateSelectionReadRepository {
|
impl SqliteMinimalCandidateSelectionReadRepository {
|
||||||
pub fn new(pool: SqlitePool) -> Self {
|
pub fn new(pool: SqlitePool) -> Self {
|
||||||
Self { pool }
|
Self { pool }
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn load_rows_for_api_format(
|
|
||||||
&self,
|
|
||||||
api_format: &str,
|
|
||||||
) -> Result<Vec<CandidateSelectionRow>, DataLayerError> {
|
|
||||||
let canonical_api_format = normalize_api_format(api_format);
|
|
||||||
let storage_aliases = api_format_aliases(&canonical_api_format);
|
|
||||||
let match_aliases = sql_match_aliases(&storage_aliases);
|
|
||||||
|
|
||||||
let mut builder = QueryBuilder::<Sqlite>::new(CANDIDATE_SELECTION_COLUMNS);
|
|
||||||
builder.push(" AND LOWER(pe.api_format) IN (");
|
|
||||||
{
|
|
||||||
let mut separated = builder.separated(", ");
|
|
||||||
for alias in &match_aliases {
|
|
||||||
separated.push_bind(alias);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
builder.push(")");
|
|
||||||
let rows = builder.build().fetch_all(&self.pool).await.map_sql_err()?;
|
|
||||||
let mut items = rows
|
|
||||||
.iter()
|
|
||||||
.map(map_candidate_selection_row)
|
|
||||||
.collect::<Result<Vec<_>, _>>()?;
|
|
||||||
items.retain(|item| {
|
|
||||||
api_format_matches(&item.row.endpoint_api_format, &canonical_api_format)
|
|
||||||
&& item.row.key_supports_api_format(&canonical_api_format)
|
|
||||||
&& key_auth_channel_matches(item, &canonical_api_format)
|
|
||||||
});
|
|
||||||
Ok(items)
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn selected_rows_for_api_format(
|
async fn selected_rows_for_api_format(
|
||||||
&self,
|
&self,
|
||||||
api_format: &str,
|
api_format: &str,
|
||||||
) -> Result<Vec<StoredMinimalCandidateSelectionRow>, DataLayerError> {
|
) -> Result<Vec<StoredMinimalCandidateSelectionRow>, DataLayerError> {
|
||||||
let rows = self.load_rows_for_api_format(api_format).await?;
|
self.load_selected_rows_for_api_format(
|
||||||
Ok(sort_rows(select_pool_rows(rows), true))
|
api_format,
|
||||||
|
SelectedRowsFilter::None,
|
||||||
|
SelectedRowsOrder::WithGlobalModel,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn load_selected_rows_for_api_format(
|
||||||
|
&self,
|
||||||
|
api_format: &str,
|
||||||
|
filter: SelectedRowsFilter<'_>,
|
||||||
|
order: SelectedRowsOrder,
|
||||||
|
page: Option<SqlPage>,
|
||||||
|
) -> Result<Vec<StoredMinimalCandidateSelectionRow>, DataLayerError> {
|
||||||
|
let canonical_api_format = normalize_api_format(api_format);
|
||||||
|
let storage_aliases = api_format_aliases(&canonical_api_format);
|
||||||
|
let match_aliases = sql_match_aliases(&storage_aliases);
|
||||||
|
let mut rows = Vec::new();
|
||||||
|
|
||||||
|
for storage_api_format in storage_aliases {
|
||||||
|
let mut builder = QueryBuilder::<Sqlite>::new("WITH candidate_rows AS (");
|
||||||
|
builder.push(CANDIDATE_SELECTION_COLUMNS);
|
||||||
|
push_candidate_sql_filters(&mut builder, &storage_api_format, &match_aliases);
|
||||||
|
match filter {
|
||||||
|
SelectedRowsFilter::None => {}
|
||||||
|
SelectedRowsFilter::GlobalModel(global_model_name) => {
|
||||||
|
builder.push(" AND gm.name = ");
|
||||||
|
builder.push_bind(global_model_name);
|
||||||
|
}
|
||||||
|
SelectedRowsFilter::RequestedModel(requested_model_name) => {
|
||||||
|
push_requested_model_sql_filter(
|
||||||
|
&mut builder,
|
||||||
|
requested_model_name,
|
||||||
|
&match_aliases,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
builder.push(
|
||||||
|
r#"
|
||||||
|
),
|
||||||
|
pool_rows AS (
|
||||||
|
SELECT candidate.*
|
||||||
|
FROM candidate_rows candidate
|
||||||
|
WHERE candidate.provider_pool_enabled = 1
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM candidate_rows other
|
||||||
|
WHERE other.provider_pool_enabled = 1
|
||||||
|
AND other.provider_id = candidate.provider_id
|
||||||
|
AND other.endpoint_id = candidate.endpoint_id
|
||||||
|
AND other.model_id = candidate.model_id
|
||||||
|
AND (
|
||||||
|
other.key_internal_priority < candidate.key_internal_priority
|
||||||
|
OR (
|
||||||
|
other.key_internal_priority = candidate.key_internal_priority
|
||||||
|
AND other.key_id < candidate.key_id
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
),
|
||||||
|
selected_rows AS (
|
||||||
|
SELECT * FROM candidate_rows WHERE provider_pool_enabled = 0
|
||||||
|
UNION ALL
|
||||||
|
SELECT * FROM pool_rows
|
||||||
|
)
|
||||||
|
SELECT * FROM selected_rows
|
||||||
|
"#,
|
||||||
|
);
|
||||||
|
push_selected_rows_order(&mut builder, order);
|
||||||
|
if let Some(page) = page {
|
||||||
|
builder.push(" LIMIT ");
|
||||||
|
builder.push_bind(page.limit);
|
||||||
|
builder.push(" OFFSET ");
|
||||||
|
builder.push_bind(page.offset);
|
||||||
|
}
|
||||||
|
|
||||||
|
let query_rows = builder.build().fetch_all(&self.pool).await.map_sql_err()?;
|
||||||
|
let mut items = query_rows
|
||||||
|
.iter()
|
||||||
|
.map(map_candidate_selection_row)
|
||||||
|
.collect::<Result<Vec<_>, _>>()?;
|
||||||
|
items.retain(|item| {
|
||||||
|
api_format_matches(&item.row.endpoint_api_format, &canonical_api_format)
|
||||||
|
&& item.row.key_supports_api_format(&canonical_api_format)
|
||||||
|
&& key_auth_channel_matches(item, &canonical_api_format)
|
||||||
|
});
|
||||||
|
rows.extend(items.into_iter().map(|item| item.row));
|
||||||
|
}
|
||||||
|
|
||||||
|
let rows = match filter {
|
||||||
|
SelectedRowsFilter::RequestedModel(requested_model_name) => rows
|
||||||
|
.into_iter()
|
||||||
|
.filter(|row| {
|
||||||
|
row_matches_requested_model(row, requested_model_name, &canonical_api_format)
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
_ => rows,
|
||||||
|
};
|
||||||
|
Ok(dedupe_candidate_selection_rows(rows))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -130,14 +225,13 @@ impl MinimalCandidateSelectionReadRepository for SqliteMinimalCandidateSelection
|
|||||||
api_format: &str,
|
api_format: &str,
|
||||||
global_model_name: &str,
|
global_model_name: &str,
|
||||||
) -> Result<Vec<StoredMinimalCandidateSelectionRow>, DataLayerError> {
|
) -> Result<Vec<StoredMinimalCandidateSelectionRow>, DataLayerError> {
|
||||||
Ok(sort_rows(
|
self.load_selected_rows_for_api_format(
|
||||||
self.selected_rows_for_api_format(api_format)
|
api_format,
|
||||||
.await?
|
SelectedRowsFilter::GlobalModel(global_model_name),
|
||||||
.into_iter()
|
SelectedRowsOrder::WithoutGlobalModel,
|
||||||
.filter(|row| row.global_model_name == global_model_name)
|
None,
|
||||||
.collect(),
|
)
|
||||||
false,
|
.await
|
||||||
))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn list_for_exact_api_format_and_requested_model(
|
async fn list_for_exact_api_format_and_requested_model(
|
||||||
@@ -145,13 +239,11 @@ impl MinimalCandidateSelectionReadRepository for SqliteMinimalCandidateSelection
|
|||||||
api_format: &str,
|
api_format: &str,
|
||||||
requested_model_name: &str,
|
requested_model_name: &str,
|
||||||
) -> Result<Vec<StoredMinimalCandidateSelectionRow>, DataLayerError> {
|
) -> Result<Vec<StoredMinimalCandidateSelectionRow>, DataLayerError> {
|
||||||
self.list_for_exact_api_format_and_requested_model_page(
|
self.load_selected_rows_for_api_format(
|
||||||
&StoredRequestedModelCandidateRowsQuery {
|
api_format,
|
||||||
api_format: api_format.to_string(),
|
SelectedRowsFilter::RequestedModel(requested_model_name),
|
||||||
requested_model_name: requested_model_name.to_string(),
|
SelectedRowsOrder::WithGlobalModel,
|
||||||
offset: 0,
|
None,
|
||||||
limit: u32::MAX,
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
@@ -160,42 +252,74 @@ impl MinimalCandidateSelectionReadRepository for SqliteMinimalCandidateSelection
|
|||||||
&self,
|
&self,
|
||||||
query: &StoredRequestedModelCandidateRowsQuery,
|
query: &StoredRequestedModelCandidateRowsQuery,
|
||||||
) -> Result<Vec<StoredMinimalCandidateSelectionRow>, DataLayerError> {
|
) -> Result<Vec<StoredMinimalCandidateSelectionRow>, DataLayerError> {
|
||||||
let rows = self
|
self.load_selected_rows_for_api_format(
|
||||||
.selected_rows_for_api_format(&query.api_format)
|
&query.api_format,
|
||||||
.await?
|
SelectedRowsFilter::RequestedModel(&query.requested_model_name),
|
||||||
.into_iter()
|
SelectedRowsOrder::WithGlobalModel,
|
||||||
.filter(|row| {
|
Some(SqlPage {
|
||||||
row_matches_requested_model(row, &query.requested_model_name, &query.api_format)
|
limit: i64::from(query.limit.max(1)),
|
||||||
})
|
offset: i64::from(query.offset),
|
||||||
.collect::<Vec<_>>();
|
}),
|
||||||
Ok(sort_rows(rows, true)
|
)
|
||||||
.into_iter()
|
.await
|
||||||
.skip(query.offset as usize)
|
|
||||||
.take(query.limit as usize)
|
|
||||||
.collect())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn list_pool_key_rows_for_group(
|
async fn list_pool_key_rows_for_group(
|
||||||
&self,
|
&self,
|
||||||
query: &StoredPoolKeyCandidateRowsQuery,
|
query: &StoredPoolKeyCandidateRowsQuery,
|
||||||
) -> Result<Vec<StoredMinimalCandidateSelectionRow>, DataLayerError> {
|
) -> Result<Vec<StoredMinimalCandidateSelectionRow>, DataLayerError> {
|
||||||
let rows = self
|
let canonical_api_format = normalize_api_format(&query.api_format);
|
||||||
.load_rows_for_api_format(&query.api_format)
|
let storage_aliases = api_format_aliases(&canonical_api_format);
|
||||||
.await?
|
let match_aliases = sql_match_aliases(&storage_aliases);
|
||||||
.into_iter()
|
let mut rows = Vec::<CandidateSelectionRow>::new();
|
||||||
.filter(|row| {
|
let page_in_sql = !matches!(query.order, StoredPoolKeyCandidateOrder::LoadBalance { .. });
|
||||||
row.row.provider_id == query.provider_id
|
|
||||||
&& row.row.endpoint_id == query.endpoint_id
|
for storage_api_format in storage_aliases {
|
||||||
&& row.row.model_id == query.model_id
|
let mut builder = QueryBuilder::<Sqlite>::new(CANDIDATE_SELECTION_COLUMNS);
|
||||||
})
|
push_candidate_sql_filters(&mut builder, &storage_api_format, &match_aliases);
|
||||||
.collect::<Vec<_>>();
|
builder.push(" AND p.id = ");
|
||||||
let mut rows = sort_pool_key_rows(rows, &query.order);
|
builder.push_bind(&query.provider_id);
|
||||||
Ok(rows
|
builder.push(" AND pe.id = ");
|
||||||
.drain(..)
|
builder.push_bind(&query.endpoint_id);
|
||||||
.skip(query.offset as usize)
|
builder.push(" AND m.id = ");
|
||||||
.take(query.limit as usize)
|
builder.push_bind(&query.model_id);
|
||||||
.map(|item| item.row)
|
if page_in_sql {
|
||||||
.collect())
|
push_pool_key_order(&mut builder, &query.order);
|
||||||
|
builder.push(" LIMIT ");
|
||||||
|
builder.push_bind(i64::from(query.limit.max(1)));
|
||||||
|
builder.push(" OFFSET ");
|
||||||
|
builder.push_bind(i64::from(query.offset));
|
||||||
|
} else {
|
||||||
|
builder.push(" ORDER BY pak.id ASC");
|
||||||
|
}
|
||||||
|
|
||||||
|
let query_rows = builder.build().fetch_all(&self.pool).await.map_sql_err()?;
|
||||||
|
let mut items = query_rows
|
||||||
|
.iter()
|
||||||
|
.map(map_candidate_selection_row)
|
||||||
|
.collect::<Result<Vec<_>, _>>()?;
|
||||||
|
items.retain(|item| {
|
||||||
|
api_format_matches(&item.row.endpoint_api_format, &canonical_api_format)
|
||||||
|
&& item.row.key_supports_api_format(&canonical_api_format)
|
||||||
|
&& key_auth_channel_matches(item, &canonical_api_format)
|
||||||
|
});
|
||||||
|
rows.extend(items);
|
||||||
|
}
|
||||||
|
|
||||||
|
if page_in_sql {
|
||||||
|
Ok(dedupe_candidate_selection_rows(
|
||||||
|
rows.into_iter().map(|item| item.row).collect(),
|
||||||
|
))
|
||||||
|
} else {
|
||||||
|
Ok(dedupe_candidate_selection_rows(
|
||||||
|
sort_pool_key_rows(rows, &query.order)
|
||||||
|
.into_iter()
|
||||||
|
.skip(query.offset as usize)
|
||||||
|
.take(query.limit as usize)
|
||||||
|
.map(|item| item.row)
|
||||||
|
.collect(),
|
||||||
|
))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn list_pool_key_rows_for_group_key_ids(
|
async fn list_pool_key_rows_for_group_key_ids(
|
||||||
@@ -211,75 +335,330 @@ impl MinimalCandidateSelectionReadRepository for SqliteMinimalCandidateSelection
|
|||||||
.enumerate()
|
.enumerate()
|
||||||
.map(|(index, key_id)| (key_id.as_str(), index))
|
.map(|(index, key_id)| (key_id.as_str(), index))
|
||||||
.collect::<BTreeMap<_, _>>();
|
.collect::<BTreeMap<_, _>>();
|
||||||
let mut rows = self
|
let canonical_api_format = normalize_api_format(&query.api_format);
|
||||||
.load_rows_for_api_format(&query.api_format)
|
let storage_aliases = api_format_aliases(&canonical_api_format);
|
||||||
.await?
|
let match_aliases = sql_match_aliases(&storage_aliases);
|
||||||
.into_iter()
|
let mut rows = Vec::new();
|
||||||
.filter(|row| {
|
|
||||||
row.row.provider_id == query.provider_id
|
for storage_api_format in storage_aliases {
|
||||||
&& row.row.endpoint_id == query.endpoint_id
|
let mut builder = QueryBuilder::<Sqlite>::new(CANDIDATE_SELECTION_COLUMNS);
|
||||||
&& row.row.model_id == query.model_id
|
push_candidate_sql_filters(&mut builder, &storage_api_format, &match_aliases);
|
||||||
&& key_order.contains_key(row.row.key_id.as_str())
|
builder.push(" AND p.id = ");
|
||||||
})
|
builder.push_bind(&query.provider_id);
|
||||||
.map(|item| item.row)
|
builder.push(" AND pe.id = ");
|
||||||
.collect::<Vec<_>>();
|
builder.push_bind(&query.endpoint_id);
|
||||||
|
builder.push(" AND m.id = ");
|
||||||
|
builder.push_bind(&query.model_id);
|
||||||
|
builder.push(" AND pak.id IN (");
|
||||||
|
{
|
||||||
|
let mut separated = builder.separated(", ");
|
||||||
|
for key_id in &query.key_ids {
|
||||||
|
separated.push_bind(key_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
builder.push(")");
|
||||||
|
builder.push(" ORDER BY CASE pak.id");
|
||||||
|
for (index, key_id) in query.key_ids.iter().enumerate() {
|
||||||
|
builder.push(" WHEN ");
|
||||||
|
builder.push_bind(key_id);
|
||||||
|
builder.push(" THEN ");
|
||||||
|
builder.push_bind(i64::try_from(index).map_err(|_| {
|
||||||
|
DataLayerError::UnexpectedValue("key id order index overflowed".to_string())
|
||||||
|
})?);
|
||||||
|
}
|
||||||
|
builder.push(" ELSE ");
|
||||||
|
builder.push_bind(i64::try_from(query.key_ids.len()).map_err(|_| {
|
||||||
|
DataLayerError::UnexpectedValue("key id order length overflowed".to_string())
|
||||||
|
})?);
|
||||||
|
builder.push(" END ASC, pak.id ASC");
|
||||||
|
|
||||||
|
let query_rows = builder.build().fetch_all(&self.pool).await.map_sql_err()?;
|
||||||
|
let mut items = query_rows
|
||||||
|
.iter()
|
||||||
|
.map(map_candidate_selection_row)
|
||||||
|
.collect::<Result<Vec<_>, _>>()?;
|
||||||
|
items.retain(|item| {
|
||||||
|
api_format_matches(&item.row.endpoint_api_format, &canonical_api_format)
|
||||||
|
&& item.row.key_supports_api_format(&canonical_api_format)
|
||||||
|
&& key_auth_channel_matches(item, &canonical_api_format)
|
||||||
|
});
|
||||||
|
rows.extend(items.into_iter().map(|item| item.row));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut rows = dedupe_candidate_selection_rows(rows);
|
||||||
rows.sort_by(|left, right| {
|
rows.sort_by(|left, right| {
|
||||||
key_order
|
key_order
|
||||||
.get(left.key_id.as_str())
|
.get(left.key_id.as_str())
|
||||||
.cmp(&key_order.get(right.key_id.as_str()))
|
.cmp(&key_order.get(right.key_id.as_str()))
|
||||||
.then(left.key_id.cmp(&right.key_id))
|
.then(left.key_id.cmp(&right.key_id))
|
||||||
});
|
});
|
||||||
Ok(dedupe_candidate_selection_rows(rows))
|
Ok(rows)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn select_pool_rows(rows: Vec<CandidateSelectionRow>) -> Vec<StoredMinimalCandidateSelectionRow> {
|
fn push_candidate_sql_filters(
|
||||||
let mut selected = Vec::new();
|
builder: &mut QueryBuilder<'_, Sqlite>,
|
||||||
let mut pool_rows =
|
storage_api_format: &str,
|
||||||
BTreeMap::<(String, String, String), StoredMinimalCandidateSelectionRow>::new();
|
match_aliases: &[String],
|
||||||
for item in rows {
|
) {
|
||||||
if !item.provider_pool_enabled {
|
builder.push(" AND LOWER(COALESCE(pe.api_format, '')) = ");
|
||||||
selected.push(item.row);
|
builder.push_bind(storage_api_format.trim().to_ascii_lowercase());
|
||||||
continue;
|
push_key_api_format_sql_filter(builder, match_aliases);
|
||||||
}
|
push_key_auth_channel_sql_filter(builder, storage_api_format);
|
||||||
let key = (
|
|
||||||
item.row.provider_id.clone(),
|
|
||||||
item.row.endpoint_id.clone(),
|
|
||||||
item.row.model_id.clone(),
|
|
||||||
);
|
|
||||||
match pool_rows.get(&key) {
|
|
||||||
Some(existing)
|
|
||||||
if (existing.key_internal_priority, existing.key_id.as_str())
|
|
||||||
<= (item.row.key_internal_priority, item.row.key_id.as_str()) => {}
|
|
||||||
_ => {
|
|
||||||
pool_rows.insert(key, item.row);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
selected.extend(pool_rows.into_values());
|
|
||||||
dedupe_candidate_selection_rows(selected)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn sort_rows(
|
fn push_key_api_format_sql_filter(
|
||||||
mut rows: Vec<StoredMinimalCandidateSelectionRow>,
|
builder: &mut QueryBuilder<'_, Sqlite>,
|
||||||
include_global_model: bool,
|
match_aliases: &[String],
|
||||||
) -> Vec<StoredMinimalCandidateSelectionRow> {
|
) {
|
||||||
rows.sort_by(|left, right| {
|
builder.push(
|
||||||
if include_global_model {
|
r#"
|
||||||
let ordering = left.global_model_name.cmp(&right.global_model_name);
|
AND (
|
||||||
if !ordering.is_eq() {
|
pak.api_formats IS NULL
|
||||||
return ordering;
|
OR TRIM(pak.api_formats) = ''
|
||||||
}
|
OR CASE
|
||||||
|
WHEN json_valid(pak.api_formats) THEN
|
||||||
|
(
|
||||||
|
(
|
||||||
|
json_type(pak.api_formats) = 'array'
|
||||||
|
AND EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM json_each(pak.api_formats) AS fmt
|
||||||
|
WHERE LOWER(TRIM(CAST(fmt.value AS TEXT))) IN (
|
||||||
|
"#,
|
||||||
|
);
|
||||||
|
push_bind_list(builder, match_aliases);
|
||||||
|
builder.push(
|
||||||
|
r#"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
OR (
|
||||||
|
json_type(pak.api_formats) = 'text'
|
||||||
|
AND LOWER(TRIM(CAST(json_extract(pak.api_formats, '$') AS TEXT))) IN (
|
||||||
|
"#,
|
||||||
|
);
|
||||||
|
push_bind_list(builder, match_aliases);
|
||||||
|
builder.push(
|
||||||
|
r#"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
OR (
|
||||||
|
json_type(pak.api_formats) = 'text'
|
||||||
|
AND EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM json_each(
|
||||||
|
CASE
|
||||||
|
WHEN json_valid(CAST(json_extract(pak.api_formats, '$') AS TEXT))
|
||||||
|
THEN CAST(json_extract(pak.api_formats, '$') AS TEXT)
|
||||||
|
ELSE '[]'
|
||||||
|
END
|
||||||
|
) AS fmt
|
||||||
|
WHERE LOWER(TRIM(CAST(fmt.value AS TEXT))) IN (
|
||||||
|
"#,
|
||||||
|
);
|
||||||
|
push_bind_list(builder, match_aliases);
|
||||||
|
builder.push(
|
||||||
|
r#"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
ELSE 0
|
||||||
|
END
|
||||||
|
OR LOWER(TRIM(pak.api_formats)) IN (
|
||||||
|
"#,
|
||||||
|
);
|
||||||
|
push_bind_list(builder, match_aliases);
|
||||||
|
builder.push(
|
||||||
|
r#"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
"#,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_key_auth_channel_sql_filter(
|
||||||
|
builder: &mut QueryBuilder<'_, Sqlite>,
|
||||||
|
storage_api_format: &str,
|
||||||
|
) {
|
||||||
|
let api_format = normalize_api_format(storage_api_format);
|
||||||
|
builder.push(
|
||||||
|
r#"
|
||||||
|
AND (
|
||||||
|
(
|
||||||
|
LOWER(TRIM(p.provider_type)) = 'codex'
|
||||||
|
AND LOWER(TRIM(pak.auth_type)) = 'oauth'
|
||||||
|
AND "#,
|
||||||
|
);
|
||||||
|
builder.push_bind(api_format.clone());
|
||||||
|
builder.push(
|
||||||
|
r#" IN ('openai:responses', 'openai:responses:compact', 'openai:image')
|
||||||
|
)
|
||||||
|
OR (
|
||||||
|
LOWER(TRIM(p.provider_type)) = 'chatgpt_web'
|
||||||
|
AND LOWER(TRIM(pak.auth_type)) IN ('oauth', 'bearer')
|
||||||
|
AND "#,
|
||||||
|
);
|
||||||
|
builder.push_bind(api_format.clone());
|
||||||
|
builder.push(
|
||||||
|
r#" = 'openai:image'
|
||||||
|
)
|
||||||
|
OR (
|
||||||
|
LOWER(TRIM(p.provider_type)) = 'claude_code'
|
||||||
|
AND LOWER(TRIM(pak.auth_type)) = 'oauth'
|
||||||
|
AND "#,
|
||||||
|
);
|
||||||
|
builder.push_bind(api_format.clone());
|
||||||
|
builder.push(
|
||||||
|
r#" = 'claude:messages'
|
||||||
|
)
|
||||||
|
OR (
|
||||||
|
LOWER(TRIM(p.provider_type)) = 'kiro'
|
||||||
|
AND "#,
|
||||||
|
);
|
||||||
|
builder.push_bind(api_format.clone());
|
||||||
|
builder.push(
|
||||||
|
r#" = 'claude:messages'
|
||||||
|
AND (
|
||||||
|
LOWER(TRIM(pak.auth_type)) = 'oauth'
|
||||||
|
OR (
|
||||||
|
LOWER(TRIM(pak.auth_type)) = 'bearer'
|
||||||
|
AND pak.auth_config IS NOT NULL
|
||||||
|
AND TRIM(pak.auth_config) <> ''
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
OR (
|
||||||
|
LOWER(TRIM(p.provider_type)) IN ('gemini_cli', 'antigravity')
|
||||||
|
AND LOWER(TRIM(pak.auth_type)) = 'oauth'
|
||||||
|
AND "#,
|
||||||
|
);
|
||||||
|
builder.push_bind(api_format.clone());
|
||||||
|
builder.push(
|
||||||
|
r#" = 'gemini:generate_content'
|
||||||
|
)
|
||||||
|
OR (
|
||||||
|
LOWER(TRIM(p.provider_type)) = 'vertex_ai'
|
||||||
|
AND (
|
||||||
|
(
|
||||||
|
LOWER(TRIM(pak.auth_type)) = 'api_key'
|
||||||
|
AND "#,
|
||||||
|
);
|
||||||
|
builder.push_bind(api_format.clone());
|
||||||
|
builder.push(
|
||||||
|
r#" = 'gemini:generate_content'
|
||||||
|
)
|
||||||
|
OR (
|
||||||
|
LOWER(TRIM(pak.auth_type)) IN ('service_account', 'vertex_ai')
|
||||||
|
AND "#,
|
||||||
|
);
|
||||||
|
builder.push_bind(api_format.clone());
|
||||||
|
builder.push(
|
||||||
|
r#" IN ('claude:messages', 'gemini:generate_content')
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
OR (
|
||||||
|
LOWER(TRIM(p.provider_type)) NOT IN (
|
||||||
|
'chatgpt_web',
|
||||||
|
'claude_code',
|
||||||
|
'codex',
|
||||||
|
'gemini_cli',
|
||||||
|
'vertex_ai',
|
||||||
|
'antigravity',
|
||||||
|
'kiro'
|
||||||
|
)
|
||||||
|
AND LOWER(TRIM(pak.auth_type)) <> 'oauth'
|
||||||
|
)
|
||||||
|
)
|
||||||
|
"#,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_requested_model_sql_filter(
|
||||||
|
builder: &mut QueryBuilder<'_, Sqlite>,
|
||||||
|
requested_model_name: &str,
|
||||||
|
_match_aliases: &[String],
|
||||||
|
) {
|
||||||
|
builder.push(
|
||||||
|
r#"
|
||||||
|
AND (
|
||||||
|
gm.name = "#,
|
||||||
|
);
|
||||||
|
builder.push_bind(requested_model_name.to_string());
|
||||||
|
builder.push(
|
||||||
|
r#"
|
||||||
|
OR m.provider_model_name = "#,
|
||||||
|
);
|
||||||
|
builder.push_bind(requested_model_name.to_string());
|
||||||
|
builder.push(
|
||||||
|
r#"
|
||||||
|
OR (
|
||||||
|
m.provider_model_mappings IS NOT NULL
|
||||||
|
AND m.provider_model_mappings LIKE "#,
|
||||||
|
);
|
||||||
|
builder.push_bind(format!(
|
||||||
|
"%{}%",
|
||||||
|
requested_model_name
|
||||||
|
.replace('\\', "\\\\")
|
||||||
|
.replace('%', "\\%")
|
||||||
|
.replace('_', "\\_")
|
||||||
|
));
|
||||||
|
builder.push(
|
||||||
|
r#"
|
||||||
|
ESCAPE '\'
|
||||||
|
)
|
||||||
|
)
|
||||||
|
"#,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_selected_rows_order(builder: &mut QueryBuilder<'_, Sqlite>, order: SelectedRowsOrder) {
|
||||||
|
builder.push(" ORDER BY ");
|
||||||
|
if matches!(order, SelectedRowsOrder::WithGlobalModel) {
|
||||||
|
builder.push("global_model_name ASC, ");
|
||||||
|
}
|
||||||
|
builder.push(
|
||||||
|
"provider_priority ASC, key_internal_priority ASC, provider_id ASC, endpoint_id ASC, key_id ASC, model_id ASC",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_pool_key_order(
|
||||||
|
builder: &mut QueryBuilder<'_, Sqlite>,
|
||||||
|
order: &StoredPoolKeyCandidateOrder,
|
||||||
|
) {
|
||||||
|
match order {
|
||||||
|
StoredPoolKeyCandidateOrder::InternalPriority => {
|
||||||
|
builder.push(" ORDER BY pak.internal_priority ASC, pak.id ASC");
|
||||||
}
|
}
|
||||||
left.provider_priority
|
StoredPoolKeyCandidateOrder::Lru => {
|
||||||
.cmp(&right.provider_priority)
|
builder.push(
|
||||||
.then(left.key_internal_priority.cmp(&right.key_internal_priority))
|
" ORDER BY pak.last_used_at IS NOT NULL ASC, pak.last_used_at ASC, pak.internal_priority ASC, pak.id ASC",
|
||||||
.then(left.provider_id.cmp(&right.provider_id))
|
);
|
||||||
.then(left.endpoint_id.cmp(&right.endpoint_id))
|
}
|
||||||
.then(left.key_id.cmp(&right.key_id))
|
StoredPoolKeyCandidateOrder::CacheAffinity => {
|
||||||
.then(left.model_id.cmp(&right.model_id))
|
builder.push(
|
||||||
});
|
" ORDER BY pak.last_used_at IS NULL ASC, pak.last_used_at DESC, pak.internal_priority ASC, pak.id ASC",
|
||||||
rows
|
);
|
||||||
|
}
|
||||||
|
StoredPoolKeyCandidateOrder::SingleAccount => {
|
||||||
|
builder.push(
|
||||||
|
" ORDER BY pak.internal_priority ASC, pak.last_used_at IS NULL ASC, pak.last_used_at DESC, pak.id ASC",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
StoredPoolKeyCandidateOrder::LoadBalance { seed } => {
|
||||||
|
let _ = seed;
|
||||||
|
builder.push(" ORDER BY pak.id ASC");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_bind_list(builder: &mut QueryBuilder<'_, Sqlite>, values: &[String]) {
|
||||||
|
let mut separated = builder.separated(", ");
|
||||||
|
for value in values {
|
||||||
|
separated.push_bind(value.clone());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn sort_pool_key_rows(
|
fn sort_pool_key_rows(
|
||||||
@@ -473,9 +852,8 @@ fn dedupe_candidate_selection_rows(
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn map_candidate_selection_row(row: &SqliteRow) -> Result<CandidateSelectionRow, DataLayerError> {
|
fn map_candidate_selection_row(row: &SqliteRow) -> Result<CandidateSelectionRow, DataLayerError> {
|
||||||
let provider_config = parse_json(row.try_get("provider_config").ok().flatten())?;
|
let _provider_config = parse_json(row.try_get("provider_config").ok().flatten())?;
|
||||||
let global_model_config = parse_json(row.try_get("global_model_config").ok().flatten())?;
|
let global_model_config = parse_json(row.try_get("global_model_config").ok().flatten())?;
|
||||||
let provider_pool_enabled = json_object_field_present(&provider_config, "pool_advanced");
|
|
||||||
let global_model_mappings = global_model_config
|
let global_model_mappings = global_model_config
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.and_then(|value| value.get("model_mappings").cloned());
|
.and_then(|value| value.get("model_mappings").cloned());
|
||||||
@@ -528,7 +906,6 @@ fn map_candidate_selection_row(row: &SqliteRow) -> Result<CandidateSelectionRow,
|
|||||||
model_is_active: row.try_get("model_is_active").map_sql_err()?,
|
model_is_active: row.try_get("model_is_active").map_sql_err()?,
|
||||||
model_is_available: row.try_get("model_is_available").map_sql_err()?,
|
model_is_available: row.try_get("model_is_available").map_sql_err()?,
|
||||||
},
|
},
|
||||||
provider_pool_enabled,
|
|
||||||
key_auth_config: row.try_get("key_auth_config").map_sql_err()?,
|
key_auth_config: row.try_get("key_auth_config").map_sql_err()?,
|
||||||
key_last_used_at_unix_secs: row
|
key_last_used_at_unix_secs: row
|
||||||
.try_get::<Option<i64>, _>("key_last_used_at_unix_secs")
|
.try_get::<Option<i64>, _>("key_last_used_at_unix_secs")
|
||||||
@@ -550,13 +927,6 @@ fn parse_json(value: Option<String>) -> Result<Option<serde_json::Value>, DataLa
|
|||||||
.transpose()
|
.transpose()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn json_object_field_present(value: &Option<serde_json::Value>, field: &str) -> bool {
|
|
||||||
value
|
|
||||||
.as_ref()
|
|
||||||
.and_then(|value| value.get(field))
|
|
||||||
.is_some_and(|value| !value.is_null())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn json_bool(value: &serde_json::Value) -> Option<bool> {
|
fn json_bool(value: &serde_json::Value) -> Option<bool> {
|
||||||
value.as_bool().or_else(|| {
|
value.as_bool().or_else(|| {
|
||||||
value
|
value
|
||||||
|
|||||||
@@ -1,16 +1,279 @@
|
|||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use sqlx::{sqlite::SqliteRow, Row};
|
use sqlx::{sqlite::SqliteRow, QueryBuilder, Row, Sqlite};
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
InMemoryProviderCatalogReadRepository, ProviderCatalogKeyListQuery,
|
ProviderCatalogKeyListOrder, ProviderCatalogKeyListQuery, ProviderCatalogReadRepository,
|
||||||
ProviderCatalogReadRepository, ProviderCatalogWriteRepository, StoredProviderCatalogEndpoint,
|
ProviderCatalogWriteRepository, StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
|
||||||
StoredProviderCatalogKey, StoredProviderCatalogKeyPage, StoredProviderCatalogKeyStats,
|
StoredProviderCatalogKeyPage, StoredProviderCatalogKeyStats, StoredProviderCatalogProvider,
|
||||||
StoredProviderCatalogProvider,
|
|
||||||
};
|
};
|
||||||
use crate::driver::sqlite::{sqlite_optional_real, SqlitePool};
|
use crate::driver::sqlite::{sqlite_optional_real, SqlitePool};
|
||||||
use crate::error::SqlResultExt;
|
use crate::error::SqlResultExt;
|
||||||
use crate::DataLayerError;
|
use crate::DataLayerError;
|
||||||
|
|
||||||
|
const LIST_PROVIDERS_BY_IDS_PREFIX: &str = r#"
|
||||||
|
SELECT
|
||||||
|
id,
|
||||||
|
name,
|
||||||
|
description,
|
||||||
|
website,
|
||||||
|
provider_type,
|
||||||
|
billing_type,
|
||||||
|
CAST(monthly_quota_usd AS REAL) AS monthly_quota_usd,
|
||||||
|
CAST(monthly_used_usd AS REAL) AS monthly_used_usd,
|
||||||
|
quota_reset_day,
|
||||||
|
quota_last_reset_at AS quota_last_reset_at_unix_secs,
|
||||||
|
quota_expires_at AS quota_expires_at_unix_secs,
|
||||||
|
provider_priority,
|
||||||
|
is_active,
|
||||||
|
keep_priority_on_conversion,
|
||||||
|
enable_format_conversion,
|
||||||
|
concurrent_limit,
|
||||||
|
max_retries,
|
||||||
|
proxy,
|
||||||
|
request_timeout,
|
||||||
|
stream_first_byte_timeout,
|
||||||
|
config,
|
||||||
|
created_at AS created_at_unix_ms,
|
||||||
|
updated_at AS updated_at_unix_secs
|
||||||
|
FROM providers
|
||||||
|
WHERE id IN (
|
||||||
|
"#;
|
||||||
|
|
||||||
|
const LIST_ENDPOINTS_BY_IDS_PREFIX: &str = r#"
|
||||||
|
SELECT
|
||||||
|
id,
|
||||||
|
provider_id,
|
||||||
|
api_format,
|
||||||
|
api_family,
|
||||||
|
endpoint_kind,
|
||||||
|
is_active,
|
||||||
|
health_score,
|
||||||
|
base_url,
|
||||||
|
header_rules,
|
||||||
|
body_rules,
|
||||||
|
max_retries,
|
||||||
|
custom_path,
|
||||||
|
config,
|
||||||
|
format_acceptance_config,
|
||||||
|
proxy,
|
||||||
|
created_at AS created_at_unix_ms,
|
||||||
|
updated_at AS updated_at_unix_secs
|
||||||
|
FROM provider_endpoints
|
||||||
|
WHERE id IN (
|
||||||
|
"#;
|
||||||
|
|
||||||
|
const LIST_ENDPOINTS_BY_PROVIDER_IDS_PREFIX: &str = r#"
|
||||||
|
SELECT
|
||||||
|
id,
|
||||||
|
provider_id,
|
||||||
|
api_format,
|
||||||
|
api_family,
|
||||||
|
endpoint_kind,
|
||||||
|
is_active,
|
||||||
|
health_score,
|
||||||
|
base_url,
|
||||||
|
header_rules,
|
||||||
|
body_rules,
|
||||||
|
max_retries,
|
||||||
|
custom_path,
|
||||||
|
config,
|
||||||
|
format_acceptance_config,
|
||||||
|
proxy,
|
||||||
|
created_at AS created_at_unix_ms,
|
||||||
|
updated_at AS updated_at_unix_secs
|
||||||
|
FROM provider_endpoints
|
||||||
|
WHERE provider_id IN (
|
||||||
|
"#;
|
||||||
|
|
||||||
|
const LIST_KEYS_BY_IDS_PREFIX: &str = r#"
|
||||||
|
SELECT
|
||||||
|
id,
|
||||||
|
provider_id,
|
||||||
|
name,
|
||||||
|
auth_type,
|
||||||
|
capabilities,
|
||||||
|
is_active,
|
||||||
|
api_formats,
|
||||||
|
auth_type_by_format,
|
||||||
|
allow_auth_channel_mismatch_formats,
|
||||||
|
COALESCE(api_key, encrypted_key) AS api_key,
|
||||||
|
auth_config,
|
||||||
|
note,
|
||||||
|
internal_priority,
|
||||||
|
rate_multipliers,
|
||||||
|
global_priority_by_format,
|
||||||
|
allowed_models,
|
||||||
|
expires_at AS expires_at_unix_secs,
|
||||||
|
cache_ttl_minutes,
|
||||||
|
max_probe_interval_minutes,
|
||||||
|
proxy,
|
||||||
|
fingerprint,
|
||||||
|
rpm_limit,
|
||||||
|
concurrent_limit,
|
||||||
|
learned_rpm_limit,
|
||||||
|
concurrent_429_count,
|
||||||
|
rpm_429_count,
|
||||||
|
last_429_at AS last_429_at_unix_secs,
|
||||||
|
last_429_type,
|
||||||
|
adjustment_history,
|
||||||
|
utilization_samples,
|
||||||
|
last_probe_increase_at AS last_probe_increase_at_unix_secs,
|
||||||
|
last_rpm_peak,
|
||||||
|
request_count,
|
||||||
|
total_tokens,
|
||||||
|
CAST(total_cost_usd AS REAL) AS total_cost_usd,
|
||||||
|
success_count,
|
||||||
|
error_count,
|
||||||
|
total_response_time_ms,
|
||||||
|
last_used_at AS last_used_at_unix_secs,
|
||||||
|
auto_fetch_models,
|
||||||
|
last_models_fetch_at AS last_models_fetch_at_unix_secs,
|
||||||
|
last_models_fetch_error,
|
||||||
|
locked_models,
|
||||||
|
model_include_patterns,
|
||||||
|
model_exclude_patterns,
|
||||||
|
upstream_metadata,
|
||||||
|
oauth_invalid_at AS oauth_invalid_at_unix_secs,
|
||||||
|
oauth_invalid_reason,
|
||||||
|
status_snapshot,
|
||||||
|
created_at AS created_at_unix_ms,
|
||||||
|
updated_at AS updated_at_unix_secs,
|
||||||
|
health_by_format,
|
||||||
|
circuit_breaker_by_format
|
||||||
|
FROM provider_api_keys
|
||||||
|
WHERE id IN (
|
||||||
|
"#;
|
||||||
|
|
||||||
|
const LIST_KEYS_BY_PROVIDER_IDS_PREFIX: &str = r#"
|
||||||
|
SELECT
|
||||||
|
id,
|
||||||
|
provider_id,
|
||||||
|
name,
|
||||||
|
auth_type,
|
||||||
|
capabilities,
|
||||||
|
is_active,
|
||||||
|
api_formats,
|
||||||
|
auth_type_by_format,
|
||||||
|
allow_auth_channel_mismatch_formats,
|
||||||
|
COALESCE(api_key, encrypted_key) AS api_key,
|
||||||
|
auth_config,
|
||||||
|
note,
|
||||||
|
internal_priority,
|
||||||
|
rate_multipliers,
|
||||||
|
global_priority_by_format,
|
||||||
|
allowed_models,
|
||||||
|
expires_at AS expires_at_unix_secs,
|
||||||
|
cache_ttl_minutes,
|
||||||
|
max_probe_interval_minutes,
|
||||||
|
proxy,
|
||||||
|
fingerprint,
|
||||||
|
rpm_limit,
|
||||||
|
concurrent_limit,
|
||||||
|
learned_rpm_limit,
|
||||||
|
concurrent_429_count,
|
||||||
|
rpm_429_count,
|
||||||
|
last_429_at AS last_429_at_unix_secs,
|
||||||
|
last_429_type,
|
||||||
|
adjustment_history,
|
||||||
|
utilization_samples,
|
||||||
|
last_probe_increase_at AS last_probe_increase_at_unix_secs,
|
||||||
|
last_rpm_peak,
|
||||||
|
request_count,
|
||||||
|
total_tokens,
|
||||||
|
CAST(total_cost_usd AS REAL) AS total_cost_usd,
|
||||||
|
success_count,
|
||||||
|
error_count,
|
||||||
|
total_response_time_ms,
|
||||||
|
last_used_at AS last_used_at_unix_secs,
|
||||||
|
auto_fetch_models,
|
||||||
|
last_models_fetch_at AS last_models_fetch_at_unix_secs,
|
||||||
|
last_models_fetch_error,
|
||||||
|
locked_models,
|
||||||
|
model_include_patterns,
|
||||||
|
model_exclude_patterns,
|
||||||
|
upstream_metadata,
|
||||||
|
oauth_invalid_at AS oauth_invalid_at_unix_secs,
|
||||||
|
oauth_invalid_reason,
|
||||||
|
status_snapshot,
|
||||||
|
created_at AS created_at_unix_ms,
|
||||||
|
updated_at AS updated_at_unix_secs,
|
||||||
|
health_by_format,
|
||||||
|
circuit_breaker_by_format
|
||||||
|
FROM provider_api_keys
|
||||||
|
WHERE provider_id IN (
|
||||||
|
"#;
|
||||||
|
|
||||||
|
const LIST_KEY_SUMMARIES_BY_PROVIDER_IDS_PREFIX: &str = r#"
|
||||||
|
SELECT
|
||||||
|
id,
|
||||||
|
provider_id,
|
||||||
|
COALESCE(NULLIF(name, ''), id) AS name,
|
||||||
|
COALESCE(NULLIF(auth_type, ''), 'summary') AS auth_type,
|
||||||
|
NULL AS capabilities,
|
||||||
|
is_active,
|
||||||
|
api_formats,
|
||||||
|
NULL AS auth_type_by_format,
|
||||||
|
NULL AS allow_auth_channel_mismatch_formats,
|
||||||
|
'summary' AS api_key,
|
||||||
|
CASE
|
||||||
|
WHEN auth_config IS NULL THEN NULL
|
||||||
|
ELSE '{}'
|
||||||
|
END AS auth_config,
|
||||||
|
NULL AS note,
|
||||||
|
NULL AS internal_priority,
|
||||||
|
NULL AS rate_multipliers,
|
||||||
|
NULL AS global_priority_by_format,
|
||||||
|
NULL AS allowed_models,
|
||||||
|
NULL AS expires_at_unix_secs,
|
||||||
|
NULL AS cache_ttl_minutes,
|
||||||
|
NULL AS max_probe_interval_minutes,
|
||||||
|
NULL AS proxy,
|
||||||
|
NULL AS fingerprint,
|
||||||
|
NULL AS rpm_limit,
|
||||||
|
NULL AS concurrent_limit,
|
||||||
|
NULL AS learned_rpm_limit,
|
||||||
|
NULL AS concurrent_429_count,
|
||||||
|
NULL AS rpm_429_count,
|
||||||
|
NULL AS last_429_at_unix_secs,
|
||||||
|
NULL AS last_429_type,
|
||||||
|
NULL AS adjustment_history,
|
||||||
|
NULL AS utilization_samples,
|
||||||
|
NULL AS last_probe_increase_at_unix_secs,
|
||||||
|
NULL AS last_rpm_peak,
|
||||||
|
NULL AS request_count,
|
||||||
|
0 AS total_tokens,
|
||||||
|
0.0 AS total_cost_usd,
|
||||||
|
NULL AS success_count,
|
||||||
|
NULL AS error_count,
|
||||||
|
NULL AS total_response_time_ms,
|
||||||
|
NULL AS last_used_at_unix_secs,
|
||||||
|
FALSE AS auto_fetch_models,
|
||||||
|
NULL AS last_models_fetch_at_unix_secs,
|
||||||
|
NULL AS last_models_fetch_error,
|
||||||
|
NULL AS locked_models,
|
||||||
|
NULL AS model_include_patterns,
|
||||||
|
NULL AS model_exclude_patterns,
|
||||||
|
NULL AS upstream_metadata,
|
||||||
|
NULL AS oauth_invalid_at_unix_secs,
|
||||||
|
NULL AS oauth_invalid_reason,
|
||||||
|
NULL AS status_snapshot,
|
||||||
|
NULL AS created_at_unix_ms,
|
||||||
|
NULL AS updated_at_unix_secs,
|
||||||
|
health_by_format,
|
||||||
|
NULL AS circuit_breaker_by_format
|
||||||
|
FROM provider_api_keys
|
||||||
|
WHERE provider_id IN (
|
||||||
|
"#;
|
||||||
|
|
||||||
|
const LIST_KEY_STATS_BY_PROVIDER_IDS_PREFIX: &str = r#"
|
||||||
|
SELECT
|
||||||
|
provider_id,
|
||||||
|
COUNT(*) AS total_keys,
|
||||||
|
SUM(CASE WHEN is_active THEN 1 ELSE 0 END) AS active_keys
|
||||||
|
FROM provider_api_keys
|
||||||
|
WHERE provider_id IN (
|
||||||
|
"#;
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct SqliteProviderCatalogReadRepository {
|
pub struct SqliteProviderCatalogReadRepository {
|
||||||
pool: SqlitePool,
|
pool: SqlitePool,
|
||||||
@@ -21,92 +284,340 @@ impl SqliteProviderCatalogReadRepository {
|
|||||||
Self { pool }
|
Self { pool }
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn load_memory(&self) -> Result<InMemoryProviderCatalogReadRepository, DataLayerError> {
|
pub async fn list_providers_by_ids(
|
||||||
Ok(InMemoryProviderCatalogReadRepository::seed(
|
&self,
|
||||||
self.load_providers().await?,
|
provider_ids: &[String],
|
||||||
self.load_endpoints().await?,
|
) -> Result<Vec<StoredProviderCatalogProvider>, DataLayerError> {
|
||||||
self.load_keys().await?,
|
if provider_ids.is_empty() {
|
||||||
))
|
return Ok(Vec::new());
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn load_providers(&self) -> Result<Vec<StoredProviderCatalogProvider>, DataLayerError> {
|
let rows = build_list_query(
|
||||||
let rows = sqlx::query(
|
LIST_PROVIDERS_BY_IDS_PREFIX,
|
||||||
r#"
|
provider_ids,
|
||||||
SELECT
|
" ORDER BY name ASC",
|
||||||
id, name, description, website, provider_type, billing_type,
|
|
||||||
CAST(monthly_quota_usd AS REAL) AS monthly_quota_usd,
|
|
||||||
CAST(monthly_used_usd AS REAL) AS monthly_used_usd,
|
|
||||||
quota_reset_day,
|
|
||||||
quota_last_reset_at AS quota_last_reset_at_unix_secs,
|
|
||||||
quota_expires_at AS quota_expires_at_unix_secs,
|
|
||||||
provider_priority, is_active, keep_priority_on_conversion,
|
|
||||||
enable_format_conversion, concurrent_limit, max_retries, proxy,
|
|
||||||
request_timeout, stream_first_byte_timeout, config,
|
|
||||||
created_at AS created_at_unix_ms,
|
|
||||||
updated_at AS updated_at_unix_secs
|
|
||||||
FROM providers
|
|
||||||
"#,
|
|
||||||
)
|
)
|
||||||
|
.build()
|
||||||
.fetch_all(&self.pool)
|
.fetch_all(&self.pool)
|
||||||
.await
|
.await
|
||||||
.map_sql_err()?;
|
.map_sql_err()?;
|
||||||
rows.iter().map(map_provider_row).collect()
|
rows.iter().map(map_provider_row).collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn load_endpoints(&self) -> Result<Vec<StoredProviderCatalogEndpoint>, DataLayerError> {
|
pub async fn list_providers(
|
||||||
|
&self,
|
||||||
|
active_only: bool,
|
||||||
|
) -> Result<Vec<StoredProviderCatalogProvider>, DataLayerError> {
|
||||||
let rows = sqlx::query(
|
let rows = sqlx::query(
|
||||||
r#"
|
r#"
|
||||||
SELECT
|
SELECT
|
||||||
id, provider_id, api_format, api_family, endpoint_kind, is_active,
|
id,
|
||||||
health_score, base_url, header_rules, body_rules, max_retries,
|
name,
|
||||||
custom_path, config, format_acceptance_config, proxy,
|
description,
|
||||||
|
website,
|
||||||
|
provider_type,
|
||||||
|
billing_type,
|
||||||
|
CAST(monthly_quota_usd AS REAL) AS monthly_quota_usd,
|
||||||
|
CAST(monthly_used_usd AS REAL) AS monthly_used_usd,
|
||||||
|
quota_reset_day,
|
||||||
|
quota_last_reset_at AS quota_last_reset_at_unix_secs,
|
||||||
|
quota_expires_at AS quota_expires_at_unix_secs,
|
||||||
|
provider_priority,
|
||||||
|
is_active,
|
||||||
|
keep_priority_on_conversion,
|
||||||
|
enable_format_conversion,
|
||||||
|
concurrent_limit,
|
||||||
|
max_retries,
|
||||||
|
proxy,
|
||||||
|
request_timeout,
|
||||||
|
stream_first_byte_timeout,
|
||||||
|
config,
|
||||||
created_at AS created_at_unix_ms,
|
created_at AS created_at_unix_ms,
|
||||||
updated_at AS updated_at_unix_secs
|
updated_at AS updated_at_unix_secs
|
||||||
FROM provider_endpoints
|
FROM providers
|
||||||
WHERE api_format IS NOT NULL
|
WHERE (? = FALSE OR is_active = TRUE)
|
||||||
|
ORDER BY provider_priority ASC, name ASC
|
||||||
"#,
|
"#,
|
||||||
)
|
)
|
||||||
|
.bind(active_only)
|
||||||
|
.fetch_all(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_sql_err()?;
|
||||||
|
rows.iter().map(map_provider_row).collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn list_endpoints_by_ids(
|
||||||
|
&self,
|
||||||
|
endpoint_ids: &[String],
|
||||||
|
) -> Result<Vec<StoredProviderCatalogEndpoint>, DataLayerError> {
|
||||||
|
if endpoint_ids.is_empty() {
|
||||||
|
return Ok(Vec::new());
|
||||||
|
}
|
||||||
|
|
||||||
|
let rows = build_list_query(
|
||||||
|
LIST_ENDPOINTS_BY_IDS_PREFIX,
|
||||||
|
endpoint_ids,
|
||||||
|
" ORDER BY api_format ASC, id ASC",
|
||||||
|
)
|
||||||
|
.build()
|
||||||
.fetch_all(&self.pool)
|
.fetch_all(&self.pool)
|
||||||
.await
|
.await
|
||||||
.map_sql_err()?;
|
.map_sql_err()?;
|
||||||
rows.iter().map(map_endpoint_row).collect()
|
rows.iter().map(map_endpoint_row).collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn load_keys(&self) -> Result<Vec<StoredProviderCatalogKey>, DataLayerError> {
|
pub async fn list_endpoints_by_provider_ids(
|
||||||
let rows = sqlx::query(
|
&self,
|
||||||
r#"
|
provider_ids: &[String],
|
||||||
SELECT
|
) -> Result<Vec<StoredProviderCatalogEndpoint>, DataLayerError> {
|
||||||
id, provider_id, name, auth_type, capabilities, is_active, api_formats,
|
if provider_ids.is_empty() {
|
||||||
auth_type_by_format, allow_auth_channel_mismatch_formats,
|
return Ok(Vec::new());
|
||||||
COALESCE(api_key, encrypted_key) AS api_key,
|
}
|
||||||
auth_config, note, internal_priority, rate_multipliers,
|
|
||||||
global_priority_by_format, allowed_models,
|
let rows = build_list_query(
|
||||||
expires_at AS expires_at_unix_secs,
|
LIST_ENDPOINTS_BY_PROVIDER_IDS_PREFIX,
|
||||||
cache_ttl_minutes, max_probe_interval_minutes, proxy, fingerprint,
|
provider_ids,
|
||||||
rpm_limit, concurrent_limit, learned_rpm_limit, concurrent_429_count,
|
" ORDER BY provider_id ASC, api_format ASC, id ASC",
|
||||||
rpm_429_count, last_429_at AS last_429_at_unix_secs, last_429_type,
|
|
||||||
adjustment_history, utilization_samples,
|
|
||||||
last_probe_increase_at AS last_probe_increase_at_unix_secs,
|
|
||||||
last_rpm_peak, request_count, total_tokens, total_cost_usd,
|
|
||||||
success_count, error_count, total_response_time_ms,
|
|
||||||
last_used_at AS last_used_at_unix_secs, auto_fetch_models,
|
|
||||||
last_models_fetch_at AS last_models_fetch_at_unix_secs,
|
|
||||||
last_models_fetch_error, locked_models, model_include_patterns,
|
|
||||||
model_exclude_patterns, upstream_metadata,
|
|
||||||
oauth_invalid_at AS oauth_invalid_at_unix_secs,
|
|
||||||
oauth_invalid_reason, status_snapshot,
|
|
||||||
created_at AS created_at_unix_ms,
|
|
||||||
updated_at AS updated_at_unix_secs,
|
|
||||||
health_by_format, circuit_breaker_by_format
|
|
||||||
FROM provider_api_keys
|
|
||||||
"#,
|
|
||||||
)
|
)
|
||||||
|
.build()
|
||||||
|
.fetch_all(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_sql_err()?;
|
||||||
|
rows.iter().map(map_endpoint_row).collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn list_keys_by_ids(
|
||||||
|
&self,
|
||||||
|
key_ids: &[String],
|
||||||
|
) -> Result<Vec<StoredProviderCatalogKey>, DataLayerError> {
|
||||||
|
if key_ids.is_empty() {
|
||||||
|
return Ok(Vec::new());
|
||||||
|
}
|
||||||
|
|
||||||
|
let rows = build_list_query(
|
||||||
|
LIST_KEYS_BY_IDS_PREFIX,
|
||||||
|
key_ids,
|
||||||
|
" ORDER BY name ASC, id ASC",
|
||||||
|
)
|
||||||
|
.build()
|
||||||
.fetch_all(&self.pool)
|
.fetch_all(&self.pool)
|
||||||
.await
|
.await
|
||||||
.map_sql_err()?;
|
.map_sql_err()?;
|
||||||
rows.iter().map(map_key_row).collect()
|
rows.iter().map(map_key_row).collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn list_keys_by_provider_ids(
|
||||||
|
&self,
|
||||||
|
provider_ids: &[String],
|
||||||
|
) -> Result<Vec<StoredProviderCatalogKey>, DataLayerError> {
|
||||||
|
if provider_ids.is_empty() {
|
||||||
|
return Ok(Vec::new());
|
||||||
|
}
|
||||||
|
|
||||||
|
let rows = build_list_query(
|
||||||
|
LIST_KEYS_BY_PROVIDER_IDS_PREFIX,
|
||||||
|
provider_ids,
|
||||||
|
" ORDER BY provider_id ASC, name ASC, id ASC",
|
||||||
|
)
|
||||||
|
.build()
|
||||||
|
.fetch_all(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_sql_err()?;
|
||||||
|
rows.iter().map(map_key_row).collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn list_key_summaries_by_provider_ids(
|
||||||
|
&self,
|
||||||
|
provider_ids: &[String],
|
||||||
|
) -> Result<Vec<StoredProviderCatalogKey>, DataLayerError> {
|
||||||
|
if provider_ids.is_empty() {
|
||||||
|
return Ok(Vec::new());
|
||||||
|
}
|
||||||
|
|
||||||
|
let rows = build_list_query(
|
||||||
|
LIST_KEY_SUMMARIES_BY_PROVIDER_IDS_PREFIX,
|
||||||
|
provider_ids,
|
||||||
|
" ORDER BY provider_id ASC, id ASC",
|
||||||
|
)
|
||||||
|
.build()
|
||||||
|
.fetch_all(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_sql_err()?;
|
||||||
|
rows.iter().map(map_key_row).collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn list_keys_page(
|
||||||
|
&self,
|
||||||
|
query: &ProviderCatalogKeyListQuery,
|
||||||
|
) -> Result<StoredProviderCatalogKeyPage, DataLayerError> {
|
||||||
|
if query.provider_id.trim().is_empty() {
|
||||||
|
return Err(DataLayerError::InvalidInput(
|
||||||
|
"provider catalog provider_id is empty".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
let offset = i64::try_from(query.offset).map_err(|_| {
|
||||||
|
DataLayerError::InvalidInput(format!(
|
||||||
|
"invalid provider catalog key offset: {}",
|
||||||
|
query.offset
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
let limit = i64::try_from(query.limit).map_err(|_| {
|
||||||
|
DataLayerError::InvalidInput(format!(
|
||||||
|
"invalid provider catalog key limit: {}",
|
||||||
|
query.limit
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
let search_pattern = query
|
||||||
|
.search
|
||||||
|
.as_deref()
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.map(|value| format!("%{}%", value.to_ascii_lowercase()));
|
||||||
|
let order_by = match query.order {
|
||||||
|
ProviderCatalogKeyListOrder::Name => "internal_priority ASC, name ASC, id ASC",
|
||||||
|
ProviderCatalogKeyListOrder::CreatedAt => {
|
||||||
|
"internal_priority ASC, COALESCE(created_at, 0) ASC, id ASC"
|
||||||
|
}
|
||||||
|
ProviderCatalogKeyListOrder::CreatedAtAsc => {
|
||||||
|
"created_at IS NULL ASC, created_at ASC, name ASC, id ASC"
|
||||||
|
}
|
||||||
|
ProviderCatalogKeyListOrder::CreatedAtDesc => {
|
||||||
|
"created_at IS NULL ASC, created_at DESC, name ASC, id ASC"
|
||||||
|
}
|
||||||
|
ProviderCatalogKeyListOrder::LastUsedAtAsc => {
|
||||||
|
"last_used_at IS NULL ASC, last_used_at ASC, name ASC, id ASC"
|
||||||
|
}
|
||||||
|
ProviderCatalogKeyListOrder::LastUsedAtDesc => {
|
||||||
|
"last_used_at IS NULL ASC, last_used_at DESC, name ASC, id ASC"
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let count_row = sqlx::query(
|
||||||
|
r#"
|
||||||
|
SELECT COUNT(*) AS total
|
||||||
|
FROM provider_api_keys
|
||||||
|
WHERE provider_id = ?
|
||||||
|
AND (? IS NULL OR LOWER(name) LIKE ? OR LOWER(id) LIKE ?)
|
||||||
|
AND (? IS NULL OR is_active = ?)
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.bind(&query.provider_id)
|
||||||
|
.bind(search_pattern.as_deref())
|
||||||
|
.bind(search_pattern.as_deref())
|
||||||
|
.bind(search_pattern.as_deref())
|
||||||
|
.bind(query.is_active)
|
||||||
|
.bind(query.is_active)
|
||||||
|
.fetch_one(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_sql_err()?;
|
||||||
|
let total = count_row.try_get::<i64, _>("total").map_sql_err()?.max(0) as usize;
|
||||||
|
|
||||||
|
let sql = format!(
|
||||||
|
r#"
|
||||||
|
SELECT
|
||||||
|
id,
|
||||||
|
provider_id,
|
||||||
|
name,
|
||||||
|
auth_type,
|
||||||
|
capabilities,
|
||||||
|
is_active,
|
||||||
|
api_formats,
|
||||||
|
auth_type_by_format,
|
||||||
|
allow_auth_channel_mismatch_formats,
|
||||||
|
COALESCE(api_key, encrypted_key) AS api_key,
|
||||||
|
auth_config,
|
||||||
|
note,
|
||||||
|
internal_priority,
|
||||||
|
rate_multipliers,
|
||||||
|
global_priority_by_format,
|
||||||
|
allowed_models,
|
||||||
|
expires_at AS expires_at_unix_secs,
|
||||||
|
cache_ttl_minutes,
|
||||||
|
max_probe_interval_minutes,
|
||||||
|
proxy,
|
||||||
|
fingerprint,
|
||||||
|
rpm_limit,
|
||||||
|
concurrent_limit,
|
||||||
|
learned_rpm_limit,
|
||||||
|
concurrent_429_count,
|
||||||
|
rpm_429_count,
|
||||||
|
last_429_at AS last_429_at_unix_secs,
|
||||||
|
last_429_type,
|
||||||
|
adjustment_history,
|
||||||
|
utilization_samples,
|
||||||
|
last_probe_increase_at AS last_probe_increase_at_unix_secs,
|
||||||
|
last_rpm_peak,
|
||||||
|
request_count,
|
||||||
|
total_tokens,
|
||||||
|
CAST(total_cost_usd AS REAL) AS total_cost_usd,
|
||||||
|
success_count,
|
||||||
|
error_count,
|
||||||
|
total_response_time_ms,
|
||||||
|
last_used_at AS last_used_at_unix_secs,
|
||||||
|
auto_fetch_models,
|
||||||
|
last_models_fetch_at AS last_models_fetch_at_unix_secs,
|
||||||
|
last_models_fetch_error,
|
||||||
|
locked_models,
|
||||||
|
model_include_patterns,
|
||||||
|
model_exclude_patterns,
|
||||||
|
upstream_metadata,
|
||||||
|
oauth_invalid_at AS oauth_invalid_at_unix_secs,
|
||||||
|
oauth_invalid_reason,
|
||||||
|
status_snapshot,
|
||||||
|
created_at AS created_at_unix_ms,
|
||||||
|
updated_at AS updated_at_unix_secs,
|
||||||
|
health_by_format,
|
||||||
|
circuit_breaker_by_format
|
||||||
|
FROM provider_api_keys
|
||||||
|
WHERE provider_id = ?
|
||||||
|
AND (? IS NULL OR LOWER(name) LIKE ? OR LOWER(id) LIKE ?)
|
||||||
|
AND (? IS NULL OR is_active = ?)
|
||||||
|
ORDER BY {order_by}
|
||||||
|
LIMIT ?
|
||||||
|
OFFSET ?
|
||||||
|
"#,
|
||||||
|
);
|
||||||
|
let rows = sqlx::query(&sql)
|
||||||
|
.bind(&query.provider_id)
|
||||||
|
.bind(search_pattern.as_deref())
|
||||||
|
.bind(search_pattern.as_deref())
|
||||||
|
.bind(search_pattern.as_deref())
|
||||||
|
.bind(query.is_active)
|
||||||
|
.bind(query.is_active)
|
||||||
|
.bind(limit)
|
||||||
|
.bind(offset)
|
||||||
|
.fetch_all(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_sql_err()?;
|
||||||
|
let items = rows
|
||||||
|
.iter()
|
||||||
|
.map(map_key_row)
|
||||||
|
.collect::<Result<Vec<_>, _>>()?;
|
||||||
|
|
||||||
|
Ok(StoredProviderCatalogKeyPage { items, total })
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn list_key_stats_by_provider_ids(
|
||||||
|
&self,
|
||||||
|
provider_ids: &[String],
|
||||||
|
) -> Result<Vec<StoredProviderCatalogKeyStats>, DataLayerError> {
|
||||||
|
if provider_ids.is_empty() {
|
||||||
|
return Ok(Vec::new());
|
||||||
|
}
|
||||||
|
|
||||||
|
let rows = build_list_query(
|
||||||
|
LIST_KEY_STATS_BY_PROVIDER_IDS_PREFIX,
|
||||||
|
provider_ids,
|
||||||
|
"\nGROUP BY provider_id\nORDER BY provider_id ASC",
|
||||||
|
)
|
||||||
|
.build()
|
||||||
|
.fetch_all(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_sql_err()?;
|
||||||
|
rows.iter().map(map_key_stats_row).collect()
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn create_provider(
|
pub async fn create_provider(
|
||||||
&self,
|
&self,
|
||||||
provider: &StoredProviderCatalogProvider,
|
provider: &StoredProviderCatalogProvider,
|
||||||
@@ -870,81 +1381,63 @@ impl ProviderCatalogReadRepository for SqliteProviderCatalogReadRepository {
|
|||||||
&self,
|
&self,
|
||||||
active_only: bool,
|
active_only: bool,
|
||||||
) -> Result<Vec<StoredProviderCatalogProvider>, DataLayerError> {
|
) -> Result<Vec<StoredProviderCatalogProvider>, DataLayerError> {
|
||||||
self.load_memory().await?.list_providers(active_only).await
|
Self::list_providers(self, active_only).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn list_providers_by_ids(
|
async fn list_providers_by_ids(
|
||||||
&self,
|
&self,
|
||||||
provider_ids: &[String],
|
provider_ids: &[String],
|
||||||
) -> Result<Vec<StoredProviderCatalogProvider>, DataLayerError> {
|
) -> Result<Vec<StoredProviderCatalogProvider>, DataLayerError> {
|
||||||
self.load_memory()
|
Self::list_providers_by_ids(self, provider_ids).await
|
||||||
.await?
|
|
||||||
.list_providers_by_ids(provider_ids)
|
|
||||||
.await
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn list_endpoints_by_ids(
|
async fn list_endpoints_by_ids(
|
||||||
&self,
|
&self,
|
||||||
endpoint_ids: &[String],
|
endpoint_ids: &[String],
|
||||||
) -> Result<Vec<StoredProviderCatalogEndpoint>, DataLayerError> {
|
) -> Result<Vec<StoredProviderCatalogEndpoint>, DataLayerError> {
|
||||||
self.load_memory()
|
Self::list_endpoints_by_ids(self, endpoint_ids).await
|
||||||
.await?
|
|
||||||
.list_endpoints_by_ids(endpoint_ids)
|
|
||||||
.await
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn list_endpoints_by_provider_ids(
|
async fn list_endpoints_by_provider_ids(
|
||||||
&self,
|
&self,
|
||||||
provider_ids: &[String],
|
provider_ids: &[String],
|
||||||
) -> Result<Vec<StoredProviderCatalogEndpoint>, DataLayerError> {
|
) -> Result<Vec<StoredProviderCatalogEndpoint>, DataLayerError> {
|
||||||
self.load_memory()
|
Self::list_endpoints_by_provider_ids(self, provider_ids).await
|
||||||
.await?
|
|
||||||
.list_endpoints_by_provider_ids(provider_ids)
|
|
||||||
.await
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn list_keys_by_ids(
|
async fn list_keys_by_ids(
|
||||||
&self,
|
&self,
|
||||||
key_ids: &[String],
|
key_ids: &[String],
|
||||||
) -> Result<Vec<StoredProviderCatalogKey>, DataLayerError> {
|
) -> Result<Vec<StoredProviderCatalogKey>, DataLayerError> {
|
||||||
self.load_memory().await?.list_keys_by_ids(key_ids).await
|
Self::list_keys_by_ids(self, key_ids).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn list_keys_by_provider_ids(
|
async fn list_keys_by_provider_ids(
|
||||||
&self,
|
&self,
|
||||||
provider_ids: &[String],
|
provider_ids: &[String],
|
||||||
) -> Result<Vec<StoredProviderCatalogKey>, DataLayerError> {
|
) -> Result<Vec<StoredProviderCatalogKey>, DataLayerError> {
|
||||||
self.load_memory()
|
Self::list_keys_by_provider_ids(self, provider_ids).await
|
||||||
.await?
|
|
||||||
.list_keys_by_provider_ids(provider_ids)
|
|
||||||
.await
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn list_key_summaries_by_provider_ids(
|
async fn list_key_summaries_by_provider_ids(
|
||||||
&self,
|
&self,
|
||||||
provider_ids: &[String],
|
provider_ids: &[String],
|
||||||
) -> Result<Vec<StoredProviderCatalogKey>, DataLayerError> {
|
) -> Result<Vec<StoredProviderCatalogKey>, DataLayerError> {
|
||||||
self.load_memory()
|
Self::list_key_summaries_by_provider_ids(self, provider_ids).await
|
||||||
.await?
|
|
||||||
.list_key_summaries_by_provider_ids(provider_ids)
|
|
||||||
.await
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn list_keys_page(
|
async fn list_keys_page(
|
||||||
&self,
|
&self,
|
||||||
query: &ProviderCatalogKeyListQuery,
|
query: &ProviderCatalogKeyListQuery,
|
||||||
) -> Result<StoredProviderCatalogKeyPage, DataLayerError> {
|
) -> Result<StoredProviderCatalogKeyPage, DataLayerError> {
|
||||||
self.load_memory().await?.list_keys_page(query).await
|
Self::list_keys_page(self, query).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn list_key_stats_by_provider_ids(
|
async fn list_key_stats_by_provider_ids(
|
||||||
&self,
|
&self,
|
||||||
provider_ids: &[String],
|
provider_ids: &[String],
|
||||||
) -> Result<Vec<StoredProviderCatalogKeyStats>, DataLayerError> {
|
) -> Result<Vec<StoredProviderCatalogKeyStats>, DataLayerError> {
|
||||||
self.load_memory()
|
Self::list_key_stats_by_provider_ids(self, provider_ids).await
|
||||||
.await?
|
|
||||||
.list_key_stats_by_provider_ids(provider_ids)
|
|
||||||
.await
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1063,6 +1556,21 @@ impl ProviderCatalogWriteRepository for SqliteProviderCatalogReadRepository {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn build_list_query<'a>(
|
||||||
|
prefix: &'static str,
|
||||||
|
ids: &'a [String],
|
||||||
|
suffix: &'static str,
|
||||||
|
) -> QueryBuilder<'a, Sqlite> {
|
||||||
|
let mut builder = QueryBuilder::<Sqlite>::new(prefix);
|
||||||
|
let mut separated = builder.separated(", ");
|
||||||
|
for id in ids {
|
||||||
|
separated.push_bind(id);
|
||||||
|
}
|
||||||
|
separated.push_unseparated(")");
|
||||||
|
builder.push(suffix);
|
||||||
|
builder
|
||||||
|
}
|
||||||
|
|
||||||
fn current_unix_secs() -> u64 {
|
fn current_unix_secs() -> u64 {
|
||||||
chrono::Utc::now().timestamp().max(0) as u64
|
chrono::Utc::now().timestamp().max(0) as u64
|
||||||
}
|
}
|
||||||
@@ -1353,6 +1861,14 @@ fn map_endpoint_row(row: &SqliteRow) -> Result<StoredProviderCatalogEndpoint, Da
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn map_key_stats_row(row: &SqliteRow) -> Result<StoredProviderCatalogKeyStats, DataLayerError> {
|
||||||
|
StoredProviderCatalogKeyStats::new(
|
||||||
|
row.try_get("provider_id").map_sql_err()?,
|
||||||
|
row.try_get("total_keys").map_sql_err()?,
|
||||||
|
row.try_get("active_keys").map_sql_err()?,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
fn map_key_row(row: &SqliteRow) -> Result<StoredProviderCatalogKey, DataLayerError> {
|
fn map_key_row(row: &SqliteRow) -> Result<StoredProviderCatalogKey, DataLayerError> {
|
||||||
let total_cost_usd = sqlite_optional_real(row, "total_cost_usd")?.unwrap_or(0.0);
|
let total_cost_usd = sqlite_optional_real(row, "total_cost_usd")?.unwrap_or(0.0);
|
||||||
if !total_cost_usd.is_finite() {
|
if !total_cost_usd.is_finite() {
|
||||||
@@ -1549,8 +2065,8 @@ mod tests {
|
|||||||
use super::SqliteProviderCatalogReadRepository;
|
use super::SqliteProviderCatalogReadRepository;
|
||||||
use crate::lifecycle::migrate::run_sqlite_migrations;
|
use crate::lifecycle::migrate::run_sqlite_migrations;
|
||||||
use crate::repository::provider_catalog::{
|
use crate::repository::provider_catalog::{
|
||||||
ProviderCatalogKeyListOrder, ProviderCatalogKeyListQuery, ProviderCatalogReadRepository,
|
ProviderCatalogKeyListOrder, ProviderCatalogKeyListQuery, StoredProviderCatalogEndpoint,
|
||||||
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
|
StoredProviderCatalogKey, StoredProviderCatalogProvider,
|
||||||
};
|
};
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
|
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -3,7 +3,7 @@ services:
|
|||||||
image: ${APP_IMAGE:-ghcr.io/fawney19/aether:latest}
|
image: ${APP_IMAGE:-ghcr.io/fawney19/aether:latest}
|
||||||
container_name: aether-app
|
container_name: aether-app
|
||||||
env_file:
|
env_file:
|
||||||
- .env
|
- ${AETHER_ENV_FILE:-.env}
|
||||||
environment:
|
environment:
|
||||||
TZ: Asia/Shanghai
|
TZ: Asia/Shanghai
|
||||||
AETHER_DATABASE_DRIVER: sqlite
|
AETHER_DATABASE_DRIVER: sqlite
|
||||||
238
docs/operations/pg-to-single-node-migration.md
Normal file
238
docs/operations/pg-to-single-node-migration.md
Normal file
@@ -0,0 +1,238 @@
|
|||||||
|
# Postgres to Aether Single Node Migration
|
||||||
|
|
||||||
|
Chinese version: [pg-to-single-node-migration.zh-CN.md](pg-to-single-node-migration.zh-CN.md)
|
||||||
|
|
||||||
|
This runbook migrates an existing Docker Compose Postgres deployment to Aether
|
||||||
|
single-node. In this repository, **single-node** means the default SQLite installer mode:
|
||||||
|
`install.sh --mode single-node`, a system service backed by SQLite. The Docker Compose
|
||||||
|
single-node template is `docker-compose.single-node.yml`, exposed through `--mode compose-single-node`.
|
||||||
|
|
||||||
|
The migration script is:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
scripts/migrate-pg-to-single-node.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
If the target should stay on Docker Compose instead of becoming a system
|
||||||
|
service, use the image-based Compose migration script:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
scripts/migrate-pg-compose-to-single-node.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
Both migration scripts pull/install the target single-node version before
|
||||||
|
downtime, stop only the source `app`, copy Postgres records directly into a
|
||||||
|
temporary SQLite DB without writing a JSONL file, replace the target
|
||||||
|
`aether.db`, and start single-node.
|
||||||
|
|
||||||
|
You can also use the installer as the unified entrypoint and let `--mode`
|
||||||
|
select the migration target:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# In interactive mode, first choose the target deployment mode:
|
||||||
|
# 1) Docker Compose standard deployment (Postgres + Redis)
|
||||||
|
# 2) Docker Compose single-node deployment (SQLite)
|
||||||
|
# 3) System service single-node deployment (SQLite)
|
||||||
|
# After choosing 2 or 3, choose the data initialization mode:
|
||||||
|
# 1) Fresh initialization (do not migrate existing data)
|
||||||
|
# 2) Migrate from an existing Docker Compose PG database
|
||||||
|
install.sh
|
||||||
|
|
||||||
|
# Migrate into a new single-node Docker Compose directory.
|
||||||
|
install.sh \
|
||||||
|
--mode compose-single-node \
|
||||||
|
--migrate-from-compose /root/Aether/docker-compose.yml \
|
||||||
|
--compose-dir /opt/aether-single \
|
||||||
|
--replace-existing
|
||||||
|
|
||||||
|
# Migrate into the system service + SQLite layout.
|
||||||
|
sudo install.sh \
|
||||||
|
--mode single-node \
|
||||||
|
--migrate-from-compose /root/Aether/docker-compose.yml \
|
||||||
|
--replace-existing
|
||||||
|
```
|
||||||
|
|
||||||
|
Interactive mode first asks for the target deployment shape. If the target is
|
||||||
|
`compose-single-node` or `single-node`, the installer then asks for the data
|
||||||
|
initialization mode: fresh initialization, or migration from an existing Docker
|
||||||
|
Compose PG database. If you choose migration, it tries to detect the source PG
|
||||||
|
Compose file from `docker compose ls`, then verifies that the Compose config
|
||||||
|
contains the default `app` and `postgres` services. If exactly one match is
|
||||||
|
found, it is used as the default prompt value. If detection is ambiguous or
|
||||||
|
fails, the installer stops; rerun it with `--migrate-from-compose` to specify
|
||||||
|
the source compose path.
|
||||||
|
|
||||||
|
The installer only normalizes the entrypoint: `compose-single-node` delegates to
|
||||||
|
`scripts/migrate-pg-compose-to-single-node.sh`, while `single-node` delegates to
|
||||||
|
`scripts/migrate-pg-to-single-node.sh`.
|
||||||
|
|
||||||
|
## What It Does
|
||||||
|
|
||||||
|
The script keeps the production cutover window short:
|
||||||
|
|
||||||
|
1. Reads the source Compose `.env`.
|
||||||
|
2. Builds a single-node env file that preserves `JWT_SECRET_KEY`, `ENCRYPTION_KEY` or
|
||||||
|
`AETHER_GATEWAY_DATA_ENCRYPTION_KEY`, admin settings, port, and app config.
|
||||||
|
3. Installs the single-node release with `install.sh --mode single-node --skip-start`.
|
||||||
|
4. Preflights SQLite migrations with the installed single-node binary.
|
||||||
|
5. Pulls the target single-node image, confirms its `copy` command is available,
|
||||||
|
and verifies that its Docker image ID matches the currently running source
|
||||||
|
`app` image ID.
|
||||||
|
6. Uses the target SQLite schema as the migration plan: same-name source
|
||||||
|
Postgres tables and columns are copied into the temporary SQLite DB.
|
||||||
|
7. Applies the compressed body and HTTP body detail policy. The default is full,
|
||||||
|
and you can opt into an omit mode for large artifacts.
|
||||||
|
8. Checks that the work directory and target SQLite directory have enough free
|
||||||
|
disk space for the temporary and final SQLite files.
|
||||||
|
9. Stops only the source `app` service, leaving Postgres and Redis running.
|
||||||
|
10. Copies source Postgres records directly into a temporary SQLite database
|
||||||
|
without generating JSONL files.
|
||||||
|
11. Replaces the target SQLite DB, including SQLite `-wal`/`-shm` sidecar files
|
||||||
|
when present, and starts the single-node service.
|
||||||
|
|
||||||
|
The image check compares Docker image IDs, not just tag strings. If both source
|
||||||
|
and target say `latest` but resolve to different image IDs, migration stops.
|
||||||
|
Upgrade the source PG Compose `app` to the target single-node version first,
|
||||||
|
verify it is healthy, then run the migration. The scripts also check that the
|
||||||
|
target image supports direct copy and the request-body omit flag; using a new
|
||||||
|
script with an old image stops before cutover to avoid missing data.
|
||||||
|
|
||||||
|
## Production Cutover
|
||||||
|
|
||||||
|
Before production cutover, take a normal server backup or snapshot. Then run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo scripts/migrate-pg-to-single-node.sh \
|
||||||
|
--source-compose /root/Aether/docker-compose.yml \
|
||||||
|
--replace-existing
|
||||||
|
```
|
||||||
|
|
||||||
|
For Docker Compose single-node cutover instead of a system service:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
scripts/migrate-pg-compose-to-single-node.sh \
|
||||||
|
--source-compose /root/Aether/docker-compose.yml \
|
||||||
|
--replace-existing
|
||||||
|
```
|
||||||
|
|
||||||
|
The source Postgres compose directory and target single-node compose directory
|
||||||
|
can be different. For example:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
install.sh \
|
||||||
|
--mode compose-single-node \
|
||||||
|
--migrate-from-compose /root/Aether/docker-compose.yml \
|
||||||
|
--compose-dir /opt/aether-single \
|
||||||
|
--replace-existing
|
||||||
|
```
|
||||||
|
|
||||||
|
Equivalently, call the lower-level script and pass each target path explicitly:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
scripts/migrate-pg-compose-to-single-node.sh \
|
||||||
|
--source-compose /root/Aether/docker-compose.yml \
|
||||||
|
--target-compose /opt/aether-single/docker-compose.single-node.yml \
|
||||||
|
--target-env /opt/aether-single/.env.single-node \
|
||||||
|
--target-db /opt/aether-single/data/aether.db \
|
||||||
|
--replace-existing
|
||||||
|
```
|
||||||
|
|
||||||
|
During cutover, the script stops and removes only the source `app` container to
|
||||||
|
free the fixed `aether-app` container name. Postgres, Redis, and their volumes
|
||||||
|
remain in place for rollback.
|
||||||
|
|
||||||
|
Defaults:
|
||||||
|
|
||||||
|
| Setting | Default |
|
||||||
|
| --- | --- |
|
||||||
|
| Source Compose | `docker-compose.yml` |
|
||||||
|
| Single Node install root | `/opt/aether` |
|
||||||
|
| Single Node config dir | `/etc/aether` |
|
||||||
|
| Target SQLite DB | `/opt/aether/data/aether.db` |
|
||||||
|
| Source app service | `app` |
|
||||||
|
| Source Postgres service | `postgres` |
|
||||||
|
| Single Node service | `aether-gateway` |
|
||||||
|
|
||||||
|
The script writes migration artifacts under `./data/pg-to-single-node-<timestamp>` next
|
||||||
|
to the source Compose file unless `--work-dir` is provided.
|
||||||
|
|
||||||
|
## Rollback
|
||||||
|
|
||||||
|
The script leaves the original Postgres and Redis volumes in place. If cutover
|
||||||
|
finishes but you need to roll back:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo systemctl stop aether-gateway
|
||||||
|
cd /root/Aether
|
||||||
|
docker compose -f docker-compose.yml up -d app
|
||||||
|
```
|
||||||
|
|
||||||
|
For the Compose single-node script, rollback is the same idea: start the app
|
||||||
|
again from the original Postgres compose file.
|
||||||
|
|
||||||
|
If the migration fails before cutover completes, the script attempts to restart
|
||||||
|
the source `app` service automatically. Pass `--keep-source-stopped-on-error` if
|
||||||
|
you want to inspect the stopped source deployment manually instead.
|
||||||
|
|
||||||
|
## Data Coverage Guard
|
||||||
|
|
||||||
|
The migration does not maintain a separate business-domain table list. The
|
||||||
|
target single-node image first builds a temporary SQLite database with its
|
||||||
|
normal migrations, then `aether-gateway copy` reads that SQLite schema and copies
|
||||||
|
matching public Postgres tables and columns.
|
||||||
|
|
||||||
|
If the source Postgres database has a non-empty public table that does not exist
|
||||||
|
in the target SQLite schema, the copy stops instead of silently dropping it. It
|
||||||
|
ignores lifecycle metadata tables such as `_sqlx_migrations` and
|
||||||
|
`schema_backfills`. Extra source columns that are absent from the target schema
|
||||||
|
are not copied.
|
||||||
|
|
||||||
|
## Request Body Detail Policy
|
||||||
|
|
||||||
|
The production migration migrates all migratable data by default. The only
|
||||||
|
optional exclusion is request body detail data.
|
||||||
|
|
||||||
|
When you choose to skip request bodies, the migration does not copy
|
||||||
|
`usage_body_blobs`, `usage_http_audits`, or legacy `usage` request body columns
|
||||||
|
such as `request_body`, `provider_request_body`, `response_body`,
|
||||||
|
`client_response_body`, and `*_body_compressed`.
|
||||||
|
|
||||||
|
Interactive installation lets you choose:
|
||||||
|
|
||||||
|
```text
|
||||||
|
1) Full migration: migrate all migratable data, including request body details
|
||||||
|
2) Skip request bodies: migrate all other data; skip only request body large fields and HTTP body detail tables; source PG is unchanged
|
||||||
|
```
|
||||||
|
|
||||||
|
For non-interactive full runs:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
scripts/migrate-pg-to-single-node.sh \
|
||||||
|
--request-body-mode full
|
||||||
|
```
|
||||||
|
|
||||||
|
For non-interactive omit runs:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
scripts/migrate-pg-to-single-node.sh \
|
||||||
|
--request-body-mode omit
|
||||||
|
```
|
||||||
|
|
||||||
|
`omit` only skips writing those large artifacts and detail tables into the
|
||||||
|
target SQLite database. It does not delete or clear the source Postgres data.
|
||||||
|
|
||||||
|
## Notes
|
||||||
|
|
||||||
|
- Single Node requires root or sudo because it writes `/opt/aether`, `/etc/aether`, and
|
||||||
|
the system service definition.
|
||||||
|
- The script does not decrypt or re-encrypt provider keys. It preserves the
|
||||||
|
original encryption key and moves encrypted data as-is.
|
||||||
|
- Existing target SQLite databases, including `-wal`/`-shm` sidecars, are not
|
||||||
|
replaced unless `--replace-existing` is provided.
|
||||||
|
- Disk space checks use `pg_database_size(current_database()) * 2 + 1 GiB` as the
|
||||||
|
conservative estimate for one SQLite copy. If the work directory and target DB
|
||||||
|
directory are on the same filesystem, the script requires enough space for both
|
||||||
|
the temporary and final SQLite files. With `--request-body-mode omit`, the
|
||||||
|
estimate subtracts `usage_body_blobs` and `usage_http_audits` relation sizes.
|
||||||
|
- For non-standard source Compose files, set `--app-service` and
|
||||||
|
`--postgres-service` to match the service names.
|
||||||
221
docs/operations/pg-to-single-node-migration.zh-CN.md
Normal file
221
docs/operations/pg-to-single-node-migration.zh-CN.md
Normal file
@@ -0,0 +1,221 @@
|
|||||||
|
# Postgres 到 Aether Single Node 迁移
|
||||||
|
|
||||||
|
英文版:[pg-to-single-node-migration.md](pg-to-single-node-migration.md)
|
||||||
|
|
||||||
|
本文档用于把现有 Docker Compose Postgres 部署迁移到 Aether
|
||||||
|
single-node。当前版本里,**single-node** 指默认 SQLite 安装模式:
|
||||||
|
`install.sh --mode single-node`,也就是系统服务加 SQLite。Docker Compose
|
||||||
|
单机模板是 `docker-compose.single-node.yml`,安装脚本入口是
|
||||||
|
`--mode compose-single-node`。
|
||||||
|
|
||||||
|
迁移脚本:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
scripts/migrate-pg-to-single-node.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
如果目标形态仍然要保持 Docker Compose,而不是系统服务,使用镜像版迁移脚本:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
scripts/migrate-pg-compose-to-single-node.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
两种迁移脚本都会先拉取/安装目标 single-node 版本,再停止源 `app`,把 Postgres
|
||||||
|
记录直接写入临时 SQLite DB,不落 JSONL 中间文件;复制成功后替换目标
|
||||||
|
`aether.db`,最后启动 single-node。
|
||||||
|
|
||||||
|
也可以直接用安装脚本作为统一入口,由 `--mode` 选择迁移目标:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 交互式执行时,先选择目标部署模式:
|
||||||
|
# 1) Docker Compose 标准部署(Postgres + Redis)
|
||||||
|
# 2) Docker Compose 单节点部署(SQLite)
|
||||||
|
# 3) 系统服务单节点部署(SQLite)
|
||||||
|
# 选择 2 或 3 后,再选择数据初始化方式:
|
||||||
|
# 1) 全新初始化(不迁移现有数据)
|
||||||
|
# 2) 从现有 Docker Compose PG 数据库迁移
|
||||||
|
install.sh
|
||||||
|
|
||||||
|
# 迁移到新的 single-node Docker Compose 目录
|
||||||
|
install.sh \
|
||||||
|
--mode compose-single-node \
|
||||||
|
--migrate-from-compose /root/Aether/docker-compose.yml \
|
||||||
|
--compose-dir /opt/aether-single \
|
||||||
|
--replace-existing
|
||||||
|
|
||||||
|
# 迁移到系统服务 + SQLite
|
||||||
|
sudo install.sh \
|
||||||
|
--mode single-node \
|
||||||
|
--migrate-from-compose /root/Aether/docker-compose.yml \
|
||||||
|
--replace-existing
|
||||||
|
```
|
||||||
|
|
||||||
|
交互模式会先选择目标部署形态。如果目标是 `compose-single-node` 或
|
||||||
|
`single-node`,安装脚本会再询问数据初始化方式:全新初始化,或从现有 Docker
|
||||||
|
Compose PG 数据库迁移。选择迁移后,脚本会通过 `docker compose ls` 自动探测源
|
||||||
|
PG Compose 文件,并确认该 Compose 配置里存在默认的 `app` 和 `postgres` 服务;
|
||||||
|
如果能唯一识别,会作为默认值带入提示。探测不到或存在多个候选时会直接中止;
|
||||||
|
此时请用 `--migrate-from-compose` 显式指定源 compose 路径。
|
||||||
|
|
||||||
|
安装脚本只是统一参数入口:`compose-single-node` 会委托给
|
||||||
|
`scripts/migrate-pg-compose-to-single-node.sh`,`single-node` 会委托给
|
||||||
|
`scripts/migrate-pg-to-single-node.sh`。
|
||||||
|
|
||||||
|
## 迁移内容
|
||||||
|
|
||||||
|
脚本会尽量缩短生产停机窗口:
|
||||||
|
|
||||||
|
1. 读取源 Compose 目录下的 `.env`。
|
||||||
|
2. 生成 single-node 环境文件,保留 `JWT_SECRET_KEY`、`ENCRYPTION_KEY` 或
|
||||||
|
`AETHER_GATEWAY_DATA_ENCRYPTION_KEY`、管理员配置、端口和应用配置。
|
||||||
|
3. 执行 `install.sh --mode single-node --skip-start`,提前安装 single-node
|
||||||
|
release,但不启动服务。
|
||||||
|
4. 使用已安装的 single-node 二进制预检 SQLite schema migration。
|
||||||
|
5. 拉取目标 single-node 镜像,确认其 `copy` 命令可用,并检查源 `app`
|
||||||
|
当前运行镜像 ID 与目标镜像 ID 一致。
|
||||||
|
6. 以目标 SQLite schema 作为迁移计划:把源 Postgres 中同名表、同名字段
|
||||||
|
复制到临时 SQLite DB。
|
||||||
|
7. 检查请求体明细迁移策略;默认全部迁移,也可以选择只跳过请求体明细。
|
||||||
|
8. 检查 work-dir 和目标 SQLite 目录是否有足够空间容纳临时库和正式库。
|
||||||
|
9. 只停止源 Compose 的 `app` 服务,保留 Postgres 和 Redis 运行,方便回滚。
|
||||||
|
10. 从源 Postgres 直接复制记录到临时 SQLite 数据库,不生成 JSONL 中间文件。
|
||||||
|
11. 复制完成后替换目标 SQLite DB,包括 SQLite `-wal`、`-shm` 边车文件,
|
||||||
|
然后启动 single-node 系统服务。
|
||||||
|
|
||||||
|
镜像一致性检查比较的是 Docker 镜像 ID,不只是 tag 字符串。即使源和目标都写着
|
||||||
|
`latest`,只要实际镜像 ID 不同,迁移也会中止。请先把源 PG Compose 的 `app`
|
||||||
|
升级到目标 single-node 相同版本,确认运行正常后再迁移。迁移脚本也会检查目标镜像
|
||||||
|
是否支持直接 copy 和请求体跳过开关;如果只是换了脚本但镜像还是旧版本,脚本会
|
||||||
|
直接中止,避免漏迁。
|
||||||
|
|
||||||
|
## 生产切换
|
||||||
|
|
||||||
|
切换前先做一次常规服务器备份或快照。确认后执行:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo scripts/migrate-pg-to-single-node.sh \
|
||||||
|
--source-compose /root/Aether/docker-compose.yml \
|
||||||
|
--replace-existing
|
||||||
|
```
|
||||||
|
|
||||||
|
如果要迁移到 Docker Compose single-node,而不是系统服务:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
scripts/migrate-pg-compose-to-single-node.sh \
|
||||||
|
--source-compose /root/Aether/docker-compose.yml \
|
||||||
|
--replace-existing
|
||||||
|
```
|
||||||
|
|
||||||
|
源 Postgres Compose 目录和目标 single-node Compose 目录可以不一样。例如:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
install.sh \
|
||||||
|
--mode compose-single-node \
|
||||||
|
--migrate-from-compose /root/Aether/docker-compose.yml \
|
||||||
|
--compose-dir /opt/aether-single \
|
||||||
|
--replace-existing
|
||||||
|
```
|
||||||
|
|
||||||
|
等价地,也可以直接调底层脚本并显式传入每个目标路径:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
scripts/migrate-pg-compose-to-single-node.sh \
|
||||||
|
--source-compose /root/Aether/docker-compose.yml \
|
||||||
|
--target-compose /opt/aether-single/docker-compose.single-node.yml \
|
||||||
|
--target-env /opt/aether-single/.env.single-node \
|
||||||
|
--target-db /opt/aether-single/data/aether.db \
|
||||||
|
--replace-existing
|
||||||
|
```
|
||||||
|
|
||||||
|
切换时脚本只会停止并移除源 `app` 容器,用来释放固定的 `aether-app`
|
||||||
|
容器名;Postgres、Redis 和它们的 volume 都会保留,方便回滚。
|
||||||
|
|
||||||
|
默认路径和服务名:
|
||||||
|
|
||||||
|
| 配置项 | 默认值 |
|
||||||
|
| --- | --- |
|
||||||
|
| 源 Compose 文件 | `docker-compose.yml` |
|
||||||
|
| single-node 安装目录 | `/opt/aether` |
|
||||||
|
| single-node 配置目录 | `/etc/aether` |
|
||||||
|
| 目标 SQLite DB | `/opt/aether/data/aether.db` |
|
||||||
|
| 源 app 服务 | `app` |
|
||||||
|
| 源 Postgres 服务 | `postgres` |
|
||||||
|
| single-node 服务 | `aether-gateway` |
|
||||||
|
|
||||||
|
除非显式传入 `--work-dir`,脚本会把迁移产物写到源 Compose 文件旁边的
|
||||||
|
`./data/pg-to-single-node-<timestamp>`。
|
||||||
|
|
||||||
|
## 回滚
|
||||||
|
|
||||||
|
脚本会保留原 Postgres 和 Redis volume。迁移已经完成但需要回滚时:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo systemctl stop aether-gateway
|
||||||
|
cd /root/Aether
|
||||||
|
docker compose -f docker-compose.yml up -d app
|
||||||
|
```
|
||||||
|
|
||||||
|
对于 Compose single-node 脚本,回滚思路相同:重新用原 Postgres compose 文件
|
||||||
|
拉起 `app`。
|
||||||
|
|
||||||
|
如果迁移在切换完成前失败,脚本默认会尝试自动拉起源 `app` 服务。需要失败后
|
||||||
|
保持源应用停止以便人工排查时,增加:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
--keep-source-stopped-on-error
|
||||||
|
```
|
||||||
|
|
||||||
|
## 数据覆盖保护
|
||||||
|
|
||||||
|
迁移不再维护一份额外的业务表清单。目标 single-node 镜像会先用正常
|
||||||
|
migrations 建出临时 SQLite 数据库,然后 `aether-gateway copy` 读取这个
|
||||||
|
SQLite schema,把源 Postgres 里同名表、同名字段复制过去。
|
||||||
|
|
||||||
|
如果源 Postgres 里存在非空 public 表,但目标 SQLite schema 中没有同名表,
|
||||||
|
copy 会直接中止,不会静默丢弃。生命周期元数据表 `_sqlx_migrations` 和
|
||||||
|
`schema_backfills` 会被忽略。源表中存在但目标 SQLite 不存在的额外字段不会复制。
|
||||||
|
|
||||||
|
## 请求体明细策略
|
||||||
|
|
||||||
|
single-node SQLite 生产迁移默认迁移所有可迁移数据,唯一可选的跳过项是请求体明细。
|
||||||
|
|
||||||
|
选择“不迁移请求体”时,不会迁移 `usage_body_blobs`、`usage_http_audits`,也不会迁移 `usage`
|
||||||
|
表里的 `request_body` / `provider_request_body` / `response_body` /
|
||||||
|
`client_response_body` / `*_body_compressed` 等请求体大字段。
|
||||||
|
|
||||||
|
交互安装时可以选择:
|
||||||
|
|
||||||
|
```text
|
||||||
|
1) 全部迁移:迁移所有可迁移数据,包括请求体明细
|
||||||
|
2) 不迁移请求体:迁移其他所有数据;仅跳过请求体大字段和 HTTP 请求体明细,源 PG 不清除
|
||||||
|
```
|
||||||
|
|
||||||
|
非交互执行时,全部迁移可以显式指定:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
scripts/migrate-pg-to-single-node.sh \
|
||||||
|
--request-body-mode full
|
||||||
|
```
|
||||||
|
|
||||||
|
不迁移请求体可以显式指定:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
scripts/migrate-pg-to-single-node.sh \
|
||||||
|
--request-body-mode omit
|
||||||
|
```
|
||||||
|
|
||||||
|
`omit` 只是不把这些大字段和明细表写进目标 SQLite,不会删除或清空源 Postgres。
|
||||||
|
|
||||||
|
## 注意事项
|
||||||
|
|
||||||
|
- single-node 安装需要 root 或 sudo 权限,因为会写入 `/opt/aether`、
|
||||||
|
`/etc/aether` 和系统服务定义。
|
||||||
|
- 脚本不会解密或重新加密供应商密钥;它会沿用源环境的加密密钥,并原样迁移已加密数据。
|
||||||
|
- 已存在的目标 SQLite DB,包括 `-wal`、`-shm` 边车文件,只有在传入
|
||||||
|
`--replace-existing` 时才会被替换。
|
||||||
|
- 空间检查会用 `pg_database_size(current_database()) * 2 + 1 GiB` 作为单份
|
||||||
|
SQLite 的保守估算。如果 work-dir 和目标 DB 目录在同一个文件系统,会要求同时
|
||||||
|
容纳临时 SQLite 和正式 SQLite。选择 `--request-body-mode omit` 时,
|
||||||
|
估算会扣除 `usage_body_blobs` 和 `usage_http_audits` 的表空间。
|
||||||
|
- 非标准 Compose 服务名需要通过 `--app-service` 和 `--postgres-service`
|
||||||
|
明确指定。
|
||||||
673
install.sh
673
install.sh
@@ -49,6 +49,19 @@ ADMIN_PASSWORD_SOURCE=""
|
|||||||
UI_LANG="${AETHER_LANG:-${AETHER_LANGUAGE:-auto}}"
|
UI_LANG="${AETHER_LANG:-${AETHER_LANGUAGE:-auto}}"
|
||||||
RELEASE_KEEP="${AETHER_RELEASE_KEEP:-3}"
|
RELEASE_KEEP="${AETHER_RELEASE_KEEP:-3}"
|
||||||
RELEASE_ARCHIVE_URL="${AETHER_RELEASE_ARCHIVE_URL:-${AETHER_DOWNLOAD_URL:-}}"
|
RELEASE_ARCHIVE_URL="${AETHER_RELEASE_ARCHIVE_URL:-${AETHER_DOWNLOAD_URL:-}}"
|
||||||
|
MIGRATE_FROM_COMPOSE=""
|
||||||
|
MIGRATE_TARGET_COMPOSE=""
|
||||||
|
MIGRATE_TARGET_ENV=""
|
||||||
|
MIGRATE_TARGET_DB=""
|
||||||
|
MIGRATE_WORK_DIR=""
|
||||||
|
MIGRATE_APP_SERVICE=""
|
||||||
|
MIGRATE_POSTGRES_SERVICE=""
|
||||||
|
MIGRATE_SINGLE_NODE_SERVICE=""
|
||||||
|
MIGRATE_REPLACE_EXISTING="false"
|
||||||
|
MIGRATE_REPLACE_TARGET_COMPOSE="false"
|
||||||
|
MIGRATE_KEEP_SOURCE_STOPPED_ON_ERROR="false"
|
||||||
|
MIGRATE_INTERACTIVE="false"
|
||||||
|
MIGRATE_REQUEST_BODY_MODE=""
|
||||||
|
|
||||||
usage() {
|
usage() {
|
||||||
cat <<'EOF'
|
cat <<'EOF'
|
||||||
@@ -57,10 +70,10 @@ Usage: install.sh [options]
|
|||||||
Install Aether Gateway.
|
Install Aether Gateway.
|
||||||
|
|
||||||
Options:
|
Options:
|
||||||
--mode MODE Deployment mode: compose, compose-sqlite, or single
|
--mode MODE Deployment mode: compose, compose-single-node, or single-node
|
||||||
compose: Docker Compose app + Postgres + Redis
|
compose: Docker Compose app + Postgres + Redis
|
||||||
compose-sqlite: Docker Compose app + SQLite
|
compose-single-node: Docker Compose single-node app
|
||||||
single: system service with SQLite
|
single-node: single-node system service
|
||||||
Linux services use systemd; macOS services use launchd
|
Linux services use systemd; macOS services use launchd
|
||||||
--channel CHANNEL Release channel to resolve when --version is omitted: stable, latest, rc, or beta
|
--channel CHANNEL Release channel to resolve when --version is omitted: stable, latest, rc, or beta
|
||||||
stable/latest resolves the latest stable tag (default)
|
stable/latest resolves the latest stable tag (default)
|
||||||
@@ -79,8 +92,31 @@ Options:
|
|||||||
--lang LANG Installer language: zh or en
|
--lang LANG Installer language: zh or en
|
||||||
--skip-start Install files, but do not start Docker Compose or restart the service
|
--skip-start Install files, but do not start Docker Compose or restart the service
|
||||||
--keep-releases N Keep the latest N releases, prune older ones (default: 3, 0=disable)
|
--keep-releases N Keep the latest N releases, prune older ones (default: 3, 0=disable)
|
||||||
|
--migrate-from-compose PATH
|
||||||
|
Migrate an existing Postgres Compose deployment into the selected single-node mode
|
||||||
|
--target-compose PATH
|
||||||
|
Migration target compose file for --mode compose-single-node
|
||||||
|
--target-env PATH Migration target env file for --mode compose-single-node
|
||||||
|
--target-db PATH Migration target SQLite DB path
|
||||||
|
--work-dir PATH Migration working directory
|
||||||
|
--app-service NAME Source compose app service for migration
|
||||||
|
--postgres-service NAME
|
||||||
|
Source compose Postgres service for migration
|
||||||
|
--single-node-service NAME
|
||||||
|
Target compose service for --mode compose-single-node migration
|
||||||
|
--replace-existing Allow replacing an existing target SQLite DB during migration
|
||||||
|
--replace-target-compose
|
||||||
|
Overwrite target compose file from the single-node template during migration
|
||||||
|
--request-body-mode MODE
|
||||||
|
Request/response body detail handling during migration: full/1 or omit/2
|
||||||
|
--keep-source-stopped-on-error
|
||||||
|
Do not auto-restart source app if migration fails after stopping it
|
||||||
-h, --help Show this help
|
-h, --help Show this help
|
||||||
|
|
||||||
|
Migration examples:
|
||||||
|
install.sh --mode compose-single-node --migrate-from-compose /root/Aether/docker-compose.yml --compose-dir /opt/aether-single --replace-existing
|
||||||
|
sudo install.sh --mode single-node --migrate-from-compose /root/Aether/docker-compose.yml --replace-existing
|
||||||
|
|
||||||
Environment overrides:
|
Environment overrides:
|
||||||
AETHER_REPO, AETHER_SOURCE_REF, AETHER_INSTALL_MODE, AETHER_CHANNEL, AETHER_VERSION
|
AETHER_REPO, AETHER_SOURCE_REF, AETHER_INSTALL_MODE, AETHER_CHANNEL, AETHER_VERSION
|
||||||
AETHER_LANG or AETHER_LANGUAGE
|
AETHER_LANG or AETHER_LANGUAGE
|
||||||
@@ -158,7 +194,7 @@ select_language() {
|
|||||||
|
|
||||||
请选择安装语言 / Choose installer language:
|
请选择安装语言 / Choose installer language:
|
||||||
1) 中文
|
1) 中文
|
||||||
2) 英语 / English
|
2) English
|
||||||
|
|
||||||
请输入选项 / Enter choice [1]:
|
请输入选项 / Enter choice [1]:
|
||||||
EOF
|
EOF
|
||||||
@@ -264,6 +300,63 @@ parse_args() {
|
|||||||
RELEASE_KEEP="$2"
|
RELEASE_KEEP="$2"
|
||||||
shift 2
|
shift 2
|
||||||
;;
|
;;
|
||||||
|
--migrate-from-compose)
|
||||||
|
[[ $# -ge 2 ]] || die "--migrate-from-compose requires a path"
|
||||||
|
MIGRATE_FROM_COMPOSE="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--target-compose)
|
||||||
|
[[ $# -ge 2 ]] || die "--target-compose requires a path"
|
||||||
|
MIGRATE_TARGET_COMPOSE="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--target-env)
|
||||||
|
[[ $# -ge 2 ]] || die "--target-env requires a path"
|
||||||
|
MIGRATE_TARGET_ENV="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--target-db)
|
||||||
|
[[ $# -ge 2 ]] || die "--target-db requires a path"
|
||||||
|
MIGRATE_TARGET_DB="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--work-dir)
|
||||||
|
[[ $# -ge 2 ]] || die "--work-dir requires a path"
|
||||||
|
MIGRATE_WORK_DIR="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--app-service)
|
||||||
|
[[ $# -ge 2 ]] || die "--app-service requires a service name"
|
||||||
|
MIGRATE_APP_SERVICE="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--postgres-service)
|
||||||
|
[[ $# -ge 2 ]] || die "--postgres-service requires a service name"
|
||||||
|
MIGRATE_POSTGRES_SERVICE="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--single-node-service)
|
||||||
|
[[ $# -ge 2 ]] || die "--single-node-service requires a service name"
|
||||||
|
MIGRATE_SINGLE_NODE_SERVICE="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--replace-existing)
|
||||||
|
MIGRATE_REPLACE_EXISTING="true"
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
--replace-target-compose)
|
||||||
|
MIGRATE_REPLACE_TARGET_COMPOSE="true"
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
--request-body-mode)
|
||||||
|
[[ $# -ge 2 ]] || die "--request-body-mode requires a value"
|
||||||
|
MIGRATE_REQUEST_BODY_MODE="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--keep-source-stopped-on-error)
|
||||||
|
MIGRATE_KEEP_SOURCE_STOPPED_ON_ERROR="true"
|
||||||
|
shift
|
||||||
|
;;
|
||||||
-h|--help)
|
-h|--help)
|
||||||
usage
|
usage
|
||||||
exit 0
|
exit 0
|
||||||
@@ -441,25 +534,25 @@ select_mode() {
|
|||||||
MODE="compose"
|
MODE="compose"
|
||||||
return
|
return
|
||||||
;;
|
;;
|
||||||
compose-sqlite|sqlite-compose|compose-solo|solo|docker-solo|docker-solo-compose)
|
compose-single-node|docker-single-node|docker-single-node-compose)
|
||||||
MODE="compose-sqlite"
|
MODE="compose-single-node"
|
||||||
return
|
return
|
||||||
;;
|
;;
|
||||||
single|service|systemd|launchd|sqlite)
|
single-node|service|systemd|launchd|sqlite)
|
||||||
MODE="single"
|
MODE="single-node"
|
||||||
return
|
return
|
||||||
;;
|
;;
|
||||||
cluster|multi|multi-node)
|
cluster|multi|multi-node)
|
||||||
if ui_is_zh; then
|
if ui_is_zh; then
|
||||||
die "集群部署模式暂未开放;请先选择 compose、compose-sqlite 或 single"
|
die "集群部署模式暂未开放;请先选择 compose、compose-single-node 或 single-node"
|
||||||
else
|
else
|
||||||
die "cluster deployment mode is temporarily disabled; choose compose, compose-sqlite, or single"
|
die "cluster deployment mode is temporarily disabled; choose compose, compose-single-node, or single-node"
|
||||||
fi
|
fi
|
||||||
;;
|
;;
|
||||||
auto|"")
|
auto|"")
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
die "unsupported install mode: ${MODE}; expected compose, compose-sqlite, or single"
|
die "unsupported install mode: ${MODE}; expected compose, compose-single-node, or single-node"
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
||||||
@@ -468,34 +561,34 @@ select_mode() {
|
|||||||
cat >/dev/tty <<EOF
|
cat >/dev/tty <<EOF
|
||||||
|
|
||||||
请选择 Aether 部署模式:
|
请选择 Aether 部署模式:
|
||||||
1) Docker Compose 应用: Postgres + Redis
|
1) Docker Compose 标准部署(Postgres + Redis)
|
||||||
3) Docker Compose 应用: 仅SQLite
|
2) Docker Compose 单节点部署(SQLite)
|
||||||
4) 系统服务: 仅SQLite
|
3) 系统服务单节点部署(SQLite)
|
||||||
|
|
||||||
请输入选项 [4]:
|
请输入选项 [3]:
|
||||||
EOF
|
EOF
|
||||||
else
|
else
|
||||||
cat >/dev/tty <<EOF
|
cat >/dev/tty <<EOF
|
||||||
|
|
||||||
Choose Aether deployment mode:
|
Choose Aether deployment mode:
|
||||||
1) Docker Compose app: Postgres + Redis
|
1) Docker Compose standard deployment (Postgres + Redis)
|
||||||
3) Docker Compose app: SQLite only
|
2) Docker Compose single-node deployment (SQLite)
|
||||||
4) System service: SQLite only
|
3) System service single-node deployment (SQLite)
|
||||||
|
|
||||||
Enter choice [4]:
|
Enter choice [3]:
|
||||||
EOF
|
EOF
|
||||||
fi
|
fi
|
||||||
local choice
|
local choice
|
||||||
IFS= read -r choice </dev/tty || choice=""
|
IFS= read -r choice </dev/tty || choice=""
|
||||||
case "${choice:-4}" in
|
case "${choice:-3}" in
|
||||||
1)
|
1)
|
||||||
MODE="compose"
|
MODE="compose"
|
||||||
;;
|
;;
|
||||||
3)
|
2)
|
||||||
MODE="compose-sqlite"
|
MODE="compose-single-node"
|
||||||
;;
|
;;
|
||||||
4)
|
3)
|
||||||
MODE="single"
|
MODE="single-node"
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
if ui_is_zh; then
|
if ui_is_zh; then
|
||||||
@@ -505,8 +598,45 @@ EOF
|
|||||||
fi
|
fi
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
if [[ -z "${MIGRATE_FROM_COMPOSE}" && "${MODE}" != "compose" ]]; then
|
||||||
|
if ui_is_zh; then
|
||||||
|
cat >/dev/tty <<'EOF'
|
||||||
|
|
||||||
|
请选择数据初始化方式:
|
||||||
|
1) 全新初始化(不迁移现有数据)
|
||||||
|
2) 从现有 Docker Compose PG 数据库迁移
|
||||||
|
|
||||||
|
请输入选项 [1]:
|
||||||
|
EOF
|
||||||
|
else
|
||||||
|
cat >/dev/tty <<'EOF'
|
||||||
|
|
||||||
|
Choose data initialization mode:
|
||||||
|
1) Fresh initialization (do not migrate existing data)
|
||||||
|
2) Migrate from an existing Docker Compose PG database
|
||||||
|
|
||||||
|
Enter choice [1]:
|
||||||
|
EOF
|
||||||
|
fi
|
||||||
|
local init_choice
|
||||||
|
IFS= read -r init_choice </dev/tty || init_choice=""
|
||||||
|
case "${init_choice:-1}" in
|
||||||
|
1)
|
||||||
|
;;
|
||||||
|
2)
|
||||||
|
MIGRATE_INTERACTIVE="true"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
if ui_is_zh; then
|
||||||
|
die "无效的数据初始化方式选项: ${init_choice}"
|
||||||
|
else
|
||||||
|
die "invalid data initialization choice: ${init_choice}"
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
fi
|
||||||
else
|
else
|
||||||
MODE="single"
|
MODE="single-node"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -857,6 +987,434 @@ start_compose_deployment() {
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
migration_options_requested() {
|
||||||
|
[[ -n "${MIGRATE_TARGET_COMPOSE}" ]] && return 0
|
||||||
|
[[ -n "${MIGRATE_TARGET_ENV}" ]] && return 0
|
||||||
|
[[ -n "${MIGRATE_TARGET_DB}" ]] && return 0
|
||||||
|
[[ -n "${MIGRATE_WORK_DIR}" ]] && return 0
|
||||||
|
[[ -n "${MIGRATE_APP_SERVICE}" ]] && return 0
|
||||||
|
[[ -n "${MIGRATE_POSTGRES_SERVICE}" ]] && return 0
|
||||||
|
[[ -n "${MIGRATE_SINGLE_NODE_SERVICE}" ]] && return 0
|
||||||
|
[[ "${MIGRATE_REPLACE_EXISTING}" == "true" ]] && return 0
|
||||||
|
[[ "${MIGRATE_REPLACE_TARGET_COMPOSE}" == "true" ]] && return 0
|
||||||
|
[[ -n "${MIGRATE_REQUEST_BODY_MODE}" ]] && return 0
|
||||||
|
[[ "${MIGRATE_KEEP_SOURCE_STOPPED_ON_ERROR}" == "true" ]] && return 0
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
normalize_migration_request_body_mode() {
|
||||||
|
case "${MIGRATE_REQUEST_BODY_MODE}" in
|
||||||
|
""|1|full|all|include)
|
||||||
|
MIGRATE_REQUEST_BODY_MODE="full"
|
||||||
|
;;
|
||||||
|
2|omit|skip)
|
||||||
|
MIGRATE_REQUEST_BODY_MODE="omit"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
die "--request-body-mode must be full/1 or omit/2"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
prompt_with_default() {
|
||||||
|
local prompt="$1"
|
||||||
|
local default_value="$2"
|
||||||
|
local value
|
||||||
|
|
||||||
|
if [[ -n "${default_value}" ]]; then
|
||||||
|
printf '%s [%s]: ' "${prompt}" "${default_value}" >/dev/tty
|
||||||
|
else
|
||||||
|
printf '%s: ' "${prompt}" >/dev/tty
|
||||||
|
fi
|
||||||
|
IFS= read -r value </dev/tty || value=""
|
||||||
|
if [[ -z "${value}" ]]; then
|
||||||
|
printf '%s\n' "${default_value}"
|
||||||
|
else
|
||||||
|
printf '%s\n' "${value}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
prompt_yes_no() {
|
||||||
|
local prompt="$1"
|
||||||
|
local default_value="$2"
|
||||||
|
local suffix choice
|
||||||
|
|
||||||
|
case "${default_value}" in
|
||||||
|
yes)
|
||||||
|
if ui_is_zh; then
|
||||||
|
suffix="[y/n,默认 y]"
|
||||||
|
else
|
||||||
|
suffix="[y/n, default y]"
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
default_value="no"
|
||||||
|
if ui_is_zh; then
|
||||||
|
suffix="[y/n,默认 n]"
|
||||||
|
else
|
||||||
|
suffix="[y/n, default n]"
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
while true; do
|
||||||
|
printf '%s %s: ' "${prompt}" "${suffix}" >/dev/tty
|
||||||
|
IFS= read -r choice </dev/tty || choice=""
|
||||||
|
choice="$(printf '%s' "${choice}" | tr '[:upper:]' '[:lower:]')"
|
||||||
|
case "${choice:-${default_value}}" in
|
||||||
|
y|yes)
|
||||||
|
return 0
|
||||||
|
;;
|
||||||
|
n|no)
|
||||||
|
return 1
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
if ui_is_zh; then
|
||||||
|
echo "请输入 y 或 n。" >/dev/tty
|
||||||
|
else
|
||||||
|
echo "Enter y or n." >/dev/tty
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
docker_compose_ls_config_files() {
|
||||||
|
local output
|
||||||
|
|
||||||
|
output="$(docker compose ls --format json 2>/dev/null || true)"
|
||||||
|
if [[ -n "${output}" && "${output}" == *ConfigFiles* ]]; then
|
||||||
|
printf '%s' "${output}" |
|
||||||
|
tr '{' '\n' |
|
||||||
|
sed -n 's/.*"ConfigFiles"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p'
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
docker compose ls 2>/dev/null | awk 'NR > 1 && NF > 0 { print $NF }'
|
||||||
|
}
|
||||||
|
|
||||||
|
compose_file_has_source_services() {
|
||||||
|
local compose_file="$1"
|
||||||
|
local app_service="${MIGRATE_APP_SERVICE:-app}"
|
||||||
|
local postgres_service="${MIGRATE_POSTGRES_SERVICE:-postgres}"
|
||||||
|
local services
|
||||||
|
|
||||||
|
services="$(docker compose -f "${compose_file}" config --services 2>/dev/null || true)"
|
||||||
|
[[ -n "${services}" ]] || return 1
|
||||||
|
printf '%s\n' "${services}" | grep -Fxq "${app_service}" || return 1
|
||||||
|
printf '%s\n' "${services}" | grep -Fxq "${postgres_service}" || return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
append_unique_candidate() {
|
||||||
|
local candidate="$1"
|
||||||
|
shift
|
||||||
|
local existing
|
||||||
|
|
||||||
|
for existing in "$@"; do
|
||||||
|
[[ "${existing}" != "${candidate}" ]] || return 1
|
||||||
|
done
|
||||||
|
printf '%s\n' "${candidate}"
|
||||||
|
}
|
||||||
|
|
||||||
|
detect_source_compose_from_docker_compose_ls() {
|
||||||
|
local config_files compose_file candidate
|
||||||
|
local -a candidates=()
|
||||||
|
|
||||||
|
command -v docker >/dev/null 2>&1 || return 0
|
||||||
|
docker compose version >/dev/null 2>&1 || return 0
|
||||||
|
|
||||||
|
while IFS= read -r config_files || [[ -n "${config_files}" ]]; do
|
||||||
|
config_files="$(trim_whitespace "${config_files}")"
|
||||||
|
[[ -n "${config_files}" ]] || continue
|
||||||
|
|
||||||
|
# The migration scripts currently accept one source compose file. If the
|
||||||
|
# source project was launched with multiple compose files, ask explicitly.
|
||||||
|
[[ "${config_files}" != *,* ]] || continue
|
||||||
|
|
||||||
|
compose_file="${config_files}"
|
||||||
|
[[ -f "${compose_file}" ]] || continue
|
||||||
|
compose_file="$(absolute_path "${compose_file}")"
|
||||||
|
compose_file_has_source_services "${compose_file}" || continue
|
||||||
|
candidate="$(append_unique_candidate "${compose_file}" "${candidates[@]}" || true)"
|
||||||
|
[[ -z "${candidate}" ]] || candidates+=("${candidate}")
|
||||||
|
done < <(docker_compose_ls_config_files)
|
||||||
|
|
||||||
|
case "${#candidates[@]}" in
|
||||||
|
0)
|
||||||
|
return 0
|
||||||
|
;;
|
||||||
|
1)
|
||||||
|
printf '%s\n' "${candidates[0]}"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
if interactive_tty_available; then
|
||||||
|
if ui_is_zh; then
|
||||||
|
echo "从 docker compose ls 找到多个可能的源 Compose,无法安全自动选择:" >/dev/tty
|
||||||
|
else
|
||||||
|
echo "docker compose ls found multiple possible source Compose files and cannot choose safely:" >/dev/tty
|
||||||
|
fi
|
||||||
|
printf ' %s\n' "${candidates[@]}" >/dev/tty
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
collect_interactive_migration_options() {
|
||||||
|
local detected_source
|
||||||
|
local prompt
|
||||||
|
local source_compose_abs
|
||||||
|
local source_compose_dir
|
||||||
|
|
||||||
|
[[ "${MIGRATE_INTERACTIVE}" == "true" ]] || return
|
||||||
|
interactive_tty_available || die "interactive migration selection requires a terminal"
|
||||||
|
|
||||||
|
if ui_is_zh; then
|
||||||
|
cat >/dev/tty <<'EOF'
|
||||||
|
|
||||||
|
迁移会先做预检并拉取/安装目标 single-node,再在切换窗口停止源 app。
|
||||||
|
源 Postgres 和 Redis 会保留,便于回滚。
|
||||||
|
|
||||||
|
EOF
|
||||||
|
else
|
||||||
|
cat >/dev/tty <<'EOF'
|
||||||
|
|
||||||
|
Migration will preflight and pull/install the target single-node release first.
|
||||||
|
During cutover it stops only the source app. Source Postgres and Redis remain for rollback.
|
||||||
|
|
||||||
|
EOF
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -z "${MIGRATE_FROM_COMPOSE}" ]]; then
|
||||||
|
detected_source="$(detect_source_compose_from_docker_compose_ls || true)"
|
||||||
|
if [[ -z "${detected_source}" ]]; then
|
||||||
|
if ui_is_zh; then
|
||||||
|
die "未能通过 docker compose ls 唯一识别源 PG Compose;请使用 --migrate-from-compose 显式指定"
|
||||||
|
else
|
||||||
|
die "could not uniquely detect source PG Compose from docker compose ls; pass --migrate-from-compose explicitly"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
if ui_is_zh; then
|
||||||
|
printf '已通过 docker compose ls 探测到源 Compose: %s\n' "${detected_source}" >/dev/tty
|
||||||
|
else
|
||||||
|
printf 'Detected source Compose from docker compose ls: %s\n' "${detected_source}" >/dev/tty
|
||||||
|
fi
|
||||||
|
if ui_is_zh; then
|
||||||
|
prompt="确认使用该源 Compose 进行迁移"
|
||||||
|
else
|
||||||
|
prompt="Use this source Compose for migration"
|
||||||
|
fi
|
||||||
|
if prompt_yes_no "${prompt}" "yes"; then
|
||||||
|
MIGRATE_FROM_COMPOSE="${detected_source}"
|
||||||
|
else
|
||||||
|
if ui_is_zh; then
|
||||||
|
die "已取消迁移;如需指定其他源 Compose,请使用 --migrate-from-compose"
|
||||||
|
else
|
||||||
|
die "migration cancelled; pass --migrate-from-compose to use another source Compose"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
[[ -n "${MIGRATE_FROM_COMPOSE}" ]] || die "--migrate-from-compose cannot be empty"
|
||||||
|
source_compose_abs="$(absolute_path "${MIGRATE_FROM_COMPOSE}")"
|
||||||
|
source_compose_dir="$(dirname "${source_compose_abs}")"
|
||||||
|
|
||||||
|
if [[ "${MODE}" == "compose-single-node" ]]; then
|
||||||
|
if [[ "${COMPOSE_DIR_EXPLICIT}" != "true" ]]; then
|
||||||
|
COMPOSE_DIR="${source_compose_dir}-single-node"
|
||||||
|
fi
|
||||||
|
if ui_is_zh; then
|
||||||
|
printf '已自动选择目标 single-node Compose 目录: %s\n' "${COMPOSE_DIR}" >/dev/tty
|
||||||
|
prompt="确认使用该目标目录"
|
||||||
|
else
|
||||||
|
printf 'Selected target single-node Compose directory: %s\n' "${COMPOSE_DIR}" >/dev/tty
|
||||||
|
prompt="Use this target directory"
|
||||||
|
fi
|
||||||
|
if ! prompt_yes_no "${prompt}" "yes"; then
|
||||||
|
if ui_is_zh; then
|
||||||
|
die "已取消迁移;如需指定其他目标目录,请使用 --compose-dir"
|
||||||
|
else
|
||||||
|
die "migration cancelled; pass --compose-dir to use another target directory"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "${MIGRATE_REPLACE_EXISTING}" != "true" ]]; then
|
||||||
|
if ui_is_zh; then
|
||||||
|
prompt="如果目标 SQLite 已存在,是否允许备份后替换"
|
||||||
|
else
|
||||||
|
prompt="If the target SQLite DB already exists, allow backup and replacement"
|
||||||
|
fi
|
||||||
|
if prompt_yes_no "${prompt}" "no"; then
|
||||||
|
MIGRATE_REPLACE_EXISTING="true"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -z "${MIGRATE_REQUEST_BODY_MODE}" ]]; then
|
||||||
|
if ui_is_zh; then
|
||||||
|
cat >/dev/tty <<'EOF'
|
||||||
|
|
||||||
|
请求体明细迁移策略:
|
||||||
|
1) 全部迁移:迁移所有可迁移数据,包括请求体明细
|
||||||
|
2) 不迁移请求体:迁移其他所有数据;仅跳过请求体大字段和 HTTP 请求体明细,源 PG 不清除
|
||||||
|
|
||||||
|
请输入选项 [1]:
|
||||||
|
EOF
|
||||||
|
else
|
||||||
|
cat >/dev/tty <<'EOF'
|
||||||
|
|
||||||
|
Request/response body detail migration mode:
|
||||||
|
1) Full migration: migrate all migratable data, including request body details
|
||||||
|
2) Skip request bodies: migrate all other data; skip only request body large fields and HTTP body detail tables; source PG is unchanged
|
||||||
|
|
||||||
|
Enter choice [1]:
|
||||||
|
EOF
|
||||||
|
fi
|
||||||
|
local body_choice
|
||||||
|
IFS= read -r body_choice </dev/tty || body_choice=""
|
||||||
|
MIGRATE_REQUEST_BODY_MODE="${body_choice:-1}"
|
||||||
|
fi
|
||||||
|
normalize_migration_request_body_mode
|
||||||
|
}
|
||||||
|
|
||||||
|
install_migration_project_file() {
|
||||||
|
local source_path="$1"
|
||||||
|
local mode="$2"
|
||||||
|
local target_path
|
||||||
|
|
||||||
|
ensure_tmp_root
|
||||||
|
target_path="${TMP_ROOT}/$(basename "${source_path}")"
|
||||||
|
install_project_file "${source_path}" "${target_path}" "${mode}"
|
||||||
|
printf '%s\n' "${target_path}"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_compose_single_node_migration() {
|
||||||
|
local migration_script
|
||||||
|
local target_template
|
||||||
|
local source_compose_abs
|
||||||
|
local source_compose_dir
|
||||||
|
local compose_dir_abs
|
||||||
|
local target_compose
|
||||||
|
local target_compose_abs
|
||||||
|
local target_compose_dir
|
||||||
|
local target_env
|
||||||
|
local target_env_abs
|
||||||
|
local table
|
||||||
|
local -a migrate_args
|
||||||
|
|
||||||
|
source_compose_abs="$(absolute_path "${MIGRATE_FROM_COMPOSE}")"
|
||||||
|
[[ -f "${source_compose_abs}" ]] || die "source compose file not found: ${MIGRATE_FROM_COMPOSE}"
|
||||||
|
source_compose_dir="$(dirname "${source_compose_abs}")"
|
||||||
|
compose_dir_abs="$(absolute_path_maybe_missing "${COMPOSE_DIR}")"
|
||||||
|
|
||||||
|
if [[ -n "${MIGRATE_TARGET_COMPOSE}" ]]; then
|
||||||
|
target_compose="${MIGRATE_TARGET_COMPOSE}"
|
||||||
|
elif [[ "${compose_dir_abs}" == "${source_compose_dir}" ]]; then
|
||||||
|
target_compose="${compose_dir_abs}/docker-compose.single-node.yml"
|
||||||
|
else
|
||||||
|
target_compose="${compose_dir_abs}/docker-compose.yml"
|
||||||
|
fi
|
||||||
|
target_compose_abs="$(absolute_path_maybe_missing "${target_compose}")"
|
||||||
|
target_compose_dir="$(dirname "${target_compose_abs}")"
|
||||||
|
|
||||||
|
if [[ -n "${MIGRATE_TARGET_ENV}" ]]; then
|
||||||
|
target_env="${MIGRATE_TARGET_ENV}"
|
||||||
|
elif [[ "$(basename "${target_compose_abs}")" == "docker-compose.yml" ]]; then
|
||||||
|
target_env="${target_compose_dir}/.env"
|
||||||
|
else
|
||||||
|
target_env="${target_compose_dir}/.env.single-node"
|
||||||
|
fi
|
||||||
|
target_env_abs="$(absolute_path_maybe_missing "${target_env}")"
|
||||||
|
|
||||||
|
[[ "${target_compose_abs}" != "${source_compose_abs}" ]] || die "target compose would overwrite the source compose file; pass --target-compose or --compose-dir"
|
||||||
|
[[ "${target_env_abs}" != "${source_compose_dir}/.env" ]] || die "target env would overwrite the source .env; pass --target-env or --compose-dir"
|
||||||
|
|
||||||
|
migration_script="$(install_migration_project_file "scripts/migrate-pg-compose-to-single-node.sh" "0755")"
|
||||||
|
target_template="$(install_migration_project_file "docker-compose.single-node.yml" "0644")"
|
||||||
|
|
||||||
|
migrate_args=(
|
||||||
|
"${migration_script}"
|
||||||
|
--source-compose "${source_compose_abs}"
|
||||||
|
--target-compose "${target_compose_abs}"
|
||||||
|
--target-template "${target_template}"
|
||||||
|
--target-env "${target_env_abs}"
|
||||||
|
--app-image "$(compose_image)"
|
||||||
|
)
|
||||||
|
[[ -z "${MIGRATE_TARGET_DB}" ]] || migrate_args+=(--target-db "${MIGRATE_TARGET_DB}")
|
||||||
|
[[ -z "${MIGRATE_WORK_DIR}" ]] || migrate_args+=(--work-dir "${MIGRATE_WORK_DIR}")
|
||||||
|
[[ -z "${MIGRATE_APP_SERVICE}" ]] || migrate_args+=(--app-service "${MIGRATE_APP_SERVICE}")
|
||||||
|
[[ -z "${MIGRATE_POSTGRES_SERVICE}" ]] || migrate_args+=(--postgres-service "${MIGRATE_POSTGRES_SERVICE}")
|
||||||
|
[[ -z "${MIGRATE_SINGLE_NODE_SERVICE}" ]] || migrate_args+=(--single-node-service "${MIGRATE_SINGLE_NODE_SERVICE}")
|
||||||
|
[[ "${MIGRATE_REPLACE_EXISTING}" != "true" ]] || migrate_args+=(--replace-existing)
|
||||||
|
[[ "${MIGRATE_REPLACE_TARGET_COMPOSE}" != "true" ]] || migrate_args+=(--replace-target-compose)
|
||||||
|
[[ -z "${MIGRATE_REQUEST_BODY_MODE}" ]] || migrate_args+=(--request-body-mode "${MIGRATE_REQUEST_BODY_MODE}")
|
||||||
|
[[ "${MIGRATE_KEEP_SOURCE_STOPPED_ON_ERROR}" != "true" ]] || migrate_args+=(--keep-source-stopped-on-error)
|
||||||
|
|
||||||
|
bash "${migrate_args[@]}"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_single_node_service_migration() {
|
||||||
|
local migration_script
|
||||||
|
local installer
|
||||||
|
local table
|
||||||
|
local -a migrate_args
|
||||||
|
|
||||||
|
[[ -z "${MIGRATE_TARGET_COMPOSE}" ]] || die "--target-compose is only valid with --mode compose-single-node"
|
||||||
|
[[ -z "${MIGRATE_TARGET_ENV}" ]] || die "--target-env is only valid with --mode compose-single-node"
|
||||||
|
[[ -z "${MIGRATE_SINGLE_NODE_SERVICE}" ]] || die "--single-node-service is only valid with --mode compose-single-node"
|
||||||
|
[[ "${MIGRATE_REPLACE_TARGET_COMPOSE}" != "true" ]] || die "--replace-target-compose is only valid with --mode compose-single-node"
|
||||||
|
|
||||||
|
migration_script="$(install_migration_project_file "scripts/migrate-pg-to-single-node.sh" "0755")"
|
||||||
|
installer="$(install_migration_project_file "install.sh" "0755")"
|
||||||
|
|
||||||
|
migrate_args=(
|
||||||
|
"${migration_script}"
|
||||||
|
--source-compose "${MIGRATE_FROM_COMPOSE}"
|
||||||
|
--installer "${installer}"
|
||||||
|
--install-root "${INSTALL_ROOT}"
|
||||||
|
--config-dir "${CONFIG_DIR}"
|
||||||
|
--service-name "${SERVICE_NAME}"
|
||||||
|
--service-user "${SERVICE_USER}"
|
||||||
|
--service-group "${SERVICE_GROUP}"
|
||||||
|
--app-image "$(compose_image)"
|
||||||
|
--install-channel "${CHANNEL}"
|
||||||
|
--install-repo "${REPO}"
|
||||||
|
--install-source-ref "${SOURCE_REF}"
|
||||||
|
)
|
||||||
|
[[ -z "${VERSION}" ]] || migrate_args+=(--install-version "${VERSION}")
|
||||||
|
[[ -z "${ARCHIVE_PATH}" ]] || migrate_args+=(--install-archive "${ARCHIVE_PATH}")
|
||||||
|
[[ -z "${RELEASE_ARCHIVE_URL}" ]] || migrate_args+=(--install-download-url "${RELEASE_ARCHIVE_URL}")
|
||||||
|
[[ -z "${MIGRATE_TARGET_DB}" ]] || migrate_args+=(--target-db "${MIGRATE_TARGET_DB}")
|
||||||
|
[[ -z "${MIGRATE_WORK_DIR}" ]] || migrate_args+=(--work-dir "${MIGRATE_WORK_DIR}")
|
||||||
|
[[ -z "${MIGRATE_APP_SERVICE}" ]] || migrate_args+=(--app-service "${MIGRATE_APP_SERVICE}")
|
||||||
|
[[ -z "${MIGRATE_POSTGRES_SERVICE}" ]] || migrate_args+=(--postgres-service "${MIGRATE_POSTGRES_SERVICE}")
|
||||||
|
[[ "${MIGRATE_REPLACE_EXISTING}" != "true" ]] || migrate_args+=(--replace-existing)
|
||||||
|
[[ -z "${MIGRATE_REQUEST_BODY_MODE}" ]] || migrate_args+=(--request-body-mode "${MIGRATE_REQUEST_BODY_MODE}")
|
||||||
|
[[ "${MIGRATE_KEEP_SOURCE_STOPPED_ON_ERROR}" != "true" ]] || migrate_args+=(--keep-source-stopped-on-error)
|
||||||
|
|
||||||
|
bash "${migrate_args[@]}"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_migration_from_compose() {
|
||||||
|
if [[ -n "${MIGRATE_REQUEST_BODY_MODE}" ]]; then
|
||||||
|
normalize_migration_request_body_mode
|
||||||
|
fi
|
||||||
|
|
||||||
|
case "${MODE}" in
|
||||||
|
compose-single-node)
|
||||||
|
run_compose_single_node_migration
|
||||||
|
;;
|
||||||
|
single-node)
|
||||||
|
run_single_node_service_migration
|
||||||
|
;;
|
||||||
|
compose)
|
||||||
|
die "--migrate-from-compose target mode must be compose-single-node or single-node"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
die "unsupported migration target mode: ${MODE}"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
resolve_version() {
|
resolve_version() {
|
||||||
if [[ -n "${VERSION}" ]]; then
|
if [[ -n "${VERSION}" ]]; then
|
||||||
echo "${VERSION}"
|
echo "${VERSION}"
|
||||||
@@ -899,6 +1457,36 @@ current_script_dir() {
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ensure_tmp_root() {
|
||||||
|
if [[ -z "${TMP_ROOT}" ]]; then
|
||||||
|
TMP_ROOT="$(mktemp -d)"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
absolute_path() {
|
||||||
|
local path="$1"
|
||||||
|
local dir
|
||||||
|
local base
|
||||||
|
|
||||||
|
if [[ "${path}" == /* ]]; then
|
||||||
|
printf '%s\n' "${path}"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
dir="$(dirname "${path}")"
|
||||||
|
base="$(basename "${path}")"
|
||||||
|
printf '%s/%s\n' "$(cd "${dir}" && pwd -P)" "${base}"
|
||||||
|
}
|
||||||
|
|
||||||
|
absolute_path_maybe_missing() {
|
||||||
|
local path="$1"
|
||||||
|
if [[ "${path}" == /* ]]; then
|
||||||
|
printf '%s\n' "${path}"
|
||||||
|
else
|
||||||
|
printf '%s/%s\n' "$(pwd -P)" "${path}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
local_bundle_dir() {
|
local_bundle_dir() {
|
||||||
local dir
|
local dir
|
||||||
dir="$(current_script_dir)"
|
dir="$(current_script_dir)"
|
||||||
@@ -1092,6 +1680,8 @@ AETHER_GATEWAY_AUTO_PREPARE_DATABASE=true
|
|||||||
AETHER_RUNTIME_BACKEND=memory
|
AETHER_RUNTIME_BACKEND=memory
|
||||||
API_KEY_PREFIX=sk
|
API_KEY_PREFIX=sk
|
||||||
|
|
||||||
|
AETHER_DATABASE_DRIVER=sqlite
|
||||||
|
AETHER_DATABASE_URL=sqlite://${INSTALL_ROOT}/data/aether.db
|
||||||
DATABASE_URL=sqlite://${INSTALL_ROOT}/data/aether.db
|
DATABASE_URL=sqlite://${INSTALL_ROOT}/data/aether.db
|
||||||
|
|
||||||
JWT_SECRET_KEY=${jwt_key}
|
JWT_SECRET_KEY=${jwt_key}
|
||||||
@@ -1192,7 +1782,7 @@ generate_compose_env() {
|
|||||||
replace_or_append_env "${output}" "AETHER_GATEWAY_AUTO_PREPARE_DATABASE" "true"
|
replace_or_append_env "${output}" "AETHER_GATEWAY_AUTO_PREPARE_DATABASE" "true"
|
||||||
}
|
}
|
||||||
|
|
||||||
generate_compose_sqlite_env() {
|
generate_compose_single_node_env() {
|
||||||
local output="$1"
|
local output="$1"
|
||||||
local jwt_key encryption_key
|
local jwt_key encryption_key
|
||||||
prompt_admin_password
|
prompt_admin_password
|
||||||
@@ -1218,6 +1808,8 @@ AETHER_GATEWAY_AUTO_PREPARE_DATABASE=true
|
|||||||
AETHER_RUNTIME_BACKEND=memory
|
AETHER_RUNTIME_BACKEND=memory
|
||||||
API_KEY_PREFIX=sk
|
API_KEY_PREFIX=sk
|
||||||
|
|
||||||
|
AETHER_DATABASE_DRIVER=sqlite
|
||||||
|
AETHER_DATABASE_URL=sqlite:///app/data/aether.db
|
||||||
DATABASE_URL=sqlite:///app/data/aether.db
|
DATABASE_URL=sqlite:///app/data/aether.db
|
||||||
|
|
||||||
JWT_SECRET_KEY=${JWT_SECRET_KEY:-${jwt_key}}
|
JWT_SECRET_KEY=${JWT_SECRET_KEY:-${jwt_key}}
|
||||||
@@ -1374,9 +1966,9 @@ ensure_env_matches_requested_mode() {
|
|||||||
topology="${topology:-single-node}"
|
topology="${topology:-single-node}"
|
||||||
|
|
||||||
if [[ "${mode}" == "cluster" ]]; then
|
if [[ "${mode}" == "cluster" ]]; then
|
||||||
[[ "${topology}" == "multi-node" ]] || die "existing env ${file} is ${topology}; set AETHER_GATEWAY_DEPLOYMENT_TOPOLOGY=multi-node or use --mode single"
|
[[ "${topology}" == "multi-node" ]] || die "existing env ${file} is ${topology}; set AETHER_GATEWAY_DEPLOYMENT_TOPOLOGY=multi-node or use --mode single-node"
|
||||||
cluster_env_has_required_backends "${file}" || die "existing multi-node env ${file} must define DATABASE_URL and REDIS_URL"
|
cluster_env_has_required_backends "${file}" || die "existing multi-node env ${file} must define DATABASE_URL and REDIS_URL"
|
||||||
elif [[ "${mode}" == "single" && "${topology}" == "multi-node" ]]; then
|
elif [[ "${mode}" == "single-node" && "${topology}" == "multi-node" ]]; then
|
||||||
die "existing env ${file} is multi-node; cluster mode is temporarily disabled, edit the env file"
|
die "existing env ${file} is multi-node; cluster mode is temporarily disabled, edit the env file"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
@@ -1580,7 +2172,7 @@ EOF
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
info "generating first-install SQLite env file"
|
info "generating first-install single-node env file"
|
||||||
generate_first_install_env "${GENERATED_ENV}"
|
generate_first_install_env "${GENERATED_ENV}"
|
||||||
fi
|
fi
|
||||||
echo "${GENERATED_ENV}"
|
echo "${GENERATED_ENV}"
|
||||||
@@ -1624,14 +2216,14 @@ EOF
|
|||||||
compose_next_steps
|
compose_next_steps
|
||||||
}
|
}
|
||||||
|
|
||||||
install_compose_sqlite_mode() {
|
install_compose_single_node_mode() {
|
||||||
resolve_compose_dir
|
resolve_compose_dir
|
||||||
info "preparing Docker Compose SQLite deployment in ${COMPOSE_DIR}"
|
info "preparing Docker Compose single-node deployment in ${COMPOSE_DIR}"
|
||||||
ensure_directory "${COMPOSE_DIR}"
|
ensure_directory "${COMPOSE_DIR}"
|
||||||
ensure_directory "${COMPOSE_DIR}/logs"
|
ensure_directory "${COMPOSE_DIR}/logs"
|
||||||
ensure_directory "${COMPOSE_DIR}/data"
|
ensure_directory "${COMPOSE_DIR}/data"
|
||||||
|
|
||||||
install_project_file "docker-compose.sqlite.yml" "${COMPOSE_DIR}/docker-compose.yml" "0644"
|
install_project_file "docker-compose.single-node.yml" "${COMPOSE_DIR}/docker-compose.yml" "0644"
|
||||||
install_project_file ".env.example" "${COMPOSE_DIR}/.env.example" "0644"
|
install_project_file ".env.example" "${COMPOSE_DIR}/.env.example" "0644"
|
||||||
install_generate_keys_script "${COMPOSE_DIR}/generate_keys.sh"
|
install_generate_keys_script "${COMPOSE_DIR}/generate_keys.sh"
|
||||||
|
|
||||||
@@ -1639,13 +2231,13 @@ install_compose_sqlite_mode() {
|
|||||||
warn "keeping existing ${COMPOSE_DIR}/.env"
|
warn "keeping existing ${COMPOSE_DIR}/.env"
|
||||||
else
|
else
|
||||||
info "generating ${COMPOSE_DIR}/.env"
|
info "generating ${COMPOSE_DIR}/.env"
|
||||||
generate_compose_sqlite_env "${COMPOSE_DIR}/.env"
|
generate_compose_single_node_env "${COMPOSE_DIR}/.env"
|
||||||
chmod 0600 "${COMPOSE_DIR}/.env"
|
chmod 0600 "${COMPOSE_DIR}/.env"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
cat <<EOF
|
cat <<EOF
|
||||||
|
|
||||||
Docker Compose SQLite files are ready:
|
Docker Compose single-node files are ready:
|
||||||
${COMPOSE_DIR}/docker-compose.yml
|
${COMPOSE_DIR}/docker-compose.yml
|
||||||
${COMPOSE_DIR}/.env
|
${COMPOSE_DIR}/.env
|
||||||
${COMPOSE_DIR}/.env.example
|
${COMPOSE_DIR}/.env.example
|
||||||
@@ -2077,11 +2669,20 @@ main() {
|
|||||||
apply_platform_defaults
|
apply_platform_defaults
|
||||||
select_version
|
select_version
|
||||||
select_mode
|
select_mode
|
||||||
|
collect_interactive_migration_options
|
||||||
|
|
||||||
|
if [[ -n "${MIGRATE_FROM_COMPOSE}" ]]; then
|
||||||
|
run_migration_from_compose
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
if migration_options_requested; then
|
||||||
|
die "migration options require --migrate-from-compose"
|
||||||
|
fi
|
||||||
|
|
||||||
if [[ "${MODE}" == "compose" ]]; then
|
if [[ "${MODE}" == "compose" ]]; then
|
||||||
install_compose_mode
|
install_compose_mode
|
||||||
elif [[ "${MODE}" == "compose-sqlite" ]]; then
|
elif [[ "${MODE}" == "compose-single-node" ]]; then
|
||||||
install_compose_sqlite_mode
|
install_compose_single_node_mode
|
||||||
else
|
else
|
||||||
require_root
|
require_root
|
||||||
require_service_manager
|
require_service_manager
|
||||||
|
|||||||
873
scripts/migrate-pg-compose-to-single-node.sh
Executable file
873
scripts/migrate-pg-compose-to-single-node.sh
Executable file
@@ -0,0 +1,873 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SOURCE_COMPOSE="docker-compose.yml"
|
||||||
|
TARGET_COMPOSE=""
|
||||||
|
TARGET_COMPOSE_TEMPLATE="./docker-compose.single-node.yml"
|
||||||
|
TARGET_ENV=""
|
||||||
|
TARGET_DB=""
|
||||||
|
WORK_DIR=""
|
||||||
|
APP_SERVICE="app"
|
||||||
|
POSTGRES_SERVICE="postgres"
|
||||||
|
SINGLE_NODE_SERVICE="app"
|
||||||
|
APP_IMAGE=""
|
||||||
|
REPLACE_EXISTING="false"
|
||||||
|
REPLACE_TARGET_COMPOSE="false"
|
||||||
|
DRY_RUN="false"
|
||||||
|
KEEP_SOURCE_STOPPED_ON_ERROR="false"
|
||||||
|
DISK_SPACE_MULTIPLIER="${AETHER_MIGRATION_DISK_SPACE_MULTIPLIER:-2}"
|
||||||
|
DISK_SPACE_MIN_FREE_BYTES="${AETHER_MIGRATION_MIN_FREE_BYTES:-1073741824}"
|
||||||
|
REQUEST_BODY_MODE="${AETHER_MIGRATION_REQUEST_BODY_MODE:-full}"
|
||||||
|
|
||||||
|
APP_STOPPED="false"
|
||||||
|
CUTOVER_COMPLETE="false"
|
||||||
|
SOURCE_COMPOSE_ABS=""
|
||||||
|
SOURCE_COMPOSE_DIR=""
|
||||||
|
SOURCE_ENV=""
|
||||||
|
SOURCE_NETWORK=""
|
||||||
|
TARGET_COMPOSE_ABS=""
|
||||||
|
TARGET_COMPOSE_DIR=""
|
||||||
|
TARGET_ENV_ABS=""
|
||||||
|
DB_USER=""
|
||||||
|
DB_NAME=""
|
||||||
|
DB_PASSWORD=""
|
||||||
|
NOW=""
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<'EOF'
|
||||||
|
Usage: scripts/migrate-pg-compose-to-single-node.sh [options]
|
||||||
|
|
||||||
|
Migrate an existing Docker Compose Postgres deployment to Docker Compose single-node.
|
||||||
|
This script pulls the single-node app image before downtime, stops only the source app,
|
||||||
|
then copies Postgres data directly into a temporary SQLite DB without writing a JSONL
|
||||||
|
intermediate file. After the copy succeeds, it starts the single-node Compose app.
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--source-compose PATH Source Postgres docker compose file (default: docker-compose.yml)
|
||||||
|
--target-compose PATH Target single-node compose file (default: SOURCE_DIR/docker-compose.single-node.yml)
|
||||||
|
--target-template PATH Template copied when target compose is missing (default: ./docker-compose.single-node.yml)
|
||||||
|
--target-env PATH Target env file (default: TARGET_COMPOSE_DIR/.env.single-node)
|
||||||
|
--target-db PATH Final SQLite DB path (default: TARGET_COMPOSE_DIR/data/aether.db)
|
||||||
|
--work-dir PATH Working directory (default: SOURCE_DIR/data/pg-compose-to-single-node-<timestamp>)
|
||||||
|
--app-service NAME Source compose app service (default: app)
|
||||||
|
--postgres-service NAME Source compose Postgres service (default: postgres)
|
||||||
|
--single-node-service NAME Target single-node compose service (default: app)
|
||||||
|
--app-image IMAGE Override APP_IMAGE for the target single-node compose env
|
||||||
|
--replace-existing Allow replacing an existing target SQLite database
|
||||||
|
--replace-target-compose Overwrite target compose file from the template
|
||||||
|
--dry-run Pull/preflight/direct-copy without stopping or switching
|
||||||
|
--request-body-mode MODE Request/response body detail handling: full/1 or omit/2
|
||||||
|
full: migrate all migratable data, including request body details
|
||||||
|
omit: migrate all other data; skip only request body large fields and HTTP body detail tables; source PG is unchanged
|
||||||
|
--keep-source-stopped-on-error
|
||||||
|
Do not auto-restart source app if migration fails after stopping it
|
||||||
|
-h, --help Show this help
|
||||||
|
|
||||||
|
Default cutover behavior:
|
||||||
|
1. Copy/prepare docker-compose.single-node.yml and .env.single-node.
|
||||||
|
2. Pull the target single-node app image while the source app is still running.
|
||||||
|
3. Verify the running source app image ID matches the target single-node image ID.
|
||||||
|
4. Preflight SQLite migrations in a temporary DB with the target image.
|
||||||
|
5. Re-check migration coverage and available disk space.
|
||||||
|
6. Stop/remove only the source app container; keep Postgres/Redis running.
|
||||||
|
7. Copy records directly into SQLite, replace TARGET_DB, and start single-node.
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
log() {
|
||||||
|
printf '>>> %s\n' "$*"
|
||||||
|
}
|
||||||
|
|
||||||
|
warn() {
|
||||||
|
printf 'WARN: %s\n' "$*" >&2
|
||||||
|
}
|
||||||
|
|
||||||
|
die() {
|
||||||
|
printf 'ERROR: %s\n' "$*" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
trim() {
|
||||||
|
local value="$1"
|
||||||
|
value="${value#"${value%%[![:space:]]*}"}"
|
||||||
|
value="${value%"${value##*[![:space:]]}"}"
|
||||||
|
printf '%s' "$value"
|
||||||
|
}
|
||||||
|
|
||||||
|
strip_optional_quotes() {
|
||||||
|
local value="$1"
|
||||||
|
if [[ "${#value}" -ge 2 ]]; then
|
||||||
|
if [[ "${value:0:1}" == "\"" && "${value: -1}" == "\"" ]]; then
|
||||||
|
printf '%s' "${value:1:${#value}-2}"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
if [[ "${value:0:1}" == "'" && "${value: -1}" == "'" ]]; then
|
||||||
|
printf '%s' "${value:1:${#value}-2}"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
printf '%s' "$value"
|
||||||
|
}
|
||||||
|
|
||||||
|
absolute_path() {
|
||||||
|
local path="$1"
|
||||||
|
local dir
|
||||||
|
local base
|
||||||
|
|
||||||
|
if [[ "$path" == /* ]]; then
|
||||||
|
printf '%s\n' "$path"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
dir="$(dirname "$path")"
|
||||||
|
base="$(basename "$path")"
|
||||||
|
printf '%s/%s\n' "$(cd "$dir" && pwd -P)" "$base"
|
||||||
|
}
|
||||||
|
|
||||||
|
absolute_path_maybe_missing() {
|
||||||
|
local path="$1"
|
||||||
|
if [[ "$path" == /* ]]; then
|
||||||
|
printf '%s\n' "$path"
|
||||||
|
else
|
||||||
|
printf '%s/%s\n' "$(pwd -P)" "$path"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
parse_args() {
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
|
case "$1" in
|
||||||
|
--source-compose)
|
||||||
|
[[ $# -ge 2 ]] || die "--source-compose requires a value"
|
||||||
|
SOURCE_COMPOSE="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--target-compose)
|
||||||
|
[[ $# -ge 2 ]] || die "--target-compose requires a value"
|
||||||
|
TARGET_COMPOSE="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--target-template)
|
||||||
|
[[ $# -ge 2 ]] || die "--target-template requires a value"
|
||||||
|
TARGET_COMPOSE_TEMPLATE="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--target-env)
|
||||||
|
[[ $# -ge 2 ]] || die "--target-env requires a value"
|
||||||
|
TARGET_ENV="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--target-db)
|
||||||
|
[[ $# -ge 2 ]] || die "--target-db requires a value"
|
||||||
|
TARGET_DB="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--work-dir)
|
||||||
|
[[ $# -ge 2 ]] || die "--work-dir requires a value"
|
||||||
|
WORK_DIR="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--app-service)
|
||||||
|
[[ $# -ge 2 ]] || die "--app-service requires a value"
|
||||||
|
APP_SERVICE="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--postgres-service)
|
||||||
|
[[ $# -ge 2 ]] || die "--postgres-service requires a value"
|
||||||
|
POSTGRES_SERVICE="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--single-node-service)
|
||||||
|
[[ $# -ge 2 ]] || die "--single-node-service requires a value"
|
||||||
|
SINGLE_NODE_SERVICE="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--app-image)
|
||||||
|
[[ $# -ge 2 ]] || die "--app-image requires a value"
|
||||||
|
APP_IMAGE="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--replace-existing)
|
||||||
|
REPLACE_EXISTING="true"
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
--replace-target-compose)
|
||||||
|
REPLACE_TARGET_COMPOSE="true"
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
--dry-run)
|
||||||
|
DRY_RUN="true"
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
--request-body-mode)
|
||||||
|
[[ $# -ge 2 ]] || die "--request-body-mode requires a value"
|
||||||
|
REQUEST_BODY_MODE="$2"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--keep-source-stopped-on-error)
|
||||||
|
KEEP_SOURCE_STOPPED_ON_ERROR="true"
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
-h|--help)
|
||||||
|
usage
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
die "unknown argument: $1"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
require_command() {
|
||||||
|
command -v "$1" >/dev/null 2>&1 || die "required command not found: $1"
|
||||||
|
}
|
||||||
|
|
||||||
|
normalize_request_body_mode() {
|
||||||
|
case "$REQUEST_BODY_MODE" in
|
||||||
|
""|1|full|all|include)
|
||||||
|
REQUEST_BODY_MODE="full"
|
||||||
|
;;
|
||||||
|
2|omit|skip)
|
||||||
|
REQUEST_BODY_MODE="omit"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
die "--request-body-mode must be full/1 or omit/2"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
env_file_get() {
|
||||||
|
local file="$1"
|
||||||
|
local wanted="$2"
|
||||||
|
local line key value
|
||||||
|
local found=""
|
||||||
|
|
||||||
|
while IFS= read -r line || [[ -n "$line" ]]; do
|
||||||
|
line="${line%$'\r'}"
|
||||||
|
line="$(trim "$line")"
|
||||||
|
[[ -z "$line" || "${line:0:1}" == "#" ]] && continue
|
||||||
|
[[ "$line" == export\ * ]] && line="${line#export }"
|
||||||
|
key="$(trim "${line%%=*}")"
|
||||||
|
[[ "$key" == "$wanted" ]] || continue
|
||||||
|
value="${line#*=}"
|
||||||
|
found="$(strip_optional_quotes "$(trim "$value")")"
|
||||||
|
done < "$file"
|
||||||
|
|
||||||
|
printf '%s' "$found"
|
||||||
|
}
|
||||||
|
|
||||||
|
validate_env_line_for_copy() {
|
||||||
|
local line="$1"
|
||||||
|
local line_no="$2"
|
||||||
|
[[ "$line" == *'${'* ]] && die "source env line ${line_no} uses variable expansion; write a concrete value before migration"
|
||||||
|
[[ "$line" == *'$('* ]] && die "source env line ${line_no} uses command substitution; write a concrete value before migration"
|
||||||
|
[[ "$line" == *'`'* ]] && die "source env line ${line_no} uses command substitution; write a concrete value before migration"
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
should_skip_single_node_env_key() {
|
||||||
|
case "$1" in
|
||||||
|
APP_IMAGE|LOCAL_APP_IMAGE|APP_PORT|DB_HOST|DB_PORT|DB_USER|DB_NAME|DB_PASSWORD|POSTGRES_*|MYSQL_*|REDIS_HOST|REDIS_PORT|REDIS_PASSWORD|REDIS_URL|AETHER_GATEWAY_DATA_REDIS_URL|AETHER_GATEWAY_DATA_REDIS_KEY_PREFIX|DATABASE_URL|AETHER_DATABASE_URL|AETHER_DATABASE_DRIVER|AETHER_GATEWAY_DATA_POSTGRES_URL|AETHER_RUNTIME_BACKEND|AETHER_RUNTIME_REDIS_URL|AETHER_RUNTIME_REDIS_KEY_PREFIX|AETHER_GATEWAY_DEPLOYMENT_TOPOLOGY|AETHER_GATEWAY_NODE_ROLE|AETHER_GATEWAY_STATIC_DIR|AETHER_LOG_DIR|AETHER_GATEWAY_AUTO_PREPARE_DATABASE)
|
||||||
|
return 0
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
return 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
write_single_node_env() {
|
||||||
|
local output="$1"
|
||||||
|
local line raw_line key
|
||||||
|
local line_no=0
|
||||||
|
local app_port app_image jwt_key encryption_key
|
||||||
|
|
||||||
|
app_port="$(env_file_get "$SOURCE_ENV" "APP_PORT")"
|
||||||
|
app_port="${app_port:-8084}"
|
||||||
|
app_image="${APP_IMAGE:-$(env_file_get "$SOURCE_ENV" "APP_IMAGE")}"
|
||||||
|
app_image="${app_image:-ghcr.io/fawney19/aether:latest}"
|
||||||
|
jwt_key="$(env_file_get "$SOURCE_ENV" "JWT_SECRET_KEY")"
|
||||||
|
encryption_key="$(env_file_get "$SOURCE_ENV" "ENCRYPTION_KEY")"
|
||||||
|
|
||||||
|
: > "$output"
|
||||||
|
{
|
||||||
|
printf '# Generated by scripts/migrate-pg-compose-to-single-node.sh from %s\n' "$SOURCE_ENV"
|
||||||
|
printf '# single-node means Docker Compose app + SQLite for this migration target.\n\n'
|
||||||
|
} >> "$output"
|
||||||
|
|
||||||
|
while IFS= read -r raw_line || [[ -n "$raw_line" ]]; do
|
||||||
|
line_no=$((line_no + 1))
|
||||||
|
line="${raw_line%$'\r'}"
|
||||||
|
line="$(trim "$line")"
|
||||||
|
[[ -z "$line" || "${line:0:1}" == "#" ]] && continue
|
||||||
|
[[ "$line" == export\ * ]] && die "source env line ${line_no} uses export; write KEY=VALUE before migration"
|
||||||
|
[[ "$line" =~ ^[A-Za-z_][A-Za-z0-9_]*= ]] || die "source env line ${line_no} must be KEY=VALUE"
|
||||||
|
validate_env_line_for_copy "$line" "$line_no"
|
||||||
|
|
||||||
|
key="${line%%=*}"
|
||||||
|
if should_skip_single_node_env_key "$key"; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
printf '%s\n' "$line" >> "$output"
|
||||||
|
done < "$SOURCE_ENV"
|
||||||
|
|
||||||
|
{
|
||||||
|
printf '\n# Single-node Compose runtime overrides\n'
|
||||||
|
printf 'APP_IMAGE=%s\n' "$app_image"
|
||||||
|
printf 'APP_PORT=%s\n' "$app_port"
|
||||||
|
printf 'AETHER_GATEWAY_STATIC_DIR=/srv/frontend\n'
|
||||||
|
printf 'AETHER_LOG_DESTINATION=both\n'
|
||||||
|
printf 'AETHER_LOG_FORMAT=pretty\n'
|
||||||
|
printf 'AETHER_LOG_DIR=/app/logs\n'
|
||||||
|
printf 'AETHER_DATABASE_DRIVER=sqlite\n'
|
||||||
|
printf 'AETHER_DATABASE_URL=sqlite://./data/aether.db\n'
|
||||||
|
printf 'DATABASE_URL=sqlite://./data/aether.db\n'
|
||||||
|
printf 'AETHER_RUNTIME_BACKEND=memory\n'
|
||||||
|
printf 'AETHER_GATEWAY_DEPLOYMENT_TOPOLOGY=single-node\n'
|
||||||
|
printf 'AETHER_GATEWAY_NODE_ROLE=all\n'
|
||||||
|
printf 'AETHER_GATEWAY_AUTO_PREPARE_DATABASE=true\n'
|
||||||
|
printf 'JWT_SECRET_KEY=%s\n' "$jwt_key"
|
||||||
|
printf 'ENCRYPTION_KEY=%s\n' "$encryption_key"
|
||||||
|
} >> "$output"
|
||||||
|
}
|
||||||
|
|
||||||
|
source_compose() {
|
||||||
|
docker compose -f "$SOURCE_COMPOSE_ABS" "$@"
|
||||||
|
}
|
||||||
|
|
||||||
|
target_compose() {
|
||||||
|
AETHER_ENV_FILE="$TARGET_ENV_ABS" docker compose --env-file "$TARGET_ENV_ABS" -f "$TARGET_COMPOSE_ABS" "$@"
|
||||||
|
}
|
||||||
|
|
||||||
|
target_run_app() {
|
||||||
|
local database_url="$1"
|
||||||
|
shift
|
||||||
|
target_compose run --rm --no-deps \
|
||||||
|
-v "${WORK_DIR}:/migration" \
|
||||||
|
-e AETHER_LOG_DESTINATION=stdout \
|
||||||
|
-e AETHER_DATABASE_DRIVER=sqlite \
|
||||||
|
-e "AETHER_DATABASE_URL=${database_url}" \
|
||||||
|
-e "DATABASE_URL=${database_url}" \
|
||||||
|
"$SINGLE_NODE_SERVICE" "$@"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_psql_stdin() {
|
||||||
|
local sql_file="$1"
|
||||||
|
source_compose exec -T \
|
||||||
|
-e "PGPASSWORD=${DB_PASSWORD}" \
|
||||||
|
"$POSTGRES_SERVICE" \
|
||||||
|
psql -h 127.0.0.1 -U "$DB_USER" -d "$DB_NAME" -v ON_ERROR_STOP=1 -At -f - < "$sql_file"
|
||||||
|
}
|
||||||
|
|
||||||
|
source_database_size_bytes() {
|
||||||
|
local sql_file
|
||||||
|
local result
|
||||||
|
|
||||||
|
sql_file="${WORK_DIR}/source-database-size.sql"
|
||||||
|
if [[ "$REQUEST_BODY_MODE" == "omit" ]]; then
|
||||||
|
cat > "$sql_file" <<'SQL'
|
||||||
|
SELECT GREATEST(
|
||||||
|
pg_database_size(current_database())
|
||||||
|
- COALESCE(pg_total_relation_size(to_regclass('public.usage_body_blobs')), 0)
|
||||||
|
- COALESCE(pg_total_relation_size(to_regclass('public.usage_http_audits')), 0),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
SQL
|
||||||
|
else
|
||||||
|
printf 'SELECT pg_database_size(current_database());\n' > "$sql_file"
|
||||||
|
fi
|
||||||
|
result="$(run_psql_stdin "$sql_file" | tr -d '[:space:]')"
|
||||||
|
[[ "$result" =~ ^[0-9]+$ ]] || die "could not determine source Postgres database size"
|
||||||
|
printf '%s\n' "$result"
|
||||||
|
}
|
||||||
|
|
||||||
|
file_size_bytes() {
|
||||||
|
local path="$1"
|
||||||
|
|
||||||
|
if [[ ! -e "$path" ]]; then
|
||||||
|
printf '0\n'
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
stat -c '%s' "$path" 2>/dev/null || stat -f '%z' "$path" 2>/dev/null || die "could not stat file: ${path}"
|
||||||
|
}
|
||||||
|
|
||||||
|
target_sqlite_size_bytes() {
|
||||||
|
local total=0
|
||||||
|
local suffix
|
||||||
|
local size
|
||||||
|
|
||||||
|
for suffix in "" "-wal" "-shm"; do
|
||||||
|
size="$(file_size_bytes "${TARGET_DB}${suffix}")"
|
||||||
|
total=$((total + size))
|
||||||
|
done
|
||||||
|
printf '%s\n' "$total"
|
||||||
|
}
|
||||||
|
|
||||||
|
available_bytes_for_path() {
|
||||||
|
local path="$1"
|
||||||
|
df -Pk "$path" | awk 'NR == 2 { printf "%.0f\n", $4 * 1024 }'
|
||||||
|
}
|
||||||
|
|
||||||
|
filesystem_key_for_path() {
|
||||||
|
local path="$1"
|
||||||
|
df -Pk "$path" | awk 'NR == 2 { print $1 }'
|
||||||
|
}
|
||||||
|
|
||||||
|
format_bytes() {
|
||||||
|
local bytes="$1"
|
||||||
|
awk -v bytes="$bytes" 'BEGIN {
|
||||||
|
if (bytes >= 1073741824) {
|
||||||
|
printf "%.1f GiB", bytes / 1073741824
|
||||||
|
} else {
|
||||||
|
printf "%.1f MiB", bytes / 1048576
|
||||||
|
}
|
||||||
|
}'
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_available_space() {
|
||||||
|
local path="$1"
|
||||||
|
local required_bytes="$2"
|
||||||
|
local label="$3"
|
||||||
|
local available_bytes
|
||||||
|
|
||||||
|
available_bytes="$(available_bytes_for_path "$path")"
|
||||||
|
[[ "$available_bytes" =~ ^[0-9]+$ ]] || die "could not determine free disk space for ${path}"
|
||||||
|
log "${label} free space: $(format_bytes "$available_bytes"); required: $(format_bytes "$required_bytes")"
|
||||||
|
|
||||||
|
if (( available_bytes < required_bytes )); then
|
||||||
|
die "${label} does not have enough free disk space; required $(format_bytes "$required_bytes"), available $(format_bytes "$available_bytes")"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
check_disk_space() {
|
||||||
|
local source_bytes
|
||||||
|
local estimated_db_bytes
|
||||||
|
local backup_bytes=0
|
||||||
|
local target_dir
|
||||||
|
local work_fs
|
||||||
|
local target_fs
|
||||||
|
local required_bytes
|
||||||
|
|
||||||
|
[[ "$DISK_SPACE_MULTIPLIER" =~ ^[1-9][0-9]*$ ]] || die "AETHER_MIGRATION_DISK_SPACE_MULTIPLIER must be a positive integer"
|
||||||
|
[[ "$DISK_SPACE_MIN_FREE_BYTES" =~ ^[0-9]+$ ]] || die "AETHER_MIGRATION_MIN_FREE_BYTES must be a non-negative integer"
|
||||||
|
|
||||||
|
source_bytes="$(source_database_size_bytes)"
|
||||||
|
estimated_db_bytes=$((source_bytes * DISK_SPACE_MULTIPLIER + DISK_SPACE_MIN_FREE_BYTES))
|
||||||
|
target_dir="$(dirname "$TARGET_DB")"
|
||||||
|
mkdir -p "$target_dir"
|
||||||
|
|
||||||
|
if [[ "$REPLACE_EXISTING" == "true" ]]; then
|
||||||
|
backup_bytes="$(target_sqlite_size_bytes)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "source Postgres size used for disk estimate: $(format_bytes "$source_bytes")"
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == "true" ]]; then
|
||||||
|
assert_available_space "$WORK_DIR" "$estimated_db_bytes" "work dir"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
work_fs="$(filesystem_key_for_path "$WORK_DIR")"
|
||||||
|
target_fs="$(filesystem_key_for_path "$target_dir")"
|
||||||
|
if [[ "$work_fs" == "$target_fs" ]]; then
|
||||||
|
required_bytes=$((estimated_db_bytes * 2 + backup_bytes))
|
||||||
|
assert_available_space "$WORK_DIR" "$required_bytes" "work/target filesystem"
|
||||||
|
else
|
||||||
|
assert_available_space "$WORK_DIR" "$((estimated_db_bytes + backup_bytes))" "work dir"
|
||||||
|
assert_available_space "$target_dir" "$estimated_db_bytes" "target DB dir"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
check_request_body_artifacts() {
|
||||||
|
local sql_file
|
||||||
|
local result_file
|
||||||
|
|
||||||
|
sql_file="${WORK_DIR}/check-request-body-artifacts.sql"
|
||||||
|
result_file="${WORK_DIR}/request-body-artifacts.txt"
|
||||||
|
|
||||||
|
cat > "$sql_file" <<'SQL'
|
||||||
|
CREATE TEMP TABLE aether_request_body_artifacts (
|
||||||
|
artifact text PRIMARY KEY
|
||||||
|
) ON COMMIT PRESERVE ROWS;
|
||||||
|
|
||||||
|
DO $$
|
||||||
|
DECLARE
|
||||||
|
candidate record;
|
||||||
|
has_rows boolean;
|
||||||
|
BEGIN
|
||||||
|
IF EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM information_schema.tables
|
||||||
|
WHERE table_schema = 'public'
|
||||||
|
AND table_name = 'usage_body_blobs'
|
||||||
|
) THEN
|
||||||
|
EXECUTE 'SELECT EXISTS (SELECT 1 FROM public.usage_body_blobs LIMIT 1)'
|
||||||
|
INTO has_rows;
|
||||||
|
IF has_rows THEN
|
||||||
|
INSERT INTO aether_request_body_artifacts(artifact)
|
||||||
|
VALUES ('usage_body_blobs')
|
||||||
|
ON CONFLICT DO NOTHING;
|
||||||
|
END IF;
|
||||||
|
END IF;
|
||||||
|
|
||||||
|
IF EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM information_schema.tables
|
||||||
|
WHERE table_schema = 'public'
|
||||||
|
AND table_name = 'usage_http_audits'
|
||||||
|
) THEN
|
||||||
|
EXECUTE 'SELECT EXISTS (SELECT 1 FROM public.usage_http_audits LIMIT 1)'
|
||||||
|
INTO has_rows;
|
||||||
|
IF has_rows THEN
|
||||||
|
INSERT INTO aether_request_body_artifacts(artifact)
|
||||||
|
VALUES ('usage_http_audits')
|
||||||
|
ON CONFLICT DO NOTHING;
|
||||||
|
END IF;
|
||||||
|
END IF;
|
||||||
|
|
||||||
|
IF EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM information_schema.tables
|
||||||
|
WHERE table_schema = 'public'
|
||||||
|
AND table_name = 'usage'
|
||||||
|
) THEN
|
||||||
|
FOR candidate IN
|
||||||
|
SELECT unnest(ARRAY[
|
||||||
|
'request_body',
|
||||||
|
'response_body',
|
||||||
|
'provider_request_body',
|
||||||
|
'client_response_body',
|
||||||
|
'request_body_compressed',
|
||||||
|
'response_body_compressed',
|
||||||
|
'provider_request_body_compressed',
|
||||||
|
'client_response_body_compressed'
|
||||||
|
]) AS column_name
|
||||||
|
LOOP
|
||||||
|
IF EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM information_schema.columns
|
||||||
|
WHERE table_schema = 'public'
|
||||||
|
AND table_name = 'usage'
|
||||||
|
AND column_name = candidate.column_name
|
||||||
|
) THEN
|
||||||
|
EXECUTE format('SELECT EXISTS (SELECT 1 FROM public.usage WHERE %I IS NOT NULL LIMIT 1)', candidate.column_name)
|
||||||
|
INTO has_rows;
|
||||||
|
IF has_rows THEN
|
||||||
|
INSERT INTO aether_request_body_artifacts(artifact)
|
||||||
|
VALUES ('usage.' || candidate.column_name)
|
||||||
|
ON CONFLICT DO NOTHING;
|
||||||
|
END IF;
|
||||||
|
END IF;
|
||||||
|
END LOOP;
|
||||||
|
END IF;
|
||||||
|
END
|
||||||
|
$$;
|
||||||
|
|
||||||
|
SELECT artifact FROM aether_request_body_artifacts ORDER BY artifact;
|
||||||
|
SQL
|
||||||
|
|
||||||
|
run_psql_stdin "$sql_file" > "$result_file"
|
||||||
|
|
||||||
|
if [[ ! -s "$result_file" ]]; then
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$REQUEST_BODY_MODE" == "omit" ]]; then
|
||||||
|
warn "source has request/response body details that will not be copied into single-node SQLite"
|
||||||
|
cat "$result_file" >&2
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
warn "source has request/response body details that will be copied into single-node SQLite"
|
||||||
|
cat "$result_file" >&2
|
||||||
|
}
|
||||||
|
|
||||||
|
source_service_is_running() {
|
||||||
|
source_compose ps --services --status running | grep -Fxq "$1"
|
||||||
|
}
|
||||||
|
|
||||||
|
source_app_container_id() {
|
||||||
|
source_compose ps -q "$APP_SERVICE"
|
||||||
|
}
|
||||||
|
|
||||||
|
docker_image_id() {
|
||||||
|
local image="$1"
|
||||||
|
docker image inspect -f '{{.Id}}' "$image" 2>/dev/null || true
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_source_and_target_images_match() {
|
||||||
|
local target_image="$1"
|
||||||
|
local source_container_id
|
||||||
|
local source_image_ref
|
||||||
|
local source_image_id
|
||||||
|
local target_image_id
|
||||||
|
|
||||||
|
source_container_id="$(source_app_container_id)"
|
||||||
|
[[ -n "$source_container_id" ]] || die "could not resolve running source app container for service: ${APP_SERVICE}"
|
||||||
|
|
||||||
|
source_image_ref="$(docker inspect -f '{{.Config.Image}}' "$source_container_id")"
|
||||||
|
source_image_id="$(docker inspect -f '{{.Image}}' "$source_container_id")"
|
||||||
|
target_image_id="$(docker_image_id "$target_image")"
|
||||||
|
[[ -n "$target_image_id" ]] || die "target single-node image is not available locally after pull: ${target_image}"
|
||||||
|
|
||||||
|
log "source app image: ${source_image_ref} (${source_image_id})"
|
||||||
|
log "target single-node image: ${target_image} (${target_image_id})"
|
||||||
|
|
||||||
|
if [[ "$source_image_id" != "$target_image_id" ]]; then
|
||||||
|
die "source app image and target single-node image are different; upgrade the source PG Compose app to ${target_image} before migration"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_target_copy_command_available() {
|
||||||
|
local target_image="$1"
|
||||||
|
local help_output
|
||||||
|
|
||||||
|
help_output="$(docker run --rm --entrypoint aether-gateway "$target_image" copy --help 2>&1 || true)"
|
||||||
|
if [[ "$help_output" != *"--source-driver"* || "$help_output" != *"--target-driver"* || "$help_output" != *"--omit-request-body-details"* ]]; then
|
||||||
|
die "target single-node image does not support direct PG-to-SQLite copy; use a matching Aether release image"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
resolve_source_network() {
|
||||||
|
local container_id
|
||||||
|
local network
|
||||||
|
|
||||||
|
container_id="$(source_compose ps -q "$POSTGRES_SERVICE")"
|
||||||
|
[[ -n "$container_id" ]] || die "could not resolve container id for source Postgres service: ${POSTGRES_SERVICE}"
|
||||||
|
|
||||||
|
SOURCE_NETWORK=""
|
||||||
|
while IFS= read -r network; do
|
||||||
|
[[ -n "$network" ]] || continue
|
||||||
|
SOURCE_NETWORK="$network"
|
||||||
|
break
|
||||||
|
done < <(docker inspect -f '{{range $name, $_ := .NetworkSettings.Networks}}{{println $name}}{{end}}' "$container_id")
|
||||||
|
|
||||||
|
[[ -n "$SOURCE_NETWORK" ]] || die "could not resolve Docker network for source Postgres service: ${POSTGRES_SERVICE}"
|
||||||
|
}
|
||||||
|
|
||||||
|
prepare_target_compose() {
|
||||||
|
local template_abs
|
||||||
|
|
||||||
|
TARGET_COMPOSE="${TARGET_COMPOSE:-${SOURCE_COMPOSE_DIR}/docker-compose.single-node.yml}"
|
||||||
|
TARGET_COMPOSE_ABS="$(absolute_path_maybe_missing "$TARGET_COMPOSE")"
|
||||||
|
TARGET_COMPOSE_DIR="$(dirname "$TARGET_COMPOSE_ABS")"
|
||||||
|
template_abs="$(absolute_path "$TARGET_COMPOSE_TEMPLATE")"
|
||||||
|
[[ -f "$template_abs" ]] || die "target compose template not found: ${TARGET_COMPOSE_TEMPLATE}"
|
||||||
|
|
||||||
|
mkdir -p "$TARGET_COMPOSE_DIR"
|
||||||
|
if [[ ! -f "$TARGET_COMPOSE_ABS" || "$REPLACE_TARGET_COMPOSE" == "true" ]]; then
|
||||||
|
log "installing target single-node compose file at ${TARGET_COMPOSE_ABS}"
|
||||||
|
install -m 0644 "$template_abs" "$TARGET_COMPOSE_ABS"
|
||||||
|
else
|
||||||
|
log "keeping existing target compose file ${TARGET_COMPOSE_ABS}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
TARGET_ENV="${TARGET_ENV:-${TARGET_COMPOSE_DIR}/.env.single-node}"
|
||||||
|
TARGET_ENV_ABS="$(absolute_path_maybe_missing "$TARGET_ENV")"
|
||||||
|
}
|
||||||
|
|
||||||
|
target_sqlite_file_exists() {
|
||||||
|
[[ -e "$TARGET_DB" || -e "${TARGET_DB}-wal" || -e "${TARGET_DB}-shm" ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
finalize_target_db() {
|
||||||
|
local temp_db="$1"
|
||||||
|
local target_dir
|
||||||
|
local backup_path
|
||||||
|
local suffix
|
||||||
|
|
||||||
|
target_dir="$(dirname "$TARGET_DB")"
|
||||||
|
mkdir -p "$target_dir"
|
||||||
|
|
||||||
|
if target_sqlite_file_exists; then
|
||||||
|
[[ "$REPLACE_EXISTING" == "true" ]] || die "target DB already exists: ${TARGET_DB}; pass --replace-existing to replace it"
|
||||||
|
for suffix in "" "-wal" "-shm"; do
|
||||||
|
if [[ -e "${TARGET_DB}${suffix}" ]]; then
|
||||||
|
backup_path="${WORK_DIR}/$(basename "$TARGET_DB")${suffix}.backup.${NOW}"
|
||||||
|
log "backing up existing target SQLite file to ${backup_path}"
|
||||||
|
cp -p "${TARGET_DB}${suffix}" "$backup_path"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "installing migrated SQLite DB at ${TARGET_DB}"
|
||||||
|
install -m 0640 "$temp_db" "$TARGET_DB"
|
||||||
|
for suffix in "-wal" "-shm"; do
|
||||||
|
if [[ -e "${temp_db}${suffix}" ]]; then
|
||||||
|
install -m 0640 "${temp_db}${suffix}" "${TARGET_DB}${suffix}"
|
||||||
|
else
|
||||||
|
rm -f "${TARGET_DB}${suffix}"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanup_on_exit() {
|
||||||
|
local status=$?
|
||||||
|
if [[ "$status" -eq 0 ]]; then
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
warn "migration failed with exit status ${status}"
|
||||||
|
if [[ "$APP_STOPPED" == "true" && "$CUTOVER_COMPLETE" != "true" && "$KEEP_SOURCE_STOPPED_ON_ERROR" != "true" ]]; then
|
||||||
|
warn "attempting to restart source compose app because cutover did not complete"
|
||||||
|
source_compose up -d "$APP_SERVICE" || warn "source app restart failed; check ${SOURCE_COMPOSE_ABS}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
preflight() {
|
||||||
|
require_command docker
|
||||||
|
require_command awk
|
||||||
|
require_command df
|
||||||
|
docker compose version >/dev/null
|
||||||
|
|
||||||
|
SOURCE_COMPOSE_ABS="$(absolute_path "$SOURCE_COMPOSE")"
|
||||||
|
[[ -f "$SOURCE_COMPOSE_ABS" ]] || die "source compose file not found: ${SOURCE_COMPOSE}"
|
||||||
|
SOURCE_COMPOSE_DIR="$(dirname "$SOURCE_COMPOSE_ABS")"
|
||||||
|
SOURCE_ENV="${SOURCE_COMPOSE_DIR}/.env"
|
||||||
|
[[ -f "$SOURCE_ENV" ]] || die "source env file not found: ${SOURCE_ENV}"
|
||||||
|
|
||||||
|
NOW="$(date +%Y%m%d%H%M%S)"
|
||||||
|
if [[ -z "$WORK_DIR" ]]; then
|
||||||
|
WORK_DIR="${SOURCE_COMPOSE_DIR}/data/pg-compose-to-single-node-${NOW}"
|
||||||
|
fi
|
||||||
|
WORK_DIR="$(absolute_path_maybe_missing "$WORK_DIR")"
|
||||||
|
mkdir -p "$WORK_DIR"
|
||||||
|
|
||||||
|
prepare_target_compose
|
||||||
|
TARGET_DB="${TARGET_DB:-${TARGET_COMPOSE_DIR}/data/aether.db}"
|
||||||
|
TARGET_DB="$(absolute_path_maybe_missing "$TARGET_DB")"
|
||||||
|
|
||||||
|
DB_USER="$(env_file_get "$SOURCE_ENV" "DB_USER")"
|
||||||
|
DB_USER="${DB_USER:-postgres}"
|
||||||
|
DB_NAME="$(env_file_get "$SOURCE_ENV" "DB_NAME")"
|
||||||
|
DB_NAME="${DB_NAME:-aether}"
|
||||||
|
DB_PASSWORD="$(env_file_get "$SOURCE_ENV" "DB_PASSWORD")"
|
||||||
|
DB_PASSWORD="${DB_PASSWORD:-aether}"
|
||||||
|
|
||||||
|
[[ -n "$(env_file_get "$SOURCE_ENV" "JWT_SECRET_KEY")" ]] || die "source env must define JWT_SECRET_KEY"
|
||||||
|
if [[ -z "$(env_file_get "$SOURCE_ENV" "ENCRYPTION_KEY")" && -z "$(env_file_get "$SOURCE_ENV" "AETHER_GATEWAY_DATA_ENCRYPTION_KEY")" ]]; then
|
||||||
|
die "source env must define ENCRYPTION_KEY or AETHER_GATEWAY_DATA_ENCRYPTION_KEY"
|
||||||
|
fi
|
||||||
|
|
||||||
|
write_single_node_env "$TARGET_ENV_ABS"
|
||||||
|
chmod 0600 "$TARGET_ENV_ABS"
|
||||||
|
|
||||||
|
log "source compose: ${SOURCE_COMPOSE_ABS}"
|
||||||
|
log "source env: ${SOURCE_ENV}"
|
||||||
|
log "target compose: ${TARGET_COMPOSE_ABS}"
|
||||||
|
log "target env: ${TARGET_ENV_ABS}"
|
||||||
|
log "target SQLite DB: ${TARGET_DB}"
|
||||||
|
log "work dir: ${WORK_DIR}"
|
||||||
|
|
||||||
|
source_service_is_running "$POSTGRES_SERVICE" || die "source Postgres service is not running: ${POSTGRES_SERVICE}"
|
||||||
|
resolve_source_network
|
||||||
|
log "source Docker network: ${SOURCE_NETWORK}"
|
||||||
|
}
|
||||||
|
|
||||||
|
source_postgres_url() {
|
||||||
|
printf 'postgresql://%s:%s@%s:5432/%s' "$DB_USER" "$DB_PASSWORD" "$POSTGRES_SERVICE" "$DB_NAME"
|
||||||
|
}
|
||||||
|
|
||||||
|
copy_source_to_sqlite() {
|
||||||
|
local target_temp_db="$1"
|
||||||
|
local target_url
|
||||||
|
local image
|
||||||
|
local -a copy_args
|
||||||
|
|
||||||
|
image="$(env_file_get "$TARGET_ENV_ABS" "APP_IMAGE")"
|
||||||
|
[[ -n "$image" ]] || die "target env must define APP_IMAGE"
|
||||||
|
target_url="sqlite:///migration/$(basename "$target_temp_db")"
|
||||||
|
|
||||||
|
rm -f "$target_temp_db" "${target_temp_db}-wal" "${target_temp_db}-shm"
|
||||||
|
target_run_app "$target_url" --migrate
|
||||||
|
copy_args=(
|
||||||
|
copy
|
||||||
|
--source-driver postgres
|
||||||
|
--source-url "$(source_postgres_url)"
|
||||||
|
--target-driver sqlite
|
||||||
|
--target-url "$target_url"
|
||||||
|
)
|
||||||
|
if [[ "$REQUEST_BODY_MODE" == "omit" ]]; then
|
||||||
|
copy_args+=(--omit-request-body-details)
|
||||||
|
fi
|
||||||
|
docker run --rm \
|
||||||
|
--network "$SOURCE_NETWORK" \
|
||||||
|
-v "${WORK_DIR}:/migration" \
|
||||||
|
--env-file "$TARGET_ENV_ABS" \
|
||||||
|
-e AETHER_LOG_DESTINATION=stdout \
|
||||||
|
"$image" \
|
||||||
|
"${copy_args[@]}"
|
||||||
|
}
|
||||||
|
|
||||||
|
main() {
|
||||||
|
local preflight_db
|
||||||
|
local dry_run_db
|
||||||
|
local target_temp_db
|
||||||
|
local target_image
|
||||||
|
|
||||||
|
parse_args "$@"
|
||||||
|
normalize_request_body_mode
|
||||||
|
trap cleanup_on_exit EXIT
|
||||||
|
preflight
|
||||||
|
|
||||||
|
preflight_db="${WORK_DIR}/single-node-preflight.db"
|
||||||
|
dry_run_db="${WORK_DIR}/dry-run-target-aether.db"
|
||||||
|
target_temp_db="${WORK_DIR}/target-aether.db"
|
||||||
|
|
||||||
|
if target_sqlite_file_exists && [[ "$REPLACE_EXISTING" != "true" && "$DRY_RUN" != "true" ]]; then
|
||||||
|
die "target DB already exists: ${TARGET_DB}; pass --replace-existing to replace it"
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "pulling target single-node image before downtime"
|
||||||
|
target_compose pull "$SINGLE_NODE_SERVICE"
|
||||||
|
target_image="$(env_file_get "$TARGET_ENV_ABS" "APP_IMAGE")"
|
||||||
|
[[ -n "$target_image" ]] || die "target env must define APP_IMAGE"
|
||||||
|
|
||||||
|
log "checking target image copy command is available"
|
||||||
|
assert_target_copy_command_available "$target_image"
|
||||||
|
|
||||||
|
log "checking source app image matches target single-node image"
|
||||||
|
assert_source_and_target_images_match "$target_image"
|
||||||
|
|
||||||
|
log "preflighting target SQLite schema migration"
|
||||||
|
rm -f "$preflight_db" "${preflight_db}-wal" "${preflight_db}-shm"
|
||||||
|
target_run_app "sqlite:///migration/$(basename "$preflight_db")" --migrate
|
||||||
|
|
||||||
|
log "checking request body detail policy"
|
||||||
|
check_request_body_artifacts
|
||||||
|
|
||||||
|
log "checking available disk space before copy"
|
||||||
|
check_disk_space
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == "true" ]]; then
|
||||||
|
warn "dry-run copy happens while the source app may still be writing; use only for rehearsal"
|
||||||
|
log "copying source Postgres tables directly into dry-run SQLite target"
|
||||||
|
copy_source_to_sqlite "$dry_run_db"
|
||||||
|
log "dry run complete; temporary SQLite DB is ${dry_run_db}"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "stopping source app service; Postgres and Redis stay running"
|
||||||
|
source_compose stop "$APP_SERVICE"
|
||||||
|
APP_STOPPED="true"
|
||||||
|
|
||||||
|
log "checking request body detail policy again after the app has stopped"
|
||||||
|
check_request_body_artifacts
|
||||||
|
|
||||||
|
log "copying source Postgres tables directly into temporary SQLite DB"
|
||||||
|
copy_source_to_sqlite "$target_temp_db"
|
||||||
|
|
||||||
|
finalize_target_db "$target_temp_db"
|
||||||
|
|
||||||
|
log "removing stopped source app container to free the app container name"
|
||||||
|
source_compose rm -f "$APP_SERVICE"
|
||||||
|
|
||||||
|
log "starting single-node compose app"
|
||||||
|
target_compose up -d "$SINGLE_NODE_SERVICE"
|
||||||
|
CUTOVER_COMPLETE="true"
|
||||||
|
|
||||||
|
log "migration complete"
|
||||||
|
log "source Postgres/Redis volumes were left in place for rollback"
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
1030
scripts/migrate-pg-to-single-node.sh
Executable file
1030
scripts/migrate-pg-to-single-node.sh
Executable file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user