mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-10 05:00:19 +08:00
Merge remote-tracking branch 'origin/main'
This commit is contained in:
@@ -261,8 +261,7 @@ jobs:
|
||||
root="package/${bundle}"
|
||||
mkdir -p \
|
||||
"${root}/bin" \
|
||||
"${root}/frontend" \
|
||||
"${root}/scripts"
|
||||
"${root}/frontend"
|
||||
|
||||
install -m 0755 "artifacts/aether-gateway-${platform}-${arch}/aether-gateway" "${root}/bin/aether-gateway"
|
||||
cp -R artifacts/frontend-dist/. "${root}/frontend/"
|
||||
@@ -274,8 +273,6 @@ jobs:
|
||||
install -m 0755 update.sh "${root}/update.sh"
|
||||
install -m 0644 docker-compose.yml "${root}/docker-compose.yml"
|
||||
install -m 0644 docker-compose.single-node.yml "${root}/docker-compose.single-node.yml"
|
||||
install -m 0755 scripts/migrate-pg-compose-to-single-node.sh "${root}/scripts/migrate-pg-compose-to-single-node.sh"
|
||||
install -m 0755 scripts/migrate-pg-to-single-node.sh "${root}/scripts/migrate-pg-to-single-node.sh"
|
||||
install -m 0644 .env.example "${root}/.env.example"
|
||||
install -m 0755 generate_keys.sh "${root}/generate_keys.sh"
|
||||
install -m 0644 README.md "${root}/README.md"
|
||||
|
||||
@@ -1,238 +0,0 @@
|
||||
# Postgres to Aether Single Node Migration
|
||||
|
||||
Chinese version: [pg-to-single-node-migration.zh-CN.md](pg-to-single-node-migration.zh-CN.md)
|
||||
|
||||
This runbook migrates an existing Docker Compose Postgres deployment to Aether
|
||||
single-node. In this repository, **single-node** means the default SQLite installer mode:
|
||||
`install.sh --mode single-node`, a system service backed by SQLite. The Docker Compose
|
||||
single-node template is `docker-compose.single-node.yml`, exposed through `--mode compose-single-node`.
|
||||
|
||||
The migration script is:
|
||||
|
||||
```bash
|
||||
scripts/migrate-pg-to-single-node.sh
|
||||
```
|
||||
|
||||
If the target should stay on Docker Compose instead of becoming a system
|
||||
service, use the image-based Compose migration script:
|
||||
|
||||
```bash
|
||||
scripts/migrate-pg-compose-to-single-node.sh
|
||||
```
|
||||
|
||||
Both migration scripts pull/install the target single-node version before
|
||||
downtime, stop only the source `app`, copy Postgres records directly into a
|
||||
temporary SQLite DB without writing a JSONL file, replace the target
|
||||
`aether.db`, and start single-node.
|
||||
|
||||
You can also use the installer as the unified entrypoint and let `--mode`
|
||||
select the migration target:
|
||||
|
||||
```bash
|
||||
# In interactive mode, first choose the target deployment mode:
|
||||
# 1) Docker Compose standard deployment (Postgres + Redis)
|
||||
# 2) Docker Compose single-node deployment (SQLite)
|
||||
# 3) System service single-node deployment (SQLite)
|
||||
# After choosing 2 or 3, choose the data initialization mode:
|
||||
# 1) Fresh initialization (do not migrate existing data)
|
||||
# 2) Migrate from an existing Docker Compose PG database
|
||||
install.sh
|
||||
|
||||
# Migrate into a new single-node Docker Compose directory.
|
||||
install.sh \
|
||||
--mode compose-single-node \
|
||||
--migrate-from-compose /root/Aether/docker-compose.yml \
|
||||
--compose-dir /opt/aether-single \
|
||||
--replace-existing
|
||||
|
||||
# Migrate into the system service + SQLite layout.
|
||||
sudo install.sh \
|
||||
--mode single-node \
|
||||
--migrate-from-compose /root/Aether/docker-compose.yml \
|
||||
--replace-existing
|
||||
```
|
||||
|
||||
Interactive mode first asks for the target deployment shape. If the target is
|
||||
`compose-single-node` or `single-node`, the installer then asks for the data
|
||||
initialization mode: fresh initialization, or migration from an existing Docker
|
||||
Compose PG database. If you choose migration, it tries to detect the source PG
|
||||
Compose file from `docker compose ls`, then verifies that the Compose config
|
||||
contains the default `app` and `postgres` services. If exactly one match is
|
||||
found, it is used as the default prompt value. If detection is ambiguous or
|
||||
fails, the installer stops; rerun it with `--migrate-from-compose` to specify
|
||||
the source compose path.
|
||||
|
||||
The installer only normalizes the entrypoint: `compose-single-node` delegates to
|
||||
`scripts/migrate-pg-compose-to-single-node.sh`, while `single-node` delegates to
|
||||
`scripts/migrate-pg-to-single-node.sh`.
|
||||
|
||||
## What It Does
|
||||
|
||||
The script keeps the production cutover window short:
|
||||
|
||||
1. Reads the source Compose `.env`.
|
||||
2. Builds a single-node env file that preserves `JWT_SECRET_KEY`, `ENCRYPTION_KEY` or
|
||||
`AETHER_GATEWAY_DATA_ENCRYPTION_KEY`, admin settings, port, and app config.
|
||||
3. Installs the single-node release with `install.sh --mode single-node --skip-start`.
|
||||
4. Preflights SQLite migrations with the installed single-node binary.
|
||||
5. Pulls the target single-node image, confirms its `copy` command is available,
|
||||
and verifies that its Docker image ID matches the currently running source
|
||||
`app` image ID.
|
||||
6. Uses the target SQLite schema as the migration plan: same-name source
|
||||
Postgres tables and columns are copied into the temporary SQLite DB.
|
||||
7. Applies the compressed body and HTTP body detail policy. The default is full,
|
||||
and you can opt into an omit mode for large artifacts.
|
||||
8. Checks that the work directory and target SQLite directory have enough free
|
||||
disk space for the temporary and final SQLite files.
|
||||
9. Stops only the source `app` service, leaving Postgres and Redis running.
|
||||
10. Copies source Postgres records directly into a temporary SQLite database
|
||||
without generating JSONL files.
|
||||
11. Replaces the target SQLite DB, including SQLite `-wal`/`-shm` sidecar files
|
||||
when present, and starts the single-node service.
|
||||
|
||||
The image check compares Docker image IDs, not just tag strings. If both source
|
||||
and target say `latest` but resolve to different image IDs, migration stops.
|
||||
Upgrade the source PG Compose `app` to the target single-node version first,
|
||||
verify it is healthy, then run the migration. The scripts also check that the
|
||||
target image supports direct copy and the request-body omit flag; using a new
|
||||
script with an old image stops before cutover to avoid missing data.
|
||||
|
||||
## Production Cutover
|
||||
|
||||
Before production cutover, take a normal server backup or snapshot. Then run:
|
||||
|
||||
```bash
|
||||
sudo scripts/migrate-pg-to-single-node.sh \
|
||||
--source-compose /root/Aether/docker-compose.yml \
|
||||
--replace-existing
|
||||
```
|
||||
|
||||
For Docker Compose single-node cutover instead of a system service:
|
||||
|
||||
```bash
|
||||
scripts/migrate-pg-compose-to-single-node.sh \
|
||||
--source-compose /root/Aether/docker-compose.yml \
|
||||
--replace-existing
|
||||
```
|
||||
|
||||
The source Postgres compose directory and target single-node compose directory
|
||||
can be different. For example:
|
||||
|
||||
```bash
|
||||
install.sh \
|
||||
--mode compose-single-node \
|
||||
--migrate-from-compose /root/Aether/docker-compose.yml \
|
||||
--compose-dir /opt/aether-single \
|
||||
--replace-existing
|
||||
```
|
||||
|
||||
Equivalently, call the lower-level script and pass each target path explicitly:
|
||||
|
||||
```bash
|
||||
scripts/migrate-pg-compose-to-single-node.sh \
|
||||
--source-compose /root/Aether/docker-compose.yml \
|
||||
--target-compose /opt/aether-single/docker-compose.single-node.yml \
|
||||
--target-env /opt/aether-single/.env.single-node \
|
||||
--target-db /opt/aether-single/data/aether.db \
|
||||
--replace-existing
|
||||
```
|
||||
|
||||
During cutover, the script stops and removes only the source `app` container to
|
||||
free the fixed `aether-app` container name. Postgres, Redis, and their volumes
|
||||
remain in place for rollback.
|
||||
|
||||
Defaults:
|
||||
|
||||
| Setting | Default |
|
||||
| --- | --- |
|
||||
| Source Compose | `docker-compose.yml` |
|
||||
| Single Node install root | `/opt/aether` |
|
||||
| Single Node config dir | `/etc/aether` |
|
||||
| Target SQLite DB | `/opt/aether/data/aether.db` |
|
||||
| Source app service | `app` |
|
||||
| Source Postgres service | `postgres` |
|
||||
| Single Node service | `aether-gateway` |
|
||||
|
||||
The script writes migration artifacts under `./data/pg-to-single-node-<timestamp>` next
|
||||
to the source Compose file unless `--work-dir` is provided.
|
||||
|
||||
## Rollback
|
||||
|
||||
The script leaves the original Postgres and Redis volumes in place. If cutover
|
||||
finishes but you need to roll back:
|
||||
|
||||
```bash
|
||||
sudo systemctl stop aether-gateway
|
||||
cd /root/Aether
|
||||
docker compose -f docker-compose.yml up -d app
|
||||
```
|
||||
|
||||
For the Compose single-node script, rollback is the same idea: start the app
|
||||
again from the original Postgres compose file.
|
||||
|
||||
If the migration fails before cutover completes, the script attempts to restart
|
||||
the source `app` service automatically. Pass `--keep-source-stopped-on-error` if
|
||||
you want to inspect the stopped source deployment manually instead.
|
||||
|
||||
## Data Coverage Guard
|
||||
|
||||
The migration does not maintain a separate business-domain table list. The
|
||||
target single-node image first builds a temporary SQLite database with its
|
||||
normal migrations, then `aether-gateway copy` reads that SQLite schema and copies
|
||||
matching public Postgres tables and columns.
|
||||
|
||||
If the source Postgres database has a non-empty public table that does not exist
|
||||
in the target SQLite schema, the copy stops instead of silently dropping it. It
|
||||
ignores lifecycle metadata tables such as `_sqlx_migrations` and
|
||||
`schema_backfills`. Extra source columns that are absent from the target schema
|
||||
are not copied.
|
||||
|
||||
## Request Body Detail Policy
|
||||
|
||||
The production migration migrates all migratable data by default. The only
|
||||
optional exclusion is request body detail data.
|
||||
|
||||
When you choose to skip request bodies, the migration does not copy
|
||||
`usage_body_blobs`, `usage_http_audits`, or legacy `usage` request body columns
|
||||
such as `request_body`, `provider_request_body`, `response_body`,
|
||||
`client_response_body`, and `*_body_compressed`.
|
||||
|
||||
Interactive installation lets you choose:
|
||||
|
||||
```text
|
||||
1) Full migration: migrate all migratable data, including request body details
|
||||
2) Skip request bodies: migrate all other data; skip only request body large fields and HTTP body detail tables; source PG is unchanged
|
||||
```
|
||||
|
||||
For non-interactive full runs:
|
||||
|
||||
```bash
|
||||
scripts/migrate-pg-to-single-node.sh \
|
||||
--request-body-mode full
|
||||
```
|
||||
|
||||
For non-interactive omit runs:
|
||||
|
||||
```bash
|
||||
scripts/migrate-pg-to-single-node.sh \
|
||||
--request-body-mode omit
|
||||
```
|
||||
|
||||
`omit` only skips writing those large artifacts and detail tables into the
|
||||
target SQLite database. It does not delete or clear the source Postgres data.
|
||||
|
||||
## Notes
|
||||
|
||||
- Single Node requires root or sudo because it writes `/opt/aether`, `/etc/aether`, and
|
||||
the system service definition.
|
||||
- The script does not decrypt or re-encrypt provider keys. It preserves the
|
||||
original encryption key and moves encrypted data as-is.
|
||||
- Existing target SQLite databases, including `-wal`/`-shm` sidecars, are not
|
||||
replaced unless `--replace-existing` is provided.
|
||||
- Disk space checks use `pg_database_size(current_database()) * 2 + 1 GiB` as the
|
||||
conservative estimate for one SQLite copy. If the work directory and target DB
|
||||
directory are on the same filesystem, the script requires enough space for both
|
||||
the temporary and final SQLite files. With `--request-body-mode omit`, the
|
||||
estimate subtracts `usage_body_blobs` and `usage_http_audits` relation sizes.
|
||||
- For non-standard source Compose files, set `--app-service` and
|
||||
`--postgres-service` to match the service names.
|
||||
@@ -1,221 +0,0 @@
|
||||
# Postgres 到 Aether Single Node 迁移
|
||||
|
||||
英文版:[pg-to-single-node-migration.md](pg-to-single-node-migration.md)
|
||||
|
||||
本文档用于把现有 Docker Compose Postgres 部署迁移到 Aether
|
||||
single-node。当前版本里,**single-node** 指默认 SQLite 安装模式:
|
||||
`install.sh --mode single-node`,也就是系统服务加 SQLite。Docker Compose
|
||||
单机模板是 `docker-compose.single-node.yml`,安装脚本入口是
|
||||
`--mode compose-single-node`。
|
||||
|
||||
迁移脚本:
|
||||
|
||||
```bash
|
||||
scripts/migrate-pg-to-single-node.sh
|
||||
```
|
||||
|
||||
如果目标形态仍然要保持 Docker Compose,而不是系统服务,使用镜像版迁移脚本:
|
||||
|
||||
```bash
|
||||
scripts/migrate-pg-compose-to-single-node.sh
|
||||
```
|
||||
|
||||
两种迁移脚本都会先拉取/安装目标 single-node 版本,再停止源 `app`,把 Postgres
|
||||
记录直接写入临时 SQLite DB,不落 JSONL 中间文件;复制成功后替换目标
|
||||
`aether.db`,最后启动 single-node。
|
||||
|
||||
也可以直接用安装脚本作为统一入口,由 `--mode` 选择迁移目标:
|
||||
|
||||
```bash
|
||||
# 交互式执行时,先选择目标部署模式:
|
||||
# 1) Docker Compose 标准部署(Postgres + Redis)
|
||||
# 2) Docker Compose 单节点部署(SQLite)
|
||||
# 3) 系统服务单节点部署(SQLite)
|
||||
# 选择 2 或 3 后,再选择数据初始化方式:
|
||||
# 1) 全新初始化(不迁移现有数据)
|
||||
# 2) 从现有 Docker Compose PG 数据库迁移
|
||||
install.sh
|
||||
|
||||
# 迁移到新的 single-node Docker Compose 目录
|
||||
install.sh \
|
||||
--mode compose-single-node \
|
||||
--migrate-from-compose /root/Aether/docker-compose.yml \
|
||||
--compose-dir /opt/aether-single \
|
||||
--replace-existing
|
||||
|
||||
# 迁移到系统服务 + SQLite
|
||||
sudo install.sh \
|
||||
--mode single-node \
|
||||
--migrate-from-compose /root/Aether/docker-compose.yml \
|
||||
--replace-existing
|
||||
```
|
||||
|
||||
交互模式会先选择目标部署形态。如果目标是 `compose-single-node` 或
|
||||
`single-node`,安装脚本会再询问数据初始化方式:全新初始化,或从现有 Docker
|
||||
Compose PG 数据库迁移。选择迁移后,脚本会通过 `docker compose ls` 自动探测源
|
||||
PG Compose 文件,并确认该 Compose 配置里存在默认的 `app` 和 `postgres` 服务;
|
||||
如果能唯一识别,会作为默认值带入提示。探测不到或存在多个候选时会直接中止;
|
||||
此时请用 `--migrate-from-compose` 显式指定源 compose 路径。
|
||||
|
||||
安装脚本只是统一参数入口:`compose-single-node` 会委托给
|
||||
`scripts/migrate-pg-compose-to-single-node.sh`,`single-node` 会委托给
|
||||
`scripts/migrate-pg-to-single-node.sh`。
|
||||
|
||||
## 迁移内容
|
||||
|
||||
脚本会尽量缩短生产停机窗口:
|
||||
|
||||
1. 读取源 Compose 目录下的 `.env`。
|
||||
2. 生成 single-node 环境文件,保留 `JWT_SECRET_KEY`、`ENCRYPTION_KEY` 或
|
||||
`AETHER_GATEWAY_DATA_ENCRYPTION_KEY`、管理员配置、端口和应用配置。
|
||||
3. 执行 `install.sh --mode single-node --skip-start`,提前安装 single-node
|
||||
release,但不启动服务。
|
||||
4. 使用已安装的 single-node 二进制预检 SQLite schema migration。
|
||||
5. 拉取目标 single-node 镜像,确认其 `copy` 命令可用,并检查源 `app`
|
||||
当前运行镜像 ID 与目标镜像 ID 一致。
|
||||
6. 以目标 SQLite schema 作为迁移计划:把源 Postgres 中同名表、同名字段
|
||||
复制到临时 SQLite DB。
|
||||
7. 检查请求体明细迁移策略;默认全部迁移,也可以选择只跳过请求体明细。
|
||||
8. 检查 work-dir 和目标 SQLite 目录是否有足够空间容纳临时库和正式库。
|
||||
9. 只停止源 Compose 的 `app` 服务,保留 Postgres 和 Redis 运行,方便回滚。
|
||||
10. 从源 Postgres 直接复制记录到临时 SQLite 数据库,不生成 JSONL 中间文件。
|
||||
11. 复制完成后替换目标 SQLite DB,包括 SQLite `-wal`、`-shm` 边车文件,
|
||||
然后启动 single-node 系统服务。
|
||||
|
||||
镜像一致性检查比较的是 Docker 镜像 ID,不只是 tag 字符串。即使源和目标都写着
|
||||
`latest`,只要实际镜像 ID 不同,迁移也会中止。请先把源 PG Compose 的 `app`
|
||||
升级到目标 single-node 相同版本,确认运行正常后再迁移。迁移脚本也会检查目标镜像
|
||||
是否支持直接 copy 和请求体跳过开关;如果只是换了脚本但镜像还是旧版本,脚本会
|
||||
直接中止,避免漏迁。
|
||||
|
||||
## 生产切换
|
||||
|
||||
切换前先做一次常规服务器备份或快照。确认后执行:
|
||||
|
||||
```bash
|
||||
sudo scripts/migrate-pg-to-single-node.sh \
|
||||
--source-compose /root/Aether/docker-compose.yml \
|
||||
--replace-existing
|
||||
```
|
||||
|
||||
如果要迁移到 Docker Compose single-node,而不是系统服务:
|
||||
|
||||
```bash
|
||||
scripts/migrate-pg-compose-to-single-node.sh \
|
||||
--source-compose /root/Aether/docker-compose.yml \
|
||||
--replace-existing
|
||||
```
|
||||
|
||||
源 Postgres Compose 目录和目标 single-node Compose 目录可以不一样。例如:
|
||||
|
||||
```bash
|
||||
install.sh \
|
||||
--mode compose-single-node \
|
||||
--migrate-from-compose /root/Aether/docker-compose.yml \
|
||||
--compose-dir /opt/aether-single \
|
||||
--replace-existing
|
||||
```
|
||||
|
||||
等价地,也可以直接调底层脚本并显式传入每个目标路径:
|
||||
|
||||
```bash
|
||||
scripts/migrate-pg-compose-to-single-node.sh \
|
||||
--source-compose /root/Aether/docker-compose.yml \
|
||||
--target-compose /opt/aether-single/docker-compose.single-node.yml \
|
||||
--target-env /opt/aether-single/.env.single-node \
|
||||
--target-db /opt/aether-single/data/aether.db \
|
||||
--replace-existing
|
||||
```
|
||||
|
||||
切换时脚本只会停止并移除源 `app` 容器,用来释放固定的 `aether-app`
|
||||
容器名;Postgres、Redis 和它们的 volume 都会保留,方便回滚。
|
||||
|
||||
默认路径和服务名:
|
||||
|
||||
| 配置项 | 默认值 |
|
||||
| --- | --- |
|
||||
| 源 Compose 文件 | `docker-compose.yml` |
|
||||
| single-node 安装目录 | `/opt/aether` |
|
||||
| single-node 配置目录 | `/etc/aether` |
|
||||
| 目标 SQLite DB | `/opt/aether/data/aether.db` |
|
||||
| 源 app 服务 | `app` |
|
||||
| 源 Postgres 服务 | `postgres` |
|
||||
| single-node 服务 | `aether-gateway` |
|
||||
|
||||
除非显式传入 `--work-dir`,脚本会把迁移产物写到源 Compose 文件旁边的
|
||||
`./data/pg-to-single-node-<timestamp>`。
|
||||
|
||||
## 回滚
|
||||
|
||||
脚本会保留原 Postgres 和 Redis volume。迁移已经完成但需要回滚时:
|
||||
|
||||
```bash
|
||||
sudo systemctl stop aether-gateway
|
||||
cd /root/Aether
|
||||
docker compose -f docker-compose.yml up -d app
|
||||
```
|
||||
|
||||
对于 Compose single-node 脚本,回滚思路相同:重新用原 Postgres compose 文件
|
||||
拉起 `app`。
|
||||
|
||||
如果迁移在切换完成前失败,脚本默认会尝试自动拉起源 `app` 服务。需要失败后
|
||||
保持源应用停止以便人工排查时,增加:
|
||||
|
||||
```bash
|
||||
--keep-source-stopped-on-error
|
||||
```
|
||||
|
||||
## 数据覆盖保护
|
||||
|
||||
迁移不再维护一份额外的业务表清单。目标 single-node 镜像会先用正常
|
||||
migrations 建出临时 SQLite 数据库,然后 `aether-gateway copy` 读取这个
|
||||
SQLite schema,把源 Postgres 里同名表、同名字段复制过去。
|
||||
|
||||
如果源 Postgres 里存在非空 public 表,但目标 SQLite schema 中没有同名表,
|
||||
copy 会直接中止,不会静默丢弃。生命周期元数据表 `_sqlx_migrations` 和
|
||||
`schema_backfills` 会被忽略。源表中存在但目标 SQLite 不存在的额外字段不会复制。
|
||||
|
||||
## 请求体明细策略
|
||||
|
||||
single-node SQLite 生产迁移默认迁移所有可迁移数据,唯一可选的跳过项是请求体明细。
|
||||
|
||||
选择“不迁移请求体”时,不会迁移 `usage_body_blobs`、`usage_http_audits`,也不会迁移 `usage`
|
||||
表里的 `request_body` / `provider_request_body` / `response_body` /
|
||||
`client_response_body` / `*_body_compressed` 等请求体大字段。
|
||||
|
||||
交互安装时可以选择:
|
||||
|
||||
```text
|
||||
1) 全部迁移:迁移所有可迁移数据,包括请求体明细
|
||||
2) 不迁移请求体:迁移其他所有数据;仅跳过请求体大字段和 HTTP 请求体明细,源 PG 不清除
|
||||
```
|
||||
|
||||
非交互执行时,全部迁移可以显式指定:
|
||||
|
||||
```bash
|
||||
scripts/migrate-pg-to-single-node.sh \
|
||||
--request-body-mode full
|
||||
```
|
||||
|
||||
不迁移请求体可以显式指定:
|
||||
|
||||
```bash
|
||||
scripts/migrate-pg-to-single-node.sh \
|
||||
--request-body-mode omit
|
||||
```
|
||||
|
||||
`omit` 只是不把这些大字段和明细表写进目标 SQLite,不会删除或清空源 Postgres。
|
||||
|
||||
## 注意事项
|
||||
|
||||
- single-node 安装需要 root 或 sudo 权限,因为会写入 `/opt/aether`、
|
||||
`/etc/aether` 和系统服务定义。
|
||||
- 脚本不会解密或重新加密供应商密钥;它会沿用源环境的加密密钥,并原样迁移已加密数据。
|
||||
- 已存在的目标 SQLite DB,包括 `-wal`、`-shm` 边车文件,只有在传入
|
||||
`--replace-existing` 时才会被替换。
|
||||
- 空间检查会用 `pg_database_size(current_database()) * 2 + 1 GiB` 作为单份
|
||||
SQLite 的保守估算。如果 work-dir 和目标 DB 目录在同一个文件系统,会要求同时
|
||||
容纳临时 SQLite 和正式 SQLite。选择 `--request-body-mode omit` 时,
|
||||
估算会扣除 `usage_body_blobs` 和 `usage_http_audits` 的表空间。
|
||||
- 非标准 Compose 服务名需要通过 `--app-service` 和 `--postgres-service`
|
||||
明确指定。
|
||||
-568
@@ -61,19 +61,6 @@ ADMIN_PASSWORD_SOURCE=""
|
||||
UI_LANG="${AETHER_LANG:-${AETHER_LANGUAGE:-auto}}"
|
||||
RELEASE_KEEP="${AETHER_RELEASE_KEEP:-3}"
|
||||
RELEASE_ARCHIVE_URL="${AETHER_RELEASE_ARCHIVE_URL:-${AETHER_DOWNLOAD_URL:-}}"
|
||||
MIGRATE_FROM_COMPOSE=""
|
||||
MIGRATE_TARGET_COMPOSE=""
|
||||
MIGRATE_TARGET_ENV=""
|
||||
MIGRATE_TARGET_DB=""
|
||||
MIGRATE_WORK_DIR=""
|
||||
MIGRATE_APP_SERVICE=""
|
||||
MIGRATE_POSTGRES_SERVICE=""
|
||||
MIGRATE_SINGLE_NODE_SERVICE=""
|
||||
MIGRATE_REPLACE_EXISTING="false"
|
||||
MIGRATE_REPLACE_TARGET_COMPOSE="false"
|
||||
MIGRATE_KEEP_SOURCE_STOPPED_ON_ERROR="false"
|
||||
MIGRATE_INTERACTIVE="false"
|
||||
MIGRATE_REQUEST_BODY_MODE=""
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
@@ -104,31 +91,8 @@ Options:
|
||||
--lang LANG Installer language: zh or en
|
||||
--skip-start Install files, but do not start Docker Compose or restart the service
|
||||
--keep-releases N Keep the latest N releases, prune older ones (default: 3, 0=disable)
|
||||
--migrate-from-compose PATH
|
||||
Migrate an existing Postgres Compose deployment into the selected single-node mode
|
||||
--target-compose PATH
|
||||
Migration target compose file for --mode compose-single-node
|
||||
--target-env PATH Migration target env file for --mode compose-single-node
|
||||
--target-db PATH Migration target SQLite DB path
|
||||
--work-dir PATH Migration working directory
|
||||
--app-service NAME Source compose app service for migration
|
||||
--postgres-service NAME
|
||||
Source compose Postgres service for migration
|
||||
--single-node-service NAME
|
||||
Target compose service for --mode compose-single-node migration
|
||||
--replace-existing Allow replacing an existing target SQLite DB during migration
|
||||
--replace-target-compose
|
||||
Overwrite target compose file from the single-node template during migration
|
||||
--request-body-mode MODE
|
||||
Request/response body detail handling during migration: full/1 or omit/2
|
||||
--keep-source-stopped-on-error
|
||||
Do not auto-restart source app if migration fails after stopping it
|
||||
-h, --help Show this help
|
||||
|
||||
Migration examples:
|
||||
install.sh --mode compose-single-node --migrate-from-compose /root/Aether/docker-compose.yml --compose-dir /opt/aether-single --replace-existing
|
||||
sudo install.sh --mode single-node --migrate-from-compose /root/Aether/docker-compose.yml --replace-existing
|
||||
|
||||
Environment overrides:
|
||||
AETHER_REPO, AETHER_SOURCE_REF, AETHER_INSTALL_MODE, AETHER_CHANNEL, AETHER_VERSION
|
||||
AETHER_LANG or AETHER_LANGUAGE
|
||||
@@ -312,63 +276,6 @@ parse_args() {
|
||||
RELEASE_KEEP="$2"
|
||||
shift 2
|
||||
;;
|
||||
--migrate-from-compose)
|
||||
[[ $# -ge 2 ]] || die "--migrate-from-compose requires a path"
|
||||
MIGRATE_FROM_COMPOSE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--target-compose)
|
||||
[[ $# -ge 2 ]] || die "--target-compose requires a path"
|
||||
MIGRATE_TARGET_COMPOSE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--target-env)
|
||||
[[ $# -ge 2 ]] || die "--target-env requires a path"
|
||||
MIGRATE_TARGET_ENV="$2"
|
||||
shift 2
|
||||
;;
|
||||
--target-db)
|
||||
[[ $# -ge 2 ]] || die "--target-db requires a path"
|
||||
MIGRATE_TARGET_DB="$2"
|
||||
shift 2
|
||||
;;
|
||||
--work-dir)
|
||||
[[ $# -ge 2 ]] || die "--work-dir requires a path"
|
||||
MIGRATE_WORK_DIR="$2"
|
||||
shift 2
|
||||
;;
|
||||
--app-service)
|
||||
[[ $# -ge 2 ]] || die "--app-service requires a service name"
|
||||
MIGRATE_APP_SERVICE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--postgres-service)
|
||||
[[ $# -ge 2 ]] || die "--postgres-service requires a service name"
|
||||
MIGRATE_POSTGRES_SERVICE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--single-node-service)
|
||||
[[ $# -ge 2 ]] || die "--single-node-service requires a service name"
|
||||
MIGRATE_SINGLE_NODE_SERVICE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--replace-existing)
|
||||
MIGRATE_REPLACE_EXISTING="true"
|
||||
shift
|
||||
;;
|
||||
--replace-target-compose)
|
||||
MIGRATE_REPLACE_TARGET_COMPOSE="true"
|
||||
shift
|
||||
;;
|
||||
--request-body-mode)
|
||||
[[ $# -ge 2 ]] || die "--request-body-mode requires a value"
|
||||
MIGRATE_REQUEST_BODY_MODE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--keep-source-stopped-on-error)
|
||||
MIGRATE_KEEP_SOURCE_STOPPED_ON_ERROR="true"
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
@@ -610,44 +517,6 @@ EOF
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ -z "${MIGRATE_FROM_COMPOSE}" && "${MODE}" != "compose" ]]; then
|
||||
if ui_is_zh; then
|
||||
cat >/dev/tty <<'EOF'
|
||||
|
||||
请选择数据初始化方式:
|
||||
1) 全新初始化(不迁移现有数据)
|
||||
2) 从现有 Docker Compose PG 数据库迁移
|
||||
|
||||
请输入选项 [1]:
|
||||
EOF
|
||||
else
|
||||
cat >/dev/tty <<'EOF'
|
||||
|
||||
Choose data initialization mode:
|
||||
1) Fresh initialization (do not migrate existing data)
|
||||
2) Migrate from an existing Docker Compose PG database
|
||||
|
||||
Enter choice [1]:
|
||||
EOF
|
||||
fi
|
||||
local init_choice
|
||||
IFS= read -r init_choice </dev/tty || init_choice=""
|
||||
case "${init_choice:-1}" in
|
||||
1)
|
||||
;;
|
||||
2)
|
||||
MIGRATE_INTERACTIVE="true"
|
||||
;;
|
||||
*)
|
||||
if ui_is_zh; then
|
||||
die "无效的数据初始化方式选项: ${init_choice}"
|
||||
else
|
||||
die "invalid data initialization choice: ${init_choice}"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
else
|
||||
MODE="single-node"
|
||||
fi
|
||||
@@ -1010,434 +879,6 @@ start_compose_deployment() {
|
||||
run_compose "${compose_args[@]}" up -d
|
||||
}
|
||||
|
||||
migration_options_requested() {
|
||||
[[ -n "${MIGRATE_TARGET_COMPOSE}" ]] && return 0
|
||||
[[ -n "${MIGRATE_TARGET_ENV}" ]] && return 0
|
||||
[[ -n "${MIGRATE_TARGET_DB}" ]] && return 0
|
||||
[[ -n "${MIGRATE_WORK_DIR}" ]] && return 0
|
||||
[[ -n "${MIGRATE_APP_SERVICE}" ]] && return 0
|
||||
[[ -n "${MIGRATE_POSTGRES_SERVICE}" ]] && return 0
|
||||
[[ -n "${MIGRATE_SINGLE_NODE_SERVICE}" ]] && return 0
|
||||
[[ "${MIGRATE_REPLACE_EXISTING}" == "true" ]] && return 0
|
||||
[[ "${MIGRATE_REPLACE_TARGET_COMPOSE}" == "true" ]] && return 0
|
||||
[[ -n "${MIGRATE_REQUEST_BODY_MODE}" ]] && return 0
|
||||
[[ "${MIGRATE_KEEP_SOURCE_STOPPED_ON_ERROR}" == "true" ]] && return 0
|
||||
return 1
|
||||
}
|
||||
|
||||
normalize_migration_request_body_mode() {
|
||||
case "${MIGRATE_REQUEST_BODY_MODE}" in
|
||||
""|1|full|all|include)
|
||||
MIGRATE_REQUEST_BODY_MODE="full"
|
||||
;;
|
||||
2|omit|skip)
|
||||
MIGRATE_REQUEST_BODY_MODE="omit"
|
||||
;;
|
||||
*)
|
||||
die "--request-body-mode must be full/1 or omit/2"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
prompt_with_default() {
|
||||
local prompt="$1"
|
||||
local default_value="$2"
|
||||
local value
|
||||
|
||||
if [[ -n "${default_value}" ]]; then
|
||||
printf '%s [%s]: ' "${prompt}" "${default_value}" >/dev/tty
|
||||
else
|
||||
printf '%s: ' "${prompt}" >/dev/tty
|
||||
fi
|
||||
IFS= read -r value </dev/tty || value=""
|
||||
if [[ -z "${value}" ]]; then
|
||||
printf '%s\n' "${default_value}"
|
||||
else
|
||||
printf '%s\n' "${value}"
|
||||
fi
|
||||
}
|
||||
|
||||
prompt_yes_no() {
|
||||
local prompt="$1"
|
||||
local default_value="$2"
|
||||
local suffix choice
|
||||
|
||||
case "${default_value}" in
|
||||
yes)
|
||||
if ui_is_zh; then
|
||||
suffix="[y/n,默认 y]"
|
||||
else
|
||||
suffix="[y/n, default y]"
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
default_value="no"
|
||||
if ui_is_zh; then
|
||||
suffix="[y/n,默认 n]"
|
||||
else
|
||||
suffix="[y/n, default n]"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
while true; do
|
||||
printf '%s %s: ' "${prompt}" "${suffix}" >/dev/tty
|
||||
IFS= read -r choice </dev/tty || choice=""
|
||||
choice="$(printf '%s' "${choice}" | tr '[:upper:]' '[:lower:]')"
|
||||
case "${choice:-${default_value}}" in
|
||||
y|yes)
|
||||
return 0
|
||||
;;
|
||||
n|no)
|
||||
return 1
|
||||
;;
|
||||
*)
|
||||
if ui_is_zh; then
|
||||
echo "请输入 y 或 n。" >/dev/tty
|
||||
else
|
||||
echo "Enter y or n." >/dev/tty
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
docker_compose_ls_config_files() {
|
||||
local output
|
||||
|
||||
output="$(docker compose ls --format json 2>/dev/null || true)"
|
||||
if [[ -n "${output}" && "${output}" == *ConfigFiles* ]]; then
|
||||
printf '%s' "${output}" |
|
||||
tr '{' '\n' |
|
||||
sed -n 's/.*"ConfigFiles"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p'
|
||||
return
|
||||
fi
|
||||
|
||||
docker compose ls 2>/dev/null | awk 'NR > 1 && NF > 0 { print $NF }'
|
||||
}
|
||||
|
||||
compose_file_has_source_services() {
|
||||
local compose_file="$1"
|
||||
local app_service="${MIGRATE_APP_SERVICE:-app}"
|
||||
local postgres_service="${MIGRATE_POSTGRES_SERVICE:-postgres}"
|
||||
local services
|
||||
|
||||
services="$(docker compose -f "${compose_file}" config --services 2>/dev/null || true)"
|
||||
[[ -n "${services}" ]] || return 1
|
||||
printf '%s\n' "${services}" | grep -Fxq "${app_service}" || return 1
|
||||
printf '%s\n' "${services}" | grep -Fxq "${postgres_service}" || return 1
|
||||
}
|
||||
|
||||
append_unique_candidate() {
|
||||
local candidate="$1"
|
||||
shift
|
||||
local existing
|
||||
|
||||
for existing in "$@"; do
|
||||
[[ "${existing}" != "${candidate}" ]] || return 1
|
||||
done
|
||||
printf '%s\n' "${candidate}"
|
||||
}
|
||||
|
||||
detect_source_compose_from_docker_compose_ls() {
|
||||
local config_files compose_file candidate
|
||||
local -a candidates=()
|
||||
|
||||
command -v docker >/dev/null 2>&1 || return 0
|
||||
docker compose version >/dev/null 2>&1 || return 0
|
||||
|
||||
while IFS= read -r config_files || [[ -n "${config_files}" ]]; do
|
||||
config_files="$(trim_whitespace "${config_files}")"
|
||||
[[ -n "${config_files}" ]] || continue
|
||||
|
||||
# The migration scripts currently accept one source compose file. If the
|
||||
# source project was launched with multiple compose files, ask explicitly.
|
||||
[[ "${config_files}" != *,* ]] || continue
|
||||
|
||||
compose_file="${config_files}"
|
||||
[[ -f "${compose_file}" ]] || continue
|
||||
compose_file="$(absolute_path "${compose_file}")"
|
||||
compose_file_has_source_services "${compose_file}" || continue
|
||||
candidate="$(append_unique_candidate "${compose_file}" "${candidates[@]}" || true)"
|
||||
[[ -z "${candidate}" ]] || candidates+=("${candidate}")
|
||||
done < <(docker_compose_ls_config_files)
|
||||
|
||||
case "${#candidates[@]}" in
|
||||
0)
|
||||
return 0
|
||||
;;
|
||||
1)
|
||||
printf '%s\n' "${candidates[0]}"
|
||||
;;
|
||||
*)
|
||||
if interactive_tty_available; then
|
||||
if ui_is_zh; then
|
||||
echo "从 docker compose ls 找到多个可能的源 Compose,无法安全自动选择:" >/dev/tty
|
||||
else
|
||||
echo "docker compose ls found multiple possible source Compose files and cannot choose safely:" >/dev/tty
|
||||
fi
|
||||
printf ' %s\n' "${candidates[@]}" >/dev/tty
|
||||
fi
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
collect_interactive_migration_options() {
|
||||
local detected_source
|
||||
local prompt
|
||||
local source_compose_abs
|
||||
local source_compose_dir
|
||||
|
||||
[[ "${MIGRATE_INTERACTIVE}" == "true" ]] || return
|
||||
interactive_tty_available || die "interactive migration selection requires a terminal"
|
||||
|
||||
if ui_is_zh; then
|
||||
cat >/dev/tty <<'EOF'
|
||||
|
||||
迁移会先做预检并拉取/安装目标 single-node,再在切换窗口停止源 app。
|
||||
源 Postgres 和 Redis 会保留,便于回滚。
|
||||
|
||||
EOF
|
||||
else
|
||||
cat >/dev/tty <<'EOF'
|
||||
|
||||
Migration will preflight and pull/install the target single-node release first.
|
||||
During cutover it stops only the source app. Source Postgres and Redis remain for rollback.
|
||||
|
||||
EOF
|
||||
fi
|
||||
|
||||
if [[ -z "${MIGRATE_FROM_COMPOSE}" ]]; then
|
||||
detected_source="$(detect_source_compose_from_docker_compose_ls || true)"
|
||||
if [[ -z "${detected_source}" ]]; then
|
||||
if ui_is_zh; then
|
||||
die "未能通过 docker compose ls 唯一识别源 PG Compose;请使用 --migrate-from-compose 显式指定"
|
||||
else
|
||||
die "could not uniquely detect source PG Compose from docker compose ls; pass --migrate-from-compose explicitly"
|
||||
fi
|
||||
fi
|
||||
if ui_is_zh; then
|
||||
printf '已通过 docker compose ls 探测到源 Compose: %s\n' "${detected_source}" >/dev/tty
|
||||
else
|
||||
printf 'Detected source Compose from docker compose ls: %s\n' "${detected_source}" >/dev/tty
|
||||
fi
|
||||
if ui_is_zh; then
|
||||
prompt="确认使用该源 Compose 进行迁移"
|
||||
else
|
||||
prompt="Use this source Compose for migration"
|
||||
fi
|
||||
if prompt_yes_no "${prompt}" "yes"; then
|
||||
MIGRATE_FROM_COMPOSE="${detected_source}"
|
||||
else
|
||||
if ui_is_zh; then
|
||||
die "已取消迁移;如需指定其他源 Compose,请使用 --migrate-from-compose"
|
||||
else
|
||||
die "migration cancelled; pass --migrate-from-compose to use another source Compose"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
[[ -n "${MIGRATE_FROM_COMPOSE}" ]] || die "--migrate-from-compose cannot be empty"
|
||||
source_compose_abs="$(absolute_path "${MIGRATE_FROM_COMPOSE}")"
|
||||
source_compose_dir="$(dirname "${source_compose_abs}")"
|
||||
|
||||
if [[ "${MODE}" == "compose-single-node" ]]; then
|
||||
if [[ "${COMPOSE_DIR_EXPLICIT}" != "true" ]]; then
|
||||
COMPOSE_DIR="${source_compose_dir}-single-node"
|
||||
fi
|
||||
if ui_is_zh; then
|
||||
printf '已自动选择目标 single-node Compose 目录: %s\n' "${COMPOSE_DIR}" >/dev/tty
|
||||
prompt="确认使用该目标目录"
|
||||
else
|
||||
printf 'Selected target single-node Compose directory: %s\n' "${COMPOSE_DIR}" >/dev/tty
|
||||
prompt="Use this target directory"
|
||||
fi
|
||||
if ! prompt_yes_no "${prompt}" "yes"; then
|
||||
if ui_is_zh; then
|
||||
die "已取消迁移;如需指定其他目标目录,请使用 --compose-dir"
|
||||
else
|
||||
die "migration cancelled; pass --compose-dir to use another target directory"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "${MIGRATE_REPLACE_EXISTING}" != "true" ]]; then
|
||||
if ui_is_zh; then
|
||||
prompt="如果目标 SQLite 已存在,是否允许备份后替换"
|
||||
else
|
||||
prompt="If the target SQLite DB already exists, allow backup and replacement"
|
||||
fi
|
||||
if prompt_yes_no "${prompt}" "no"; then
|
||||
MIGRATE_REPLACE_EXISTING="true"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -z "${MIGRATE_REQUEST_BODY_MODE}" ]]; then
|
||||
if ui_is_zh; then
|
||||
cat >/dev/tty <<'EOF'
|
||||
|
||||
请求体明细迁移策略:
|
||||
1) 全部迁移:迁移所有可迁移数据,包括请求体明细
|
||||
2) 不迁移请求体:迁移其他所有数据;仅跳过请求体大字段和 HTTP 请求体明细,源 PG 不清除
|
||||
|
||||
请输入选项 [1]:
|
||||
EOF
|
||||
else
|
||||
cat >/dev/tty <<'EOF'
|
||||
|
||||
Request/response body detail migration mode:
|
||||
1) Full migration: migrate all migratable data, including request body details
|
||||
2) Skip request bodies: migrate all other data; skip only request body large fields and HTTP body detail tables; source PG is unchanged
|
||||
|
||||
Enter choice [1]:
|
||||
EOF
|
||||
fi
|
||||
local body_choice
|
||||
IFS= read -r body_choice </dev/tty || body_choice=""
|
||||
MIGRATE_REQUEST_BODY_MODE="${body_choice:-1}"
|
||||
fi
|
||||
normalize_migration_request_body_mode
|
||||
}
|
||||
|
||||
install_migration_project_file() {
|
||||
local source_path="$1"
|
||||
local mode="$2"
|
||||
local target_path
|
||||
|
||||
ensure_tmp_root
|
||||
target_path="${TMP_ROOT}/$(basename "${source_path}")"
|
||||
install_project_file "${source_path}" "${target_path}" "${mode}"
|
||||
printf '%s\n' "${target_path}"
|
||||
}
|
||||
|
||||
run_compose_single_node_migration() {
|
||||
local migration_script
|
||||
local target_template
|
||||
local source_compose_abs
|
||||
local source_compose_dir
|
||||
local compose_dir_abs
|
||||
local target_compose
|
||||
local target_compose_abs
|
||||
local target_compose_dir
|
||||
local target_env
|
||||
local target_env_abs
|
||||
local table
|
||||
local -a migrate_args
|
||||
|
||||
source_compose_abs="$(absolute_path "${MIGRATE_FROM_COMPOSE}")"
|
||||
[[ -f "${source_compose_abs}" ]] || die "source compose file not found: ${MIGRATE_FROM_COMPOSE}"
|
||||
source_compose_dir="$(dirname "${source_compose_abs}")"
|
||||
compose_dir_abs="$(absolute_path_maybe_missing "${COMPOSE_DIR}")"
|
||||
|
||||
if [[ -n "${MIGRATE_TARGET_COMPOSE}" ]]; then
|
||||
target_compose="${MIGRATE_TARGET_COMPOSE}"
|
||||
elif [[ "${compose_dir_abs}" == "${source_compose_dir}" ]]; then
|
||||
target_compose="${compose_dir_abs}/docker-compose.single-node.yml"
|
||||
else
|
||||
target_compose="${compose_dir_abs}/docker-compose.yml"
|
||||
fi
|
||||
target_compose_abs="$(absolute_path_maybe_missing "${target_compose}")"
|
||||
target_compose_dir="$(dirname "${target_compose_abs}")"
|
||||
|
||||
if [[ -n "${MIGRATE_TARGET_ENV}" ]]; then
|
||||
target_env="${MIGRATE_TARGET_ENV}"
|
||||
elif [[ "$(basename "${target_compose_abs}")" == "docker-compose.yml" ]]; then
|
||||
target_env="${target_compose_dir}/.env"
|
||||
else
|
||||
target_env="${target_compose_dir}/.env.single-node"
|
||||
fi
|
||||
target_env_abs="$(absolute_path_maybe_missing "${target_env}")"
|
||||
|
||||
[[ "${target_compose_abs}" != "${source_compose_abs}" ]] || die "target compose would overwrite the source compose file; pass --target-compose or --compose-dir"
|
||||
[[ "${target_env_abs}" != "${source_compose_dir}/.env" ]] || die "target env would overwrite the source .env; pass --target-env or --compose-dir"
|
||||
|
||||
migration_script="$(install_migration_project_file "scripts/migrate-pg-compose-to-single-node.sh" "0755")"
|
||||
target_template="$(install_migration_project_file "docker-compose.single-node.yml" "0644")"
|
||||
|
||||
migrate_args=(
|
||||
"${migration_script}"
|
||||
--source-compose "${source_compose_abs}"
|
||||
--target-compose "${target_compose_abs}"
|
||||
--target-template "${target_template}"
|
||||
--target-env "${target_env_abs}"
|
||||
--app-image "$(compose_image)"
|
||||
)
|
||||
[[ -z "${MIGRATE_TARGET_DB}" ]] || migrate_args+=(--target-db "${MIGRATE_TARGET_DB}")
|
||||
[[ -z "${MIGRATE_WORK_DIR}" ]] || migrate_args+=(--work-dir "${MIGRATE_WORK_DIR}")
|
||||
[[ -z "${MIGRATE_APP_SERVICE}" ]] || migrate_args+=(--app-service "${MIGRATE_APP_SERVICE}")
|
||||
[[ -z "${MIGRATE_POSTGRES_SERVICE}" ]] || migrate_args+=(--postgres-service "${MIGRATE_POSTGRES_SERVICE}")
|
||||
[[ -z "${MIGRATE_SINGLE_NODE_SERVICE}" ]] || migrate_args+=(--single-node-service "${MIGRATE_SINGLE_NODE_SERVICE}")
|
||||
[[ "${MIGRATE_REPLACE_EXISTING}" != "true" ]] || migrate_args+=(--replace-existing)
|
||||
[[ "${MIGRATE_REPLACE_TARGET_COMPOSE}" != "true" ]] || migrate_args+=(--replace-target-compose)
|
||||
[[ -z "${MIGRATE_REQUEST_BODY_MODE}" ]] || migrate_args+=(--request-body-mode "${MIGRATE_REQUEST_BODY_MODE}")
|
||||
[[ "${MIGRATE_KEEP_SOURCE_STOPPED_ON_ERROR}" != "true" ]] || migrate_args+=(--keep-source-stopped-on-error)
|
||||
|
||||
bash "${migrate_args[@]}"
|
||||
}
|
||||
|
||||
run_single_node_service_migration() {
|
||||
local migration_script
|
||||
local installer
|
||||
local table
|
||||
local -a migrate_args
|
||||
|
||||
[[ -z "${MIGRATE_TARGET_COMPOSE}" ]] || die "--target-compose is only valid with --mode compose-single-node"
|
||||
[[ -z "${MIGRATE_TARGET_ENV}" ]] || die "--target-env is only valid with --mode compose-single-node"
|
||||
[[ -z "${MIGRATE_SINGLE_NODE_SERVICE}" ]] || die "--single-node-service is only valid with --mode compose-single-node"
|
||||
[[ "${MIGRATE_REPLACE_TARGET_COMPOSE}" != "true" ]] || die "--replace-target-compose is only valid with --mode compose-single-node"
|
||||
|
||||
migration_script="$(install_migration_project_file "scripts/migrate-pg-to-single-node.sh" "0755")"
|
||||
installer="$(install_migration_project_file "install.sh" "0755")"
|
||||
|
||||
migrate_args=(
|
||||
"${migration_script}"
|
||||
--source-compose "${MIGRATE_FROM_COMPOSE}"
|
||||
--installer "${installer}"
|
||||
--install-root "${INSTALL_ROOT}"
|
||||
--config-dir "${CONFIG_DIR}"
|
||||
--service-name "${SERVICE_NAME}"
|
||||
--service-user "${SERVICE_USER}"
|
||||
--service-group "${SERVICE_GROUP}"
|
||||
--app-image "$(compose_image)"
|
||||
--install-channel "${CHANNEL}"
|
||||
--install-repo "${REPO}"
|
||||
--install-source-ref "${SOURCE_REF}"
|
||||
)
|
||||
[[ -z "${VERSION}" ]] || migrate_args+=(--install-version "${VERSION}")
|
||||
[[ -z "${ARCHIVE_PATH}" ]] || migrate_args+=(--install-archive "${ARCHIVE_PATH}")
|
||||
[[ -z "${RELEASE_ARCHIVE_URL}" ]] || migrate_args+=(--install-download-url "${RELEASE_ARCHIVE_URL}")
|
||||
[[ -z "${MIGRATE_TARGET_DB}" ]] || migrate_args+=(--target-db "${MIGRATE_TARGET_DB}")
|
||||
[[ -z "${MIGRATE_WORK_DIR}" ]] || migrate_args+=(--work-dir "${MIGRATE_WORK_DIR}")
|
||||
[[ -z "${MIGRATE_APP_SERVICE}" ]] || migrate_args+=(--app-service "${MIGRATE_APP_SERVICE}")
|
||||
[[ -z "${MIGRATE_POSTGRES_SERVICE}" ]] || migrate_args+=(--postgres-service "${MIGRATE_POSTGRES_SERVICE}")
|
||||
[[ "${MIGRATE_REPLACE_EXISTING}" != "true" ]] || migrate_args+=(--replace-existing)
|
||||
[[ -z "${MIGRATE_REQUEST_BODY_MODE}" ]] || migrate_args+=(--request-body-mode "${MIGRATE_REQUEST_BODY_MODE}")
|
||||
[[ "${MIGRATE_KEEP_SOURCE_STOPPED_ON_ERROR}" != "true" ]] || migrate_args+=(--keep-source-stopped-on-error)
|
||||
|
||||
bash "${migrate_args[@]}"
|
||||
}
|
||||
|
||||
run_migration_from_compose() {
|
||||
if [[ -n "${MIGRATE_REQUEST_BODY_MODE}" ]]; then
|
||||
normalize_migration_request_body_mode
|
||||
fi
|
||||
|
||||
case "${MODE}" in
|
||||
compose-single-node)
|
||||
run_compose_single_node_migration
|
||||
;;
|
||||
single-node)
|
||||
run_single_node_service_migration
|
||||
;;
|
||||
compose)
|
||||
die "--migrate-from-compose target mode must be compose-single-node or single-node"
|
||||
;;
|
||||
*)
|
||||
die "unsupported migration target mode: ${MODE}"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
resolve_version() {
|
||||
if [[ -n "${VERSION}" ]]; then
|
||||
echo "${VERSION}"
|
||||
@@ -2725,15 +2166,6 @@ main() {
|
||||
apply_platform_defaults
|
||||
select_version
|
||||
select_mode
|
||||
collect_interactive_migration_options
|
||||
|
||||
if [[ -n "${MIGRATE_FROM_COMPOSE}" ]]; then
|
||||
run_migration_from_compose
|
||||
return
|
||||
fi
|
||||
if migration_options_requested; then
|
||||
die "migration options require --migrate-from-compose"
|
||||
fi
|
||||
|
||||
if [[ "${MODE}" == "compose" ]]; then
|
||||
install_compose_mode
|
||||
|
||||
@@ -1,877 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SOURCE_COMPOSE="docker-compose.yml"
|
||||
TARGET_COMPOSE=""
|
||||
TARGET_COMPOSE_TEMPLATE="./docker-compose.single-node.yml"
|
||||
TARGET_ENV=""
|
||||
TARGET_DB=""
|
||||
WORK_DIR=""
|
||||
APP_SERVICE="app"
|
||||
POSTGRES_SERVICE="postgres"
|
||||
SINGLE_NODE_SERVICE="app"
|
||||
APP_IMAGE=""
|
||||
REPLACE_EXISTING="false"
|
||||
REPLACE_TARGET_COMPOSE="false"
|
||||
DRY_RUN="false"
|
||||
KEEP_SOURCE_STOPPED_ON_ERROR="false"
|
||||
DISK_SPACE_MULTIPLIER="${AETHER_MIGRATION_DISK_SPACE_MULTIPLIER:-2}"
|
||||
DISK_SPACE_MIN_FREE_BYTES="${AETHER_MIGRATION_MIN_FREE_BYTES:-1073741824}"
|
||||
REQUEST_BODY_MODE="${AETHER_MIGRATION_REQUEST_BODY_MODE:-full}"
|
||||
|
||||
APP_STOPPED="false"
|
||||
CUTOVER_COMPLETE="false"
|
||||
SOURCE_COMPOSE_ABS=""
|
||||
SOURCE_COMPOSE_DIR=""
|
||||
SOURCE_ENV=""
|
||||
SOURCE_NETWORK=""
|
||||
TARGET_COMPOSE_ABS=""
|
||||
TARGET_COMPOSE_DIR=""
|
||||
TARGET_ENV_ABS=""
|
||||
DB_USER=""
|
||||
DB_NAME=""
|
||||
DB_PASSWORD=""
|
||||
NOW=""
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage: scripts/migrate-pg-compose-to-single-node.sh [options]
|
||||
|
||||
Migrate an existing Docker Compose Postgres deployment to Docker Compose single-node.
|
||||
This script pulls the single-node app image before downtime, stops only the source app,
|
||||
then copies Postgres data directly into a temporary SQLite DB without writing a JSONL
|
||||
intermediate file. After the copy succeeds, it starts the single-node Compose app.
|
||||
|
||||
Options:
|
||||
--source-compose PATH Source Postgres docker compose file (default: docker-compose.yml)
|
||||
--target-compose PATH Target single-node compose file (default: SOURCE_DIR/docker-compose.single-node.yml)
|
||||
--target-template PATH Template copied when target compose is missing (default: ./docker-compose.single-node.yml)
|
||||
--target-env PATH Target env file (default: TARGET_COMPOSE_DIR/.env.single-node)
|
||||
--target-db PATH Final SQLite DB path (default: TARGET_COMPOSE_DIR/data/aether.db)
|
||||
--work-dir PATH Working directory (default: SOURCE_DIR/data/pg-compose-to-single-node-<timestamp>)
|
||||
--app-service NAME Source compose app service (default: app)
|
||||
--postgres-service NAME Source compose Postgres service (default: postgres)
|
||||
--single-node-service NAME Target single-node compose service (default: app)
|
||||
--app-image IMAGE Override APP_IMAGE for the target single-node compose env
|
||||
--replace-existing Allow replacing an existing target SQLite database
|
||||
--replace-target-compose Overwrite target compose file from the template
|
||||
--dry-run Pull/preflight/direct-copy without stopping or switching
|
||||
--request-body-mode MODE Request/response body detail handling: full/1 or omit/2
|
||||
full: migrate all migratable data, including request body details
|
||||
omit: migrate all other data; skip only request body large fields and HTTP body detail tables; source PG is unchanged
|
||||
--keep-source-stopped-on-error
|
||||
Do not auto-restart source app if migration fails after stopping it
|
||||
-h, --help Show this help
|
||||
|
||||
Default cutover behavior:
|
||||
1. Copy/prepare docker-compose.single-node.yml and .env.single-node.
|
||||
2. Pull the target single-node app image while the source app is still running.
|
||||
3. Verify the running source app image ID matches the target single-node image ID.
|
||||
4. Preflight SQLite migrations in a temporary DB with the target image.
|
||||
5. Re-check migration coverage and available disk space.
|
||||
6. Stop/remove only the source app container; keep Postgres/Redis running.
|
||||
7. Copy records directly into SQLite, replace TARGET_DB, and start single-node.
|
||||
EOF
|
||||
}
|
||||
|
||||
log() {
|
||||
printf '>>> %s\n' "$*"
|
||||
}
|
||||
|
||||
warn() {
|
||||
printf 'WARN: %s\n' "$*" >&2
|
||||
}
|
||||
|
||||
die() {
|
||||
printf 'ERROR: %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
trim() {
|
||||
local value="$1"
|
||||
value="${value#"${value%%[![:space:]]*}"}"
|
||||
value="${value%"${value##*[![:space:]]}"}"
|
||||
printf '%s' "$value"
|
||||
}
|
||||
|
||||
strip_optional_quotes() {
|
||||
local value="$1"
|
||||
if [[ "${#value}" -ge 2 ]]; then
|
||||
if [[ "${value:0:1}" == "\"" && "${value: -1}" == "\"" ]]; then
|
||||
printf '%s' "${value:1:${#value}-2}"
|
||||
return
|
||||
fi
|
||||
if [[ "${value:0:1}" == "'" && "${value: -1}" == "'" ]]; then
|
||||
printf '%s' "${value:1:${#value}-2}"
|
||||
return
|
||||
fi
|
||||
fi
|
||||
printf '%s' "$value"
|
||||
}
|
||||
|
||||
absolute_path() {
|
||||
local path="$1"
|
||||
local dir
|
||||
local base
|
||||
|
||||
if [[ "$path" == /* ]]; then
|
||||
printf '%s\n' "$path"
|
||||
return
|
||||
fi
|
||||
|
||||
dir="$(dirname "$path")"
|
||||
base="$(basename "$path")"
|
||||
printf '%s/%s\n' "$(cd "$dir" && pwd -P)" "$base"
|
||||
}
|
||||
|
||||
absolute_path_maybe_missing() {
|
||||
local path="$1"
|
||||
if [[ "$path" == /* ]]; then
|
||||
printf '%s\n' "$path"
|
||||
else
|
||||
printf '%s/%s\n' "$(pwd -P)" "$path"
|
||||
fi
|
||||
}
|
||||
|
||||
parse_args() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--source-compose)
|
||||
[[ $# -ge 2 ]] || die "--source-compose requires a value"
|
||||
SOURCE_COMPOSE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--target-compose)
|
||||
[[ $# -ge 2 ]] || die "--target-compose requires a value"
|
||||
TARGET_COMPOSE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--target-template)
|
||||
[[ $# -ge 2 ]] || die "--target-template requires a value"
|
||||
TARGET_COMPOSE_TEMPLATE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--target-env)
|
||||
[[ $# -ge 2 ]] || die "--target-env requires a value"
|
||||
TARGET_ENV="$2"
|
||||
shift 2
|
||||
;;
|
||||
--target-db)
|
||||
[[ $# -ge 2 ]] || die "--target-db requires a value"
|
||||
TARGET_DB="$2"
|
||||
shift 2
|
||||
;;
|
||||
--work-dir)
|
||||
[[ $# -ge 2 ]] || die "--work-dir requires a value"
|
||||
WORK_DIR="$2"
|
||||
shift 2
|
||||
;;
|
||||
--app-service)
|
||||
[[ $# -ge 2 ]] || die "--app-service requires a value"
|
||||
APP_SERVICE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--postgres-service)
|
||||
[[ $# -ge 2 ]] || die "--postgres-service requires a value"
|
||||
POSTGRES_SERVICE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--single-node-service)
|
||||
[[ $# -ge 2 ]] || die "--single-node-service requires a value"
|
||||
SINGLE_NODE_SERVICE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--app-image)
|
||||
[[ $# -ge 2 ]] || die "--app-image requires a value"
|
||||
APP_IMAGE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--replace-existing)
|
||||
REPLACE_EXISTING="true"
|
||||
shift
|
||||
;;
|
||||
--replace-target-compose)
|
||||
REPLACE_TARGET_COMPOSE="true"
|
||||
shift
|
||||
;;
|
||||
--dry-run)
|
||||
DRY_RUN="true"
|
||||
shift
|
||||
;;
|
||||
--request-body-mode)
|
||||
[[ $# -ge 2 ]] || die "--request-body-mode requires a value"
|
||||
REQUEST_BODY_MODE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--keep-source-stopped-on-error)
|
||||
KEEP_SOURCE_STOPPED_ON_ERROR="true"
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
die "unknown argument: $1"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
require_command() {
|
||||
command -v "$1" >/dev/null 2>&1 || die "required command not found: $1"
|
||||
}
|
||||
|
||||
normalize_request_body_mode() {
|
||||
case "$REQUEST_BODY_MODE" in
|
||||
""|1|full|all|include)
|
||||
REQUEST_BODY_MODE="full"
|
||||
;;
|
||||
2|omit|skip)
|
||||
REQUEST_BODY_MODE="omit"
|
||||
;;
|
||||
*)
|
||||
die "--request-body-mode must be full/1 or omit/2"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
env_file_get() {
|
||||
local file="$1"
|
||||
local wanted="$2"
|
||||
local line key value
|
||||
local found=""
|
||||
|
||||
while IFS= read -r line || [[ -n "$line" ]]; do
|
||||
line="${line%$'\r'}"
|
||||
line="$(trim "$line")"
|
||||
[[ -z "$line" || "${line:0:1}" == "#" ]] && continue
|
||||
[[ "$line" == export\ * ]] && line="${line#export }"
|
||||
key="$(trim "${line%%=*}")"
|
||||
[[ "$key" == "$wanted" ]] || continue
|
||||
value="${line#*=}"
|
||||
found="$(strip_optional_quotes "$(trim "$value")")"
|
||||
done < "$file"
|
||||
|
||||
printf '%s' "$found"
|
||||
}
|
||||
|
||||
validate_env_line_for_copy() {
|
||||
local line="$1"
|
||||
local line_no="$2"
|
||||
[[ "$line" == *'${'* ]] && die "source env line ${line_no} uses variable expansion; write a concrete value before migration"
|
||||
[[ "$line" == *'$('* ]] && die "source env line ${line_no} uses command substitution; write a concrete value before migration"
|
||||
[[ "$line" == *'`'* ]] && die "source env line ${line_no} uses command substitution; write a concrete value before migration"
|
||||
return 0
|
||||
}
|
||||
|
||||
should_skip_single_node_env_key() {
|
||||
case "$1" in
|
||||
APP_IMAGE|LOCAL_APP_IMAGE|APP_PORT|DB_HOST|DB_PORT|DB_USER|DB_NAME|DB_PASSWORD|POSTGRES_*|MYSQL_*|REDIS_HOST|REDIS_PORT|REDIS_PASSWORD|REDIS_URL|AETHER_GATEWAY_DATA_REDIS_URL|AETHER_GATEWAY_DATA_REDIS_KEY_PREFIX|DATABASE_URL|AETHER_DATABASE_URL|AETHER_DATABASE_DRIVER|AETHER_GATEWAY_DATA_POSTGRES_URL|AETHER_RUNTIME_BACKEND|AETHER_RUNTIME_REDIS_URL|AETHER_RUNTIME_REDIS_KEY_PREFIX|AETHER_GATEWAY_DEPLOYMENT_TOPOLOGY|AETHER_GATEWAY_NODE_ROLE|AETHER_GATEWAY_STATIC_DIR|AETHER_UPDATE_STRATEGY|AETHER_DOCKER_UPDATE_COMMAND|AETHER_LOG_DIR|AETHER_GATEWAY_AUTO_PREPARE_DATABASE)
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
write_single_node_env() {
|
||||
local output="$1"
|
||||
local line raw_line key
|
||||
local line_no=0
|
||||
local app_port app_image jwt_key encryption_key
|
||||
|
||||
app_port="$(env_file_get "$SOURCE_ENV" "APP_PORT")"
|
||||
app_port="${app_port:-8084}"
|
||||
app_image="${APP_IMAGE:-$(env_file_get "$SOURCE_ENV" "APP_IMAGE")}"
|
||||
app_image="${app_image:-ghcr.io/fawney19/aether:latest}"
|
||||
jwt_key="$(env_file_get "$SOURCE_ENV" "JWT_SECRET_KEY")"
|
||||
encryption_key="$(env_file_get "$SOURCE_ENV" "ENCRYPTION_KEY")"
|
||||
|
||||
: > "$output"
|
||||
{
|
||||
printf '# Generated by scripts/migrate-pg-compose-to-single-node.sh from %s\n' "$SOURCE_ENV"
|
||||
printf '# single-node means Docker Compose app + SQLite for this migration target.\n\n'
|
||||
} >> "$output"
|
||||
|
||||
while IFS= read -r raw_line || [[ -n "$raw_line" ]]; do
|
||||
line_no=$((line_no + 1))
|
||||
line="${raw_line%$'\r'}"
|
||||
line="$(trim "$line")"
|
||||
[[ -z "$line" || "${line:0:1}" == "#" ]] && continue
|
||||
[[ "$line" == export\ * ]] && die "source env line ${line_no} uses export; write KEY=VALUE before migration"
|
||||
[[ "$line" =~ ^[A-Za-z_][A-Za-z0-9_]*= ]] || die "source env line ${line_no} must be KEY=VALUE"
|
||||
validate_env_line_for_copy "$line" "$line_no"
|
||||
|
||||
key="${line%%=*}"
|
||||
if should_skip_single_node_env_key "$key"; then
|
||||
continue
|
||||
fi
|
||||
printf '%s\n' "$line" >> "$output"
|
||||
done < "$SOURCE_ENV"
|
||||
|
||||
{
|
||||
printf '\n# Single-node Compose runtime overrides\n'
|
||||
printf 'APP_IMAGE=%s\n' "$app_image"
|
||||
printf 'APP_PORT=%s\n' "$app_port"
|
||||
printf 'AETHER_GATEWAY_STATIC_DIR=/opt/aether/current/frontend\n'
|
||||
printf 'AETHER_UPDATE_STRATEGY=docker\n'
|
||||
printf 'AETHER_DOCKER_UPDATE_COMMAND=./update.sh\n'
|
||||
printf 'AETHER_LOG_DESTINATION=stdout\n'
|
||||
printf 'AETHER_LOG_FORMAT=pretty\n'
|
||||
printf 'AETHER_LOG_DIR=/opt/aether/logs\n'
|
||||
printf 'AETHER_DATABASE_DRIVER=sqlite\n'
|
||||
printf 'AETHER_DATABASE_URL=sqlite:///opt/aether/data/aether.db\n'
|
||||
printf 'DATABASE_URL=sqlite:///opt/aether/data/aether.db\n'
|
||||
printf 'AETHER_RUNTIME_BACKEND=memory\n'
|
||||
printf 'AETHER_GATEWAY_DEPLOYMENT_TOPOLOGY=single-node\n'
|
||||
printf 'AETHER_GATEWAY_NODE_ROLE=all\n'
|
||||
printf 'AETHER_GATEWAY_AUTO_PREPARE_DATABASE=true\n'
|
||||
printf 'JWT_SECRET_KEY=%s\n' "$jwt_key"
|
||||
printf 'ENCRYPTION_KEY=%s\n' "$encryption_key"
|
||||
} >> "$output"
|
||||
}
|
||||
|
||||
source_compose() {
|
||||
docker compose -f "$SOURCE_COMPOSE_ABS" "$@"
|
||||
}
|
||||
|
||||
target_compose() {
|
||||
AETHER_ENV_FILE="$TARGET_ENV_ABS" docker compose --env-file "$TARGET_ENV_ABS" -f "$TARGET_COMPOSE_ABS" "$@"
|
||||
}
|
||||
|
||||
target_run_app() {
|
||||
local database_url="$1"
|
||||
shift
|
||||
target_compose run --rm --no-deps \
|
||||
-v "${WORK_DIR}:/migration" \
|
||||
-e AETHER_LOG_DESTINATION=stdout \
|
||||
-e AETHER_DATABASE_DRIVER=sqlite \
|
||||
-e "AETHER_DATABASE_URL=${database_url}" \
|
||||
-e "DATABASE_URL=${database_url}" \
|
||||
"$SINGLE_NODE_SERVICE" "$@"
|
||||
}
|
||||
|
||||
run_psql_stdin() {
|
||||
local sql_file="$1"
|
||||
source_compose exec -T \
|
||||
-e "PGPASSWORD=${DB_PASSWORD}" \
|
||||
"$POSTGRES_SERVICE" \
|
||||
psql -h 127.0.0.1 -U "$DB_USER" -d "$DB_NAME" -v ON_ERROR_STOP=1 -At -f - < "$sql_file"
|
||||
}
|
||||
|
||||
source_database_size_bytes() {
|
||||
local sql_file
|
||||
local result
|
||||
|
||||
sql_file="${WORK_DIR}/source-database-size.sql"
|
||||
if [[ "$REQUEST_BODY_MODE" == "omit" ]]; then
|
||||
cat > "$sql_file" <<'SQL'
|
||||
SELECT GREATEST(
|
||||
pg_database_size(current_database())
|
||||
- COALESCE(pg_total_relation_size(to_regclass('public.usage_body_blobs')), 0)
|
||||
- COALESCE(pg_total_relation_size(to_regclass('public.usage_http_audits')), 0),
|
||||
0
|
||||
);
|
||||
SQL
|
||||
else
|
||||
printf 'SELECT pg_database_size(current_database());\n' > "$sql_file"
|
||||
fi
|
||||
result="$(run_psql_stdin "$sql_file" | tr -d '[:space:]')"
|
||||
[[ "$result" =~ ^[0-9]+$ ]] || die "could not determine source Postgres database size"
|
||||
printf '%s\n' "$result"
|
||||
}
|
||||
|
||||
file_size_bytes() {
|
||||
local path="$1"
|
||||
|
||||
if [[ ! -e "$path" ]]; then
|
||||
printf '0\n'
|
||||
return
|
||||
fi
|
||||
stat -c '%s' "$path" 2>/dev/null || stat -f '%z' "$path" 2>/dev/null || die "could not stat file: ${path}"
|
||||
}
|
||||
|
||||
target_sqlite_size_bytes() {
|
||||
local total=0
|
||||
local suffix
|
||||
local size
|
||||
|
||||
for suffix in "" "-wal" "-shm"; do
|
||||
size="$(file_size_bytes "${TARGET_DB}${suffix}")"
|
||||
total=$((total + size))
|
||||
done
|
||||
printf '%s\n' "$total"
|
||||
}
|
||||
|
||||
available_bytes_for_path() {
|
||||
local path="$1"
|
||||
df -Pk "$path" | awk 'NR == 2 { printf "%.0f\n", $4 * 1024 }'
|
||||
}
|
||||
|
||||
filesystem_key_for_path() {
|
||||
local path="$1"
|
||||
df -Pk "$path" | awk 'NR == 2 { print $1 }'
|
||||
}
|
||||
|
||||
format_bytes() {
|
||||
local bytes="$1"
|
||||
awk -v bytes="$bytes" 'BEGIN {
|
||||
if (bytes >= 1073741824) {
|
||||
printf "%.1f GiB", bytes / 1073741824
|
||||
} else {
|
||||
printf "%.1f MiB", bytes / 1048576
|
||||
}
|
||||
}'
|
||||
}
|
||||
|
||||
assert_available_space() {
|
||||
local path="$1"
|
||||
local required_bytes="$2"
|
||||
local label="$3"
|
||||
local available_bytes
|
||||
|
||||
available_bytes="$(available_bytes_for_path "$path")"
|
||||
[[ "$available_bytes" =~ ^[0-9]+$ ]] || die "could not determine free disk space for ${path}"
|
||||
log "${label} free space: $(format_bytes "$available_bytes"); required: $(format_bytes "$required_bytes")"
|
||||
|
||||
if (( available_bytes < required_bytes )); then
|
||||
die "${label} does not have enough free disk space; required $(format_bytes "$required_bytes"), available $(format_bytes "$available_bytes")"
|
||||
fi
|
||||
}
|
||||
|
||||
check_disk_space() {
|
||||
local source_bytes
|
||||
local estimated_db_bytes
|
||||
local backup_bytes=0
|
||||
local target_dir
|
||||
local work_fs
|
||||
local target_fs
|
||||
local required_bytes
|
||||
|
||||
[[ "$DISK_SPACE_MULTIPLIER" =~ ^[1-9][0-9]*$ ]] || die "AETHER_MIGRATION_DISK_SPACE_MULTIPLIER must be a positive integer"
|
||||
[[ "$DISK_SPACE_MIN_FREE_BYTES" =~ ^[0-9]+$ ]] || die "AETHER_MIGRATION_MIN_FREE_BYTES must be a non-negative integer"
|
||||
|
||||
source_bytes="$(source_database_size_bytes)"
|
||||
estimated_db_bytes=$((source_bytes * DISK_SPACE_MULTIPLIER + DISK_SPACE_MIN_FREE_BYTES))
|
||||
target_dir="$(dirname "$TARGET_DB")"
|
||||
mkdir -p "$target_dir"
|
||||
|
||||
if [[ "$REPLACE_EXISTING" == "true" ]]; then
|
||||
backup_bytes="$(target_sqlite_size_bytes)"
|
||||
fi
|
||||
|
||||
log "source Postgres size used for disk estimate: $(format_bytes "$source_bytes")"
|
||||
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
assert_available_space "$WORK_DIR" "$estimated_db_bytes" "work dir"
|
||||
return
|
||||
fi
|
||||
|
||||
work_fs="$(filesystem_key_for_path "$WORK_DIR")"
|
||||
target_fs="$(filesystem_key_for_path "$target_dir")"
|
||||
if [[ "$work_fs" == "$target_fs" ]]; then
|
||||
required_bytes=$((estimated_db_bytes * 2 + backup_bytes))
|
||||
assert_available_space "$WORK_DIR" "$required_bytes" "work/target filesystem"
|
||||
else
|
||||
assert_available_space "$WORK_DIR" "$((estimated_db_bytes + backup_bytes))" "work dir"
|
||||
assert_available_space "$target_dir" "$estimated_db_bytes" "target DB dir"
|
||||
fi
|
||||
}
|
||||
|
||||
check_request_body_artifacts() {
|
||||
local sql_file
|
||||
local result_file
|
||||
|
||||
sql_file="${WORK_DIR}/check-request-body-artifacts.sql"
|
||||
result_file="${WORK_DIR}/request-body-artifacts.txt"
|
||||
|
||||
cat > "$sql_file" <<'SQL'
|
||||
CREATE TEMP TABLE aether_request_body_artifacts (
|
||||
artifact text PRIMARY KEY
|
||||
) ON COMMIT PRESERVE ROWS;
|
||||
|
||||
DO $$
|
||||
DECLARE
|
||||
candidate record;
|
||||
has_rows boolean;
|
||||
BEGIN
|
||||
IF EXISTS (
|
||||
SELECT 1
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'public'
|
||||
AND table_name = 'usage_body_blobs'
|
||||
) THEN
|
||||
EXECUTE 'SELECT EXISTS (SELECT 1 FROM public.usage_body_blobs LIMIT 1)'
|
||||
INTO has_rows;
|
||||
IF has_rows THEN
|
||||
INSERT INTO aether_request_body_artifacts(artifact)
|
||||
VALUES ('usage_body_blobs')
|
||||
ON CONFLICT DO NOTHING;
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
IF EXISTS (
|
||||
SELECT 1
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'public'
|
||||
AND table_name = 'usage_http_audits'
|
||||
) THEN
|
||||
EXECUTE 'SELECT EXISTS (SELECT 1 FROM public.usage_http_audits LIMIT 1)'
|
||||
INTO has_rows;
|
||||
IF has_rows THEN
|
||||
INSERT INTO aether_request_body_artifacts(artifact)
|
||||
VALUES ('usage_http_audits')
|
||||
ON CONFLICT DO NOTHING;
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
IF EXISTS (
|
||||
SELECT 1
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'public'
|
||||
AND table_name = 'usage'
|
||||
) THEN
|
||||
FOR candidate IN
|
||||
SELECT unnest(ARRAY[
|
||||
'request_body',
|
||||
'response_body',
|
||||
'provider_request_body',
|
||||
'client_response_body',
|
||||
'request_body_compressed',
|
||||
'response_body_compressed',
|
||||
'provider_request_body_compressed',
|
||||
'client_response_body_compressed'
|
||||
]) AS column_name
|
||||
LOOP
|
||||
IF EXISTS (
|
||||
SELECT 1
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'public'
|
||||
AND table_name = 'usage'
|
||||
AND column_name = candidate.column_name
|
||||
) THEN
|
||||
EXECUTE format('SELECT EXISTS (SELECT 1 FROM public.usage WHERE %I IS NOT NULL LIMIT 1)', candidate.column_name)
|
||||
INTO has_rows;
|
||||
IF has_rows THEN
|
||||
INSERT INTO aether_request_body_artifacts(artifact)
|
||||
VALUES ('usage.' || candidate.column_name)
|
||||
ON CONFLICT DO NOTHING;
|
||||
END IF;
|
||||
END IF;
|
||||
END LOOP;
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
|
||||
SELECT artifact FROM aether_request_body_artifacts ORDER BY artifact;
|
||||
SQL
|
||||
|
||||
run_psql_stdin "$sql_file" > "$result_file"
|
||||
|
||||
if [[ ! -s "$result_file" ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ "$REQUEST_BODY_MODE" == "omit" ]]; then
|
||||
warn "source has request/response body details that will not be copied into single-node SQLite"
|
||||
cat "$result_file" >&2
|
||||
return
|
||||
fi
|
||||
|
||||
warn "source has request/response body details that will be copied into single-node SQLite"
|
||||
cat "$result_file" >&2
|
||||
}
|
||||
|
||||
source_service_is_running() {
|
||||
source_compose ps --services --status running | grep -Fxq "$1"
|
||||
}
|
||||
|
||||
source_app_container_id() {
|
||||
source_compose ps -q "$APP_SERVICE"
|
||||
}
|
||||
|
||||
docker_image_id() {
|
||||
local image="$1"
|
||||
docker image inspect -f '{{.Id}}' "$image" 2>/dev/null || true
|
||||
}
|
||||
|
||||
assert_source_and_target_images_match() {
|
||||
local target_image="$1"
|
||||
local source_container_id
|
||||
local source_image_ref
|
||||
local source_image_id
|
||||
local target_image_id
|
||||
|
||||
source_container_id="$(source_app_container_id)"
|
||||
[[ -n "$source_container_id" ]] || die "could not resolve running source app container for service: ${APP_SERVICE}"
|
||||
|
||||
source_image_ref="$(docker inspect -f '{{.Config.Image}}' "$source_container_id")"
|
||||
source_image_id="$(docker inspect -f '{{.Image}}' "$source_container_id")"
|
||||
target_image_id="$(docker_image_id "$target_image")"
|
||||
[[ -n "$target_image_id" ]] || die "target single-node image is not available locally after pull: ${target_image}"
|
||||
|
||||
log "source app image: ${source_image_ref} (${source_image_id})"
|
||||
log "target single-node image: ${target_image} (${target_image_id})"
|
||||
|
||||
if [[ "$source_image_id" != "$target_image_id" ]]; then
|
||||
die "source app image and target single-node image are different; upgrade the source PG Compose app to ${target_image} before migration"
|
||||
fi
|
||||
}
|
||||
|
||||
assert_target_copy_command_available() {
|
||||
local target_image="$1"
|
||||
local help_output
|
||||
|
||||
help_output="$(docker run --rm --entrypoint aether-gateway "$target_image" copy --help 2>&1 || true)"
|
||||
if [[ "$help_output" != *"--source-driver"* || "$help_output" != *"--target-driver"* || "$help_output" != *"--omit-request-body-details"* ]]; then
|
||||
die "target single-node image does not support direct PG-to-SQLite copy; use a matching Aether release image"
|
||||
fi
|
||||
}
|
||||
|
||||
resolve_source_network() {
|
||||
local container_id
|
||||
local network
|
||||
|
||||
container_id="$(source_compose ps -q "$POSTGRES_SERVICE")"
|
||||
[[ -n "$container_id" ]] || die "could not resolve container id for source Postgres service: ${POSTGRES_SERVICE}"
|
||||
|
||||
SOURCE_NETWORK=""
|
||||
while IFS= read -r network; do
|
||||
[[ -n "$network" ]] || continue
|
||||
SOURCE_NETWORK="$network"
|
||||
break
|
||||
done < <(docker inspect -f '{{range $name, $_ := .NetworkSettings.Networks}}{{println $name}}{{end}}' "$container_id")
|
||||
|
||||
[[ -n "$SOURCE_NETWORK" ]] || die "could not resolve Docker network for source Postgres service: ${POSTGRES_SERVICE}"
|
||||
}
|
||||
|
||||
prepare_target_compose() {
|
||||
local template_abs
|
||||
|
||||
TARGET_COMPOSE="${TARGET_COMPOSE:-${SOURCE_COMPOSE_DIR}/docker-compose.single-node.yml}"
|
||||
TARGET_COMPOSE_ABS="$(absolute_path_maybe_missing "$TARGET_COMPOSE")"
|
||||
TARGET_COMPOSE_DIR="$(dirname "$TARGET_COMPOSE_ABS")"
|
||||
template_abs="$(absolute_path "$TARGET_COMPOSE_TEMPLATE")"
|
||||
[[ -f "$template_abs" ]] || die "target compose template not found: ${TARGET_COMPOSE_TEMPLATE}"
|
||||
|
||||
mkdir -p "$TARGET_COMPOSE_DIR"
|
||||
if [[ ! -f "$TARGET_COMPOSE_ABS" || "$REPLACE_TARGET_COMPOSE" == "true" ]]; then
|
||||
log "installing target single-node compose file at ${TARGET_COMPOSE_ABS}"
|
||||
install -m 0644 "$template_abs" "$TARGET_COMPOSE_ABS"
|
||||
else
|
||||
log "keeping existing target compose file ${TARGET_COMPOSE_ABS}"
|
||||
fi
|
||||
|
||||
TARGET_ENV="${TARGET_ENV:-${TARGET_COMPOSE_DIR}/.env.single-node}"
|
||||
TARGET_ENV_ABS="$(absolute_path_maybe_missing "$TARGET_ENV")"
|
||||
}
|
||||
|
||||
target_sqlite_file_exists() {
|
||||
[[ -e "$TARGET_DB" || -e "${TARGET_DB}-wal" || -e "${TARGET_DB}-shm" ]]
|
||||
}
|
||||
|
||||
finalize_target_db() {
|
||||
local temp_db="$1"
|
||||
local target_dir
|
||||
local backup_path
|
||||
local suffix
|
||||
|
||||
target_dir="$(dirname "$TARGET_DB")"
|
||||
mkdir -p "$target_dir"
|
||||
|
||||
if target_sqlite_file_exists; then
|
||||
[[ "$REPLACE_EXISTING" == "true" ]] || die "target DB already exists: ${TARGET_DB}; pass --replace-existing to replace it"
|
||||
for suffix in "" "-wal" "-shm"; do
|
||||
if [[ -e "${TARGET_DB}${suffix}" ]]; then
|
||||
backup_path="${WORK_DIR}/$(basename "$TARGET_DB")${suffix}.backup.${NOW}"
|
||||
log "backing up existing target SQLite file to ${backup_path}"
|
||||
cp -p "${TARGET_DB}${suffix}" "$backup_path"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
log "installing migrated SQLite DB at ${TARGET_DB}"
|
||||
install -m 0640 "$temp_db" "$TARGET_DB"
|
||||
for suffix in "-wal" "-shm"; do
|
||||
if [[ -e "${temp_db}${suffix}" ]]; then
|
||||
install -m 0640 "${temp_db}${suffix}" "${TARGET_DB}${suffix}"
|
||||
else
|
||||
rm -f "${TARGET_DB}${suffix}"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
cleanup_on_exit() {
|
||||
local status=$?
|
||||
if [[ "$status" -eq 0 ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
warn "migration failed with exit status ${status}"
|
||||
if [[ "$APP_STOPPED" == "true" && "$CUTOVER_COMPLETE" != "true" && "$KEEP_SOURCE_STOPPED_ON_ERROR" != "true" ]]; then
|
||||
warn "attempting to restart source compose app because cutover did not complete"
|
||||
source_compose up -d "$APP_SERVICE" || warn "source app restart failed; check ${SOURCE_COMPOSE_ABS}"
|
||||
fi
|
||||
}
|
||||
|
||||
preflight() {
|
||||
require_command docker
|
||||
require_command awk
|
||||
require_command df
|
||||
docker compose version >/dev/null
|
||||
|
||||
SOURCE_COMPOSE_ABS="$(absolute_path "$SOURCE_COMPOSE")"
|
||||
[[ -f "$SOURCE_COMPOSE_ABS" ]] || die "source compose file not found: ${SOURCE_COMPOSE}"
|
||||
SOURCE_COMPOSE_DIR="$(dirname "$SOURCE_COMPOSE_ABS")"
|
||||
SOURCE_ENV="${SOURCE_COMPOSE_DIR}/.env"
|
||||
[[ -f "$SOURCE_ENV" ]] || die "source env file not found: ${SOURCE_ENV}"
|
||||
|
||||
NOW="$(date +%Y%m%d%H%M%S)"
|
||||
if [[ -z "$WORK_DIR" ]]; then
|
||||
WORK_DIR="${SOURCE_COMPOSE_DIR}/data/pg-compose-to-single-node-${NOW}"
|
||||
fi
|
||||
WORK_DIR="$(absolute_path_maybe_missing "$WORK_DIR")"
|
||||
mkdir -p "$WORK_DIR"
|
||||
|
||||
prepare_target_compose
|
||||
mkdir -p "${TARGET_COMPOSE_DIR}/logs"
|
||||
mkdir -p "${TARGET_COMPOSE_DIR}/data"
|
||||
TARGET_DB="${TARGET_DB:-${TARGET_COMPOSE_DIR}/data/aether.db}"
|
||||
TARGET_DB="$(absolute_path_maybe_missing "$TARGET_DB")"
|
||||
|
||||
DB_USER="$(env_file_get "$SOURCE_ENV" "DB_USER")"
|
||||
DB_USER="${DB_USER:-postgres}"
|
||||
DB_NAME="$(env_file_get "$SOURCE_ENV" "DB_NAME")"
|
||||
DB_NAME="${DB_NAME:-aether}"
|
||||
DB_PASSWORD="$(env_file_get "$SOURCE_ENV" "DB_PASSWORD")"
|
||||
DB_PASSWORD="${DB_PASSWORD:-aether}"
|
||||
|
||||
[[ -n "$(env_file_get "$SOURCE_ENV" "JWT_SECRET_KEY")" ]] || die "source env must define JWT_SECRET_KEY"
|
||||
if [[ -z "$(env_file_get "$SOURCE_ENV" "ENCRYPTION_KEY")" && -z "$(env_file_get "$SOURCE_ENV" "AETHER_GATEWAY_DATA_ENCRYPTION_KEY")" ]]; then
|
||||
die "source env must define ENCRYPTION_KEY or AETHER_GATEWAY_DATA_ENCRYPTION_KEY"
|
||||
fi
|
||||
|
||||
write_single_node_env "$TARGET_ENV_ABS"
|
||||
chmod 0600 "$TARGET_ENV_ABS"
|
||||
|
||||
log "source compose: ${SOURCE_COMPOSE_ABS}"
|
||||
log "source env: ${SOURCE_ENV}"
|
||||
log "target compose: ${TARGET_COMPOSE_ABS}"
|
||||
log "target env: ${TARGET_ENV_ABS}"
|
||||
log "target SQLite DB: ${TARGET_DB}"
|
||||
log "work dir: ${WORK_DIR}"
|
||||
|
||||
source_service_is_running "$POSTGRES_SERVICE" || die "source Postgres service is not running: ${POSTGRES_SERVICE}"
|
||||
resolve_source_network
|
||||
log "source Docker network: ${SOURCE_NETWORK}"
|
||||
}
|
||||
|
||||
source_postgres_url() {
|
||||
printf 'postgresql://%s:%s@%s:5432/%s' "$DB_USER" "$DB_PASSWORD" "$POSTGRES_SERVICE" "$DB_NAME"
|
||||
}
|
||||
|
||||
copy_source_to_sqlite() {
|
||||
local target_temp_db="$1"
|
||||
local target_url
|
||||
local image
|
||||
local -a copy_args
|
||||
|
||||
image="$(env_file_get "$TARGET_ENV_ABS" "APP_IMAGE")"
|
||||
[[ -n "$image" ]] || die "target env must define APP_IMAGE"
|
||||
target_url="sqlite:///migration/$(basename "$target_temp_db")"
|
||||
|
||||
rm -f "$target_temp_db" "${target_temp_db}-wal" "${target_temp_db}-shm"
|
||||
target_run_app "$target_url" --migrate
|
||||
copy_args=(
|
||||
copy
|
||||
--source-driver postgres
|
||||
--source-url "$(source_postgres_url)"
|
||||
--target-driver sqlite
|
||||
--target-url "$target_url"
|
||||
)
|
||||
if [[ "$REQUEST_BODY_MODE" == "omit" ]]; then
|
||||
copy_args+=(--omit-request-body-details)
|
||||
fi
|
||||
docker run --rm \
|
||||
--network "$SOURCE_NETWORK" \
|
||||
-v "${WORK_DIR}:/migration" \
|
||||
--env-file "$TARGET_ENV_ABS" \
|
||||
-e AETHER_LOG_DESTINATION=stdout \
|
||||
"$image" \
|
||||
"${copy_args[@]}"
|
||||
}
|
||||
|
||||
main() {
|
||||
local preflight_db
|
||||
local dry_run_db
|
||||
local target_temp_db
|
||||
local target_image
|
||||
|
||||
parse_args "$@"
|
||||
normalize_request_body_mode
|
||||
trap cleanup_on_exit EXIT
|
||||
preflight
|
||||
|
||||
preflight_db="${WORK_DIR}/single-node-preflight.db"
|
||||
dry_run_db="${WORK_DIR}/dry-run-target-aether.db"
|
||||
target_temp_db="${WORK_DIR}/target-aether.db"
|
||||
|
||||
if target_sqlite_file_exists && [[ "$REPLACE_EXISTING" != "true" && "$DRY_RUN" != "true" ]]; then
|
||||
die "target DB already exists: ${TARGET_DB}; pass --replace-existing to replace it"
|
||||
fi
|
||||
|
||||
log "pulling target single-node image before downtime"
|
||||
target_compose pull "$SINGLE_NODE_SERVICE"
|
||||
target_image="$(env_file_get "$TARGET_ENV_ABS" "APP_IMAGE")"
|
||||
[[ -n "$target_image" ]] || die "target env must define APP_IMAGE"
|
||||
|
||||
log "checking target image copy command is available"
|
||||
assert_target_copy_command_available "$target_image"
|
||||
|
||||
log "checking source app image matches target single-node image"
|
||||
assert_source_and_target_images_match "$target_image"
|
||||
|
||||
log "preflighting target SQLite schema migration"
|
||||
rm -f "$preflight_db" "${preflight_db}-wal" "${preflight_db}-shm"
|
||||
target_run_app "sqlite:///migration/$(basename "$preflight_db")" --migrate
|
||||
|
||||
log "checking request body detail policy"
|
||||
check_request_body_artifacts
|
||||
|
||||
log "checking available disk space before copy"
|
||||
check_disk_space
|
||||
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
warn "dry-run copy happens while the source app may still be writing; use only for rehearsal"
|
||||
log "copying source Postgres tables directly into dry-run SQLite target"
|
||||
copy_source_to_sqlite "$dry_run_db"
|
||||
log "dry run complete; temporary SQLite DB is ${dry_run_db}"
|
||||
return
|
||||
fi
|
||||
|
||||
log "stopping source app service; Postgres and Redis stay running"
|
||||
source_compose stop "$APP_SERVICE"
|
||||
APP_STOPPED="true"
|
||||
|
||||
log "checking request body detail policy again after the app has stopped"
|
||||
check_request_body_artifacts
|
||||
|
||||
log "copying source Postgres tables directly into temporary SQLite DB"
|
||||
copy_source_to_sqlite "$target_temp_db"
|
||||
|
||||
finalize_target_db "$target_temp_db"
|
||||
|
||||
log "removing stopped source app container to free the app container name"
|
||||
source_compose rm -f "$APP_SERVICE"
|
||||
|
||||
log "starting single-node compose app"
|
||||
target_compose up -d "$SINGLE_NODE_SERVICE"
|
||||
CUTOVER_COMPLETE="true"
|
||||
|
||||
log "migration complete"
|
||||
log "source Postgres/Redis volumes were left in place for rollback"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user