Add multi-database data layer

Introduce aether-data-schema and driver-specific schema generation for Postgres, MySQL, and SQLite.

Split data backends, lifecycle, repositories, and gateway runtime integration across database drivers.

Verified with cargo fmt --all --check, cargo clippy --workspace --all-targets -- -D warnings, and cargo test --workspace.

crates/aether-data-schema/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "aether-data-schema"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+description = "Logical schema parser and SQL generator for Aether data storage"
+
+[dependencies]
+clap = { version = "4", features = ["derive"] }
+serde = { workspace = true, features = ["derive"] }
+thiserror.workspace = true
+toml = "0.8"

crates/aether-data-schema/README.md

@@ -0,0 +1,76 @@
+# aether-data-schema
+
+`aether-data-schema` is the logical schema generator for `aether-data`.
+
+It owns:
+
+- parsing `crates/aether-data/schema/logical/*.toml`
+- validating table, column, index, unique constraint, and foreign-key metadata
+- emitting driver-specific DDL for Postgres, MySQL, and SQLite
+- checking that generated schema artifacts are current
+- cleaning generated driver baseline directories before writing fresh output
+
+It does not own runtime migrations or repository SQL. `aether-data` still owns
+`sqlx::migrate!`, backfills, export/import, and repository implementations.
+
+## Commands
+
+From the workspace root:
+
+```bash
+cargo run -p aether-data-schema --bin aether-schema -- check
+cargo run -p aether-data-schema --bin aether-schema -- generate
+cargo run -p aether-data-schema --bin aether-schema -- print --driver postgres
+cargo run -p aether-data-schema --bin aether-schema -- print --driver mysql
+cargo run -p aether-data-schema --bin aether-schema -- print --driver sqlite
+```
+
+The normal `aether-data` maintenance entrypoint wraps these:
+
+```bash
+bash crates/aether-data/schema/compose_schema.sh generate
+bash crates/aether-data/schema/compose_schema.sh check
+```
+
+## Source And Output
+
+Input:
+
+```text
+crates/aether-data/schema/logical/*.toml
+```
+
+Output:
+
+```text
+crates/aether-data/schema/generated/README.md
+crates/aether-data/schema/generated/{postgres,mysql,sqlite}/baseline/
+```
+
+Each logical TOML file becomes one generated `.sql` file per driver, and each
+driver output directory gets a generated `manifest.txt`.
+
+`generated/**` is machine-written. The generated directory README and each
+generated SQL/manifest file state that the content should not be edited by hand.
+Edit `logical/*.toml` instead.
+
+The generator writes one manifest per driver output directory. A stale README,
+stale generated file, or extra generated driver file is treated as an error by
+`aether-schema check`.
+
+## Current Coverage
+
+Logical schema now covers the clean baseline table set plus MySQL/SQLite table
+creation migrations:
+
+- identity, API keys, audit logs, announcements, management tokens, preferences,
+  and sessions
+- provider catalog, provider keys/endpoints, model catalog, request candidates,
+  Gemini file mappings, and video tasks
+- auth config, OAuth providers, LDAP config, and OAuth links
+- proxy nodes and proxy events
+- wallet, payment, refund, redeem-code, and settlement snapshot tables
+- usage capture and portable stats aggregation tables
+
+Postgres-only historical follow-up migrations still live in driver-specific SQL
+until their shape is normalized or intentionally kept as overrides.

crates/aether-data-schema/src/bin/aether-schema.rs

@@ -0,0 +1,76 @@
+use std::path::PathBuf;
+
+use aether_data_schema::dialect::{mysql, postgres, sqlite};
+use aether_data_schema::{check_generated_dir, generate_loaded_to_dir, load_schema_sources};
+use clap::{Parser, Subcommand, ValueEnum};
+
+#[derive(Debug, Parser)]
+#[command(name = "aether-schema")]
+#[command(about = "Generate SQL from Aether logical schema definitions")]
+struct Cli {
+    #[command(subcommand)]
+    command: Command,
+}
+
+#[derive(Debug, Subcommand)]
+enum Command {
+    Generate {
+        #[arg(long, default_value = "crates/aether-data/schema/logical")]
+        schema_dir: PathBuf,
+        #[arg(long, default_value = "crates/aether-data/schema/generated")]
+        output_dir: PathBuf,
+    },
+    Check {
+        #[arg(long, default_value = "crates/aether-data/schema/logical")]
+        schema_dir: PathBuf,
+        #[arg(long, default_value = "crates/aether-data/schema/generated")]
+        output_dir: PathBuf,
+        #[arg(long = "require-tables-from")]
+        require_tables_from: Vec<PathBuf>,
+    },
+    Print {
+        #[arg(long, default_value = "crates/aether-data/schema/logical")]
+        schema_dir: PathBuf,
+        #[arg(long)]
+        driver: Driver,
+    },
+}
+
+#[derive(Debug, Clone, Copy, ValueEnum)]
+enum Driver {
+    Postgres,
+    Mysql,
+    Sqlite,
+}
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let cli = Cli::parse();
+    match cli.command {
+        Command::Generate {
+            schema_dir,
+            output_dir,
+        } => {
+            let loaded = load_schema_sources(schema_dir)?;
+            generate_loaded_to_dir(&loaded, output_dir)?;
+        }
+        Command::Check {
+            schema_dir,
+            output_dir,
+            require_tables_from,
+        } => {
+            let loaded = load_schema_sources(schema_dir)?;
+            check_generated_dir(&loaded, output_dir)?;
+            aether_data_schema::check_required_tables(&loaded.schema, &require_tables_from)?;
+        }
+        Command::Print { schema_dir, driver } => {
+            let schema = load_schema_sources(schema_dir)?.schema;
+            let output = match driver {
+                Driver::Postgres => postgres::emit_schema(&schema),
+                Driver::Mysql => mysql::emit_schema(&schema),
+                Driver::Sqlite => sqlite::emit_schema(&schema),
+            };
+            print!("{output}");
+        }
+    }
+    Ok(())
+}

crates/aether-data-schema/src/dialect/mod.rs

@@ -0,0 +1,148 @@
+use crate::{Column, DefaultValue, DriverColumnOverride, LogicalType, ReferentialAction};
+
+pub mod mysql;
+pub mod postgres;
+pub mod sqlite;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Dialect {
+    Postgres,
+    Mysql,
+    Sqlite,
+}
+
+impl Dialect {
+    pub const fn as_str(self) -> &'static str {
+        match self {
+            Self::Postgres => "postgres",
+            Self::Mysql => "mysql",
+            Self::Sqlite => "sqlite",
+        }
+    }
+}
+
+fn quote_string(value: &str) -> String {
+    format!("'{}'", value.replace('\'', "''"))
+}
+
+fn default_sql(default: &DefaultValue) -> String {
+    match default {
+        DefaultValue::String(value) => quote_string(value),
+        DefaultValue::Integer(value) => value.to_string(),
+        DefaultValue::Bool(value) => {
+            if *value {
+                "1".to_string()
+            } else {
+                "0".to_string()
+            }
+        }
+        DefaultValue::Raw { raw } => raw.clone(),
+    }
+}
+
+fn default_sql_bool_keywords(default: &DefaultValue) -> String {
+    match default {
+        DefaultValue::Bool(true) => "true".to_string(),
+        DefaultValue::Bool(false) => "false".to_string(),
+        _ => default_sql(default),
+    }
+}
+
+fn column_default<'a>(
+    column: &'a Column,
+    override_: Option<&'a DriverColumnOverride>,
+) -> Option<&'a DefaultValue> {
+    override_
+        .and_then(|override_| override_.default.as_ref())
+        .or(column.default.as_ref())
+}
+
+fn column_nullable(column: &Column, override_: Option<&DriverColumnOverride>) -> bool {
+    override_
+        .and_then(|override_| override_.nullable)
+        .unwrap_or(column.nullable)
+}
+
+fn override_type(override_: Option<&DriverColumnOverride>) -> Option<&str> {
+    override_?.sql_type.as_deref()
+}
+
+fn postgres_type(column: &Column) -> String {
+    let override_ = column.driver.postgres.as_ref();
+    if let Some(sql_type) = override_type(override_) {
+        return sql_type.to_string();
+    }
+    if column.auto_increment {
+        return "bigserial".to_string();
+    }
+    match column.logical_type {
+        LogicalType::TextId | LogicalType::Text => match column.length {
+            Some(length) => format!("character varying({length})"),
+            None => "text".to_string(),
+        },
+        LogicalType::LongText => "text".to_string(),
+        LogicalType::Bool => "boolean".to_string(),
+        LogicalType::Int32 => "integer".to_string(),
+        LogicalType::Int64 | LogicalType::UnixSeconds | LogicalType::UnixMillis => {
+            "bigint".to_string()
+        }
+        LogicalType::Float64 => "double precision".to_string(),
+        LogicalType::DecimalMoney => "numeric".to_string(),
+        LogicalType::Timestamp => "timestamp with time zone".to_string(),
+        LogicalType::Json => "jsonb".to_string(),
+        LogicalType::Bytes => "bytea".to_string(),
+    }
+}
+
+fn mysql_type(column: &Column) -> String {
+    let override_ = column.driver.mysql.as_ref();
+    if let Some(sql_type) = override_type(override_) {
+        return sql_type.to_string();
+    }
+    match column.logical_type {
+        LogicalType::TextId | LogicalType::Text => match column.length {
+            Some(length) => format!("VARCHAR({length})"),
+            None => "TEXT".to_string(),
+        },
+        LogicalType::LongText => "LONGTEXT".to_string(),
+        LogicalType::Bool => "TINYINT(1)".to_string(),
+        LogicalType::Int32 => "INT".to_string(),
+        LogicalType::Int64 | LogicalType::UnixSeconds | LogicalType::UnixMillis => {
+            "BIGINT".to_string()
+        }
+        LogicalType::Float64 => "DOUBLE".to_string(),
+        LogicalType::DecimalMoney => "DECIMAL(20,8)".to_string(),
+        LogicalType::Timestamp => "BIGINT".to_string(),
+        LogicalType::Json => "JSON".to_string(),
+        LogicalType::Bytes => "LONGBLOB".to_string(),
+    }
+}
+
+fn sqlite_type(column: &Column) -> String {
+    let override_ = column.driver.sqlite.as_ref();
+    if let Some(sql_type) = override_type(override_) {
+        return sql_type.to_string();
+    }
+    match column.logical_type {
+        LogicalType::TextId | LogicalType::Text | LogicalType::LongText | LogicalType::Json => {
+            "TEXT".to_string()
+        }
+        LogicalType::Bool
+        | LogicalType::Int32
+        | LogicalType::Int64
+        | LogicalType::UnixSeconds
+        | LogicalType::UnixMillis
+        | LogicalType::Timestamp => "INTEGER".to_string(),
+        LogicalType::Float64 | LogicalType::DecimalMoney => "REAL".to_string(),
+        LogicalType::Bytes => "BLOB".to_string(),
+    }
+}
+
+fn referential_action_sql(action: &ReferentialAction) -> &'static str {
+    match action {
+        ReferentialAction::Cascade => "CASCADE",
+        ReferentialAction::SetNull => "SET NULL",
+        ReferentialAction::Restrict => "RESTRICT",
+        ReferentialAction::NoAction => "NO ACTION",
+    }
+}

crates/aether-data-schema/src/dialect/mysql.rs

@@ -0,0 +1,117 @@
+use crate::dialect::{
+    column_default, column_nullable, default_sql, mysql_type, referential_action_sql,
+};
+use crate::LogicalSchema;
+
+pub fn emit_schema(schema: &LogicalSchema) -> String {
+    emit_named_schema(schema, &schema.ordered_table_names())
+}
+
+pub fn emit_named_schema(schema: &LogicalSchema, table_names: &[String]) -> String {
+    let mut out = String::new();
+    for table_name in table_names {
+        let table = schema
+            .tables
+            .get(table_name)
+            .expect("named schema table should exist");
+        let mut definitions = Vec::new();
+        for column in &table.columns {
+            let mut definition = format!("    `{}` {}", column.name, mysql_type(column));
+            if !column_nullable(column, column.driver.mysql.as_ref()) {
+                definition.push_str(" NOT NULL");
+            }
+            if column.auto_increment {
+                definition.push_str(" AUTO_INCREMENT");
+            }
+            if let Some(default) = column_default(column, column.driver.mysql.as_ref()) {
+                definition.push_str(" DEFAULT ");
+                definition.push_str(&default_sql(default));
+            }
+            definitions.push(definition);
+        }
+        if !table.primary_key.is_empty() {
+            definitions.push(format!(
+                "    PRIMARY KEY ({})",
+                table
+                    .primary_key
+                    .iter()
+                    .map(|column| format!("`{column}`"))
+                    .collect::<Vec<_>>()
+                    .join(", ")
+            ));
+        }
+        for unique in &table.uniques {
+            definitions.push(format!(
+                "    UNIQUE KEY {} ({})",
+                unique.name,
+                unique
+                    .columns
+                    .iter()
+                    .map(|column| format!("`{column}`"))
+                    .collect::<Vec<_>>()
+                    .join(", ")
+            ));
+        }
+        for index in &table.indexes {
+            let unique = if index.unique { "UNIQUE " } else { "" };
+            definitions.push(format!(
+                "    {unique}KEY {} ({})",
+                index.name,
+                index
+                    .columns
+                    .iter()
+                    .map(|column| format!("`{column}`"))
+                    .collect::<Vec<_>>()
+                    .join(", ")
+            ));
+        }
+        for foreign_key in &table.foreign_keys {
+            let mut definition = format!(
+                "    CONSTRAINT {} FOREIGN KEY ({}) REFERENCES {} ({})",
+                foreign_key.name,
+                foreign_key
+                    .columns
+                    .iter()
+                    .map(|column| format!("`{column}`"))
+                    .collect::<Vec<_>>()
+                    .join(", "),
+                foreign_key.references_table,
+                foreign_key
+                    .references_columns
+                    .iter()
+                    .map(|column| format!("`{column}`"))
+                    .collect::<Vec<_>>()
+                    .join(", ")
+            );
+            if let Some(action) = &foreign_key.on_delete {
+                definition.push_str(" ON DELETE ");
+                definition.push_str(referential_action_sql(action));
+            }
+            definitions.push(definition);
+        }
+
+        out.push_str(&format!(
+            "CREATE TABLE IF NOT EXISTS {} (\n",
+            quote_identifier_if_needed(table_name)
+        ));
+        out.push_str(&definitions.join(",\n"));
+        out.push_str("\n);\n\n");
+    }
+    out
+}
+
+fn quote_identifier_if_needed(identifier: &str) -> String {
+    if needs_quoting(identifier) {
+        quote_identifier(identifier)
+    } else {
+        identifier.to_string()
+    }
+}
+
+fn needs_quoting(identifier: &str) -> bool {
+    matches!(identifier, "date" | "usage")
+}
+
+fn quote_identifier(identifier: &str) -> String {
+    format!("`{}`", identifier.replace('`', "``"))
+}

crates/aether-data-schema/src/dialect/postgres.rs

@@ -0,0 +1,81 @@
+use crate::dialect::{
+    column_default, column_nullable, default_sql_bool_keywords, postgres_type,
+    referential_action_sql,
+};
+use crate::LogicalSchema;
+
+pub fn emit_schema(schema: &LogicalSchema) -> String {
+    emit_named_schema(schema, &schema.ordered_table_names())
+}
+
+pub fn emit_named_schema(schema: &LogicalSchema, table_names: &[String]) -> String {
+    let mut out = String::new();
+    for table_name in table_names {
+        let table = schema
+            .tables
+            .get(table_name)
+            .expect("named schema table should exist");
+        out.push_str(&format!(
+            "CREATE TABLE IF NOT EXISTS public.{table_name} (\n"
+        ));
+        for (index, column) in table.columns.iter().enumerate() {
+            let comma = if index + 1 == table.columns.len() {
+                ""
+            } else {
+                ","
+            };
+            out.push_str("    ");
+            out.push_str(&column.name);
+            out.push(' ');
+            out.push_str(&postgres_type(column));
+            if let Some(default) = column_default(column, column.driver.postgres.as_ref()) {
+                out.push_str(" DEFAULT ");
+                out.push_str(&default_sql_bool_keywords(default));
+            }
+            if !column_nullable(column, column.driver.postgres.as_ref()) {
+                out.push_str(" NOT NULL");
+            }
+            out.push_str(comma);
+            out.push('\n');
+        }
+        out.push_str(");\n\n");
+
+        if !table.primary_key.is_empty() {
+            out.push_str(&format!(
+                "ALTER TABLE ONLY public.{table_name} ADD CONSTRAINT {table_name}_pkey PRIMARY KEY ({});\n",
+                table.primary_key.join(", ")
+            ));
+        }
+        for unique in &table.uniques {
+            out.push_str(&format!(
+                "ALTER TABLE ONLY public.{table_name} ADD CONSTRAINT {} UNIQUE ({});\n",
+                unique.name,
+                unique.columns.join(", ")
+            ));
+        }
+        for index in &table.indexes {
+            let unique = if index.unique { "UNIQUE " } else { "" };
+            out.push_str(&format!(
+                "CREATE {unique}INDEX IF NOT EXISTS {} ON public.{table_name} USING btree ({});\n",
+                index.name,
+                index.columns.join(", ")
+            ));
+        }
+        for foreign_key in &table.foreign_keys {
+            out.push_str(&format!(
+                "ALTER TABLE ONLY public.{table_name} ADD CONSTRAINT {} FOREIGN KEY ({}) REFERENCES public.{}({})",
+                foreign_key.name,
+                foreign_key.columns.join(", "),
+                foreign_key.references_table,
+                foreign_key.references_columns.join(", ")
+            ));
+            if let Some(action) = &foreign_key.on_delete {
+                out.push_str(" ON DELETE ");
+                out.push_str(referential_action_sql(action));
+            }
+            out.push_str(";\n");
+        }
+        out.push('\n');
+    }
+    out
+}

crates/aether-data-schema/src/dialect/sqlite.rs

@@ -0,0 +1,92 @@
+use crate::dialect::{
+    column_default, column_nullable, default_sql, referential_action_sql, sqlite_type,
+};
+use crate::LogicalSchema;
+
+pub fn emit_schema(schema: &LogicalSchema) -> String {
+    emit_named_schema(schema, &schema.ordered_table_names())
+}
+
+pub fn emit_named_schema(schema: &LogicalSchema, table_names: &[String]) -> String {
+    let mut out = String::new();
+    for table_name in table_names {
+        let table = schema
+            .tables
+            .get(table_name)
+            .expect("named schema table should exist");
+        let mut definitions = Vec::new();
+        for column in &table.columns {
+            let mut definition = format!("    {} {}", column.name, sqlite_type(column));
+            if table.primary_key.len() == 1 && table.primary_key[0] == column.name {
+                definition.push_str(" PRIMARY KEY");
+                if column.auto_increment {
+                    definition.push_str(" AUTOINCREMENT");
+                }
+            }
+            if !column.auto_increment && !column_nullable(column, column.driver.sqlite.as_ref()) {
+                definition.push_str(" NOT NULL");
+            }
+            if let Some(default) = column_default(column, column.driver.sqlite.as_ref()) {
+                definition.push_str(" DEFAULT ");
+                definition.push_str(&default_sql(default));
+            }
+            definitions.push(definition);
+        }
+        if table.primary_key.len() > 1 {
+            definitions.push(format!(
+                "    PRIMARY KEY ({})",
+                table.primary_key.join(", ")
+            ));
+        }
+        for unique in &table.uniques {
+            definitions.push(format!("    UNIQUE ({})", unique.columns.join(", ")));
+        }
+        for foreign_key in &table.foreign_keys {
+            let mut definition = format!(
+                "    CONSTRAINT {} FOREIGN KEY ({}) REFERENCES {} ({})",
+                foreign_key.name,
+                foreign_key.columns.join(", "),
+                foreign_key.references_table,
+                foreign_key.references_columns.join(", ")
+            );
+            if let Some(action) = &foreign_key.on_delete {
+                definition.push_str(" ON DELETE ");
+                definition.push_str(referential_action_sql(action));
+            }
+            definitions.push(definition);
+        }
+
+        let quoted_table_name = quote_identifier_if_needed(table_name);
+        out.push_str(&format!(
+            "CREATE TABLE IF NOT EXISTS {quoted_table_name} (\n"
+        ));
+        out.push_str(&definitions.join(",\n"));
+        out.push_str("\n);\n");
+        for index in &table.indexes {
+            let unique = if index.unique { "UNIQUE " } else { "" };
+            out.push_str(&format!(
+                "CREATE {unique}INDEX IF NOT EXISTS {} ON {quoted_table_name} ({});\n",
+                index.name,
+                index.columns.join(", ")
+            ));
+        }
+        out.push('\n');
+    }
+    out
+}
+
+fn quote_identifier_if_needed(identifier: &str) -> String {
+    if needs_quoting(identifier) {
+        quote_identifier(identifier)
+    } else {
+        identifier.to_string()
+    }
+}
+
+fn needs_quoting(identifier: &str) -> bool {
+    matches!(identifier, "date" | "usage")
+}
+
+fn quote_identifier(identifier: &str) -> String {
+    format!("\"{}\"", identifier.replace('"', "\"\""))
+}