i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 17:30:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(gateway): 重构 usage 数据层、迁移系统与系统导入

Browse Source 
数据库迁移:
- 引入 baseline v2 bootstrap，空库首次启动自动初始化
- 服务启动不再自动执行迁移，需显式 `--migrate` 运行
- 新增 pending migration 检测，schema 落后时拒绝启动

Usage 数据层:
- usage body 存储外部化为独立 blob 表
- 新增 HTTP audit 表拆分存储请求/响应头与 body ref
- 后台清理任务支持 legacy body ref 元数据迁移
- usage runtime 写入迁移到专用 tokio runtime（独立线程池, 8MB 栈）

系统导入/导出:
- 支持用户、API Keys、钱包数据的完整导入
- 兼容 legacy 与 v1.3+ 两种导出格式

其他改进:
- executor outcome 增加 runtime miss 诊断上下文
- 主 tokio runtime 栈大小调整为 8MB
- 前端 provider 管理支持 base URL 配置
- dev.sh 支持 --migrate 参数
This commit is contained in:
fawney19
2026-04-13 14:01:22 +08:00
parent 3698e5a833
commit 5bb08e6aa4
106 changed files with 21736 additions and 1529 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
crates/aether-data/src/repository/settlement/memory.rs
View File

@@ -46,6 +46,7 @@ impl InMemorySettlementWalletStore {
pub struct InMemorySettlementRepository {
wallets: InMemorySettlementWalletStore,
provider_monthly_used: RwLock<BTreeMap<String, f64>>,
settlements: RwLock<BTreeMap<String, StoredUsageSettlement>>,
}
impl InMemorySettlementRepository {
@@ -56,6 +57,7 @@ impl InMemorySettlementRepository {
Self {
wallets: InMemorySettlementWalletStore::seeded(items),
provider_monthly_used: RwLock::new(BTreeMap::new()),
settlements: RwLock::new(BTreeMap::new()),
}
}
@@ -63,6 +65,7 @@ impl InMemorySettlementRepository {
Self {
wallets: InMemorySettlementWalletStore::Shared(wallet_repository),
provider_monthly_used: RwLock::new(BTreeMap::new()),
settlements: RwLock::new(BTreeMap::new()),
}
}
}
@@ -75,7 +78,13 @@ impl SettlementWriteRepository for InMemorySettlementRepository {
) -> Result<Option<StoredUsageSettlement>, DataLayerError> {
input.validate()?;
if input.billing_status != "pending" {
return Ok(Some(StoredUsageSettlement {
let existing = self
.settlements
.read()
.expect("settlement snapshot lock")
.get(&input.request_id)
.cloned();
return Ok(Some(existing.unwrap_or(StoredUsageSettlement {
request_id: input.request_id,
wallet_id: None,
billing_status: input.billing_status,
@@ -87,7 +96,7 @@ impl SettlementWriteRepository for InMemorySettlementRepository {
wallet_gift_balance_after: None,
provider_monthly_used_usd: None,
finalized_at_unix_secs: input.finalized_at_unix_secs,
}));
})));
}
let final_billing_status = if input.status == "completed" {
@@ -172,6 +181,11 @@ impl SettlementWriteRepository for InMemorySettlementRepository {
}
}
self.settlements
.write()
.expect("settlement snapshot lock")
.insert(settlement.request_id.clone(), settlement.clone());
Ok(Some(settlement))
}
}
@@ -225,4 +239,42 @@ mod tests {
assert_eq!(settlement.wallet_balance_after, Some(9.0));
assert_eq!(settlement.provider_monthly_used_usd, Some(1.5));
}
#[tokio::test]
async fn returns_stored_snapshot_when_usage_is_already_finalized() {
let repository = InMemorySettlementRepository::seed(vec![sample_wallet()]);
let settled = repository
.settle_usage(UsageSettlementInput {
request_id: "req-2".to_string(),
user_id: Some("user-1".to_string()),
api_key_id: Some("key-1".to_string()),
provider_id: Some("provider-1".to_string()),
status: "completed".to_string(),
billing_status: "pending".to_string(),
total_cost_usd: 2.0,
actual_total_cost_usd: 1.0,
finalized_at_unix_secs: Some(250),
})
.await
.expect("settlement should succeed")
.expect("settlement should exist");
let replay = repository
.settle_usage(UsageSettlementInput {
request_id: "req-2".to_string(),
user_id: Some("user-1".to_string()),
api_key_id: Some("key-1".to_string()),
provider_id: Some("provider-1".to_string()),
status: "completed".to_string(),
billing_status: "settled".to_string(),
total_cost_usd: 2.0,
actual_total_cost_usd: 1.0,
finalized_at_unix_secs: Some(250),
})
.await
.expect("replay should succeed")
.expect("snapshot should exist");
assert_eq!(replay, settled);
}
}

272
crates/aether-data/src/repository/settlement/sql.rs
View File

@@ -6,6 +6,49 @@ use crate::error::SqlxResultExt;
use crate::postgres::PostgresTransactionRunner;
use crate::DataLayerError;
const FIND_USAGE_FOR_SETTLEMENT_SQL: &str = r#"
SELECT
usage_record.request_id,
COALESCE(usage_settlement_snapshots.wallet_id, usage_record.wallet_id) AS wallet_id,
usage_record.billing_status,
COALESCE(
CAST(usage_settlement_snapshots.wallet_balance_before AS DOUBLE PRECISION),
CAST(usage_record.wallet_balance_before AS DOUBLE PRECISION)
) AS wallet_balance_before,
COALESCE(
CAST(usage_settlement_snapshots.wallet_balance_after AS DOUBLE PRECISION),
CAST(usage_record.wallet_balance_after AS DOUBLE PRECISION)
) AS wallet_balance_after,
COALESCE(
CAST(usage_settlement_snapshots.wallet_recharge_balance_before AS DOUBLE PRECISION),
CAST(usage_record.wallet_recharge_balance_before AS DOUBLE PRECISION)
) AS wallet_recharge_balance_before,
COALESCE(
CAST(usage_settlement_snapshots.wallet_recharge_balance_after AS DOUBLE PRECISION),
CAST(usage_record.wallet_recharge_balance_after AS DOUBLE PRECISION)
) AS wallet_recharge_balance_after,
COALESCE(
CAST(usage_settlement_snapshots.wallet_gift_balance_before AS DOUBLE PRECISION),
CAST(usage_record.wallet_gift_balance_before AS DOUBLE PRECISION)
) AS wallet_gift_balance_before,
COALESCE(
CAST(usage_settlement_snapshots.wallet_gift_balance_after AS DOUBLE PRECISION),
CAST(usage_record.wallet_gift_balance_after AS DOUBLE PRECISION)
) AS wallet_gift_balance_after,
CAST(usage_settlement_snapshots.provider_monthly_used_usd AS DOUBLE PRECISION) AS provider_monthly_used_usd,
usage_record.provider_id,
CAST(
EXTRACT(
EPOCH FROM COALESCE(usage_settlement_snapshots.finalized_at, usage_record.finalized_at)
) AS BIGINT
) AS finalized_at_unix_secs
FROM "usage" AS usage_record
LEFT JOIN usage_settlement_snapshots
ON usage_settlement_snapshots.request_id = usage_record.request_id
WHERE usage_record.request_id = $1
FOR UPDATE OF usage_record
"#;
const FINALIZE_USAGE_BILLING_SQL: &str = r#"
UPDATE "usage"
SET
@@ -14,6 +57,71 @@ SET
WHERE request_id = $1
"#;
const UPSERT_USAGE_SETTLEMENT_SNAPSHOT_SQL: &str = r#"
INSERT INTO usage_settlement_snapshots (
request_id,
billing_status,
wallet_id,
wallet_balance_before,
wallet_balance_after,
wallet_recharge_balance_before,
wallet_recharge_balance_after,
wallet_gift_balance_before,
wallet_gift_balance_after,
provider_monthly_used_usd,
finalized_at
) VALUES (
$1,
$2,
$3,
$4,
$5,
$6,
$7,
$8,
$9,
$10,
CASE
WHEN $11 IS NULL THEN NULL
ELSE TO_TIMESTAMP($11::double precision)
END
)
ON CONFLICT (request_id)
DO UPDATE SET
billing_status = EXCLUDED.billing_status,
wallet_id = COALESCE(EXCLUDED.wallet_id, usage_settlement_snapshots.wallet_id),
wallet_balance_before = COALESCE(
EXCLUDED.wallet_balance_before,
usage_settlement_snapshots.wallet_balance_before
),
wallet_balance_after = COALESCE(
EXCLUDED.wallet_balance_after,
usage_settlement_snapshots.wallet_balance_after
),
wallet_recharge_balance_before = COALESCE(
EXCLUDED.wallet_recharge_balance_before,
usage_settlement_snapshots.wallet_recharge_balance_before
),
wallet_recharge_balance_after = COALESCE(
EXCLUDED.wallet_recharge_balance_after,
usage_settlement_snapshots.wallet_recharge_balance_after
),
wallet_gift_balance_before = COALESCE(
EXCLUDED.wallet_gift_balance_before,
usage_settlement_snapshots.wallet_gift_balance_before
),
wallet_gift_balance_after = COALESCE(
EXCLUDED.wallet_gift_balance_after,
usage_settlement_snapshots.wallet_gift_balance_after
),
provider_monthly_used_usd = COALESCE(
EXCLUDED.provider_monthly_used_usd,
usage_settlement_snapshots.provider_monthly_used_usd
),
finalized_at = COALESCE(EXCLUDED.finalized_at, usage_settlement_snapshots.finalized_at),
updated_at = NOW()
"#;
#[derive(Debug, Clone)]
pub struct SqlxSettlementRepository {
tx_runner: PostgresTransactionRunner,
@@ -26,6 +134,62 @@ impl SqlxSettlementRepository {
}
}
fn settlement_from_row(
row: &sqlx::postgres::PgRow,
) -> Result<StoredUsageSettlement, DataLayerError> {
Ok(StoredUsageSettlement {
request_id: row.try_get("request_id").map_postgres_err()?,
wallet_id: row.try_get("wallet_id").map_postgres_err()?,
billing_status: row.try_get("billing_status").map_postgres_err()?,
wallet_balance_before: row.try_get("wallet_balance_before").map_postgres_err()?,
wallet_balance_after: row.try_get("wallet_balance_after").map_postgres_err()?,
wallet_recharge_balance_before: row
.try_get("wallet_recharge_balance_before")
.map_postgres_err()?,
wallet_recharge_balance_after: row
.try_get("wallet_recharge_balance_after")
.map_postgres_err()?,
wallet_gift_balance_before: row
.try_get("wallet_gift_balance_before")
.map_postgres_err()?,
wallet_gift_balance_after: row
.try_get("wallet_gift_balance_after")
.map_postgres_err()?,
provider_monthly_used_usd: row
.try_get("provider_monthly_used_usd")
.map_postgres_err()?,
finalized_at_unix_secs: row
.try_get::<Option<i64>, _>("finalized_at_unix_secs")
.map_postgres_err()?
.map(|value| value as u64),
})
}
async fn sync_usage_settlement_snapshot<'e, E>(
executor: E,
settlement: &StoredUsageSettlement,
) -> Result<(), DataLayerError>
where
E: sqlx::Executor<'e, Database = sqlx::Postgres>,
{
sqlx::query(UPSERT_USAGE_SETTLEMENT_SNAPSHOT_SQL)
.bind(&settlement.request_id)
.bind(&settlement.billing_status)
.bind(settlement.wallet_id.as_deref())
.bind(settlement.wallet_balance_before)
.bind(settlement.wallet_balance_after)
.bind(settlement.wallet_recharge_balance_before)
.bind(settlement.wallet_recharge_balance_after)
.bind(settlement.wallet_gift_balance_before)
.bind(settlement.wallet_gift_balance_after)
.bind(settlement.provider_monthly_used_usd)
.bind(settlement.finalized_at_unix_secs.map(|value| value as f64))
.execute(executor)
.await
.map_postgres_err()?;
Ok(())
}
#[async_trait]
impl SettlementWriteRepository for SqlxSettlementRepository {
async fn settle_usage(
@@ -36,29 +200,11 @@ impl SettlementWriteRepository for SqlxSettlementRepository {
self.tx_runner
.run_read_write(|tx| {
Box::pin(async move {
let row = sqlx::query(
r#"
SELECT
request_id,
wallet_id,
billing_status,
CAST(wallet_balance_before AS DOUBLE PRECISION) AS wallet_balance_before,
CAST(wallet_balance_after AS DOUBLE PRECISION) AS wallet_balance_after,
CAST(wallet_recharge_balance_before AS DOUBLE PRECISION) AS wallet_recharge_balance_before,
CAST(wallet_recharge_balance_after AS DOUBLE PRECISION) AS wallet_recharge_balance_after,
CAST(wallet_gift_balance_before AS DOUBLE PRECISION) AS wallet_gift_balance_before,
CAST(wallet_gift_balance_after AS DOUBLE PRECISION) AS wallet_gift_balance_after,
provider_id,
CAST(EXTRACT(EPOCH FROM finalized_at) AS BIGINT) AS finalized_at_unix_secs
FROM "usage"
WHERE request_id = $1
FOR UPDATE
"#,
)
.bind(&input.request_id)
.fetch_optional(&mut **tx)
.await
.map_postgres_err()?;
let row = sqlx::query(FIND_USAGE_FOR_SETTLEMENT_SQL)
.bind(&input.request_id)
.fetch_optional(&mut **tx)
.await
.map_postgres_err()?;
let Some(usage_row) = row else {
return Ok(None);
@@ -67,34 +213,7 @@ FOR UPDATE
let current_billing_status: String =
usage_row.try_get("billing_status").map_postgres_err()?;
if current_billing_status == "settled" || current_billing_status == "void" {
return Ok(Some(StoredUsageSettlement {
request_id: usage_row.try_get("request_id").map_postgres_err()?,
wallet_id: usage_row.try_get("wallet_id").map_postgres_err()?,
billing_status: current_billing_status,
wallet_balance_before: usage_row
.try_get("wallet_balance_before")
.map_postgres_err()?,
wallet_balance_after: usage_row
.try_get("wallet_balance_after")
.map_postgres_err()?,
wallet_recharge_balance_before: usage_row
.try_get("wallet_recharge_balance_before")
.map_postgres_err()?,
wallet_recharge_balance_after: usage_row
.try_get("wallet_recharge_balance_after")
.map_postgres_err()?,
wallet_gift_balance_before: usage_row
.try_get("wallet_gift_balance_before")
.map_postgres_err()?,
wallet_gift_balance_after: usage_row
.try_get("wallet_gift_balance_after")
.map_postgres_err()?,
provider_monthly_used_usd: None,
finalized_at_unix_secs: usage_row
.try_get::<Option<i64>, _>("finalized_at_unix_secs")
.map_postgres_err()?
.map(|value| value as u64),
}));
return settlement_from_row(&usage_row).map(Some);
}
let final_billing_status = if input.status == "completed" {
@@ -223,32 +342,6 @@ WHERE id = $1
settlement.wallet_recharge_balance_after = Some(after_recharge);
settlement.wallet_gift_balance_before = Some(before_gift);
settlement.wallet_gift_balance_after = Some(after_gift);
sqlx::query(
r#"
UPDATE "usage"
SET
wallet_id = $2,
wallet_balance_before = $3,
wallet_balance_after = $4,
wallet_recharge_balance_before = $5,
wallet_recharge_balance_after = $6,
wallet_gift_balance_before = $7,
wallet_gift_balance_after = $8
WHERE request_id = $1
"#,
)
.bind(&input.request_id)
.bind(&wallet_id)
.bind(before_total)
.bind(after_recharge + after_gift)
.bind(before_recharge)
.bind(after_recharge)
.bind(before_gift)
.bind(after_gift)
.execute(&mut **tx)
.await
.map_postgres_err()?;
}
if let Some(provider_id) = input
@@ -283,6 +376,7 @@ RETURNING CAST(monthly_used_usd AS DOUBLE PRECISION) AS monthly_used_usd
.execute(&mut **tx)
.await
.map_postgres_err()?;
sync_usage_settlement_snapshot(&mut **tx, &settlement).await?;
Ok(Some(settlement))
})
@@ -297,4 +391,28 @@ mod tests {
fn finalize_usage_billing_sql_does_not_require_usage_updated_at_column() {
assert!(!super::FINALIZE_USAGE_BILLING_SQL.contains("updated_at"));
}
#[test]
fn settlement_sql_reads_settlement_snapshots_before_legacy_usage_columns() {
assert!(
super::FIND_USAGE_FOR_SETTLEMENT_SQL.contains("LEFT JOIN usage_settlement_snapshots")
);
assert!(super::FIND_USAGE_FOR_SETTLEMENT_SQL.contains("COALESCE("));
assert!(super::FIND_USAGE_FOR_SETTLEMENT_SQL.contains("FOR UPDATE OF usage_record"));
}
#[test]
fn settlement_sql_dual_writes_usage_settlement_snapshots() {
assert!(super::UPSERT_USAGE_SETTLEMENT_SNAPSHOT_SQL
.contains("INSERT INTO usage_settlement_snapshots"));
assert!(super::UPSERT_USAGE_SETTLEMENT_SNAPSHOT_SQL.contains("provider_monthly_used_usd"));
assert!(super::UPSERT_USAGE_SETTLEMENT_SNAPSHOT_SQL
.contains("TO_TIMESTAMP($11::double precision)"));
}
#[test]
fn settlement_sql_no_longer_dual_writes_wallet_snapshots_to_usage_rows() {
let source = include_str!("sql.rs");
assert!(!source.contains("UPDATE \"usage\"\nSET\n wallet_id = $2"));
}
}