i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-03 09:50:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: 拆分 gateway 单体为独立 crate，新增 systemd 部署方案

Browse Source 
将 gateway 内部的 model-fetch、provider-transport、scheduler-core、
usage-runtime、video-tasks-core 模块提取为独立 crate；重构 gateway
内部模块结构（state/router/cache/data/query 等）；移除大量遗留模块
文件；新增 systemd 二进制部署骨架及相关文档；更新前端 usage 相关
API 和组件。
This commit is contained in:
fawney19
2026-04-05 20:23:16 +08:00
parent cbc811f6ce
commit 763ff03a7b
777 changed files with 42659 additions and 21469 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
apps/aether-gateway/src/maintenance/runtime/audit_cleanup.rs
View File

@@ -1,6 +1,6 @@
use chrono::{DateTime, Utc};
use crate::gateway::gateway_data::GatewayDataState;
use crate::data::GatewayDataState;
use super::{
system_config_bool, system_config_u64, system_config_usize, DELETE_AUDIT_LOGS_BEFORE_SQL,

2
apps/aether-gateway/src/maintenance/runtime/config.rs
View File

@@ -1,7 +1,7 @@
use chrono::{DateTime, Utc};
use serde_json::Value;
use crate::gateway::gateway_data::GatewayDataState;
use crate::data::GatewayDataState;
use super::{UsageCleanupSettings, UsageCleanupWindow};

2
apps/aether-gateway/src/maintenance/runtime/db_maintenance.rs
View File

@@ -1,6 +1,6 @@
use tracing::warn;
use crate::gateway::gateway_data::GatewayDataState;
use crate::data::GatewayDataState;
use super::{system_config_bool, DB_MAINTENANCE_TABLES};

2
apps/aether-gateway/src/maintenance/runtime/pending_cleanup.rs
View File

@@ -3,7 +3,7 @@ use std::collections::HashSet;
use chrono::Utc;
use sqlx::Row;
use crate::gateway::gateway_data::GatewayDataState;
use crate::data::GatewayDataState;
use super::{
pending_cleanup_batch_size, pending_cleanup_timeout_minutes,

4
apps/aether-gateway/src/maintenance/runtime/provider_checkin.rs
View File

@@ -6,8 +6,8 @@ use aether_data::repository::provider_catalog::{
use futures_util::stream::{self, StreamExt};
use tracing::{debug, warn};
use crate::gateway::handlers::admin::admin_provider_ops_local_action_response;
use crate::gateway::{AppState, GatewayError};
use crate::handlers::admin::provider_ops::admin_provider_ops_local_action_response;
use crate::{AppState, GatewayError};
use super::{system_config_bool, PROVIDER_CHECKIN_CONCURRENCY};

2
apps/aether-gateway/src/maintenance/runtime/request_candidate_cleanup.rs
View File

@@ -1,6 +1,6 @@
use tracing::info;
use crate::gateway::gateway_data::GatewayDataState;
use crate::data::GatewayDataState;
use super::{now_unix_secs, system_config_bool, system_config_u64, system_config_usize};

44
apps/aether-gateway/src/maintenance/runtime/runners.rs
View File

@@ -1,7 +1,7 @@
use tracing::info;
use crate::gateway::gateway_data::GatewayDataState;
use crate::gateway::{AppState, GatewayError};
use crate::data::GatewayDataState;
use crate::{AppState, GatewayError};
use super::{
cleanup_audit_logs_once, cleanup_expired_gemini_file_mappings_once,
@@ -16,7 +16,13 @@ pub(super) async fn run_audit_cleanup_once(
) -> Result<(), aether_data::DataLayerError> {
let deleted = cleanup_audit_logs_once(data).await?;
if deleted > 0 {
info!(deleted, "gateway deleted expired audit logs");
info!(
event_name = "audit_cleanup_completed",
log_type = "ops",
worker = "audit_cleanup",
deleted,
"gateway deleted expired audit logs"
);
}
Ok(())
}
@@ -26,7 +32,13 @@ pub(super) async fn run_gemini_file_mapping_cleanup_once(
) -> Result<(), aether_data::DataLayerError> {
let deleted = cleanup_expired_gemini_file_mappings_once(data).await?;
if deleted > 0 {
info!(deleted, "gateway deleted expired gemini file mappings");
info!(
event_name = "gemini_file_mapping_cleanup_completed",
log_type = "ops",
worker = "gemini_file_mapping_cleanup",
deleted,
"gateway deleted expired gemini file mappings"
);
}
Ok(())
}
@@ -37,6 +49,9 @@ pub(super) async fn run_db_maintenance_once(
let summary = perform_db_maintenance_once(data).await?;
if summary.attempted > 0 {
info!(
event_name = "db_maintenance_completed",
log_type = "ops",
worker = "db_maintenance",
attempted = summary.attempted,
succeeded = summary.succeeded,
failed = summary.attempted.saturating_sub(summary.succeeded),
@@ -51,6 +66,9 @@ pub(super) async fn run_wallet_daily_usage_aggregation_once(
) -> Result<(), aether_data::DataLayerError> {
let summary = perform_wallet_daily_usage_aggregation_once(data).await?;
info!(
event_name = "wallet_daily_usage_aggregation_completed",
log_type = "ops",
worker = "wallet_daily_usage_aggregation",
billing_date = %summary.billing_date,
billing_timezone = %summary.billing_timezone,
wallets = summary.aggregated_wallets,
@@ -68,6 +86,9 @@ pub(super) async fn run_stats_aggregation_once(
};
info!(
event_name = "stats_daily_aggregation_completed",
log_type = "ops",
worker = "stats_daily_aggregation",
day_start_utc = %summary.day_start_utc,
total_requests = summary.total_requests,
model_rows = summary.model_rows,
@@ -91,6 +112,9 @@ pub(super) async fn run_usage_cleanup_once(
|| summary.records_deleted > 0
{
info!(
event_name = "usage_cleanup_completed",
log_type = "ops",
worker = "usage_cleanup",
body_compressed = summary.body_compressed,
body_cleaned = summary.body_cleaned,
header_cleaned = summary.header_cleaned,
@@ -108,6 +132,9 @@ pub(super) fn run_pool_monitor_once(data: &GatewayDataState) {
};
info!(
event_name = "postgres_pool_sampled",
log_type = "ops",
worker = "pool_monitor",
checked_out = summary.checked_out,
pool_size = summary.pool_size,
idle = summary.idle,
@@ -123,6 +150,9 @@ pub(super) async fn run_pending_cleanup_once(
let summary = cleanup_stale_pending_requests_once(data).await?;
if summary.failed > 0 || summary.recovered > 0 {
info!(
event_name = "pending_cleanup_completed",
log_type = "ops",
worker = "pending_cleanup",
failed = summary.failed,
recovered = summary.recovered,
"gateway cleaned stale pending and streaming requests"
@@ -139,6 +169,9 @@ pub(super) async fn run_stats_hourly_aggregation_once(
};
info!(
event_name = "stats_hourly_aggregation_completed",
log_type = "ops",
worker = "stats_hourly_aggregation",
hour_utc = %summary.hour_utc,
total_requests = summary.total_requests,
user_rows = summary.user_rows,
@@ -153,6 +186,9 @@ pub(super) async fn run_provider_checkin_once(state: &AppState) -> Result<(), Ga
let summary = perform_provider_checkin_once(state).await?;
if summary.attempted > 0 {
info!(
event_name = "provider_checkin_completed",
log_type = "ops",
worker = "provider_checkin",
attempted = summary.attempted,
succeeded = summary.succeeded,
failed = summary.failed,

2
apps/aether-gateway/src/maintenance/runtime/schedule.rs
View File

@@ -4,7 +4,7 @@ use chrono::{DateTime, Datelike, TimeZone, Timelike, Utc, Weekday};
use chrono_tz::Tz;
use tracing::warn;
use crate::gateway::gateway_data::GatewayDataState;
use crate::data::GatewayDataState;
use super::{
system_config_string, WalletDailyUsageAggregationTarget, DB_MAINTENANCE_HOUR,

2
apps/aether-gateway/src/maintenance/runtime/stats_daily.rs
View File

@@ -4,7 +4,7 @@ use chrono::{DateTime, Utc};
use sqlx::Row;
use uuid::Uuid;
use crate::gateway::gateway_data::GatewayDataState;
use crate::data::GatewayDataState;
use super::{
stats_aggregation_target_day, system_config_bool, PercentileSummary, StatsAggregationSummary,

2
apps/aether-gateway/src/maintenance/runtime/stats_hourly.rs
View File

@@ -2,7 +2,7 @@ use chrono::{DateTime, Utc};
use sqlx::Row;
use uuid::Uuid;
use crate::gateway::gateway_data::GatewayDataState;
use crate::data::GatewayDataState;
use super::{
stats_hourly_aggregation_target_hour, system_config_bool, SELECT_STATS_HOURLY_AGGREGATE_SQL,

2
apps/aether-gateway/src/maintenance/runtime/tests.rs
View File

@@ -276,7 +276,7 @@ fn usage_cleanup_window_uses_non_overlapping_ranges() {
#[tokio::test]
async fn summarize_postgres_pool_uses_busy_connections_for_usage_rate() {
let data = GatewayDataState::from_config(
crate::gateway::gateway_data::GatewayDataConfig::from_postgres_config(
crate::data::GatewayDataConfig::from_postgres_config(
aether_data::postgres::PostgresPoolConfig {
database_url: "postgres://localhost/aether".to_string(),
min_connections: 1,

2
apps/aether-gateway/src/maintenance/runtime/usage_cleanup.rs
View File

@@ -6,7 +6,7 @@ use serde_json::Value;
use sqlx::Row;
use tracing::warn;
use crate::gateway::gateway_data::GatewayDataState;
use crate::data::GatewayDataState;
use super::{
system_config_bool, usage_cleanup_settings, usage_cleanup_window, ExpiredApiKeyRow,

2
apps/aether-gateway/src/maintenance/runtime/wallet_daily_usage.rs
View File

@@ -2,7 +2,7 @@ use chrono::{DateTime, Utc};
use sqlx::Row;
use uuid::Uuid;
use crate::gateway::gateway_data::GatewayDataState;
use crate::data::GatewayDataState;
use super::{
maintenance_timezone, wallet_daily_usage_aggregation_target,

51
apps/aether-gateway/src/maintenance/runtime/workers.rs
View File

@@ -3,8 +3,8 @@ use std::sync::Arc;
use chrono::Utc;
use tracing::warn;
use crate::gateway::gateway_data::GatewayDataState;
use crate::gateway::AppState;
use crate::data::GatewayDataState;
use crate::AppState;
use super::{
duration_until_next_daily_run, duration_until_next_db_maintenance_run,
@@ -20,6 +20,21 @@ use super::{
WALLET_DAILY_USAGE_AGGREGATION_MINUTE,
};
fn log_maintenance_worker_failure(
worker: &'static str,
phase: &'static str,
error: &impl std::fmt::Debug,
) {
warn!(
event_name = "maintenance_worker_failed",
log_type = "ops",
worker,
phase,
error = ?error,
"gateway maintenance worker failed"
);
}
pub(crate) fn spawn_audit_cleanup_worker(
data: Arc<GatewayDataState>,
) -> Option<tokio::task::JoinHandle<()>> {
@@ -29,7 +44,7 @@ pub(crate) fn spawn_audit_cleanup_worker(
Some(tokio::spawn(async move {
if let Err(err) = run_audit_cleanup_once(&data).await {
warn!(error = %err, "gateway audit cleanup startup failed");
log_maintenance_worker_failure("audit_cleanup", "startup", &err);
}
let mut interval = tokio::time::interval(AUDIT_LOG_CLEANUP_INTERVAL);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
@@ -37,7 +52,7 @@ pub(crate) fn spawn_audit_cleanup_worker(
loop {
interval.tick().await;
if let Err(err) = run_audit_cleanup_once(&data).await {
warn!(error = %err, "gateway audit cleanup tick failed");
log_maintenance_worker_failure("audit_cleanup", "tick", &err);
}
}
}))
@@ -55,7 +70,7 @@ pub(crate) fn spawn_db_maintenance_worker(
loop {
tokio::time::sleep(duration_until_next_db_maintenance_run(Utc::now(), timezone)).await;
if let Err(err) = run_db_maintenance_once(&data).await {
warn!(error = %err, "gateway db maintenance tick failed");
log_maintenance_worker_failure("db_maintenance", "tick", &err);
}
}
}))
@@ -79,7 +94,7 @@ pub(crate) fn spawn_wallet_daily_usage_aggregation_worker(
))
.await;
if let Err(err) = run_wallet_daily_usage_aggregation_once(&data).await {
warn!(error = %err, "gateway wallet daily usage aggregation tick failed");
log_maintenance_worker_failure("wallet_daily_usage_aggregation", "tick", &err);
}
}
}))
@@ -96,7 +111,7 @@ pub(crate) fn spawn_stats_aggregation_worker(
loop {
tokio::time::sleep(duration_until_next_stats_aggregation_run(Utc::now())).await;
if let Err(err) = run_stats_aggregation_once(&data).await {
warn!(error = %err, "gateway stats aggregation tick failed");
log_maintenance_worker_failure("stats_daily_aggregation", "tick", &err);
}
}
}))
@@ -120,7 +135,7 @@ pub(crate) fn spawn_usage_cleanup_worker(
))
.await;
if let Err(err) = run_usage_cleanup_once(&data).await {
warn!(error = %err, "gateway usage cleanup tick failed");
log_maintenance_worker_failure("usage_cleanup", "tick", &err);
}
}
}))
@@ -140,6 +155,10 @@ pub(crate) fn spawn_provider_checkin_worker(
Ok(schedule) => schedule,
Err(err) => {
warn!(
event_name = "maintenance_schedule_lookup_failed",
log_type = "ops",
worker = "provider_checkin",
phase = "schedule_lookup",
error = %err,
fallback = PROVIDER_CHECKIN_DEFAULT_TIME,
"gateway provider checkin schedule lookup failed; falling back"
@@ -156,7 +175,7 @@ pub(crate) fn spawn_provider_checkin_worker(
))
.await;
if let Err(err) = run_provider_checkin_once(&state).await {
warn!(error = ?err, "gateway provider checkin tick failed");
log_maintenance_worker_failure("provider_checkin", "tick", &err);
}
}
}))
@@ -171,7 +190,7 @@ pub(crate) fn spawn_gemini_file_mapping_cleanup_worker(
Some(tokio::spawn(async move {
if let Err(err) = run_gemini_file_mapping_cleanup_once(&data).await {
warn!(error = %err, "gateway gemini file mapping cleanup startup failed");
log_maintenance_worker_failure("gemini_file_mapping_cleanup", "startup", &err);
}
let mut interval = tokio::time::interval(GEMINI_FILE_MAPPING_CLEANUP_INTERVAL);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
@@ -179,7 +198,7 @@ pub(crate) fn spawn_gemini_file_mapping_cleanup_worker(
loop {
interval.tick().await;
if let Err(err) = run_gemini_file_mapping_cleanup_once(&data).await {
warn!(error = %err, "gateway gemini file mapping cleanup tick failed");
log_maintenance_worker_failure("gemini_file_mapping_cleanup", "tick", &err);
}
}
}))
@@ -194,7 +213,7 @@ pub(crate) fn spawn_pending_cleanup_worker(
Some(tokio::spawn(async move {
if let Err(err) = run_pending_cleanup_once(&data).await {
warn!(error = %err, "gateway pending cleanup startup failed");
log_maintenance_worker_failure("pending_cleanup", "startup", &err);
}
let mut interval = tokio::time::interval(PENDING_CLEANUP_INTERVAL);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
@@ -202,7 +221,7 @@ pub(crate) fn spawn_pending_cleanup_worker(
loop {
interval.tick().await;
if let Err(err) = run_pending_cleanup_once(&data).await {
warn!(error = %err, "gateway pending cleanup tick failed");
log_maintenance_worker_failure("pending_cleanup", "tick", &err);
}
}
}))
@@ -237,7 +256,7 @@ pub(crate) fn spawn_stats_hourly_aggregation_worker(
loop {
tokio::time::sleep(duration_until_next_stats_hourly_aggregation_run(Utc::now())).await;
if let Err(err) = run_stats_hourly_aggregation_once(&data).await {
warn!(error = %err, "gateway stats hourly aggregation tick failed");
log_maintenance_worker_failure("stats_hourly_aggregation", "tick", &err);
}
}
}))
@@ -252,7 +271,7 @@ pub(crate) fn spawn_request_candidate_cleanup_worker(
Some(tokio::spawn(async move {
if let Err(err) = run_request_candidate_cleanup_once(&data).await {
warn!(error = %err, "gateway request candidate cleanup startup failed");
log_maintenance_worker_failure("request_candidate_cleanup", "startup", &err);
}
let mut interval = tokio::time::interval(REQUEST_CANDIDATE_CLEANUP_INTERVAL);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
@@ -260,7 +279,7 @@ pub(crate) fn spawn_request_candidate_cleanup_worker(
loop {
interval.tick().await;
if let Err(err) = run_request_candidate_cleanup_once(&data).await {
warn!(error = %err, "gateway request candidate cleanup tick failed");
log_maintenance_worker_failure("request_candidate_cleanup", "tick", &err);
}
}
}))