refactor: 拆分 gateway 单体为独立 crate,新增 systemd 部署方案

将 gateway 内部的 model-fetch、provider-transport、scheduler-core、
usage-runtime、video-tasks-core 模块提取为独立 crate;重构 gateway
内部模块结构(state/router/cache/data/query 等);移除大量遗留模块
文件;新增 systemd 二进制部署骨架及相关文档;更新前端 usage 相关
API 和组件。
This commit is contained in:
fawney19
2026-04-05 20:23:16 +08:00
parent cbc811f6ce
commit 763ff03a7b
777 changed files with 42654 additions and 21464 deletions
+3 -151
View File
@@ -1,152 +1,4 @@
use std::sync::Arc;
use std::time::Duration;
use aether_data::redis::{RedisConsumerName, RedisStreamEntry, RedisStreamRunner};
use tracing::warn;
use super::config::UsageRuntimeConfig;
use super::event::UsageEvent;
use super::queue::UsageQueue;
use super::write::build_upsert_usage_record_from_event;
use crate::gateway::gateway_data::GatewayDataState;
use crate::gateway::wallet_runtime::settle_usage_if_needed;
pub(crate) struct UsageQueueWorker {
queue: UsageQueue,
data: Arc<GatewayDataState>,
consumer: RedisConsumerName,
config: UsageRuntimeConfig,
}
impl UsageQueueWorker {
pub(crate) fn new(
runner: RedisStreamRunner,
data: Arc<GatewayDataState>,
config: UsageRuntimeConfig,
) -> Result<Self, aether_data::DataLayerError> {
let queue = UsageQueue::new(runner, config.clone())?;
let consumer = RedisConsumerName(consumer_name());
Ok(Self {
queue,
data,
consumer,
config,
})
}
pub(crate) fn spawn(self) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move { self.run_forever().await })
}
async fn run_forever(self) {
if let Err(err) = self.queue.ensure_consumer_group().await {
warn!(error = %err, "usage worker failed to ensure consumer group");
return;
}
let mut reclaim_interval =
tokio::time::interval(Duration::from_millis(self.config.reclaim_interval_ms));
reclaim_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
reclaim_interval.tick().await;
loop {
tokio::select! {
_ = reclaim_interval.tick() => {
match self.queue.claim_stale(&self.consumer, "0-0").await {
Ok(entries) => {
if let Err(err) = self.process_entries(entries).await {
warn!(error = %err, "usage worker failed while reclaiming stale entries");
}
}
Err(err) => warn!(error = %err, "usage worker failed to reclaim stale entries"),
}
}
result = self.queue.read_group(&self.consumer) => {
match result {
Ok(entries) => {
if let Err(err) = self.process_entries(entries).await {
warn!(error = %err, "usage worker failed to process queue entries");
tokio::time::sleep(Duration::from_millis(250)).await;
}
}
Err(err) => {
warn!(error = %err, "usage worker failed to read queue");
tokio::time::sleep(Duration::from_millis(500)).await;
}
}
}
}
}
}
async fn process_entries(
&self,
entries: Vec<RedisStreamEntry>,
) -> Result<(), aether_data::DataLayerError> {
if entries.is_empty() {
return Ok(());
}
let mut ack_ids = Vec::new();
for entry in entries {
match self.process_entry(&entry).await {
Ok(should_ack) => {
if should_ack {
ack_ids.push(entry.id.clone());
}
}
Err(err) => {
if !ack_ids.is_empty() {
let _ = self.queue.ack_and_delete(&ack_ids).await;
}
return Err(err);
}
}
}
if !ack_ids.is_empty() {
self.queue.ack_and_delete(&ack_ids).await?;
}
Ok(())
}
async fn process_entry(
&self,
entry: &RedisStreamEntry,
) -> Result<bool, aether_data::DataLayerError> {
let event = match UsageEvent::from_stream_fields(&entry.fields) {
Ok(event) => event,
Err(err) => {
self.queue.push_dead_letter(entry, &err.to_string()).await?;
return Ok(true);
}
};
write_event_record(self.data.as_ref(), &event).await?;
Ok(true)
}
}
async fn write_event_record(
data: &GatewayDataState,
event: &UsageEvent,
) -> Result<(), aether_data::DataLayerError> {
let record = build_upsert_usage_record_from_event(event)?;
if let Some(stored) = data.upsert_usage(record).await? {
settle_usage_if_needed(data, &stored).await?;
}
Ok(())
}
fn consumer_name() -> String {
let host = std::env::var("HOSTNAME")
.ok()
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty())
.unwrap_or_else(|| "aether-gateway".to_string());
format!("{host}:{}", std::process::id())
}
pub(crate) use aether_usage_runtime::{build_usage_queue_worker, write_event_record};
#[cfg(test)]
mod tests {
@@ -155,8 +7,8 @@ mod tests {
use aether_data::repository::usage::{InMemoryUsageReadRepository, UsageReadRepository};
use super::write_event_record;
use crate::gateway::gateway_data::GatewayDataState;
use crate::gateway::usage::{UsageEvent, UsageEventData, UsageEventType};
use crate::data::GatewayDataState;
use crate::usage::event::{UsageEvent, UsageEventData, UsageEventType};
#[tokio::test]
async fn worker_writes_usage_record_from_terminal_event() {