i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 17:30:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: 引入 aether-runtime/cache/data/http/testkit 基础 crate，完善并发门控与审计系统

Browse Source 
新增 crate：
- aether-runtime: 服务运行时基础设施（并发门控、分布式并发、指标、队列、优雅关闭、tracing）
- aether-cache: 通用 TTL 缓存与命名空间抽象
- aether-data: 数据访问层（PostgreSQL/Redis 后端、repository 模式）
- aether-http: HTTP 客户端封装（重试、配置）
- aether-testkit: 集成测试工具集（gateway/executor/hub/proxy fixture、等待、负载测试）

gateway 扩展：
- 引入 audit 模块（shadow 执行审计、决策链路追踪、请求审计 bundle）
- 引入 cache 模块（AuthContext 缓存、direct-plan bypass 缓存）
- 引入 data 模块（auth/candidates/config/usage/video_tasks 数据访问）
- 集成 ConcurrencyGate/DistributedConcurrencyGate 请求门控
- 新增本地 auth 拒绝、过载响应构建器
- 补充 control/auth_cache/video/concurrency 集成测试

aether-proxy 扩展：
- AppState 集成 stream_gate / distributed_stream_gate 并发门控
- 新增 ProxyAdmissionError 及准入拒绝流程
- stream_handler 补充门控饱和/不可用场景测试
- 配置与注册客户端逻辑完善

aether-hub 扩展：
- main.rs 引入运行时初始化、指标端点、健康检查
- local_relay 重构为 lib.rs 暴露公共接口
This commit is contained in:
fawney19
2026-03-24 15:12:56 +08:00
parent eaf8475f9e
commit b5a0070023
157 changed files with 22097 additions and 448 deletions
Download Patch File Download Diff File Expand all files Collapse all files

232
crates/aether-data/src/repository/video_tasks/memory.rs Normal file
View File

@@ -0,0 +1,232 @@
use std::collections::BTreeMap;
use std::sync::RwLock;
use async_trait::async_trait;
use super::types::{
StoredVideoTask, UpsertVideoTask, VideoTaskLookupKey, VideoTaskReadRepository,
VideoTaskWriteRepository,
};
use crate::DataLayerError;
#[derive(Debug, Default)]
struct MemoryVideoTaskIndex {
by_id: BTreeMap<String, StoredVideoTask>,
short_to_id: BTreeMap<String, String>,
user_external_to_id: BTreeMap<(String, String), String>,
}
#[derive(Debug, Default)]
pub struct InMemoryVideoTaskRepository {
index: RwLock<MemoryVideoTaskIndex>,
}
impl InMemoryVideoTaskRepository {
fn store_locked(index: &mut MemoryVideoTaskIndex, task: StoredVideoTask) -> StoredVideoTask {
if let Some(previous) = index.by_id.insert(task.id.clone(), task.clone()) {
if let Some(short_id) = previous.short_id {
index.short_to_id.remove(&short_id);
}
if let (Some(user_id), Some(external_task_id)) =
(previous.user_id, previous.external_task_id)
{
index
.user_external_to_id
.remove(&(user_id, external_task_id));
}
}
if let Some(short_id) = &task.short_id {
index.short_to_id.insert(short_id.clone(), task.id.clone());
}
if let (Some(user_id), Some(external_task_id)) = (&task.user_id, &task.external_task_id) {
index
.user_external_to_id
.insert((user_id.clone(), external_task_id.clone()), task.id.clone());
}
task
}
}
#[async_trait]
impl VideoTaskReadRepository for InMemoryVideoTaskRepository {
async fn find(
&self,
key: VideoTaskLookupKey<'_>,
) -> Result<Option<StoredVideoTask>, DataLayerError> {
let index = self.index.read().expect("video task repository lock");
Ok(match key {
VideoTaskLookupKey::Id(id) => index.by_id.get(id).cloned(),
VideoTaskLookupKey::ShortId(short_id) => index
.short_to_id
.get(short_id)
.and_then(|id| index.by_id.get(id))
.cloned(),
VideoTaskLookupKey::UserExternal {
user_id,
external_task_id,
} => index
.user_external_to_id
.get(&(user_id.to_string(), external_task_id.to_string()))
.and_then(|id| index.by_id.get(id))
.cloned(),
})
}
async fn list_active(&self, limit: usize) -> Result<Vec<StoredVideoTask>, DataLayerError> {
if limit == 0 {
return Ok(Vec::new());
}
let mut tasks = self
.index
.read()
.expect("video task repository lock")
.by_id
.values()
.filter(|task| task.status.is_active())
.cloned()
.collect::<Vec<_>>();
tasks.sort_by(|left, right| right.updated_at_unix_secs.cmp(&left.updated_at_unix_secs));
tasks.truncate(limit);
Ok(tasks)
}
}
#[async_trait]
impl VideoTaskWriteRepository for InMemoryVideoTaskRepository {
async fn upsert(&self, task: UpsertVideoTask) -> Result<StoredVideoTask, DataLayerError> {
let mut index = self.index.write().expect("video task repository lock");
Ok(Self::store_locked(&mut index, task.into_stored()))
}
}
#[cfg(test)]
mod tests {
use super::InMemoryVideoTaskRepository;
use crate::repository::video_tasks::{
UpsertVideoTask, VideoTaskLookupKey, VideoTaskReadRepository, VideoTaskStatus,
VideoTaskWriteRepository,
};
fn sample_task(
id: &str,
status: VideoTaskStatus,
updated_at_unix_secs: u64,
) -> UpsertVideoTask {
UpsertVideoTask {
id: id.to_string(),
short_id: Some(format!("short-{id}")),
user_id: Some("user-1".to_string()),
external_task_id: Some(format!("ext-{id}")),
provider_api_format: Some("openai:video".to_string()),
model: Some("sora-2".to_string()),
prompt: Some("hello".to_string()),
size: Some("1280x720".to_string()),
status,
progress_percent: 0,
created_at_unix_secs: updated_at_unix_secs.saturating_sub(10),
updated_at_unix_secs,
error_code: None,
error_message: None,
video_url: None,
}
}
#[tokio::test]
async fn reads_task_by_all_supported_lookup_keys() {
let repo = InMemoryVideoTaskRepository::default();
repo.upsert(sample_task("task-1", VideoTaskStatus::Submitted, 100))
.await
.expect("upsert should succeed");
assert!(repo
.find(VideoTaskLookupKey::Id("task-1"))
.await
.expect("find by id should succeed")
.is_some());
assert!(repo
.find(VideoTaskLookupKey::ShortId("short-task-1"))
.await
.expect("find by short id should succeed")
.is_some());
assert!(repo
.find(VideoTaskLookupKey::UserExternal {
user_id: "user-1",
external_task_id: "ext-task-1",
})
.await
.expect("find by user/external should succeed")
.is_some());
}
#[tokio::test]
async fn list_active_only_returns_active_tasks_in_descending_update_order() {
let repo = InMemoryVideoTaskRepository::default();
repo.upsert(sample_task("task-1", VideoTaskStatus::Completed, 100))
.await
.expect("upsert should succeed");
repo.upsert(sample_task("task-2", VideoTaskStatus::Processing, 200))
.await
.expect("upsert should succeed");
repo.upsert(sample_task("task-3", VideoTaskStatus::Queued, 150))
.await
.expect("upsert should succeed");
let active = repo
.list_active(10)
.await
.expect("list active should succeed");
assert_eq!(active.len(), 2);
assert_eq!(active[0].id, "task-2");
assert_eq!(active[1].id, "task-3");
}
#[tokio::test]
async fn upsert_replaces_secondary_indexes() {
let repo = InMemoryVideoTaskRepository::default();
repo.upsert(sample_task("task-1", VideoTaskStatus::Submitted, 100))
.await
.expect("upsert should succeed");
repo.upsert(UpsertVideoTask {
id: "task-1".to_string(),
short_id: Some("short-task-1b".to_string()),
user_id: Some("user-2".to_string()),
external_task_id: Some("ext-task-1b".to_string()),
provider_api_format: Some("gemini:video".to_string()),
model: Some("veo-3".to_string()),
prompt: Some("remix".to_string()),
size: Some("720p".to_string()),
status: VideoTaskStatus::Processing,
progress_percent: 50,
created_at_unix_secs: 150,
updated_at_unix_secs: 200,
error_code: None,
error_message: None,
video_url: None,
})
.await
.expect("upsert should succeed");
assert!(repo
.find(VideoTaskLookupKey::ShortId("short-task-1"))
.await
.expect("find should succeed")
.is_none());
assert!(repo
.find(VideoTaskLookupKey::UserExternal {
user_id: "user-1",
external_task_id: "ext-task-1",
})
.await
.expect("find should succeed")
.is_none());
assert!(repo
.find(VideoTaskLookupKey::ShortId("short-task-1b"))
.await
.expect("find should succeed")
.is_some());
}
}

10
crates/aether-data/src/repository/video_tasks/mod.rs Normal file
View File

@@ -0,0 +1,10 @@
mod memory;
mod sql;
mod types;
pub use memory::InMemoryVideoTaskRepository;
pub use sql::SqlxVideoTaskReadRepository;
pub use types::{
StoredVideoTask, UpsertVideoTask, VideoTaskLookupKey, VideoTaskReadRepository,
VideoTaskRepository, VideoTaskStatus, VideoTaskWriteRepository,
};

259
crates/aether-data/src/repository/video_tasks/sql.rs Normal file
View File

@@ -0,0 +1,259 @@
use sqlx::{PgPool, Row};
use async_trait::async_trait;
use crate::repository::video_tasks::{
StoredVideoTask, VideoTaskLookupKey, VideoTaskReadRepository, VideoTaskStatus,
};
use crate::DataLayerError;
const FIND_BY_ID_SQL: &str = r#"
SELECT
id,
short_id,
user_id,
external_task_id,
provider_api_format,
model,
prompt,
size,
status,
progress_percent,
CAST(EXTRACT(EPOCH FROM created_at) AS BIGINT) AS created_at_unix_secs,
CAST(EXTRACT(EPOCH FROM updated_at) AS BIGINT) AS updated_at_unix_secs
,
error_code,
error_message,
video_url
FROM video_tasks
WHERE id = $1
LIMIT 1
"#;
const FIND_BY_SHORT_ID_SQL: &str = r#"
SELECT
id,
short_id,
user_id,
external_task_id,
provider_api_format,
model,
prompt,
size,
status,
progress_percent,
CAST(EXTRACT(EPOCH FROM created_at) AS BIGINT) AS created_at_unix_secs,
CAST(EXTRACT(EPOCH FROM updated_at) AS BIGINT) AS updated_at_unix_secs
,
error_code,
error_message,
video_url
FROM video_tasks
WHERE short_id = $1
LIMIT 1
"#;
const FIND_BY_USER_EXTERNAL_SQL: &str = r#"
SELECT
id,
short_id,
user_id,
external_task_id,
provider_api_format,
model,
prompt,
size,
status,
progress_percent,
CAST(EXTRACT(EPOCH FROM created_at) AS BIGINT) AS created_at_unix_secs,
CAST(EXTRACT(EPOCH FROM updated_at) AS BIGINT) AS updated_at_unix_secs
,
error_code,
error_message,
video_url
FROM video_tasks
WHERE user_id = $1 AND external_task_id = $2
LIMIT 1
"#;
const LIST_ACTIVE_SQL: &str = r#"
SELECT
id,
short_id,
user_id,
external_task_id,
provider_api_format,
model,
prompt,
size,
status,
progress_percent,
CAST(EXTRACT(EPOCH FROM created_at) AS BIGINT) AS created_at_unix_secs,
CAST(EXTRACT(EPOCH FROM updated_at) AS BIGINT) AS updated_at_unix_secs
,
error_code,
error_message,
video_url
FROM video_tasks
WHERE status = ANY($1)
ORDER BY updated_at DESC
LIMIT $2
"#;
#[derive(Debug, Clone)]
pub struct SqlxVideoTaskReadRepository {
pool: PgPool,
}
impl SqlxVideoTaskReadRepository {
pub fn new(pool: PgPool) -> Self {
Self { pool }
}
pub fn pool(&self) -> &PgPool {
&self.pool
}
pub async fn find(
&self,
key: VideoTaskLookupKey<'_>,
) -> Result<Option<StoredVideoTask>, DataLayerError> {
match key {
VideoTaskLookupKey::Id(id) => self.find_by_id(id).await,
VideoTaskLookupKey::ShortId(short_id) => self.find_by_short_id(short_id).await,
VideoTaskLookupKey::UserExternal {
user_id,
external_task_id,
} => self.find_by_user_external(user_id, external_task_id).await,
}
}
pub async fn find_by_id(&self, id: &str) -> Result<Option<StoredVideoTask>, DataLayerError> {
let row = sqlx::query(FIND_BY_ID_SQL)
.bind(id)
.fetch_optional(&self.pool)
.await?;
row.as_ref().map(map_video_task_row).transpose()
}
pub async fn find_by_short_id(
&self,
short_id: &str,
) -> Result<Option<StoredVideoTask>, DataLayerError> {
let row = sqlx::query(FIND_BY_SHORT_ID_SQL)
.bind(short_id)
.fetch_optional(&self.pool)
.await?;
row.as_ref().map(map_video_task_row).transpose()
}
pub async fn find_by_user_external(
&self,
user_id: &str,
external_task_id: &str,
) -> Result<Option<StoredVideoTask>, DataLayerError> {
let row = sqlx::query(FIND_BY_USER_EXTERNAL_SQL)
.bind(user_id)
.bind(external_task_id)
.fetch_optional(&self.pool)
.await?;
row.as_ref().map(map_video_task_row).transpose()
}
pub async fn list_active(&self, limit: usize) -> Result<Vec<StoredVideoTask>, DataLayerError> {
if limit == 0 {
return Ok(Vec::new());
}
let active_statuses = vec!["pending", "submitted", "queued", "processing"];
let rows = sqlx::query(LIST_ACTIVE_SQL)
.bind(active_statuses)
.bind(i64::try_from(limit).map_err(|_| {
DataLayerError::UnexpectedValue(format!("invalid active task limit: {limit}"))
})?)
.fetch_all(&self.pool)
.await?;
rows.iter().map(map_video_task_row).collect()
}
}
#[async_trait]
impl VideoTaskReadRepository for SqlxVideoTaskReadRepository {
async fn find(
&self,
key: VideoTaskLookupKey<'_>,
) -> Result<Option<StoredVideoTask>, DataLayerError> {
Self::find(self, key).await
}
async fn list_active(&self, limit: usize) -> Result<Vec<StoredVideoTask>, DataLayerError> {
Self::list_active(self, limit).await
}
}
fn map_video_task_row(row: &sqlx::postgres::PgRow) -> Result<StoredVideoTask, DataLayerError> {
let status = VideoTaskStatus::from_database(row.try_get::<String, _>("status")?.as_str())?;
StoredVideoTask::new(
row.try_get("id")?,
row.try_get("short_id")?,
row.try_get("user_id")?,
row.try_get("external_task_id")?,
row.try_get("provider_api_format")?,
row.try_get("model")?,
row.try_get("prompt")?,
row.try_get("size")?,
status,
row.try_get("progress_percent")?,
row.try_get("created_at_unix_secs")?,
row.try_get("updated_at_unix_secs")?,
row.try_get("error_code")?,
row.try_get("error_message")?,
row.try_get("video_url")?,
)
}
#[cfg(test)]
mod tests {
use super::SqlxVideoTaskReadRepository;
use crate::postgres::{PostgresPoolConfig, PostgresPoolFactory};
use crate::repository::video_tasks::{VideoTaskLookupKey, VideoTaskReadRepository};
#[tokio::test]
async fn repository_constructs_from_lazy_pool() {
let factory = PostgresPoolFactory::new(PostgresPoolConfig {
database_url: "postgres://localhost/aether".to_string(),
min_connections: 1,
max_connections: 4,
acquire_timeout_ms: 1_000,
idle_timeout_ms: 5_000,
max_lifetime_ms: 30_000,
statement_cache_capacity: 64,
require_ssl: false,
})
.expect("factory should build");
let pool = factory.connect_lazy().expect("pool should build");
let repository = SqlxVideoTaskReadRepository::new(pool);
let _ = repository.pool();
}
#[tokio::test]
async fn read_trait_delegates_to_sqlx_repository() {
let factory = PostgresPoolFactory::new(PostgresPoolConfig {
database_url: "postgres://localhost/aether".to_string(),
min_connections: 1,
max_connections: 4,
acquire_timeout_ms: 1_000,
idle_timeout_ms: 5_000,
max_lifetime_ms: 30_000,
statement_cache_capacity: 64,
require_ssl: false,
})
.expect("factory should build");
let pool = factory.connect_lazy().expect("pool should build");
let repository = SqlxVideoTaskReadRepository::new(pool);
let _ = VideoTaskReadRepository::find(&repository, VideoTaskLookupKey::Id("task-1")).await;
}
}

277
crates/aether-data/src/repository/video_tasks/types.rs Normal file
View File

@@ -0,0 +1,277 @@
use async_trait::async_trait;
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub enum VideoTaskStatus {
Pending,
Submitted,
Queued,
Processing,
Completed,
Failed,
Cancelled,
Expired,
Deleted,
}
impl VideoTaskStatus {
pub fn from_database(value: &str) -> Result<Self, crate::DataLayerError> {
match value.trim().to_ascii_lowercase().as_str() {
"pending" => Ok(Self::Pending),
"submitted" => Ok(Self::Submitted),
"queued" => Ok(Self::Queued),
"processing" => Ok(Self::Processing),
"completed" => Ok(Self::Completed),
"failed" => Ok(Self::Failed),
"cancelled" => Ok(Self::Cancelled),
"expired" => Ok(Self::Expired),
"deleted" => Ok(Self::Deleted),
other => Err(crate::DataLayerError::UnexpectedValue(format!(
"unsupported video_tasks.status: {other}"
))),
}
}
pub fn is_active(self) -> bool {
matches!(
self,
Self::Pending | Self::Submitted | Self::Queued | Self::Processing
)
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct StoredVideoTask {
pub id: String,
pub short_id: Option<String>,
pub user_id: Option<String>,
pub external_task_id: Option<String>,
pub provider_api_format: Option<String>,
pub model: Option<String>,
pub prompt: Option<String>,
pub size: Option<String>,
pub status: VideoTaskStatus,
pub progress_percent: u16,
pub created_at_unix_secs: u64,
pub updated_at_unix_secs: u64,
pub error_code: Option<String>,
pub error_message: Option<String>,
pub video_url: Option<String>,
}
impl StoredVideoTask {
pub fn new(
id: String,
short_id: Option<String>,
user_id: Option<String>,
external_task_id: Option<String>,
provider_api_format: Option<String>,
model: Option<String>,
prompt: Option<String>,
size: Option<String>,
status: VideoTaskStatus,
progress_percent: i32,
created_at_unix_secs: i64,
updated_at_unix_secs: i64,
error_code: Option<String>,
error_message: Option<String>,
video_url: Option<String>,
) -> Result<Self, crate::DataLayerError> {
let progress_percent = u16::try_from(progress_percent).map_err(|_| {
crate::DataLayerError::UnexpectedValue(format!(
"invalid progress_percent: {progress_percent}"
))
})?;
let created_at_unix_secs = u64::try_from(created_at_unix_secs).map_err(|_| {
crate::DataLayerError::UnexpectedValue(format!(
"invalid created_at_unix_secs: {created_at_unix_secs}"
))
})?;
let updated_at_unix_secs = u64::try_from(updated_at_unix_secs).map_err(|_| {
crate::DataLayerError::UnexpectedValue(format!(
"invalid updated_at_unix_secs: {updated_at_unix_secs}"
))
})?;
Ok(Self {
id,
short_id,
user_id,
external_task_id,
provider_api_format,
model,
prompt,
size,
status,
progress_percent,
created_at_unix_secs,
updated_at_unix_secs,
error_code,
error_message,
video_url,
})
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct UpsertVideoTask {
pub id: String,
pub short_id: Option<String>,
pub user_id: Option<String>,
pub external_task_id: Option<String>,
pub provider_api_format: Option<String>,
pub model: Option<String>,
pub prompt: Option<String>,
pub size: Option<String>,
pub status: VideoTaskStatus,
pub progress_percent: u16,
pub created_at_unix_secs: u64,
pub updated_at_unix_secs: u64,
pub error_code: Option<String>,
pub error_message: Option<String>,
pub video_url: Option<String>,
}
impl UpsertVideoTask {
pub fn into_stored(self) -> StoredVideoTask {
StoredVideoTask {
id: self.id,
short_id: self.short_id,
user_id: self.user_id,
external_task_id: self.external_task_id,
provider_api_format: self.provider_api_format,
model: self.model,
prompt: self.prompt,
size: self.size,
status: self.status,
progress_percent: self.progress_percent,
created_at_unix_secs: self.created_at_unix_secs,
updated_at_unix_secs: self.updated_at_unix_secs,
error_code: self.error_code,
error_message: self.error_message,
video_url: self.video_url,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum VideoTaskLookupKey<'a> {
Id(&'a str),
ShortId(&'a str),
UserExternal {
user_id: &'a str,
external_task_id: &'a str,
},
}
#[async_trait]
pub trait VideoTaskReadRepository: Send + Sync {
async fn find(
&self,
key: VideoTaskLookupKey<'_>,
) -> Result<Option<StoredVideoTask>, crate::DataLayerError>;
async fn list_active(
&self,
limit: usize,
) -> Result<Vec<StoredVideoTask>, crate::DataLayerError>;
}
#[async_trait]
pub trait VideoTaskWriteRepository: Send + Sync {
async fn upsert(&self, task: UpsertVideoTask)
-> Result<StoredVideoTask, crate::DataLayerError>;
}
pub trait VideoTaskRepository:
VideoTaskReadRepository + VideoTaskWriteRepository + Send + Sync
{
}
impl<T> VideoTaskRepository for T where
T: VideoTaskReadRepository + VideoTaskWriteRepository + Send + Sync
{
}
#[cfg(test)]
mod tests {
use super::{StoredVideoTask, VideoTaskStatus};
#[test]
fn parses_status_from_database_text() {
assert_eq!(
VideoTaskStatus::from_database("processing").expect("status should parse"),
VideoTaskStatus::Processing
);
}
#[test]
fn rejects_invalid_database_status() {
assert!(VideoTaskStatus::from_database("mystery").is_err());
}
#[test]
fn rejects_invalid_numeric_fields() {
assert!(StoredVideoTask::new(
"task-1".to_string(),
None,
None,
None,
None,
None,
None,
None,
VideoTaskStatus::Submitted,
-1,
1,
1,
None,
None,
None
)
.is_err());
}
#[test]
fn rejects_negative_updated_at_values() {
assert!(StoredVideoTask::new(
"task-1".to_string(),
None,
None,
None,
None,
None,
None,
None,
VideoTaskStatus::Submitted,
10,
1,
-1,
None,
None,
None
)
.is_err());
}
#[test]
fn rejects_negative_created_at_values() {
assert!(StoredVideoTask::new(
"task-1".to_string(),
None,
None,
None,
None,
None,
None,
None,
VideoTaskStatus::Submitted,
10,
-1,
1,
None,
None,
None
)
.is_err());
}
}