i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 01:10:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(gateway): unify background task runtime and storage

Browse Source
This commit is contained in:
fawney19
2026-05-09 21:19:29 +08:00
parent 4a64d078f3
commit 209322b499
64 changed files with 4602 additions and 280 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
Cargo.lock generated
View File

@@ -180,6 +180,7 @@ dependencies = [
"aether-runtime",
"aether-runtime-state",
"aether-scheduler-core",
"aether-task-runtime",
"aether-testkit",
"aether-usage-runtime",
"aether-video-tasks-core",
@@ -384,6 +385,18 @@ dependencies = [
"sha2",
]
[[package]]
name = "aether-task-runtime"
version = "0.1.0"
dependencies = [
"aether-runtime",
"serde",
"serde_json",
"tokio",
"tokio-util",
"tracing",
]
[[package]]
name = "aether-testkit"
version = "0.1.0"

2
Cargo.toml
View File

@@ -17,6 +17,7 @@ members = [
"crates/aether-provider-transport",
"crates/aether-scheduler-core",
"crates/aether-runtime-state",
"crates/aether-task-runtime",
"crates/aether-usage-runtime",
"crates/aether-video-tasks-core",
"apps/aether-gateway",
@@ -48,6 +49,7 @@ aether-oauth = { path = "crates/aether-oauth" }
aether-provider-transport = { path = "crates/aether-provider-transport" }
aether-scheduler-core = { path = "crates/aether-scheduler-core" }
aether-runtime-state = { path = "crates/aether-runtime-state" }
aether-task-runtime = { path = "crates/aether-task-runtime" }
aether-usage-runtime = { path = "crates/aether-usage-runtime" }
aether-video-tasks-core = { path = "crates/aether-video-tasks-core" }
aether-gateway = { path = "apps/aether-gateway" }

1
apps/aether-gateway/Cargo.toml
View File

@@ -23,6 +23,7 @@ aether-provider-transport.workspace = true
aether-scheduler-core.workspace = true
aether-runtime.workspace = true
aether-runtime-state.workspace = true
aether-task-runtime.workspace = true
aether-usage-runtime.workspace = true
aether-video-tasks-core.workspace = true
aether-wallet.workspace = true

9
apps/aether-gateway/src/control/management_token_permissions.rs
View File

@@ -152,6 +152,11 @@ const PERMISSION_GROUPS: &[PermissionGroup] = &[
label: "系统",
assignable: true,
},
PermissionGroup {
scope: "tasks",
label: "后台任务",
assignable: true,
},
PermissionGroup {
scope: "usage",
label: "用量",
@@ -434,6 +439,9 @@ fn permission_key(scope: &str, access: &str) -> &'static str {
("system", "read") => "admin:system:read",
("system", "write") => "admin:system:write",
("system", "admin") => "admin:system:admin",
("tasks", "read") => "admin:tasks:read",
("tasks", "write") => "admin:tasks:write",
("tasks", "admin") => "admin:tasks:admin",
("usage", "read") => "admin:usage:read",
("usage", "write") => "admin:usage:write",
("usage", "admin") => "admin:usage:admin",
@@ -492,6 +500,7 @@ mod tests {
"security",
"stats",
"system",
"tasks",
"usage",
"users",
"video_tasks",

77
apps/aether-gateway/src/control/route/admin/operations_families.rs
View File

@@ -46,6 +46,83 @@ pub(super) fn classify_admin_operations_family_route(
"admin:video_tasks",
false,
))
} else if method == http::Method::GET
&& matches!(normalized_path, "/api/admin/tasks" | "/api/admin/tasks/")
{
Some(classified(
"admin_proxy",
"tasks_manage",
"list_tasks",
"admin:tasks",
false,
))
} else if method == http::Method::GET
&& matches!(
normalized_path,
"/api/admin/tasks/stats" | "/api/admin/tasks/stats/"
)
{
Some(classified(
"admin_proxy",
"tasks_manage",
"stats",
"admin:tasks",
false,
))
} else if method == http::Method::GET
&& normalized_path.starts_with("/api/admin/tasks/")
&& normalized_path.ends_with("/events")
&& normalized_path.matches('/').count() == 5
{
Some(classified(
"admin_proxy",
"tasks_manage",
"events",
"admin:tasks",
false,
))
} else if method == http::Method::POST
&& normalized_path.starts_with("/api/admin/tasks/")
&& normalized_path.ends_with("/cancel")
&& normalized_path.matches('/').count() == 5
{
Some(classified(
"admin_proxy",
"tasks_manage",
"cancel",
"admin:tasks",
false,
))
} else if method == http::Method::POST
&& normalized_path.starts_with("/api/admin/tasks/")
&& normalized_path.ends_with("/trigger")
&& normalized_path.matches('/').count() == 5
{
Some(classified(
"admin_proxy",
"tasks_manage",
"trigger",
"admin:tasks",
false,
))
} else if method == http::Method::GET
&& normalized_path.starts_with("/api/admin/tasks/")
&& normalized_path["/api/admin/tasks/".len()..]
.split('/')
.count()
== 1
&& !matches!(
normalized_path,
"/api/admin/tasks/stats" | "/api/admin/tasks/stats/"
)
{
Some(classified(
"admin_proxy",
"tasks_manage",
"detail",
"admin:tasks",
false,
))
} else if method == http::Method::GET
&& normalized_path.starts_with("/api/admin/video-tasks/")
&& normalized_path.ends_with("/video")

14
apps/aether-gateway/src/data/state/core.rs
View File

@@ -34,6 +34,8 @@ impl GatewayDataState {
proxy_node_reader: None,
proxy_node_writer: None,
billing_reader: None,
background_task_reader: None,
background_task_writer: None,
gemini_file_mapping_reader: None,
gemini_file_mapping_writer: None,
global_model_reader: None,
@@ -73,6 +75,8 @@ impl GatewayDataState {
let proxy_node_reader = backends.read().proxy_nodes();
let proxy_node_writer = backends.write().proxy_nodes();
let billing_reader = backends.read().billing();
let background_task_reader = backends.read().background_tasks();
let background_task_writer = backends.write().background_tasks();
let gemini_file_mapping_reader = backends.read().gemini_file_mappings();
let global_model_reader = backends.read().global_models();
let global_model_writer = backends.write().global_models();
@@ -110,6 +114,8 @@ impl GatewayDataState {
proxy_node_reader,
proxy_node_writer,
billing_reader,
background_task_reader,
background_task_writer,
gemini_file_mapping_reader,
gemini_file_mapping_writer,
global_model_reader,
@@ -197,6 +203,14 @@ impl GatewayDataState {
self.announcement_writer.is_some()
}
pub(crate) fn has_background_task_reader(&self) -> bool {
self.background_task_reader.is_some()
}
pub(crate) fn has_background_task_writer(&self) -> bool {
self.background_task_writer.is_some()
}
pub(crate) fn has_audit_log_reader(&self) -> bool {
self.backends
.as_ref()

15
apps/aether-gateway/src/data/state/mod.rs
View File

@@ -81,6 +81,11 @@ use aether_data::{
DataBackends, DataLayerError, DatabaseMaintenanceSummary, WalletDailyUsageAggregationInput,
WalletDailyUsageAggregationResult,
};
use aether_data_contracts::repository::background_tasks::{
BackgroundTaskListQuery, BackgroundTaskReadRepository, BackgroundTaskSummary,
BackgroundTaskWriteRepository, StoredBackgroundTaskEvent, StoredBackgroundTaskRun,
StoredBackgroundTaskRunPage, UpsertBackgroundTaskEvent, UpsertBackgroundTaskRun,
};
use aether_data_contracts::repository::billing::{
AdminBillingCollectorRecord, AdminBillingCollectorWriteInput, AdminBillingMutationOutcome,
AdminBillingPresetApplyResult, AdminBillingRuleRecord, AdminBillingRuleWriteInput,
@@ -140,6 +145,8 @@ pub(crate) struct GatewayDataState {
proxy_node_reader: Option<Arc<dyn ProxyNodeReadRepository>>,
proxy_node_writer: Option<Arc<dyn ProxyNodeWriteRepository>>,
billing_reader: Option<Arc<dyn BillingReadRepository>>,
background_task_reader: Option<Arc<dyn BackgroundTaskReadRepository>>,
background_task_writer: Option<Arc<dyn BackgroundTaskWriteRepository>>,
gemini_file_mapping_reader: Option<Arc<dyn GeminiFileMappingReadRepository>>,
gemini_file_mapping_writer: Option<Arc<dyn GeminiFileMappingWriteRepository>>,
global_model_reader: Option<Arc<dyn GlobalModelReadRepository>>,
@@ -206,6 +213,14 @@ impl fmt::Debug for GatewayDataState {
.field("has_proxy_node_reader", &self.proxy_node_reader.is_some())
.field("has_proxy_node_writer", &self.proxy_node_writer.is_some())
.field("has_billing_reader", &self.billing_reader.is_some())
.field(
"has_background_task_reader",
&self.background_task_reader.is_some(),
)
.field(
"has_background_task_writer",
&self.background_task_writer.is_some(),
)
.field(
"has_gemini_file_mapping_reader",
&self.gemini_file_mapping_reader.is_some(),

98
apps/aether-gateway/src/data/state/runtime.rs
View File

@@ -5,11 +5,12 @@ use super::{
AdminBillingRuleWriteInput, AdminPaymentOrderListQuery, AdminRedeemCodeBatchListQuery,
AdminRedeemCodeListQuery, AdminWalletLedgerQuery, AdminWalletListQuery,
AdminWalletRefundRequestListQuery, AnnouncementListQuery, AuditLogListQuery,
CompleteAdminWalletRefundInput, CreateAdminRedeemCodeBatchInput,
CreateAdminRedeemCodeBatchResult, CreateAnnouncementRecord, CreateManualWalletRechargeInput,
CreateWalletRechargeOrderInput, CreateWalletRechargeOrderOutcome,
CreateWalletRefundRequestInput, CreateWalletRefundRequestOutcome, CreditAdminPaymentOrderInput,
DataLayerError, DatabaseMaintenanceSummary, DecisionTrace, DeleteAdminRedeemCodeBatchInput,
BackgroundTaskListQuery, BackgroundTaskSummary, CompleteAdminWalletRefundInput,
CreateAdminRedeemCodeBatchInput, CreateAdminRedeemCodeBatchResult, CreateAnnouncementRecord,
CreateManualWalletRechargeInput, CreateWalletRechargeOrderInput,
CreateWalletRechargeOrderOutcome, CreateWalletRefundRequestInput,
CreateWalletRefundRequestOutcome, CreditAdminPaymentOrderInput, DataLayerError,
DatabaseMaintenanceSummary, DecisionTrace, DeleteAdminRedeemCodeBatchInput,
DisableAdminRedeemCodeBatchInput, DisableAdminRedeemCodeInput, FailAdminWalletRefundInput,
GatewayDataState, GatewayProviderTransportSnapshot, LocalVideoTaskReadResponse,
ProcessAdminWalletRefundInput, ProcessPaymentCallbackInput, ProcessPaymentCallbackOutcome,
@@ -19,15 +20,16 @@ use super::{
StoredAdminRedeemCodePage, StoredAdminWalletLedgerPage, StoredAdminWalletListPage,
StoredAdminWalletRefund, StoredAdminWalletRefundPage, StoredAdminWalletRefundRequestPage,
StoredAdminWalletTransaction, StoredAdminWalletTransactionPage, StoredAnnouncement,
StoredAnnouncementPage, StoredBillingModelContext, StoredProviderQuotaSnapshot,
StoredAnnouncementPage, StoredBackgroundTaskEvent, StoredBackgroundTaskRun,
StoredBackgroundTaskRunPage, StoredBillingModelContext, StoredProviderQuotaSnapshot,
StoredProviderUsageSummary, StoredRequestUsageAudit, StoredSuspiciousActivity,
StoredUsageSettlement, StoredUserAuditLogPage, StoredUserAuthRecord, StoredUserExportRow,
StoredUserSummary, StoredVideoTask, StoredWalletDailyUsageLedger,
StoredWalletDailyUsageLedgerPage, StoredWalletSnapshot, UpdateAnnouncementRecord,
UpsertUsageRecord, UpsertVideoTask, UsageSettlementInput, VideoTaskLookupKey,
VideoTaskModelCount, VideoTaskQueryFilter, VideoTaskStatusCount,
WalletDailyUsageAggregationInput, WalletDailyUsageAggregationResult, WalletLookupKey,
WalletMutationOutcome,
UpsertBackgroundTaskEvent, UpsertBackgroundTaskRun, UpsertUsageRecord, UpsertVideoTask,
UsageSettlementInput, VideoTaskLookupKey, VideoTaskModelCount, VideoTaskQueryFilter,
VideoTaskStatusCount, WalletDailyUsageAggregationInput, WalletDailyUsageAggregationResult,
WalletLookupKey, WalletMutationOutcome,
};
use aether_data_contracts::repository::usage::{
PendingUsageCleanupSummary, ProviderApiKeyWindowUsageRequest,
@@ -1716,6 +1718,82 @@ impl GatewayDataState {
) -> Result<Option<LocalVideoTaskReadResponse>, DataLayerError> {
read_data_backed_video_task_response(self, route_family, request_path).await
}
pub(crate) async fn find_background_task_run(
&self,
run_id: &str,
) -> Result<Option<StoredBackgroundTaskRun>, DataLayerError> {
match &self.background_task_reader {
Some(repository) => repository.find_run(run_id).await,
None => Ok(None),
}
}
pub(crate) async fn list_background_task_runs(
&self,
query: &BackgroundTaskListQuery,
) -> Result<StoredBackgroundTaskRunPage, DataLayerError> {
match &self.background_task_reader {
Some(repository) => repository.list_runs(query).await,
None => Ok(StoredBackgroundTaskRunPage::default()),
}
}
pub(crate) async fn list_background_task_events(
&self,
run_id: &str,
offset: usize,
limit: usize,
) -> Result<Vec<StoredBackgroundTaskEvent>, DataLayerError> {
match &self.background_task_reader {
Some(repository) => repository.list_events(run_id, offset, limit).await,
None => Ok(Vec::new()),
}
}
pub(crate) async fn summarize_background_task_runs(
&self,
) -> Result<BackgroundTaskSummary, DataLayerError> {
match &self.background_task_reader {
Some(repository) => repository.summarize_runs().await,
None => Ok(BackgroundTaskSummary::default()),
}
}
pub(crate) async fn upsert_background_task_run(
&self,
run: UpsertBackgroundTaskRun,
) -> Result<Option<StoredBackgroundTaskRun>, DataLayerError> {
match &self.background_task_writer {
Some(repository) => repository.upsert_run(run).await.map(Some),
None => Ok(None),
}
}
pub(crate) async fn request_cancel_background_task_run(
&self,
run_id: &str,
updated_at_unix_secs: u64,
) -> Result<bool, DataLayerError> {
match &self.background_task_writer {
Some(repository) => {
repository
.request_cancel(run_id, updated_at_unix_secs)
.await
}
None => Ok(false),
}
}
pub(crate) async fn upsert_background_task_event(
&self,
event: UpsertBackgroundTaskEvent,
) -> Result<Option<StoredBackgroundTaskEvent>, DataLayerError> {
match &self.background_task_writer {
Some(repository) => repository.upsert_event(event).await.map(Some),
None => Ok(None),
}
}
}
#[cfg(test)]

4
apps/aether-gateway/src/data/state/testing/announcements.rs
View File

@@ -43,6 +43,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -93,6 +95,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,

88
apps/aether-gateway/src/data/state/testing/mod.rs
View File

@@ -76,6 +76,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -125,6 +127,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -170,6 +174,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -303,6 +309,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -364,6 +372,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -418,6 +428,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -481,6 +493,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -526,6 +540,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -572,6 +588,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -629,6 +647,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -688,6 +708,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -731,6 +753,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -789,6 +813,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -840,6 +866,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: Some(wallet_reader),
wallet_writer: Some(wallet_writer),
settlement_writer: None,
@@ -896,6 +924,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: Some(wallet_reader),
wallet_writer: Some(wallet_writer),
settlement_writer: None,
@@ -953,6 +983,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: Some(wallet_reader),
wallet_writer: Some(wallet_writer),
settlement_writer: None,
@@ -1009,6 +1041,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -1054,6 +1088,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -1099,6 +1135,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -1156,6 +1194,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -1218,6 +1258,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -1263,6 +1305,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -1313,6 +1357,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -1380,6 +1426,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -1442,6 +1490,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -1488,6 +1538,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -1534,6 +1586,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -1582,6 +1636,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -1628,6 +1684,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -1674,6 +1732,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -1728,6 +1788,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -1783,6 +1845,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -1841,6 +1905,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -1905,6 +1971,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -1970,6 +2038,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -2039,6 +2109,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -2115,6 +2187,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: Some(wallet_reader),
wallet_writer: Some(wallet_writer),
settlement_writer: None,
@@ -2173,6 +2247,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -2222,6 +2298,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -2267,6 +2345,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -2318,6 +2398,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: Some(wallet_reader),
wallet_writer: Some(wallet_writer),
settlement_writer: None,
@@ -2373,6 +2455,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: Some(wallet_reader),
wallet_writer: Some(wallet_writer),
settlement_writer: None,
@@ -2429,6 +2513,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: Some(wallet_reader),
wallet_writer: Some(wallet_writer),
settlement_writer: None,
@@ -2478,6 +2564,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: None,
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,

12
apps/aether-gateway/src/data/state/testing/video_tasks.rs
View File

@@ -47,6 +47,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: Some(repository),
video_task_writer: None,
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -99,6 +101,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: Some(video_task_reader),
video_task_writer: Some(video_task_writer),
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -148,6 +152,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: Some(video_task_reader),
video_task_writer: Some(video_task_writer),
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -201,6 +207,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: Some(video_task_reader),
video_task_writer: Some(video_task_writer),
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -258,6 +266,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: Some(video_task_reader),
video_task_writer: Some(video_task_writer),
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
@@ -324,6 +334,8 @@ impl GatewayDataState {
usage_worker_queue: None,
video_task_reader: Some(video_task_reader),
video_task_writer: Some(video_task_writer),
background_task_reader: None,
background_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,

15
apps/aether-gateway/src/handlers/admin/features/background_tasks/mod.rs Normal file
View File

@@ -0,0 +1,15 @@
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::GatewayError;
use axum::body::{Body, Bytes};
use axum::response::Response;
mod routes;
pub(crate) async fn maybe_build_local_admin_background_tasks_response(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
request_body: Option<&Bytes>,
) -> Result<Option<Response<Body>>, GatewayError> {
routes::maybe_build_local_admin_background_tasks_response(state, request_context, request_body)
.await
}

373
apps/aether-gateway/src/handlers/admin/features/background_tasks/routes.rs Normal file
View File

@@ -0,0 +1,373 @@
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::handlers::admin::shared::{
attach_admin_audit_response, query_param_value, unix_secs_to_rfc3339,
};
use crate::task_runtime::{
self, set_cancel_signal, TASK_KEY_PROVIDER_DELETE, TASK_KEY_PROVIDER_OAUTH_BATCH_IMPORT,
};
use crate::GatewayError;
use aether_data_contracts::repository::background_tasks::{
BackgroundTaskKind, BackgroundTaskListQuery, BackgroundTaskStatus,
};
use axum::{
body::{Body, Bytes},
http,
response::{IntoResponse, Response},
Json,
};
use serde_json::json;
const DEFAULT_PAGE_SIZE: usize = 20;
const MAX_PAGE_SIZE: usize = 100;
const DEFAULT_EVENTS_PAGE_SIZE: usize = 50;
pub(super) async fn maybe_build_local_admin_background_tasks_response(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
request_body: Option<&Bytes>,
) -> Result<Option<Response<Body>>, GatewayError> {
if request_context.route_family() != Some("tasks_manage") {
return Ok(None);
}
match request_context.route_kind() {
Some("list_tasks") if request_context.method() == http::Method::GET => {
let query = request_context.query_string();
let page = query_param_value(query, "page")
.and_then(|value| value.parse::<usize>().ok())
.unwrap_or(1)
.max(1);
let page_size = query_param_value(query, "page_size")
.and_then(|value| value.parse::<usize>().ok())
.unwrap_or(DEFAULT_PAGE_SIZE)
.clamp(1, MAX_PAGE_SIZE);
let kind = query_param_value(query, "kind")
.map(|value| BackgroundTaskKind::from_database(value.as_str()))
.transpose()
.map_err(|err| GatewayError::Internal(err.to_string()))?;
let status = query_param_value(query, "status")
.map(|value| BackgroundTaskStatus::from_database(value.as_str()))
.transpose()
.map_err(|err| GatewayError::Internal(err.to_string()))?;
let trigger = query_param_value(query, "trigger");
let task_key_substring = query_param_value(query, "task_key");
let offset = (page - 1).saturating_mul(page_size);
let response = state
.list_background_task_runs(&BackgroundTaskListQuery {
task_key_substring,
kind,
status,
trigger,
offset,
limit: page_size,
})
.await?;
let pages = if response.total == 0 {
0
} else {
(response.total + page_size - 1) / page_size
};
let items = response
.items
.iter()
.map(|run| {
json!({
"id": run.id,
"task_key": run.task_key,
"kind": run.kind.as_database(),
"trigger": run.trigger,
"status": run.status.as_database(),
"attempt": run.attempt,
"max_attempts": run.max_attempts,
"owner_instance": run.owner_instance,
"progress_percent": run.progress_percent,
"progress_message": run.progress_message,
"payload": run.payload_json,
"result": run.result_json,
"error_message": run.error_message,
"cancel_requested": run.cancel_requested,
"created_by": run.created_by,
"created_at": unix_secs_to_rfc3339(run.created_at_unix_secs),
"started_at": run.started_at_unix_secs.and_then(unix_secs_to_rfc3339),
"finished_at": run.finished_at_unix_secs.and_then(unix_secs_to_rfc3339),
"updated_at": unix_secs_to_rfc3339(run.updated_at_unix_secs),
})
})
.collect::<Vec<_>>();
let definitions = task_runtime::task_definitions()
.iter()
.map(|definition| {
json!({
"task_key": definition.key,
"kind": definition.kind.as_str(),
"trigger": definition.trigger,
"max_attempts": definition.retry_policy.max_attempts,
"singleton": definition.singleton,
"persist_history": definition.persist_history,
})
})
.collect::<Vec<_>>();
return Ok(Some(
Json(json!({
"items": items,
"total": response.total,
"page": page,
"page_size": page_size,
"pages": pages,
"definitions": definitions,
}))
.into_response(),
));
}
Some("stats") if request_context.method() == http::Method::GET => {
let stats = state.summarize_background_task_runs().await?;
return Ok(Some(
Json(json!({
"total": stats.total,
"running_count": stats.running_count,
"by_status": stats.by_status,
"by_kind": stats.by_kind,
"registered_tasks": task_runtime::task_definitions().len(),
}))
.into_response(),
));
}
Some("detail") if request_context.method() == http::Method::GET => {
let Some(run_id) = task_id_from_path(request_context.path()) else {
return Ok(Some(
(
http::StatusCode::NOT_FOUND,
Json(json!({"detail":"Task not found"})),
)
.into_response(),
));
};
let Some(run) = state.find_background_task_run(run_id).await? else {
return Ok(Some(
(
http::StatusCode::NOT_FOUND,
Json(json!({"detail":"Task not found"})),
)
.into_response(),
));
};
return Ok(Some(attach_admin_audit_response(
Json(json!({
"id": run.id,
"task_key": run.task_key,
"kind": run.kind.as_database(),
"trigger": run.trigger,
"status": run.status.as_database(),
"attempt": run.attempt,
"max_attempts": run.max_attempts,
"owner_instance": run.owner_instance,
"progress_percent": run.progress_percent,
"progress_message": run.progress_message,
"payload": run.payload_json,
"result": run.result_json,
"error_message": run.error_message,
"cancel_requested": run.cancel_requested,
"created_by": run.created_by,
"created_at": unix_secs_to_rfc3339(run.created_at_unix_secs),
"started_at": run.started_at_unix_secs.and_then(unix_secs_to_rfc3339),
"finished_at": run.finished_at_unix_secs.and_then(unix_secs_to_rfc3339),
"updated_at": unix_secs_to_rfc3339(run.updated_at_unix_secs),
}))
.into_response(),
"admin_task_detail_viewed",
"view_task_detail",
"background_task",
run_id,
)));
}
Some("events") if request_context.method() == http::Method::GET => {
let Some(run_id) = nested_task_id_from_path(request_context.path(), "/events") else {
return Ok(Some(
(
http::StatusCode::NOT_FOUND,
Json(json!({"detail":"Task not found"})),
)
.into_response(),
));
};
let query = request_context.query_string();
let page = query_param_value(query, "page")
.and_then(|value| value.parse::<usize>().ok())
.unwrap_or(1)
.max(1);
let page_size = query_param_value(query, "page_size")
.and_then(|value| value.parse::<usize>().ok())
.unwrap_or(DEFAULT_EVENTS_PAGE_SIZE)
.clamp(1, MAX_PAGE_SIZE);
let offset = (page - 1).saturating_mul(page_size);
let events = state
.list_background_task_events(run_id, offset, page_size)
.await?;
return Ok(Some(
Json(json!({
"items": events.into_iter().map(|event| {
json!({
"id": event.id,
"run_id": event.run_id,
"event_type": event.event_type,
"message": event.message,
"payload": event.payload_json,
"created_at": unix_secs_to_rfc3339(event.created_at_unix_secs),
})
}).collect::<Vec<_>>(),
"page": page,
"page_size": page_size,
}))
.into_response(),
));
}
Some("cancel") if request_context.method() == http::Method::POST => {
let Some(run_id) = nested_task_id_from_path(request_context.path(), "/cancel") else {
return Ok(Some(
(
http::StatusCode::NOT_FOUND,
Json(json!({"detail":"Task not found"})),
)
.into_response(),
));
};
let now = task_runtime::now_unix_secs();
let cancelled = state
.request_cancel_background_task_run(run_id, now)
.await?;
if !cancelled {
return Ok(Some(
(
http::StatusCode::NOT_FOUND,
Json(json!({ "detail": "Task not found" })),
)
.into_response(),
));
}
let _ = set_cancel_signal(state.app(), run_id).await;
task_runtime::append_event_with_logging(
state.app(),
run_id,
"cancel_requested",
"cancel requested by admin",
None,
)
.await;
return Ok(Some(attach_admin_audit_response(
Json(json!({
"id": run_id,
"status": "cancel_requested",
"message": "Task cancellation requested",
}))
.into_response(),
"admin_task_cancel_requested",
"cancel_task",
"background_task",
run_id,
)));
}
Some("trigger") if request_context.method() == http::Method::POST => {
let Some(task_key) = nested_task_id_from_path(request_context.path(), "/trigger")
else {
return Ok(Some(
(
http::StatusCode::NOT_FOUND,
Json(json!({"detail":"Task not found"})),
)
.into_response(),
));
};
let payload = parse_json_payload(request_body)?;
if task_key == TASK_KEY_PROVIDER_DELETE {
let provider_id = payload
.get("provider_id")
.and_then(serde_json::Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.ok_or_else(|| {
GatewayError::Internal(
"admin task trigger provider delete requires provider_id".to_string(),
)
})?;
let Some(run_id) =
task_runtime::submit_provider_delete_task(state, provider_id, Some("admin"))
.await?
else {
return Ok(Some(
(
http::StatusCode::NOT_FOUND,
Json(json!({"detail":"Provider 不存在"})),
)
.into_response(),
));
};
return Ok(Some(attach_admin_audit_response(
Json(json!({
"task_key": task_key,
"run_id": run_id,
"status": "queued",
}))
.into_response(),
"admin_task_triggered",
"trigger_task",
"background_task",
task_key,
)));
}
if task_key == TASK_KEY_PROVIDER_OAUTH_BATCH_IMPORT {
return Ok(Some(
(
http::StatusCode::BAD_REQUEST,
Json(json!({
"detail": "请使用 provider oauth batch import 专用接口触发该任务",
})),
)
.into_response(),
));
}
return Ok(Some(
(
http::StatusCode::BAD_REQUEST,
Json(json!({
"detail": format!("Unsupported task_key: {task_key}"),
})),
)
.into_response(),
));
}
_ => {}
}
Ok(None)
}
fn task_id_from_path(request_path: &str) -> Option<&str> {
let task_id = request_path.strip_prefix("/api/admin/tasks/")?;
if task_id.is_empty() || task_id.contains('/') || task_id == "stats" {
return None;
}
Some(task_id)
}
fn nested_task_id_from_path<'a>(request_path: &'a str, suffix: &str) -> Option<&'a str> {
let task_id = request_path
.strip_prefix("/api/admin/tasks/")?
.strip_suffix(suffix)?;
if task_id.is_empty() || task_id.contains('/') {
return None;
}
Some(task_id)
}
fn parse_json_payload(request_body: Option<&Bytes>) -> Result<serde_json::Value, GatewayError> {
let Some(body) = request_body else {
return Ok(json!({}));
};
if body.is_empty() {
return Ok(json!({}));
}
serde_json::from_slice::<serde_json::Value>(body)
.map_err(|err| GatewayError::Internal(format!("invalid json body: {err}")))
}

2
apps/aether-gateway/src/handlers/admin/features/mod.rs
View File

@@ -1,7 +1,9 @@
mod background_tasks;
mod gemini_files;
mod routes;
mod video_tasks;
pub(super) use self::background_tasks::maybe_build_local_admin_background_tasks_response;
pub(super) use self::gemini_files::maybe_build_local_admin_gemini_files_response;
pub(super) use self::routes::maybe_build_local_admin_features_response;
pub(crate) use self::video_tasks::maybe_build_local_admin_video_tasks_response;

12
apps/aether-gateway/src/handlers/admin/features/routes.rs
View File

@@ -1,9 +1,19 @@
use super::{gemini_files, video_tasks};
use super::{background_tasks, gemini_files, video_tasks};
use crate::handlers::admin::request::{AdminRouteRequest, AdminRouteResult};
pub(crate) async fn maybe_build_local_admin_features_response(
request: AdminRouteRequest<'_>,
) -> AdminRouteResult {
if let Some(response) = background_tasks::maybe_build_local_admin_background_tasks_response(
&request.state(),
&request.request_context(),
request.request_body(),
)
.await?
{
return Ok(Some(response));
}
if let Some(response) = video_tasks::maybe_build_local_admin_video_tasks_response(
&request.state(),
&request.request_context(),

49
apps/aether-gateway/src/handlers/admin/provider/crud/delete_task.rs
View File

@@ -1,11 +1,10 @@
use crate::handlers::admin::provider::delete_task::run_admin_provider_delete_task;
use crate::handlers::admin::provider::shared::paths::{
admin_provider_delete_task_parts, admin_provider_id_for_manage_path,
};
use crate::handlers::admin::provider::shared::support::build_admin_provider_delete_task_payload;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::handlers::admin::shared::attach_admin_audit_response;
use crate::{GatewayError, LocalProviderDeleteTaskState};
use crate::GatewayError;
use axum::{
body::Body,
http,
@@ -13,8 +12,6 @@ use axum::{
Json,
};
use serde_json::json;
use tracing::warn;
use uuid::Uuid;
fn build_admin_provider_not_found_response(detail: impl Into<String>) -> Response<Body> {
(
@@ -60,52 +57,18 @@ pub(crate) async fn maybe_build_local_admin_provider_delete_task_response(
"Provider 不存在",
)));
};
let Some(provider) = state
.read_provider_catalog_providers_by_ids(std::slice::from_ref(&provider_id))
.await?
.into_iter()
.next()
let Some(task_id) =
crate::task_runtime::submit_provider_delete_task(state, &provider_id, Some("admin"))
.await?
else {
return Ok(Some(build_admin_provider_not_found_response(
"提供商不存在",
)));
};
let task_id = Uuid::new_v4().simple().to_string()[..16].to_string();
let pending_task = LocalProviderDeleteTaskState {
task_id: task_id.clone(),
provider_id: provider.id.clone(),
status: "pending".to_string(),
stage: "queued".to_string(),
total_keys: 0,
deleted_keys: 0,
total_endpoints: 0,
deleted_endpoints: 0,
message: "delete task submitted".to_string(),
};
state.put_provider_delete_task(pending_task.clone());
if let Err(err) = state
.run_admin_provider_delete_task(&provider.id, &task_id)
.await
{
warn!(
"gateway admin provider delete task failed for provider {}: {:?}",
provider.id, err
);
state.put_provider_delete_task(LocalProviderDeleteTaskState {
task_id: task_id.clone(),
provider_id: provider.id.clone(),
status: "failed".to_string(),
stage: "failed".to_string(),
total_keys: 0,
deleted_keys: 0,
total_endpoints: 0,
deleted_endpoints: 0,
message: format!("provider delete failed: {err:?}"),
});
}
return Ok(Some(attach_admin_audit_response(
Json(json!({
"task_id": task_id,
"run_id": task_id,
"status": "pending",
"message": "删除任务已提交，提供商已进入后台删除队列",
}))
@@ -113,7 +76,7 @@ pub(crate) async fn maybe_build_local_admin_provider_delete_task_response(
"admin_provider_delete_queued",
"delete_provider",
"provider",
&provider.id,
&provider_id,
)));
}

126
apps/aether-gateway/src/handlers/admin/provider/oauth/dispatch/batch/task.rs
View File

@@ -15,7 +15,14 @@ use crate::handlers::admin::provider::oauth::state::{
};
use crate::handlers::admin::provider::shared::paths::admin_provider_oauth_batch_import_task_provider_id;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::task_runtime::{
append_event_with_logging, now_unix_secs, task_definition, update_run_status,
upsert_run_with_logging, TASK_KEY_PROVIDER_OAUTH_BATCH_IMPORT,
};
use crate::GatewayError;
use aether_data_contracts::repository::background_tasks::{
BackgroundTaskKind, BackgroundTaskStatus, UpsertBackgroundTaskRun,
};
use axum::{
body::Bytes,
http,
@@ -133,11 +140,7 @@ pub(in super::super) async fn handle_admin_provider_oauth_start_batch_import_tas
}
let task_id = Uuid::new_v4().to_string();
let created_at = SystemTime::now()
.duration_since(UNIX_EPOCH)
.ok()
.map(|duration| duration.as_secs())
.unwrap_or(0);
let created_at = now_unix_secs();
let submitted_state = build_admin_provider_oauth_batch_task_state(
&task_id,
&provider_id,
@@ -167,6 +170,50 @@ pub(in super::super) async fn handle_admin_provider_oauth_start_batch_import_tas
));
}
if state.has_background_task_data_writer() {
let max_attempts = task_definition(TASK_KEY_PROVIDER_OAUTH_BATCH_IMPORT)
.map(|item| item.retry_policy.max_attempts)
.unwrap_or(1);
let run = UpsertBackgroundTaskRun {
id: task_id.clone(),
task_key: TASK_KEY_PROVIDER_OAUTH_BATCH_IMPORT.to_string(),
kind: BackgroundTaskKind::OnDemand,
trigger: "manual".to_string(),
status: BackgroundTaskStatus::Queued,
attempt: 1,
max_attempts,
owner_instance: Some(state.app().tunnel.local_instance_id().to_string()),
progress_percent: 0,
progress_message: Some("provider oauth batch import queued".to_string()),
payload_json: Some(json!({
"provider_id": provider_id.clone(),
"provider_type": provider_type.clone(),
"total": total,
})),
result_json: None,
error_message: None,
cancel_requested: false,
created_by: Some("admin".to_string()),
created_at_unix_secs: created_at,
started_at_unix_secs: None,
finished_at_unix_secs: None,
updated_at_unix_secs: created_at,
};
let _ = upsert_run_with_logging(state.app(), run).await;
append_event_with_logging(
state.app(),
&task_id,
"queued",
"provider oauth batch import queued",
Some(json!({
"provider_id": provider_id.clone(),
"provider_type": provider_type.clone(),
"total": total,
})),
)
.await;
}
let task_state = state.cloned_app();
let task_id_for_worker = task_id.clone();
let provider_id_for_worker = provider_id.clone();
@@ -201,6 +248,27 @@ pub(in super::super) async fn handle_admin_provider_oauth_start_batch_import_tas
.save_provider_oauth_batch_task_payload(&task_id_for_worker, &processing_state)
.await;
let _ = update_run_status(
&task_state,
&task_id_for_worker,
BackgroundTaskStatus::Running,
Some(1),
Some("provider oauth batch import started".to_string()),
None,
None,
Some(started_at),
None,
)
.await;
append_event_with_logging(
&task_state,
&task_id_for_worker,
"running",
"provider oauth batch import started",
None,
)
.await;
let mut progress_reporter = BatchTaskProgressReporter {
app: task_state.clone(),
task_id: task_id_for_worker.clone(),
@@ -270,6 +338,34 @@ pub(in super::super) async fn handle_admin_provider_oauth_start_batch_import_tas
let _ = AdminAppState::new(&task_state)
.save_provider_oauth_batch_task_payload(&task_id_for_worker, &completed_state)
.await;
let _ = update_run_status(
&task_state,
&task_id_for_worker,
BackgroundTaskStatus::Succeeded,
Some(100),
Some(message),
Some(json!({
"provider_id": provider_id_for_worker,
"provider_type": provider_type_for_worker,
"total": outcome.total,
"success": outcome.success,
"failed": outcome.failed,
"created_count": created_count,
"replaced_count": replaced_count,
})),
None,
None,
Some(finished_at),
)
.await;
append_event_with_logging(
&task_state,
&task_id_for_worker,
"succeeded",
"provider oauth batch import completed",
None,
)
.await;
}
Err(err) => {
let finished_at = SystemTime::now()
@@ -299,6 +395,26 @@ pub(in super::super) async fn handle_admin_provider_oauth_start_batch_import_tas
let _ = AdminAppState::new(&task_state)
.save_provider_oauth_batch_task_payload(&task_id_for_worker, &failed_state)
.await;
let _ = update_run_status(
&task_state,
&task_id_for_worker,
BackgroundTaskStatus::Failed,
Some(100),
Some("provider oauth batch import failed".to_string()),
None,
Some(error_message.clone()),
None,
Some(finished_at),
)
.await;
append_event_with_logging(
&task_state,
&task_id_for_worker,
"failed",
"provider oauth batch import failed",
Some(json!({ "error": error_message.clone() })),
)
.await;
tracing::warn!(
task_id = %task_id_for_worker,
provider_id = %provider_id_for_worker,

3
apps/aether-gateway/src/handlers/admin/provider/oauth/runtime.rs
View File

@@ -4,6 +4,7 @@ use super::quota::codex::refresh_codex_provider_quota_locally;
use super::quota::kiro::refresh_kiro_provider_quota_locally;
use crate::handlers::admin::request::AdminAppState;
use crate::provider_key_auth::provider_key_is_oauth_managed;
use crate::task_runtime::{spawn_fire_and_forget, TASK_KEY_PROVIDER_OAUTH_ACCOUNT_REFRESH};
use crate::{AppState, GatewayError};
use aether_contracts::ProxySnapshot;
use aether_data_contracts::repository::provider_catalog::{
@@ -170,7 +171,7 @@ pub(crate) fn spawn_provider_oauth_account_state_refresh_after_update(
key_id: String,
proxy_override: Option<ProxySnapshot>,
) {
tokio::spawn(async move {
spawn_fire_and_forget(TASK_KEY_PROVIDER_OAUTH_ACCOUNT_REFRESH, async move {
let _ = refresh_provider_oauth_account_state_after_update(
&AdminAppState::new(&app),
&provider,

3
apps/aether-gateway/src/handlers/admin/provider/ops/providers/balance_cache.rs
View File

@@ -1,5 +1,6 @@
use super::actions::admin_provider_ops_local_action_response;
use crate::handlers::admin::request::AdminAppState;
use crate::task_runtime::{spawn_fire_and_forget, TASK_KEY_PROVIDER_BALANCE_REFRESH};
use serde_json::{json, Value};
use std::collections::HashSet;
use std::time::Duration;
@@ -130,7 +131,7 @@ pub(super) async fn spawn_admin_provider_ops_balance_refresh(
let app = state.cloned_app();
let provider_id = provider_id.to_string();
tokio::spawn(async move {
spawn_fire_and_forget(TASK_KEY_PROVIDER_BALANCE_REFRESH, async move {
let permit = match tokio::time::timeout(
Duration::from_secs(5),
ADMIN_PROVIDER_OPS_BALANCE_REFRESH_SEMAPHORE.acquire(),

8
apps/aether-gateway/src/handlers/admin/request/capabilities.rs
View File

@@ -76,6 +76,14 @@ impl<'a> AdminAppState<'a> {
self.app.has_gemini_file_mapping_data_writer()
}
pub(crate) fn has_background_task_data_reader(&self) -> bool {
self.app.has_background_task_data_reader()
}
pub(crate) fn has_background_task_data_writer(&self) -> bool {
self.app.has_background_task_data_writer()
}
pub(crate) fn has_auth_api_key_data_reader(&self) -> bool {
self.app.has_auth_api_key_data_reader()
}

53
apps/aether-gateway/src/handlers/admin/request/features.rs
View File

@@ -2,6 +2,59 @@ use super::{AdminAppState, AdminCancelVideoTaskError};
use crate::GatewayError;
impl<'a> AdminAppState<'a> {
pub(crate) async fn list_background_task_runs(
&self,
query: &aether_data_contracts::repository::background_tasks::BackgroundTaskListQuery,
) -> Result<
aether_data_contracts::repository::background_tasks::StoredBackgroundTaskRunPage,
GatewayError,
> {
self.app.list_background_task_runs(query).await
}
pub(crate) async fn find_background_task_run(
&self,
run_id: &str,
) -> Result<
Option<aether_data_contracts::repository::background_tasks::StoredBackgroundTaskRun>,
GatewayError,
> {
self.app.find_background_task_run(run_id).await
}
pub(crate) async fn list_background_task_events(
&self,
run_id: &str,
offset: usize,
limit: usize,
) -> Result<
Vec<aether_data_contracts::repository::background_tasks::StoredBackgroundTaskEvent>,
GatewayError,
> {
self.app
.list_background_task_events(run_id, offset, limit)
.await
}
pub(crate) async fn summarize_background_task_runs(
&self,
) -> Result<
aether_data_contracts::repository::background_tasks::BackgroundTaskSummary,
GatewayError,
> {
self.app.summarize_background_task_runs().await
}
pub(crate) async fn request_cancel_background_task_run(
&self,
run_id: &str,
updated_at_unix_secs: u64,
) -> Result<bool, GatewayError> {
self.app
.request_cancel_background_task_run(run_id, updated_at_unix_secs)
.await
}
pub(crate) async fn list_gemini_file_mappings(
&self,
query: &aether_data::repository::gemini_file_mappings::GeminiFileMappingListQuery,

1
apps/aether-gateway/src/lib.rs
View File

@@ -58,6 +58,7 @@ mod router;
mod scheduler;
mod state;
mod system_features;
mod task_runtime;
mod tunnel;
mod usage;
mod video_tasks;

8
apps/aether-gateway/src/main.rs
View File

@@ -1202,13 +1202,13 @@ async fn run() -> Result<(), Box<dyn std::error::Error>> {
state.bootstrap_admin_from_env().await?;
let background_tasks = if args.node_role.spawns_background_tasks() {
state.spawn_background_tasks()
Some(state.spawn_background_tasks())
} else {
info!(
node_role = args.node_role.as_str(),
"background workers disabled for this node role"
);
Vec::new()
None
};
let listener = tokio::net::TcpListener::bind(bind_addr).await?;
let public_base_url = resolve_local_http_base_url(app_port)?;
@@ -1241,8 +1241,8 @@ async fn run() -> Result<(), Box<dyn std::error::Error>> {
router.into_make_service_with_connect_info::<std::net::SocketAddr>(),
)
.await?;
for handle in background_tasks {
handle.abort();
if let Some(background_tasks) = background_tasks {
background_tasks.shutdown().await;
}
Ok(())
}

4
apps/aether-gateway/src/maintenance/tests.rs
View File

@@ -119,9 +119,7 @@ async fn gateway_background_request_candidate_cleanup_deletes_expired_entries_in
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
for handle in background_tasks {
handle.abort();
}
background_tasks.shutdown().await;
}
#[tokio::test]

4
apps/aether-gateway/src/model_fetch/tests.rs
View File

@@ -539,8 +539,6 @@ async fn gateway_background_model_fetch_updates_key_and_syncs_provider_model_whi
.expect("execution runtime plan should be captured");
assert_eq!(seen_plan.url, "https://api.openai.example/v1/models");
for handle in background_tasks {
handle.abort();
}
background_tasks.shutdown().await;
execution_runtime_handle.abort();
}

172
apps/aether-gateway/src/state/core.rs
View File

@@ -19,7 +19,6 @@ use aether_runtime_state::{
RuntimeSemaphoreSnapshot, RuntimeState,
};
use aether_scheduler_core::PROVIDER_KEY_RPM_WINDOW_SECS;
use tokio::task::JoinHandle;
use super::{AppState, FrontdoorCorsConfig, LocalExecutionRuntimeMissDiagnostic};
@@ -997,71 +996,112 @@ impl AppState {
Ok(self)
}
pub fn spawn_background_tasks(&self) -> Vec<JoinHandle<()>> {
let mut tasks = Vec::new();
if let Some(handle) = self.usage_runtime.spawn_worker(self.data.clone()) {
tasks.push(handle);
}
if let Some(handle) =
crate::wallet_runtime::spawn_provider_quota_reset_worker(self.data.clone())
{
tasks.push(handle);
}
if let Some(handle) = spawn_audit_cleanup_worker(self.data.clone()) {
tasks.push(handle);
}
if let Some(handle) = spawn_db_maintenance_worker(self.data.clone()) {
tasks.push(handle);
}
if let Some(handle) = spawn_wallet_daily_usage_aggregation_worker(self.data.clone()) {
tasks.push(handle);
}
if let Some(handle) = spawn_stats_aggregation_worker(self.data.clone()) {
tasks.push(handle);
}
if let Some(handle) = spawn_usage_cleanup_worker(self.data.clone()) {
tasks.push(handle);
}
if let Some(handle) = spawn_pool_monitor_worker(self.data.clone()) {
tasks.push(handle);
}
if let Some(handle) = spawn_pool_quota_probe_worker(self.clone()) {
tasks.push(handle);
}
if let Some(handle) = spawn_stats_hourly_aggregation_worker(self.data.clone()) {
tasks.push(handle);
}
if let Some(handle) = spawn_pending_cleanup_worker(self.data.clone()) {
tasks.push(handle);
}
if let Some(handle) = spawn_proxy_node_stale_cleanup_worker(self.data.clone()) {
tasks.push(handle);
}
if let Some(handle) = spawn_proxy_node_metrics_cleanup_worker(self.data.clone()) {
tasks.push(handle);
}
if let Some(handle) = spawn_proxy_upgrade_rollout_worker(self.clone()) {
tasks.push(handle);
}
if let Some(handle) = spawn_provider_checkin_worker(self.clone()) {
tasks.push(handle);
}
if let Some(handle) = spawn_oauth_token_refresh_worker(self.clone()) {
tasks.push(handle);
}
if let Some(handle) = spawn_request_candidate_cleanup_worker(self.data.clone()) {
tasks.push(handle);
}
if let Some(handle) = spawn_gemini_file_mapping_cleanup_worker(self.data.clone()) {
tasks.push(handle);
}
if let Some(handle) = spawn_model_fetch_worker(self.clone()) {
tasks.push(handle);
}
if let Some(handle) = spawn_video_task_poller(self.clone()) {
tasks.push(handle);
}
tasks
pub fn spawn_background_tasks(&self) -> crate::task_runtime::TaskSupervisor {
let mut supervisor = crate::task_runtime::TaskSupervisor::new();
let record_boot = |task_key: &'static str| {
if !self.has_background_task_data_writer() {
return;
}
let Some(definition) = crate::task_runtime::task_definition(task_key) else {
return;
};
std::mem::drop(crate::task_runtime::spawn_record_worker_boot(
self.clone(),
task_key,
crate::task_runtime::background_task_kind(definition.kind),
definition.trigger,
));
};
let mut supervise_worker =
|task_key: &'static str, handle: Option<tokio::task::JoinHandle<()>>| {
if let Some(handle) = handle {
supervisor.supervise_handle(task_key, handle);
record_boot(task_key);
}
};
supervise_worker(
crate::task_runtime::TASK_KEY_USAGE_QUEUE_WORKER,
self.usage_runtime.spawn_worker(self.data.clone()),
);
supervise_worker(
crate::task_runtime::TASK_KEY_PROVIDER_QUOTA_RESET,
crate::wallet_runtime::spawn_provider_quota_reset_worker(self.data.clone()),
);
supervise_worker(
crate::task_runtime::TASK_KEY_AUDIT_CLEANUP,
spawn_audit_cleanup_worker(self.data.clone()),
);
supervise_worker(
crate::task_runtime::TASK_KEY_DB_MAINTENANCE,
spawn_db_maintenance_worker(self.data.clone()),
);
supervise_worker(
crate::task_runtime::TASK_KEY_WALLET_DAILY_USAGE_AGG,
spawn_wallet_daily_usage_aggregation_worker(self.data.clone()),
);
supervise_worker(
crate::task_runtime::TASK_KEY_STATS_DAILY_AGG,
spawn_stats_aggregation_worker(self.data.clone()),
);
supervise_worker(
crate::task_runtime::TASK_KEY_USAGE_CLEANUP,
spawn_usage_cleanup_worker(self.data.clone()),
);
supervise_worker(
crate::task_runtime::TASK_KEY_POOL_MONITOR,
spawn_pool_monitor_worker(self.data.clone()),
);
supervise_worker(
crate::task_runtime::TASK_KEY_POOL_QUOTA_PROBE,
spawn_pool_quota_probe_worker(self.clone()),
);
supervise_worker(
crate::task_runtime::TASK_KEY_STATS_HOURLY_AGG,
spawn_stats_hourly_aggregation_worker(self.data.clone()),
);
supervise_worker(
crate::task_runtime::TASK_KEY_PENDING_CLEANUP,
spawn_pending_cleanup_worker(self.data.clone()),
);
supervise_worker(
crate::task_runtime::TASK_KEY_PROXY_NODE_STALE_CLEANUP,
spawn_proxy_node_stale_cleanup_worker(self.data.clone()),
);
supervise_worker(
crate::task_runtime::TASK_KEY_PROXY_NODE_METRICS_CLEANUP,
spawn_proxy_node_metrics_cleanup_worker(self.data.clone()),
);
supervise_worker(
crate::task_runtime::TASK_KEY_PROXY_UPGRADE_ROLLOUT,
spawn_proxy_upgrade_rollout_worker(self.clone()),
);
supervise_worker(
crate::task_runtime::TASK_KEY_PROVIDER_CHECKIN,
spawn_provider_checkin_worker(self.clone()),
);
supervise_worker(
crate::task_runtime::TASK_KEY_OAUTH_TOKEN_REFRESH,
spawn_oauth_token_refresh_worker(self.clone()),
);
supervise_worker(
crate::task_runtime::TASK_KEY_REQUEST_CANDIDATE_CLEANUP,
spawn_request_candidate_cleanup_worker(self.data.clone()),
);
supervise_worker(
crate::task_runtime::TASK_KEY_GEMINI_FILES_CLEANUP,
spawn_gemini_file_mapping_cleanup_worker(self.data.clone()),
);
supervise_worker(
crate::task_runtime::TASK_KEY_MODEL_FETCH_WORKER,
spawn_model_fetch_worker(self.clone()),
);
supervise_worker(
crate::task_runtime::TASK_KEY_VIDEO_TASK_POLLER,
spawn_video_task_poller(self.clone()),
);
supervisor
}
}

80
apps/aether-gateway/src/state/runtime/background_tasks.rs Normal file
View File

@@ -0,0 +1,80 @@
use crate::{AppState, GatewayError};
use aether_data_contracts::repository::background_tasks::{
BackgroundTaskListQuery, BackgroundTaskSummary, StoredBackgroundTaskEvent,
StoredBackgroundTaskRun, StoredBackgroundTaskRunPage, UpsertBackgroundTaskEvent,
UpsertBackgroundTaskRun,
};
impl AppState {
pub(crate) async fn find_background_task_run(
&self,
run_id: &str,
) -> Result<Option<StoredBackgroundTaskRun>, GatewayError> {
self.data
.find_background_task_run(run_id)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))
}
pub(crate) async fn list_background_task_runs(
&self,
query: &BackgroundTaskListQuery,
) -> Result<StoredBackgroundTaskRunPage, GatewayError> {
self.data
.list_background_task_runs(query)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))
}
pub(crate) async fn list_background_task_events(
&self,
run_id: &str,
offset: usize,
limit: usize,
) -> Result<Vec<StoredBackgroundTaskEvent>, GatewayError> {
self.data
.list_background_task_events(run_id, offset, limit)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))
}
pub(crate) async fn summarize_background_task_runs(
&self,
) -> Result<BackgroundTaskSummary, GatewayError> {
self.data
.summarize_background_task_runs()
.await
.map_err(|err| GatewayError::Internal(err.to_string()))
}
pub(crate) async fn upsert_background_task_run(
&self,
run: UpsertBackgroundTaskRun,
) -> Result<Option<StoredBackgroundTaskRun>, GatewayError> {
self.data
.upsert_background_task_run(run)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))
}
pub(crate) async fn request_cancel_background_task_run(
&self,
run_id: &str,
updated_at_unix_secs: u64,
) -> Result<bool, GatewayError> {
self.data
.request_cancel_background_task_run(run_id, updated_at_unix_secs)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))
}
pub(crate) async fn upsert_background_task_event(
&self,
event: UpsertBackgroundTaskEvent,
) -> Result<Option<StoredBackgroundTaskEvent>, GatewayError> {
self.data
.upsert_background_task_event(event)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))
}
}

9
apps/aether-gateway/src/state/runtime/mod.rs
View File

@@ -10,6 +10,7 @@ use super::{
mod announcements;
mod api_key_exports;
mod auth;
mod background_tasks;
mod billing;
mod candidate_queries;
mod gemini_files;
@@ -29,6 +30,14 @@ impl AppState {
self.data.has_announcement_writer()
}
pub fn has_background_task_data_reader(&self) -> bool {
self.data.has_background_task_reader()
}
pub fn has_background_task_data_writer(&self) -> bool {
self.data.has_background_task_writer()
}
pub fn has_video_task_data_reader(&self) -> bool {
self.data.has_video_task_reader()
}

647
apps/aether-gateway/src/task_runtime/mod.rs Normal file
View File

@@ -0,0 +1,647 @@
use std::future::Future;
use std::time::{SystemTime, UNIX_EPOCH};
use aether_data_contracts::repository::background_tasks::{
BackgroundTaskKind, BackgroundTaskStatus, StoredBackgroundTaskRun, UpsertBackgroundTaskEvent,
UpsertBackgroundTaskRun,
};
use aether_runtime::task::spawn_named;
pub(crate) use aether_task_runtime::TaskSupervisor;
use aether_task_runtime::{RetryPolicy, TaskDefinition, TaskKind};
use serde_json::Value;
use tokio::task::JoinHandle;
use tracing::warn;
use uuid::Uuid;
use crate::{AppState, GatewayError};
pub(crate) const TASK_KEY_PROVIDER_DELETE: &str = "admin.provider.delete";
pub(crate) const TASK_KEY_PROVIDER_OAUTH_BATCH_IMPORT: &str = "admin.provider.oauth.batch_import";
pub(crate) const TASK_KEY_USAGE_QUEUE_WORKER: &str = "usage.queue.worker";
pub(crate) const TASK_KEY_VIDEO_TASK_POLLER: &str = "video.task.poller";
pub(crate) const TASK_KEY_MODEL_FETCH_WORKER: &str = "model.fetch.worker";
pub(crate) const TASK_KEY_PROVIDER_QUOTA_RESET: &str = "provider.quota.reset.worker";
pub(crate) const TASK_KEY_POOL_QUOTA_PROBE: &str = "pool.quota.probe.worker";
pub(crate) const TASK_KEY_POOL_MONITOR: &str = "pool.monitor.worker";
pub(crate) const TASK_KEY_AUDIT_CLEANUP: &str = "maintenance.audit.cleanup";
pub(crate) const TASK_KEY_DB_MAINTENANCE: &str = "maintenance.database";
pub(crate) const TASK_KEY_PENDING_CLEANUP: &str = "maintenance.pending.cleanup";
pub(crate) const TASK_KEY_REQUEST_CANDIDATE_CLEANUP: &str = "maintenance.request.candidate.cleanup";
pub(crate) const TASK_KEY_GEMINI_FILES_CLEANUP: &str = "maintenance.gemini.files.cleanup";
pub(crate) const TASK_KEY_OAUTH_TOKEN_REFRESH: &str = "maintenance.oauth.token.refresh";
pub(crate) const TASK_KEY_PROXY_NODE_STALE_CLEANUP: &str = "maintenance.proxy.node.stale.cleanup";
pub(crate) const TASK_KEY_PROXY_NODE_METRICS_CLEANUP: &str =
"maintenance.proxy.node.metrics.cleanup";
pub(crate) const TASK_KEY_PROXY_UPGRADE_ROLLOUT: &str = "maintenance.proxy.upgrade.rollout";
pub(crate) const TASK_KEY_PROVIDER_CHECKIN: &str = "maintenance.provider.checkin";
pub(crate) const TASK_KEY_USAGE_CLEANUP: &str = "maintenance.usage.cleanup";
pub(crate) const TASK_KEY_WALLET_DAILY_USAGE_AGG: &str = "maintenance.wallet.daily.usage.agg";
pub(crate) const TASK_KEY_STATS_DAILY_AGG: &str = "maintenance.stats.daily.agg";
pub(crate) const TASK_KEY_STATS_HOURLY_AGG: &str = "maintenance.stats.hourly.agg";
pub(crate) const TASK_KEY_USAGE_SYNC_REPORT: &str = "usage.sync.report";
pub(crate) const TASK_KEY_PROVIDER_OAUTH_ACCOUNT_REFRESH: &str = "provider.oauth.account.refresh";
pub(crate) const TASK_KEY_PROVIDER_BALANCE_REFRESH: &str = "provider.ops.balance.refresh";
const RETRY_ONCE: RetryPolicy = RetryPolicy { max_attempts: 1 };
const TASK_DEFINITIONS: &[TaskDefinition] = &[
TaskDefinition::new(
TASK_KEY_PROVIDER_DELETE,
TaskKind::OnDemand,
"manual",
false,
true,
RETRY_ONCE,
),
TaskDefinition::new(
TASK_KEY_PROVIDER_OAUTH_BATCH_IMPORT,
TaskKind::OnDemand,
"manual",
false,
true,
RETRY_ONCE,
),
TaskDefinition::new(
TASK_KEY_USAGE_QUEUE_WORKER,
TaskKind::Daemon,
"daemon",
true,
true,
RETRY_ONCE,
),
TaskDefinition::new(
TASK_KEY_VIDEO_TASK_POLLER,
TaskKind::Scheduled,
"interval",
true,
true,
RETRY_ONCE,
),
TaskDefinition::new(
TASK_KEY_MODEL_FETCH_WORKER,
TaskKind::Scheduled,
"interval",
true,
true,
RETRY_ONCE,
),
TaskDefinition::new(
TASK_KEY_PROVIDER_QUOTA_RESET,
TaskKind::Scheduled,
"daily",
true,
true,
RETRY_ONCE,
),
TaskDefinition::new(
TASK_KEY_POOL_QUOTA_PROBE,
TaskKind::Scheduled,
"interval",
true,
true,
RETRY_ONCE,
),
TaskDefinition::new(
TASK_KEY_POOL_MONITOR,
TaskKind::Scheduled,
"interval",
true,
true,
RETRY_ONCE,
),
TaskDefinition::new(
TASK_KEY_AUDIT_CLEANUP,
TaskKind::Scheduled,
"interval",
true,
true,
RETRY_ONCE,
),
TaskDefinition::new(
TASK_KEY_DB_MAINTENANCE,
TaskKind::Scheduled,
"interval",
true,
true,
RETRY_ONCE,
),
TaskDefinition::new(
TASK_KEY_PENDING_CLEANUP,
TaskKind::Scheduled,
"interval",
true,
true,
RETRY_ONCE,
),
TaskDefinition::new(
TASK_KEY_REQUEST_CANDIDATE_CLEANUP,
TaskKind::Scheduled,
"interval",
true,
true,
RETRY_ONCE,
),
TaskDefinition::new(
TASK_KEY_GEMINI_FILES_CLEANUP,
TaskKind::Scheduled,
"interval",
true,
true,
RETRY_ONCE,
),
TaskDefinition::new(
TASK_KEY_OAUTH_TOKEN_REFRESH,
TaskKind::Scheduled,
"interval",
true,
true,
RETRY_ONCE,
),
TaskDefinition::new(
TASK_KEY_PROXY_NODE_STALE_CLEANUP,
TaskKind::Scheduled,
"interval",
true,
true,
RETRY_ONCE,
),
TaskDefinition::new(
TASK_KEY_PROXY_NODE_METRICS_CLEANUP,
TaskKind::Scheduled,
"interval",
true,
true,
RETRY_ONCE,
),
TaskDefinition::new(
TASK_KEY_PROXY_UPGRADE_ROLLOUT,
TaskKind::Scheduled,
"interval",
true,
true,
RETRY_ONCE,
),
TaskDefinition::new(
TASK_KEY_PROVIDER_CHECKIN,
TaskKind::Scheduled,
"interval",
true,
true,
RETRY_ONCE,
),
TaskDefinition::new(
TASK_KEY_USAGE_CLEANUP,
TaskKind::Scheduled,
"interval",
true,
true,
RETRY_ONCE,
),
TaskDefinition::new(
TASK_KEY_WALLET_DAILY_USAGE_AGG,
TaskKind::Scheduled,
"interval",
true,
true,
RETRY_ONCE,
),
TaskDefinition::new(
TASK_KEY_STATS_DAILY_AGG,
TaskKind::Scheduled,
"interval",
true,
true,
RETRY_ONCE,
),
TaskDefinition::new(
TASK_KEY_STATS_HOURLY_AGG,
TaskKind::Scheduled,
"interval",
true,
true,
RETRY_ONCE,
),
TaskDefinition::new(
TASK_KEY_USAGE_SYNC_REPORT,
TaskKind::FireAndForget,
"internal",
false,
false,
RETRY_ONCE,
),
TaskDefinition::new(
TASK_KEY_PROVIDER_OAUTH_ACCOUNT_REFRESH,
TaskKind::FireAndForget,
"internal",
false,
false,
RETRY_ONCE,
),
TaskDefinition::new(
TASK_KEY_PROVIDER_BALANCE_REFRESH,
TaskKind::FireAndForget,
"internal",
false,
false,
RETRY_ONCE,
),
];
pub(crate) fn task_definitions() -> &'static [TaskDefinition] {
TASK_DEFINITIONS
}
pub(crate) fn task_definition(task_key: &str) -> Option<TaskDefinition> {
task_definitions()
.iter()
.copied()
.find(|definition| definition.key == task_key)
}
pub(crate) const fn background_task_kind(kind: TaskKind) -> BackgroundTaskKind {
match kind {
TaskKind::Scheduled => BackgroundTaskKind::Scheduled,
TaskKind::Daemon => BackgroundTaskKind::Daemon,
TaskKind::OnDemand => BackgroundTaskKind::OnDemand,
TaskKind::FireAndForget => BackgroundTaskKind::FireAndForget,
}
}
pub(crate) fn task_cancel_kv_key(run_id: &str) -> String {
format!("task_runtime:run:{run_id}:cancel")
}
pub(crate) fn now_unix_secs() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_secs())
.unwrap_or_default()
}
pub(crate) fn build_task_run_id() -> String {
Uuid::new_v4().to_string()
}
pub(crate) fn spawn_fire_and_forget<F>(task_name: &'static str, future: F) -> JoinHandle<()>
where
F: Future<Output = ()> + Send + 'static,
{
spawn_named(task_name, future)
}
fn stored_run_to_upsert(run: StoredBackgroundTaskRun) -> UpsertBackgroundTaskRun {
UpsertBackgroundTaskRun {
id: run.id,
task_key: run.task_key,
kind: run.kind,
trigger: run.trigger,
status: run.status,
attempt: run.attempt,
max_attempts: run.max_attempts,
owner_instance: run.owner_instance,
progress_percent: run.progress_percent,
progress_message: run.progress_message,
payload_json: run.payload_json,
result_json: run.result_json,
error_message: run.error_message,
cancel_requested: run.cancel_requested,
created_by: run.created_by,
created_at_unix_secs: run.created_at_unix_secs,
started_at_unix_secs: run.started_at_unix_secs,
finished_at_unix_secs: run.finished_at_unix_secs,
updated_at_unix_secs: run.updated_at_unix_secs,
}
}
pub(crate) async fn upsert_run_with_logging(
app: &AppState,
run: UpsertBackgroundTaskRun,
) -> Option<StoredBackgroundTaskRun> {
match app.upsert_background_task_run(run).await {
Ok(result) => result,
Err(error) => {
warn!(error = ?error, "failed to upsert background task run");
None
}
}
}
pub(crate) async fn update_run_status(
app: &AppState,
run_id: &str,
status: BackgroundTaskStatus,
progress_percent: Option<u16>,
progress_message: Option<String>,
result_json: Option<Value>,
error_message: Option<String>,
started_at_unix_secs: Option<u64>,
finished_at_unix_secs: Option<u64>,
) -> Option<StoredBackgroundTaskRun> {
let Some(mut existing) = app.find_background_task_run(run_id).await.ok().flatten() else {
return None;
};
existing.status = status;
if let Some(progress_percent) = progress_percent {
existing.progress_percent = progress_percent.min(100);
}
if let Some(progress_message) = progress_message {
existing.progress_message = Some(progress_message);
}
if result_json.is_some() {
existing.result_json = result_json;
}
if error_message.is_some() {
existing.error_message = error_message;
}
if started_at_unix_secs.is_some() {
existing.started_at_unix_secs = started_at_unix_secs;
}
if finished_at_unix_secs.is_some() {
existing.finished_at_unix_secs = finished_at_unix_secs;
}
existing.updated_at_unix_secs = now_unix_secs();
upsert_run_with_logging(app, stored_run_to_upsert(existing)).await
}
pub(crate) async fn append_event_with_logging(
app: &AppState,
run_id: &str,
event_type: &str,
message: &str,
payload_json: Option<Value>,
) {
let event = UpsertBackgroundTaskEvent {
id: Uuid::new_v4().to_string(),
run_id: run_id.to_string(),
event_type: event_type.to_string(),
message: message.to_string(),
payload_json,
created_at_unix_secs: now_unix_secs(),
};
if let Err(error) = app.upsert_background_task_event(event).await {
warn!(error = ?error, run_id = %run_id, "failed to upsert background task event");
}
}
pub(crate) fn spawn_record_worker_boot(
app: AppState,
task_key: &'static str,
kind: BackgroundTaskKind,
trigger: &'static str,
) -> JoinHandle<()> {
spawn_named("task-runtime-record-worker-boot", async move {
let now = now_unix_secs();
let run_id = format!("boot:{}:{}", task_key, app.tunnel.local_instance_id());
let run = UpsertBackgroundTaskRun {
id: run_id.clone(),
task_key: task_key.to_string(),
kind,
trigger: trigger.to_string(),
status: BackgroundTaskStatus::Running,
attempt: 1,
max_attempts: 1,
owner_instance: Some(app.tunnel.local_instance_id().to_string()),
progress_percent: 0,
progress_message: Some("worker booted".to_string()),
payload_json: None,
result_json: None,
error_message: None,
cancel_requested: false,
created_by: Some("system".to_string()),
created_at_unix_secs: now,
started_at_unix_secs: Some(now),
finished_at_unix_secs: None,
updated_at_unix_secs: now,
};
let _ = upsert_run_with_logging(&app, run).await;
append_event_with_logging(
&app,
&run_id,
"worker_boot",
"background worker started",
None,
)
.await;
})
}
pub(crate) async fn set_cancel_signal(app: &AppState, run_id: &str) -> Result<(), GatewayError> {
app.runtime_kv_setex(&task_cancel_kv_key(run_id), "1", 60 * 60)
.await
}
pub(crate) async fn is_cancel_requested(app: &AppState, run_id: &str) -> bool {
if let Ok(Some(run)) = app.find_background_task_run(run_id).await {
if run.cancel_requested {
return true;
}
}
app.runtime_kv_exists(&task_cancel_kv_key(run_id))
.await
.unwrap_or(false)
}
pub(crate) async fn submit_provider_delete_task(
state: &crate::admin_api::AdminAppState<'_>,
provider_id: &str,
created_by: Option<&str>,
) -> Result<Option<String>, GatewayError> {
let Some(provider) = state
.read_provider_catalog_providers_by_ids(&[provider_id.to_string()])
.await?
.into_iter()
.next()
else {
return Ok(None);
};
let task_id = Uuid::new_v4().simple().to_string()[..16].to_string();
state.put_provider_delete_task(crate::LocalProviderDeleteTaskState {
task_id: task_id.clone(),
provider_id: provider.id.clone(),
status: "pending".to_string(),
stage: "queued".to_string(),
total_keys: 0,
deleted_keys: 0,
total_endpoints: 0,
deleted_endpoints: 0,
message: "delete task submitted".to_string(),
});
let app = state.cloned_app();
let provider_id = provider.id.clone();
let run_id = task_id.clone();
let created_at = now_unix_secs();
let max_attempts = task_definition(TASK_KEY_PROVIDER_DELETE)
.map(|item| item.retry_policy.max_attempts)
.unwrap_or(1);
if state.has_background_task_data_writer() {
let run = UpsertBackgroundTaskRun {
id: run_id.clone(),
task_key: TASK_KEY_PROVIDER_DELETE.to_string(),
kind: BackgroundTaskKind::OnDemand,
trigger: "manual".to_string(),
status: BackgroundTaskStatus::Queued,
attempt: 1,
max_attempts,
owner_instance: Some(app.tunnel.local_instance_id().to_string()),
progress_percent: 0,
progress_message: Some("delete task queued".to_string()),
payload_json: Some(serde_json::json!({ "provider_id": provider_id.clone() })),
result_json: None,
error_message: None,
cancel_requested: false,
created_by: Some(created_by.unwrap_or("admin").to_string()),
created_at_unix_secs: created_at,
started_at_unix_secs: None,
finished_at_unix_secs: None,
updated_at_unix_secs: created_at,
};
let _ = upsert_run_with_logging(&app, run).await;
append_event_with_logging(
&app,
&run_id,
"queued",
"provider delete task queued",
Some(serde_json::json!({ "provider_id": provider_id.clone() })),
)
.await;
}
spawn_named("task-runtime-provider-delete", async move {
let lock_key = format!("task_runtime:lock:{TASK_KEY_PROVIDER_DELETE}:{provider_id}");
let lock_ttl = std::time::Duration::from_secs(60 * 15);
let lock = app
.runtime_state
.lock_try_acquire(&lock_key, app.tunnel.local_instance_id(), lock_ttl)
.await
.ok()
.flatten();
if lock.is_none() {
let _ = update_run_status(
&app,
&run_id,
BackgroundTaskStatus::Skipped,
Some(0),
Some("provider delete skipped: another node is running this task".to_string()),
None,
None,
None,
Some(now_unix_secs()),
)
.await;
append_event_with_logging(
&app,
&run_id,
"skipped",
"provider delete skipped by singleton lock",
None,
)
.await;
return;
}
let started_at = now_unix_secs();
let _ = update_run_status(
&app,
&run_id,
BackgroundTaskStatus::Running,
Some(5),
Some("provider delete task started".to_string()),
None,
None,
Some(started_at),
None,
)
.await;
append_event_with_logging(
&app,
&run_id,
"running",
"provider delete task started",
None,
)
.await;
let admin_state = crate::admin_api::AdminAppState::new(&app);
let result = admin_state
.run_admin_provider_delete_task(&provider_id, &run_id)
.await;
match result {
Ok(task_state) => {
let _ = update_run_status(
&app,
&run_id,
BackgroundTaskStatus::Succeeded,
Some(100),
Some(task_state.message.clone()),
Some(serde_json::json!({
"provider_id": task_state.provider_id,
"status": task_state.status,
"stage": task_state.stage,
"deleted_keys": task_state.deleted_keys,
"total_keys": task_state.total_keys,
"deleted_endpoints": task_state.deleted_endpoints,
"total_endpoints": task_state.total_endpoints,
"message": task_state.message,
})),
None,
None,
Some(now_unix_secs()),
)
.await;
append_event_with_logging(
&app,
&run_id,
"succeeded",
"provider delete task completed",
None,
)
.await;
}
Err(err) => {
warn!(
"gateway admin provider delete task failed for provider {}: {:?}",
provider_id, err
);
app.put_provider_delete_task(crate::LocalProviderDeleteTaskState {
task_id: run_id.clone(),
provider_id: provider_id.clone(),
status: "failed".to_string(),
stage: "failed".to_string(),
total_keys: 0,
deleted_keys: 0,
total_endpoints: 0,
deleted_endpoints: 0,
message: format!("provider delete failed: {err:?}"),
});
let _ = update_run_status(
&app,
&run_id,
BackgroundTaskStatus::Failed,
Some(100),
Some("provider delete task failed".to_string()),
None,
Some(format!("{err:?}")),
None,
Some(now_unix_secs()),
)
.await;
append_event_with_logging(
&app,
&run_id,
"failed",
"provider delete task failed",
Some(serde_json::json!({ "error": format!("{err:?}") })),
)
.await;
}
}
if let Some(lock) = lock {
let _ = app.runtime_state.lock_release(&lock).await;
}
});
Ok(Some(task_id))
}

4
apps/aether-gateway/src/tests/files/registry_cleanup.rs
View File

@@ -72,7 +72,5 @@ async fn gateway_background_gemini_file_mapping_cleanup_deletes_expired_entries(
"active mapping should remain"
);
for handle in background_tasks {
handle.abort();
}
background_tasks.shutdown().await;
}

8
apps/aether-gateway/src/tests/video/registry_poller.rs
View File

@@ -231,9 +231,7 @@ async fn gateway_background_video_task_poller_refreshes_due_openai_task_from_rep
}]
);
for handle in background_tasks {
handle.abort();
}
background_tasks.shutdown().await;
execution_runtime_handle.abort();
}
@@ -321,8 +319,6 @@ async fn gateway_background_video_task_poller_refreshes_due_openai_task_from_rep
}]
);
for handle in background_tasks {
handle.abort();
}
background_tasks.shutdown().await;
upstream_handle.abort();
}

3
apps/aether-gateway/src/usage/reporting/mod.rs
View File

@@ -9,6 +9,7 @@ use crate::clock::current_unix_ms;
use crate::log_ids::short_request_id;
use crate::orchestration::{apply_local_report_effect, LocalReportEffect};
use crate::request_candidate_runtime::record_report_request_candidate_status;
use crate::task_runtime::{spawn_fire_and_forget, TASK_KEY_USAGE_SYNC_REPORT};
use crate::{AppState, GatewayError};
mod context;
@@ -142,7 +143,7 @@ pub(crate) async fn submit_sync_report(
pub(crate) fn spawn_sync_report(state: AppState, payload: GatewaySyncReportRequest) {
let report_request_id_for_log =
short_request_id(report_request_id(payload.report_context.as_ref()));
tokio::spawn(async move {
spawn_fire_and_forget(TASK_KEY_USAGE_SYNC_REPORT, async move {
let trace_id = payload.trace_id.clone();
if let Err(err) = submit_sync_report(&state, payload).await {
warn!(

8
crates/aether-data-contracts/src/repository/background_tasks/mod.rs Normal file
View File

@@ -0,0 +1,8 @@
mod types;
pub use types::{
BackgroundTaskKind, BackgroundTaskListQuery, BackgroundTaskReadRepository,
BackgroundTaskRepository, BackgroundTaskStatus, BackgroundTaskSummary,
BackgroundTaskWriteRepository, StoredBackgroundTaskEvent, StoredBackgroundTaskRun,
StoredBackgroundTaskRunPage, UpsertBackgroundTaskEvent, UpsertBackgroundTaskRun,
};

290
crates/aether-data-contracts/src/repository/background_tasks/types.rs Normal file
View File

@@ -0,0 +1,290 @@
use std::collections::BTreeMap;
use async_trait::async_trait;
use serde_json::Value;
#[derive(
Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize,
)]
pub enum BackgroundTaskKind {
Scheduled,
Daemon,
OnDemand,
FireAndForget,
}
impl BackgroundTaskKind {
pub fn as_database(self) -> &'static str {
match self {
Self::Scheduled => "scheduled",
Self::Daemon => "daemon",
Self::OnDemand => "on_demand",
Self::FireAndForget => "fire_and_forget",
}
}
pub fn from_database(value: &str) -> Result<Self, crate::DataLayerError> {
match value.trim().to_ascii_lowercase().as_str() {
"scheduled" => Ok(Self::Scheduled),
"daemon" => Ok(Self::Daemon),
"on_demand" => Ok(Self::OnDemand),
"fire_and_forget" => Ok(Self::FireAndForget),
other => Err(crate::DataLayerError::UnexpectedValue(format!(
"unsupported background_tasks.kind: {other}"
))),
}
}
}
#[derive(
Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize,
)]
pub enum BackgroundTaskStatus {
Queued,
Running,
Retrying,
Succeeded,
Failed,
Cancelled,
Skipped,
}
impl BackgroundTaskStatus {
pub fn as_database(self) -> &'static str {
match self {
Self::Queued => "queued",
Self::Running => "running",
Self::Retrying => "retrying",
Self::Succeeded => "succeeded",
Self::Failed => "failed",
Self::Cancelled => "cancelled",
Self::Skipped => "skipped",
}
}
pub fn from_database(value: &str) -> Result<Self, crate::DataLayerError> {
match value.trim().to_ascii_lowercase().as_str() {
"queued" => Ok(Self::Queued),
"running" => Ok(Self::Running),
"retrying" => Ok(Self::Retrying),
"succeeded" => Ok(Self::Succeeded),
"failed" => Ok(Self::Failed),
"cancelled" => Ok(Self::Cancelled),
"skipped" => Ok(Self::Skipped),
other => Err(crate::DataLayerError::UnexpectedValue(format!(
"unsupported background_tasks.status: {other}"
))),
}
}
}
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
pub struct StoredBackgroundTaskRun {
pub id: String,
pub task_key: String,
pub kind: BackgroundTaskKind,
pub trigger: String,
pub status: BackgroundTaskStatus,
pub attempt: u32,
pub max_attempts: u32,
pub owner_instance: Option<String>,
pub progress_percent: u16,
pub progress_message: Option<String>,
pub payload_json: Option<Value>,
pub result_json: Option<Value>,
pub error_message: Option<String>,
pub cancel_requested: bool,
pub created_by: Option<String>,
pub created_at_unix_secs: u64,
pub started_at_unix_secs: Option<u64>,
pub finished_at_unix_secs: Option<u64>,
pub updated_at_unix_secs: u64,
}
#[derive(Debug, Clone, PartialEq)]
pub struct UpsertBackgroundTaskRun {
pub id: String,
pub task_key: String,
pub kind: BackgroundTaskKind,
pub trigger: String,
pub status: BackgroundTaskStatus,
pub attempt: u32,
pub max_attempts: u32,
pub owner_instance: Option<String>,
pub progress_percent: u16,
pub progress_message: Option<String>,
pub payload_json: Option<Value>,
pub result_json: Option<Value>,
pub error_message: Option<String>,
pub cancel_requested: bool,
pub created_by: Option<String>,
pub created_at_unix_secs: u64,
pub started_at_unix_secs: Option<u64>,
pub finished_at_unix_secs: Option<u64>,
pub updated_at_unix_secs: u64,
}
impl UpsertBackgroundTaskRun {
pub fn validate(&self) -> Result<(), crate::DataLayerError> {
if self.id.trim().is_empty()
|| self.task_key.trim().is_empty()
|| self.trigger.trim().is_empty()
{
return Err(crate::DataLayerError::UnexpectedValue(
"background task run identity is empty".to_string(),
));
}
if self.progress_percent > 100 {
return Err(crate::DataLayerError::UnexpectedValue(format!(
"background task progress_percent out of range: {}",
self.progress_percent
)));
}
Ok(())
}
pub fn into_stored(self) -> StoredBackgroundTaskRun {
StoredBackgroundTaskRun {
id: self.id,
task_key: self.task_key,
kind: self.kind,
trigger: self.trigger,
status: self.status,
attempt: self.attempt,
max_attempts: self.max_attempts,
owner_instance: self.owner_instance,
progress_percent: self.progress_percent,
progress_message: self.progress_message,
payload_json: self.payload_json,
result_json: self.result_json,
error_message: self.error_message,
cancel_requested: self.cancel_requested,
created_by: self.created_by,
created_at_unix_secs: self.created_at_unix_secs,
started_at_unix_secs: self.started_at_unix_secs,
finished_at_unix_secs: self.finished_at_unix_secs,
updated_at_unix_secs: self.updated_at_unix_secs,
}
}
}
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
pub struct StoredBackgroundTaskEvent {
pub id: String,
pub run_id: String,
pub event_type: String,
pub message: String,
pub payload_json: Option<Value>,
pub created_at_unix_secs: u64,
}
#[derive(Debug, Clone, PartialEq)]
pub struct UpsertBackgroundTaskEvent {
pub id: String,
pub run_id: String,
pub event_type: String,
pub message: String,
pub payload_json: Option<Value>,
pub created_at_unix_secs: u64,
}
impl UpsertBackgroundTaskEvent {
pub fn validate(&self) -> Result<(), crate::DataLayerError> {
if self.id.trim().is_empty()
|| self.run_id.trim().is_empty()
|| self.event_type.trim().is_empty()
|| self.message.trim().is_empty()
{
return Err(crate::DataLayerError::UnexpectedValue(
"background task event identity is empty".to_string(),
));
}
Ok(())
}
pub fn into_stored(self) -> StoredBackgroundTaskEvent {
StoredBackgroundTaskEvent {
id: self.id,
run_id: self.run_id,
event_type: self.event_type,
message: self.message,
payload_json: self.payload_json,
created_at_unix_secs: self.created_at_unix_secs,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)]
pub struct BackgroundTaskListQuery {
pub task_key_substring: Option<String>,
pub kind: Option<BackgroundTaskKind>,
pub status: Option<BackgroundTaskStatus>,
pub trigger: Option<String>,
pub offset: usize,
pub limit: usize,
}
#[derive(Debug, Clone, PartialEq, Default, serde::Serialize, serde::Deserialize)]
pub struct StoredBackgroundTaskRunPage {
pub items: Vec<StoredBackgroundTaskRun>,
pub total: usize,
}
#[derive(Debug, Clone, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)]
pub struct BackgroundTaskSummary {
pub total: u64,
pub running_count: u64,
pub by_status: BTreeMap<String, u64>,
pub by_kind: BTreeMap<String, u64>,
}
#[async_trait]
pub trait BackgroundTaskReadRepository: Send + Sync {
async fn find_run(
&self,
run_id: &str,
) -> Result<Option<StoredBackgroundTaskRun>, crate::DataLayerError>;
async fn list_runs(
&self,
query: &BackgroundTaskListQuery,
) -> Result<StoredBackgroundTaskRunPage, crate::DataLayerError>;
async fn list_events(
&self,
run_id: &str,
offset: usize,
limit: usize,
) -> Result<Vec<StoredBackgroundTaskEvent>, crate::DataLayerError>;
async fn summarize_runs(&self) -> Result<BackgroundTaskSummary, crate::DataLayerError>;
}
#[async_trait]
pub trait BackgroundTaskWriteRepository: Send + Sync {
async fn upsert_run(
&self,
run: UpsertBackgroundTaskRun,
) -> Result<StoredBackgroundTaskRun, crate::DataLayerError>;
async fn request_cancel(
&self,
run_id: &str,
updated_at_unix_secs: u64,
) -> Result<bool, crate::DataLayerError>;
async fn upsert_event(
&self,
event: UpsertBackgroundTaskEvent,
) -> Result<StoredBackgroundTaskEvent, crate::DataLayerError>;
}
pub trait BackgroundTaskRepository:
BackgroundTaskReadRepository + BackgroundTaskWriteRepository + Send + Sync
{
}
impl<T> BackgroundTaskRepository for T where
T: BackgroundTaskReadRepository + BackgroundTaskWriteRepository + Send + Sync
{
}

1
crates/aether-data-contracts/src/repository/mod.rs
View File

@@ -1,3 +1,4 @@
pub mod background_tasks;
pub mod billing;
pub mod candidate_selection;
pub mod candidates;

37
crates/aether-data/migrations/mysql/20260509000000_add_background_tasks.sql Normal file
View File

@@ -0,0 +1,37 @@
CREATE TABLE IF NOT EXISTS background_task_runs (
id VARCHAR(64) PRIMARY KEY,
task_key VARCHAR(200) NOT NULL,
kind VARCHAR(32) NOT NULL,
`trigger` VARCHAR(64) NOT NULL,
status VARCHAR(32) NOT NULL,
attempt INT NOT NULL DEFAULT 0,
max_attempts INT NOT NULL DEFAULT 0,
owner_instance VARCHAR(200),
progress_percent INT NOT NULL DEFAULT 0,
progress_message TEXT,
payload_json JSON,
result_json JSON,
error_message TEXT,
cancel_requested TINYINT(1) NOT NULL DEFAULT 0,
created_by VARCHAR(200),
created_at_unix_secs BIGINT NOT NULL,
started_at_unix_secs BIGINT NULL,
finished_at_unix_secs BIGINT NULL,
updated_at_unix_secs BIGINT NOT NULL,
INDEX idx_background_task_runs_task_key (task_key),
INDEX idx_background_task_runs_status (status),
INDEX idx_background_task_runs_kind (kind),
INDEX idx_background_task_runs_created_at (created_at_unix_secs)
);
CREATE TABLE IF NOT EXISTS background_task_events (
id VARCHAR(64) PRIMARY KEY,
run_id VARCHAR(64) NOT NULL,
event_type VARCHAR(64) NOT NULL,
message TEXT NOT NULL,
payload_json JSON,
created_at_unix_secs BIGINT NOT NULL,
INDEX idx_background_task_events_run_id (run_id, created_at_unix_secs),
CONSTRAINT fk_background_task_events_run
FOREIGN KEY (run_id) REFERENCES background_task_runs(id) ON DELETE CASCADE
);

42
crates/aether-data/migrations/postgres/20260509000000_add_background_tasks.sql Normal file
View File

@@ -0,0 +1,42 @@
CREATE TABLE IF NOT EXISTS background_task_runs (
id character varying(64) PRIMARY KEY,
task_key character varying(200) NOT NULL,
kind character varying(32) NOT NULL,
"trigger" character varying(64) NOT NULL,
status character varying(32) NOT NULL,
attempt integer NOT NULL DEFAULT 0,
max_attempts integer NOT NULL DEFAULT 0,
owner_instance character varying(200),
progress_percent integer NOT NULL DEFAULT 0,
progress_message text,
payload_json jsonb,
result_json jsonb,
error_message text,
cancel_requested boolean NOT NULL DEFAULT false,
created_by character varying(200),
created_at_unix_secs bigint NOT NULL,
started_at_unix_secs bigint,
finished_at_unix_secs bigint,
updated_at_unix_secs bigint NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_background_task_runs_task_key
ON background_task_runs (task_key);
CREATE INDEX IF NOT EXISTS idx_background_task_runs_status
ON background_task_runs (status);
CREATE INDEX IF NOT EXISTS idx_background_task_runs_kind
ON background_task_runs (kind);
CREATE INDEX IF NOT EXISTS idx_background_task_runs_created_at
ON background_task_runs (created_at_unix_secs DESC);
CREATE TABLE IF NOT EXISTS background_task_events (
id character varying(64) PRIMARY KEY,
run_id character varying(64) NOT NULL REFERENCES background_task_runs(id) ON DELETE CASCADE,
event_type character varying(64) NOT NULL,
message text NOT NULL,
payload_json jsonb,
created_at_unix_secs bigint NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_background_task_events_run_id
ON background_task_events (run_id, created_at_unix_secs ASC);

43
crates/aether-data/migrations/sqlite/20260509000000_add_background_tasks.sql Normal file
View File

@@ -0,0 +1,43 @@
CREATE TABLE IF NOT EXISTS background_task_runs (
id TEXT PRIMARY KEY,
task_key TEXT NOT NULL,
kind TEXT NOT NULL,
"trigger" TEXT NOT NULL,
status TEXT NOT NULL,
attempt INTEGER NOT NULL DEFAULT 0,
max_attempts INTEGER NOT NULL DEFAULT 0,
owner_instance TEXT,
progress_percent INTEGER NOT NULL DEFAULT 0,
progress_message TEXT,
payload_json TEXT,
result_json TEXT,
error_message TEXT,
cancel_requested INTEGER NOT NULL DEFAULT 0,
created_by TEXT,
created_at_unix_secs INTEGER NOT NULL,
started_at_unix_secs INTEGER,
finished_at_unix_secs INTEGER,
updated_at_unix_secs INTEGER NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_background_task_runs_task_key
ON background_task_runs (task_key);
CREATE INDEX IF NOT EXISTS idx_background_task_runs_status
ON background_task_runs (status);
CREATE INDEX IF NOT EXISTS idx_background_task_runs_kind
ON background_task_runs (kind);
CREATE INDEX IF NOT EXISTS idx_background_task_runs_created_at
ON background_task_runs (created_at_unix_secs DESC);
CREATE TABLE IF NOT EXISTS background_task_events (
id TEXT PRIMARY KEY,
run_id TEXT NOT NULL,
event_type TEXT NOT NULL,
message TEXT NOT NULL,
payload_json TEXT,
created_at_unix_secs INTEGER NOT NULL,
FOREIGN KEY (run_id) REFERENCES background_task_runs(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_background_task_events_run_id
ON background_task_events (run_id, created_at_unix_secs ASC);

42
crates/aether-data/schema/bootstrap/postgres/150_background_tasks.sql Normal file
View File

@@ -0,0 +1,42 @@
CREATE TABLE IF NOT EXISTS public.background_task_runs (
id character varying(64) PRIMARY KEY,
task_key character varying(200) NOT NULL,
kind character varying(32) NOT NULL,
"trigger" character varying(64) NOT NULL,
status character varying(32) NOT NULL,
attempt integer NOT NULL DEFAULT 0,
max_attempts integer NOT NULL DEFAULT 0,
owner_instance character varying(200),
progress_percent integer NOT NULL DEFAULT 0,
progress_message text,
payload_json jsonb,
result_json jsonb,
error_message text,
cancel_requested boolean NOT NULL DEFAULT false,
created_by character varying(200),
created_at_unix_secs bigint NOT NULL,
started_at_unix_secs bigint,
finished_at_unix_secs bigint,
updated_at_unix_secs bigint NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_background_task_runs_task_key
ON public.background_task_runs (task_key);
CREATE INDEX IF NOT EXISTS idx_background_task_runs_status
ON public.background_task_runs (status);
CREATE INDEX IF NOT EXISTS idx_background_task_runs_kind
ON public.background_task_runs (kind);
CREATE INDEX IF NOT EXISTS idx_background_task_runs_created_at
ON public.background_task_runs (created_at_unix_secs DESC);
CREATE TABLE IF NOT EXISTS public.background_task_events (
id character varying(64) PRIMARY KEY,
run_id character varying(64) NOT NULL REFERENCES public.background_task_runs(id) ON DELETE CASCADE,
event_type character varying(64) NOT NULL,
message text NOT NULL,
payload_json jsonb,
created_at_unix_secs bigint NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_background_task_events_run_id
ON public.background_task_events (run_id, created_at_unix_secs ASC);

1
crates/aether-data/schema/bootstrap/postgres/manifest.txt
View File

@@ -9,3 +9,4 @@
120_stats_rollups.sql
130_stats_cost_savings.sql
140_proxy_node_metrics.sql
150_background_tasks.sql

42
crates/aether-data/schema/generated/mysql/baseline/008_background_tasks.sql Normal file
View File

@@ -0,0 +1,42 @@
-- Generated by aether-data-schema from schema/logical/*.toml.
-- Do not edit generated files directly; edit logical schema or explicit overrides instead.
CREATE TABLE IF NOT EXISTS background_task_runs (
`id` VARCHAR(64) NOT NULL,
`task_key` VARCHAR(200) NOT NULL,
`kind` VARCHAR(32) NOT NULL,
`trigger` VARCHAR(64) NOT NULL,
`status` VARCHAR(32) NOT NULL,
`attempt` INT NOT NULL DEFAULT 0,
`max_attempts` INT NOT NULL DEFAULT 0,
`owner_instance` VARCHAR(200),
`progress_percent` INT NOT NULL DEFAULT 0,
`progress_message` TEXT,
`payload_json` JSON,
`result_json` JSON,
`error_message` TEXT,
`cancel_requested` TINYINT(1) NOT NULL DEFAULT 0,
`created_by` VARCHAR(200),
`created_at_unix_secs` BIGINT NOT NULL,
`started_at_unix_secs` BIGINT,
`finished_at_unix_secs` BIGINT,
`updated_at_unix_secs` BIGINT NOT NULL,
PRIMARY KEY (`id`),
KEY idx_background_task_runs_task_key (`task_key`),
KEY idx_background_task_runs_status (`status`),
KEY idx_background_task_runs_kind (`kind`),
KEY idx_background_task_runs_created_at (`created_at_unix_secs`)
);
CREATE TABLE IF NOT EXISTS background_task_events (
`id` VARCHAR(64) NOT NULL,
`run_id` VARCHAR(64) NOT NULL,
`event_type` VARCHAR(64) NOT NULL,
`message` TEXT NOT NULL,
`payload_json` JSON,
`created_at_unix_secs` BIGINT NOT NULL,
PRIMARY KEY (`id`),
KEY idx_background_task_events_run_id (`run_id`, `created_at_unix_secs`),
CONSTRAINT fk_background_task_events_run FOREIGN KEY (`run_id`) REFERENCES background_task_runs (`id`) ON DELETE CASCADE
);

1
crates/aether-data/schema/generated/mysql/baseline/manifest.txt
View File

@@ -8,3 +8,4 @@
005_wallet_billing.sql
006_usage.sql
007_stats.sql
008_background_tasks.sql

44
crates/aether-data/schema/generated/postgres/baseline/008_background_tasks.sql Normal file
View File

@@ -0,0 +1,44 @@
-- Generated by aether-data-schema from schema/logical/*.toml.
-- Do not edit generated files directly; edit logical schema or explicit overrides instead.
CREATE TABLE IF NOT EXISTS public.background_task_runs (
id character varying(64) NOT NULL,
task_key character varying(200) NOT NULL,
kind character varying(32) NOT NULL,
trigger character varying(64) NOT NULL,
status character varying(32) NOT NULL,
attempt integer DEFAULT 0 NOT NULL,
max_attempts integer DEFAULT 0 NOT NULL,
owner_instance character varying(200),
progress_percent integer DEFAULT 0 NOT NULL,
progress_message text,
payload_json jsonb,
result_json jsonb,
error_message text,
cancel_requested boolean DEFAULT false NOT NULL,
created_by character varying(200),
created_at_unix_secs bigint NOT NULL,
started_at_unix_secs bigint,
finished_at_unix_secs bigint,
updated_at_unix_secs bigint NOT NULL
);
ALTER TABLE ONLY public.background_task_runs ADD CONSTRAINT background_task_runs_pkey PRIMARY KEY (id);
CREATE INDEX IF NOT EXISTS idx_background_task_runs_task_key ON public.background_task_runs USING btree (task_key);
CREATE INDEX IF NOT EXISTS idx_background_task_runs_status ON public.background_task_runs USING btree (status);
CREATE INDEX IF NOT EXISTS idx_background_task_runs_kind ON public.background_task_runs USING btree (kind);
CREATE INDEX IF NOT EXISTS idx_background_task_runs_created_at ON public.background_task_runs USING btree (created_at_unix_secs);
CREATE TABLE IF NOT EXISTS public.background_task_events (
id character varying(64) NOT NULL,
run_id character varying(64) NOT NULL,
event_type character varying(64) NOT NULL,
message text NOT NULL,
payload_json jsonb,
created_at_unix_secs bigint NOT NULL
);
ALTER TABLE ONLY public.background_task_events ADD CONSTRAINT background_task_events_pkey PRIMARY KEY (id);
CREATE INDEX IF NOT EXISTS idx_background_task_events_run_id ON public.background_task_events USING btree (run_id, created_at_unix_secs);
ALTER TABLE ONLY public.background_task_events ADD CONSTRAINT fk_background_task_events_run FOREIGN KEY (run_id) REFERENCES public.background_task_runs(id) ON DELETE CASCADE;

1
crates/aether-data/schema/generated/postgres/baseline/manifest.txt
View File

@@ -8,3 +8,4 @@
005_wallet_billing.sql
006_usage.sql
007_stats.sql
008_background_tasks.sql

40
crates/aether-data/schema/generated/sqlite/baseline/008_background_tasks.sql Normal file
View File

@@ -0,0 +1,40 @@
-- Generated by aether-data-schema from schema/logical/*.toml.
-- Do not edit generated files directly; edit logical schema or explicit overrides instead.
CREATE TABLE IF NOT EXISTS background_task_runs (
id TEXT PRIMARY KEY NOT NULL,
task_key TEXT NOT NULL,
kind TEXT NOT NULL,
trigger TEXT NOT NULL,
status TEXT NOT NULL,
attempt INTEGER NOT NULL DEFAULT 0,
max_attempts INTEGER NOT NULL DEFAULT 0,
owner_instance TEXT,
progress_percent INTEGER NOT NULL DEFAULT 0,
progress_message TEXT,
payload_json TEXT,
result_json TEXT,
error_message TEXT,
cancel_requested INTEGER NOT NULL DEFAULT 0,
created_by TEXT,
created_at_unix_secs INTEGER NOT NULL,
started_at_unix_secs INTEGER,
finished_at_unix_secs INTEGER,
updated_at_unix_secs INTEGER NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_background_task_runs_task_key ON background_task_runs (task_key);
CREATE INDEX IF NOT EXISTS idx_background_task_runs_status ON background_task_runs (status);
CREATE INDEX IF NOT EXISTS idx_background_task_runs_kind ON background_task_runs (kind);
CREATE INDEX IF NOT EXISTS idx_background_task_runs_created_at ON background_task_runs (created_at_unix_secs);
CREATE TABLE IF NOT EXISTS background_task_events (
id TEXT PRIMARY KEY NOT NULL,
run_id TEXT NOT NULL,
event_type TEXT NOT NULL,
message TEXT NOT NULL,
payload_json TEXT,
created_at_unix_secs INTEGER NOT NULL,
CONSTRAINT fk_background_task_events_run FOREIGN KEY (run_id) REFERENCES background_task_runs (id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_background_task_events_run_id ON background_task_events (run_id, created_at_unix_secs);

1
crates/aether-data/schema/generated/sqlite/baseline/manifest.txt
View File

@@ -8,3 +8,4 @@
005_wallet_billing.sql
006_usage.sql
007_stats.sql
008_background_tasks.sql

159
crates/aether-data/schema/logical/008_background_tasks.toml Normal file
View File

@@ -0,0 +1,159 @@
[table.background_task_runs]
domain = "background_tasks"
order = 10
primary_key = ["id"]
[[table.background_task_runs.columns]]
name = "id"
type = "text_id"
length = 64
[[table.background_task_runs.columns]]
name = "task_key"
type = "text"
length = 200
[[table.background_task_runs.columns]]
name = "kind"
type = "text"
length = 32
[[table.background_task_runs.columns]]
name = "trigger"
type = "text"
length = 64
[[table.background_task_runs.columns]]
name = "status"
type = "text"
length = 32
[[table.background_task_runs.columns]]
name = "attempt"
type = "int32"
default = 0
[[table.background_task_runs.columns]]
name = "max_attempts"
type = "int32"
default = 0
[[table.background_task_runs.columns]]
name = "owner_instance"
type = "text"
length = 200
nullable = true
[[table.background_task_runs.columns]]
name = "progress_percent"
type = "int32"
default = 0
[[table.background_task_runs.columns]]
name = "progress_message"
type = "text"
nullable = true
[[table.background_task_runs.columns]]
name = "payload_json"
type = "json"
nullable = true
[[table.background_task_runs.columns]]
name = "result_json"
type = "json"
nullable = true
[[table.background_task_runs.columns]]
name = "error_message"
type = "text"
nullable = true
[[table.background_task_runs.columns]]
name = "cancel_requested"
type = "bool"
default = false
[[table.background_task_runs.columns]]
name = "created_by"
type = "text"
length = 200
nullable = true
[[table.background_task_runs.columns]]
name = "created_at_unix_secs"
type = "unix_seconds"
[[table.background_task_runs.columns]]
name = "started_at_unix_secs"
type = "unix_seconds"
nullable = true
[[table.background_task_runs.columns]]
name = "finished_at_unix_secs"
type = "unix_seconds"
nullable = true
[[table.background_task_runs.columns]]
name = "updated_at_unix_secs"
type = "unix_seconds"
[[table.background_task_runs.indexes]]
name = "idx_background_task_runs_task_key"
columns = ["task_key"]
[[table.background_task_runs.indexes]]
name = "idx_background_task_runs_status"
columns = ["status"]
[[table.background_task_runs.indexes]]
name = "idx_background_task_runs_kind"
columns = ["kind"]
[[table.background_task_runs.indexes]]
name = "idx_background_task_runs_created_at"
columns = ["created_at_unix_secs"]
[table.background_task_events]
domain = "background_tasks"
order = 20
primary_key = ["id"]
[[table.background_task_events.columns]]
name = "id"
type = "text_id"
length = 64
[[table.background_task_events.columns]]
name = "run_id"
type = "text_id"
length = 64
[[table.background_task_events.columns]]
name = "event_type"
type = "text"
length = 64
[[table.background_task_events.columns]]
name = "message"
type = "text"
[[table.background_task_events.columns]]
name = "payload_json"
type = "json"
nullable = true
[[table.background_task_events.columns]]
name = "created_at_unix_secs"
type = "unix_seconds"
[[table.background_task_events.indexes]]
name = "idx_background_task_events_run_id"
columns = ["run_id", "created_at_unix_secs"]
[[table.background_task_events.foreign_keys]]
name = "fk_background_task_events_run"
columns = ["run_id"]
references_table = "background_task_runs"
references_columns = ["id"]
on_delete = "cascade"

11
crates/aether-data/src/backend/mysql.rs
View File

@@ -13,6 +13,9 @@ use crate::repository::auth_modules::{
AuthModuleReadRepository, AuthModuleWriteRepository, MysqlAuthModuleReadRepository,
MysqlAuthModuleRepository,
};
use crate::repository::background_tasks::{
BackgroundTaskReadRepository, BackgroundTaskWriteRepository, MysqlBackgroundTaskRepository,
};
use crate::repository::billing::{BillingReadRepository, MysqlBillingReadRepository};
use crate::repository::candidate_selection::{
MinimalCandidateSelectionReadRepository, MysqlMinimalCandidateSelectionReadRepository,
@@ -123,6 +126,14 @@ impl MysqlBackend {
Arc::new(MysqlBillingReadRepository::new(self.pool_clone()))
}
pub fn background_task_read_repository(&self) -> Arc<dyn BackgroundTaskReadRepository> {
Arc::new(MysqlBackgroundTaskRepository::new(self.pool_clone()))
}
pub fn background_task_write_repository(&self) -> Arc<dyn BackgroundTaskWriteRepository> {
Arc::new(MysqlBackgroundTaskRepository::new(self.pool_clone()))
}
pub fn request_candidate_read_repository(&self) -> Arc<dyn RequestCandidateReadRepository> {
Arc::new(MysqlRequestCandidateRepository::new(self.pool_clone()))
}

11
crates/aether-data/src/backend/postgres.rs
View File

@@ -15,6 +15,9 @@ use crate::repository::auth_modules::{
AuthModuleReadRepository, AuthModuleWriteRepository, SqlxAuthModuleReadRepository,
SqlxAuthModuleRepository,
};
use crate::repository::background_tasks::{
BackgroundTaskReadRepository, BackgroundTaskWriteRepository, SqlxBackgroundTaskRepository,
};
use crate::repository::billing::{BillingReadRepository, SqlxBillingReadRepository};
use crate::repository::candidate_selection::{
MinimalCandidateSelectionReadRepository, SqlxMinimalCandidateSelectionReadRepository,
@@ -118,6 +121,14 @@ impl PostgresBackend {
Arc::new(SqlxBillingReadRepository::new(self.pool_clone()))
}
pub fn background_task_read_repository(&self) -> Arc<dyn BackgroundTaskReadRepository> {
Arc::new(SqlxBackgroundTaskRepository::new(self.pool_clone()))
}
pub fn background_task_write_repository(&self) -> Arc<dyn BackgroundTaskWriteRepository> {
Arc::new(SqlxBackgroundTaskRepository::new(self.pool_clone()))
}
pub fn minimal_candidate_selection_read_repository(
&self,
) -> Arc<dyn MinimalCandidateSelectionReadRepository> {

12
crates/aether-data/src/backend/read.rs
View File

@@ -6,6 +6,7 @@ use crate::repository::announcements::AnnouncementReadRepository;
use crate::repository::audit::AuditLogReadRepository;
use crate::repository::auth::AuthApiKeyReadRepository;
use crate::repository::auth_modules::AuthModuleReadRepository;
use crate::repository::background_tasks::BackgroundTaskReadRepository;
use crate::repository::billing::BillingReadRepository;
use crate::repository::candidate_selection::MinimalCandidateSelectionReadRepository;
use crate::repository::candidates::RequestCandidateReadRepository;
@@ -27,6 +28,7 @@ pub struct DataReadRepositories {
audit_logs: Option<Arc<dyn AuditLogReadRepository>>,
auth_api_keys: Option<Arc<dyn AuthApiKeyReadRepository>>,
auth_modules: Option<Arc<dyn AuthModuleReadRepository>>,
background_tasks: Option<Arc<dyn BackgroundTaskReadRepository>>,
billing: Option<Arc<dyn BillingReadRepository>>,
gemini_file_mappings: Option<Arc<dyn GeminiFileMappingReadRepository>>,
global_models: Option<Arc<dyn GlobalModelReadRepository>>,
@@ -50,6 +52,7 @@ impl fmt::Debug for DataReadRepositories {
.field("has_announcements", &self.announcements.is_some())
.field("has_audit_logs", &self.audit_logs.is_some())
.field("has_auth_modules", &self.auth_modules.is_some())
.field("has_background_tasks", &self.background_tasks.is_some())
.field("has_billing", &self.billing.is_some())
.field(
"has_gemini_file_mappings",
@@ -97,6 +100,10 @@ impl DataReadRepositories {
.map(PostgresBackend::auth_module_read_repository)
.or_else(|| mysql.map(MysqlBackend::auth_module_read_repository))
.or_else(|| sqlite.map(SqliteBackend::auth_module_read_repository)),
background_tasks: postgres
.map(PostgresBackend::background_task_read_repository)
.or_else(|| mysql.map(MysqlBackend::background_task_read_repository))
.or_else(|| sqlite.map(SqliteBackend::background_task_read_repository)),
billing: postgres
.map(PostgresBackend::billing_read_repository)
.or_else(|| mysql.map(MysqlBackend::billing_read_repository))
@@ -177,6 +184,10 @@ impl DataReadRepositories {
self.auth_modules.clone()
}
pub fn background_tasks(&self) -> Option<Arc<dyn BackgroundTaskReadRepository>> {
self.background_tasks.clone()
}
pub fn billing(&self) -> Option<Arc<dyn BillingReadRepository>> {
self.billing.clone()
}
@@ -240,6 +251,7 @@ impl DataReadRepositories {
|| self.announcements.is_some()
|| self.audit_logs.is_some()
|| self.auth_modules.is_some()
|| self.background_tasks.is_some()
|| self.billing.is_some()
|| self.gemini_file_mappings.is_some()
|| self.global_models.is_some()

11
crates/aether-data/src/backend/sqlite.rs
View File

@@ -13,6 +13,9 @@ use crate::repository::auth_modules::{
AuthModuleReadRepository, AuthModuleWriteRepository, SqliteAuthModuleReadRepository,
SqliteAuthModuleRepository,
};
use crate::repository::background_tasks::{
BackgroundTaskReadRepository, BackgroundTaskWriteRepository, SqliteBackgroundTaskRepository,
};
use crate::repository::billing::{BillingReadRepository, SqliteBillingReadRepository};
use crate::repository::candidate_selection::{
MinimalCandidateSelectionReadRepository, SqliteMinimalCandidateSelectionReadRepository,
@@ -124,6 +127,14 @@ impl SqliteBackend {
Arc::new(SqliteBillingReadRepository::new(self.pool_clone()))
}
pub fn background_task_read_repository(&self) -> Arc<dyn BackgroundTaskReadRepository> {
Arc::new(SqliteBackgroundTaskRepository::new(self.pool_clone()))
}
pub fn background_task_write_repository(&self) -> Arc<dyn BackgroundTaskWriteRepository> {
Arc::new(SqliteBackgroundTaskRepository::new(self.pool_clone()))
}
pub fn request_candidate_read_repository(&self) -> Arc<dyn RequestCandidateReadRepository> {
Arc::new(SqliteRequestCandidateRepository::new(self.pool_clone()))
}

12
crates/aether-data/src/backend/write.rs
View File

@@ -5,6 +5,7 @@ use super::{MysqlBackend, PostgresBackend, SqliteBackend};
use crate::repository::announcements::AnnouncementWriteRepository;
use crate::repository::auth::AuthApiKeyWriteRepository;
use crate::repository::auth_modules::AuthModuleWriteRepository;
use crate::repository::background_tasks::BackgroundTaskWriteRepository;
use crate::repository::candidates::RequestCandidateWriteRepository;
use crate::repository::gemini_file_mappings::GeminiFileMappingWriteRepository;
use crate::repository::global_models::GlobalModelWriteRepository;
@@ -23,6 +24,7 @@ pub struct DataWriteRepositories {
announcements: Option<Arc<dyn AnnouncementWriteRepository>>,
auth_api_keys: Option<Arc<dyn AuthApiKeyWriteRepository>>,
auth_modules: Option<Arc<dyn AuthModuleWriteRepository>>,
background_tasks: Option<Arc<dyn BackgroundTaskWriteRepository>>,
request_candidates: Option<Arc<dyn RequestCandidateWriteRepository>>,
gemini_file_mappings: Option<Arc<dyn GeminiFileMappingWriteRepository>>,
global_models: Option<Arc<dyn GlobalModelWriteRepository>>,
@@ -43,6 +45,7 @@ impl fmt::Debug for DataWriteRepositories {
.field("has_announcements", &self.announcements.is_some())
.field("has_auth_api_keys", &self.auth_api_keys.is_some())
.field("has_auth_modules", &self.auth_modules.is_some())
.field("has_background_tasks", &self.background_tasks.is_some())
.field("has_request_candidates", &self.request_candidates.is_some())
.field(
"has_gemini_file_mappings",
@@ -81,6 +84,10 @@ impl DataWriteRepositories {
.map(PostgresBackend::auth_module_write_repository)
.or_else(|| mysql.map(MysqlBackend::auth_module_write_repository))
.or_else(|| sqlite.map(SqliteBackend::auth_module_write_repository)),
background_tasks: postgres
.map(PostgresBackend::background_task_write_repository)
.or_else(|| mysql.map(MysqlBackend::background_task_write_repository))
.or_else(|| sqlite.map(SqliteBackend::background_task_write_repository)),
request_candidates: postgres
.map(PostgresBackend::request_candidate_write_repository)
.or_else(|| mysql.map(MysqlBackend::request_candidate_write_repository))
@@ -149,6 +156,10 @@ impl DataWriteRepositories {
self.auth_modules.clone()
}
pub fn background_tasks(&self) -> Option<Arc<dyn BackgroundTaskWriteRepository>> {
self.background_tasks.clone()
}
pub fn usage(&self) -> Option<Arc<dyn UsageWriteRepository>> {
self.usage.clone()
}
@@ -201,6 +212,7 @@ impl DataWriteRepositories {
self.announcements.is_some()
|| self.auth_api_keys.is_some()
|| self.auth_modules.is_some()
|| self.background_tasks.is_some()
|| self.request_candidates.is_some()
|| self.gemini_file_mappings.is_some()
|| self.global_models.is_some()

2
crates/aether-data/src/lifecycle/bootstrap/postgres.rs
View File

@@ -7,7 +7,7 @@ use tracing::info;
// Generated by build.rs from schema/bootstrap/postgres.
pub(crate) static EMPTY_DATABASE_SNAPSHOT_SQL: &str =
include_str!(concat!(env!("OUT_DIR"), "/empty_database_snapshot.sql"));
pub(crate) const EMPTY_DATABASE_SNAPSHOT_CUTOFF_VERSION: i64 = 20260508000000;
pub(crate) const EMPTY_DATABASE_SNAPSHOT_CUTOFF_VERSION: i64 = 20260509000000;
const PUBLIC_BASE_TABLE_COUNT_SQL: &str = r#"
SELECT COUNT(*)::BIGINT

16
crates/aether-data/src/lifecycle/migrate/tests.rs
View File

@@ -294,6 +294,7 @@ fn empty_database_snapshot_covers_current_cutoff_versions() {
20260507000000,
20260507120000,
20260508000000,
20260509000000,
]
);
}
@@ -513,11 +514,21 @@ fn mysql_and_sqlite_migrations_include_enabled_incrementals() {
assert_eq!(
mysql_versions,
vec![20260403000000, 20260507120000, 20260508000000]
vec![
20260403000000,
20260507120000,
20260508000000,
20260509000000
]
);
assert_eq!(
sqlite_versions,
vec![20260403000000, 20260507120000, 20260508000000]
vec![
20260403000000,
20260507120000,
20260508000000,
20260509000000
]
);
}
@@ -1022,6 +1033,7 @@ fn pending_migrations_from_applied_skips_versions_already_applied() {
20260507000000,
20260507120000,
20260508000000,
20260509000000,
]
);
}

218
crates/aether-data/src/repository/background_tasks/memory.rs Normal file
View File

@@ -0,0 +1,218 @@
use std::collections::{BTreeMap, BTreeSet};
use std::sync::RwLock;
use async_trait::async_trait;
use super::{
BackgroundTaskListQuery, BackgroundTaskReadRepository, BackgroundTaskStatus,
BackgroundTaskSummary, BackgroundTaskWriteRepository, StoredBackgroundTaskEvent,
StoredBackgroundTaskRun, StoredBackgroundTaskRunPage, UpsertBackgroundTaskEvent,
UpsertBackgroundTaskRun,
};
use crate::DataLayerError;
#[derive(Debug, Default)]
struct InMemoryBackgroundTaskIndex {
runs: BTreeMap<String, StoredBackgroundTaskRun>,
events_by_run: BTreeMap<String, Vec<StoredBackgroundTaskEvent>>,
}
#[derive(Debug, Default)]
pub struct InMemoryBackgroundTaskRepository {
index: RwLock<InMemoryBackgroundTaskIndex>,
}
impl InMemoryBackgroundTaskRepository {
fn matches_filter(run: &StoredBackgroundTaskRun, query: &BackgroundTaskListQuery) -> bool {
if let Some(kind) = query.kind {
if run.kind != kind {
return false;
}
}
if let Some(status) = query.status {
if run.status != status {
return false;
}
}
if let Some(trigger) = query.trigger.as_deref() {
if run.trigger != trigger {
return false;
}
}
if let Some(task_key_substring) = query.task_key_substring.as_deref() {
let needle = task_key_substring.to_ascii_lowercase();
if !run.task_key.to_ascii_lowercase().contains(&needle) {
return false;
}
}
true
}
pub fn seed_runs<I>(runs: I) -> Self
where
I: IntoIterator<Item = StoredBackgroundTaskRun>,
{
let mut index = InMemoryBackgroundTaskIndex::default();
for run in runs {
index.runs.insert(run.id.clone(), run);
}
Self {
index: RwLock::new(index),
}
}
}
#[async_trait]
impl BackgroundTaskReadRepository for InMemoryBackgroundTaskRepository {
async fn find_run(
&self,
run_id: &str,
) -> Result<Option<StoredBackgroundTaskRun>, DataLayerError> {
Ok(self
.index
.read()
.expect("background task repository lock")
.runs
.get(run_id)
.cloned())
}
async fn list_runs(
&self,
query: &BackgroundTaskListQuery,
) -> Result<StoredBackgroundTaskRunPage, DataLayerError> {
let mut items = self
.index
.read()
.expect("background task repository lock")
.runs
.values()
.filter(|run| Self::matches_filter(run, query))
.cloned()
.collect::<Vec<_>>();
items.sort_by(|left, right| {
right
.created_at_unix_secs
.cmp(&left.created_at_unix_secs)
.then_with(|| right.updated_at_unix_secs.cmp(&left.updated_at_unix_secs))
});
let total = items.len();
let limit = query.limit.max(1);
let items = items
.into_iter()
.skip(query.offset)
.take(limit)
.collect::<Vec<_>>();
Ok(StoredBackgroundTaskRunPage { items, total })
}
async fn list_events(
&self,
run_id: &str,
offset: usize,
limit: usize,
) -> Result<Vec<StoredBackgroundTaskEvent>, DataLayerError> {
let Some(events) = self
.index
.read()
.expect("background task repository lock")
.events_by_run
.get(run_id)
.cloned()
else {
return Ok(Vec::new());
};
let limit = limit.max(1);
Ok(events.into_iter().skip(offset).take(limit).collect())
}
async fn summarize_runs(&self) -> Result<BackgroundTaskSummary, DataLayerError> {
let runs = self
.index
.read()
.expect("background task repository lock")
.runs
.values()
.cloned()
.collect::<Vec<_>>();
let mut by_status = BTreeMap::new();
let mut by_kind = BTreeMap::new();
let mut running_count = 0_u64;
for run in runs {
*by_status
.entry(run.status.as_database().to_string())
.or_insert(0) += 1;
*by_kind
.entry(run.kind.as_database().to_string())
.or_insert(0) += 1;
if run.status == BackgroundTaskStatus::Running {
running_count += 1;
}
}
let total = by_status.values().copied().sum();
Ok(BackgroundTaskSummary {
total,
running_count,
by_status,
by_kind,
})
}
}
#[async_trait]
impl BackgroundTaskWriteRepository for InMemoryBackgroundTaskRepository {
async fn upsert_run(
&self,
run: UpsertBackgroundTaskRun,
) -> Result<StoredBackgroundTaskRun, DataLayerError> {
run.validate()?;
let stored = run.into_stored();
self.index
.write()
.expect("background task repository lock")
.runs
.insert(stored.id.clone(), stored.clone());
Ok(stored)
}
async fn request_cancel(
&self,
run_id: &str,
updated_at_unix_secs: u64,
) -> Result<bool, DataLayerError> {
let mut guard = self.index.write().expect("background task repository lock");
let Some(run) = guard.runs.get_mut(run_id) else {
return Ok(false);
};
run.cancel_requested = true;
run.updated_at_unix_secs = updated_at_unix_secs;
Ok(true)
}
async fn upsert_event(
&self,
event: UpsertBackgroundTaskEvent,
) -> Result<StoredBackgroundTaskEvent, DataLayerError> {
event.validate()?;
let stored = event.into_stored();
let mut guard = self.index.write().expect("background task repository lock");
let entries = guard
.events_by_run
.entry(stored.run_id.clone())
.or_default();
if let Some(position) = entries.iter().position(|value| value.id == stored.id) {
entries[position] = stored.clone();
} else {
entries.push(stored.clone());
}
let mut seen = BTreeSet::new();
entries.retain(|entry| seen.insert(entry.id.clone()));
entries.sort_by(|left, right| {
left.created_at_unix_secs
.cmp(&right.created_at_unix_secs)
.then_with(|| left.id.cmp(&right.id))
});
Ok(stored)
}
}

17
crates/aether-data/src/repository/background_tasks/mod.rs Normal file
View File

@@ -0,0 +1,17 @@
mod memory;
mod mysql;
mod postgres;
mod sqlite;
#[allow(unused_imports)]
pub(crate) use aether_data_contracts::repository::background_tasks::{
BackgroundTaskKind, BackgroundTaskListQuery, BackgroundTaskReadRepository,
BackgroundTaskRepository, BackgroundTaskStatus, BackgroundTaskSummary,
BackgroundTaskWriteRepository, StoredBackgroundTaskEvent, StoredBackgroundTaskRun,
StoredBackgroundTaskRunPage, UpsertBackgroundTaskEvent, UpsertBackgroundTaskRun,
};
pub use memory::InMemoryBackgroundTaskRepository;
pub use mysql::MysqlBackgroundTaskRepository;
pub use postgres::SqlxBackgroundTaskRepository;
pub use sqlite::SqliteBackgroundTaskRepository;

449
crates/aether-data/src/repository/background_tasks/mysql.rs Normal file
View File

@@ -0,0 +1,449 @@
use async_trait::async_trait;
use sqlx::{mysql::MySqlRow, MySql, QueryBuilder, Row};
use super::{
BackgroundTaskKind, BackgroundTaskListQuery, BackgroundTaskReadRepository,
BackgroundTaskStatus, BackgroundTaskSummary, BackgroundTaskWriteRepository,
StoredBackgroundTaskEvent, StoredBackgroundTaskRun, StoredBackgroundTaskRunPage,
UpsertBackgroundTaskEvent, UpsertBackgroundTaskRun,
};
use crate::driver::mysql::MysqlPool;
use crate::error::SqlResultExt;
use crate::DataLayerError;
const RUN_COLUMNS: &str = r#"
SELECT
id,
task_key,
kind,
`trigger`,
status,
attempt,
max_attempts,
owner_instance,
progress_percent,
progress_message,
payload_json,
result_json,
error_message,
cancel_requested,
created_by,
created_at_unix_secs,
started_at_unix_secs,
finished_at_unix_secs,
updated_at_unix_secs
FROM background_task_runs
"#;
const EVENT_COLUMNS: &str = r#"
SELECT
id,
run_id,
event_type,
message,
payload_json,
created_at_unix_secs
FROM background_task_events
"#;
#[derive(Debug, Clone)]
pub struct MysqlBackgroundTaskRepository {
pool: MysqlPool,
}
impl MysqlBackgroundTaskRepository {
pub fn new(pool: MysqlPool) -> Self {
Self { pool }
}
fn apply_run_filter(builder: &mut QueryBuilder<'_, MySql>, query: &BackgroundTaskListQuery) {
let mut has_where = false;
if let Some(kind) = query.kind {
if !has_where {
builder.push(" WHERE ");
has_where = true;
} else {
builder.push(" AND ");
}
builder.push("kind = ").push_bind(kind.as_database());
}
if let Some(status) = query.status {
if !has_where {
builder.push(" WHERE ");
has_where = true;
} else {
builder.push(" AND ");
}
builder.push("status = ").push_bind(status.as_database());
}
if let Some(trigger) = query.trigger.as_deref() {
if !has_where {
builder.push(" WHERE ");
has_where = true;
} else {
builder.push(" AND ");
}
builder.push("`trigger` = ").push_bind(trigger.to_string());
}
if let Some(task_key_substring) = query.task_key_substring.as_deref() {
if !has_where {
builder.push(" WHERE ");
} else {
builder.push(" AND ");
}
builder.push("LOWER(task_key) LIKE ").push_bind(format!(
"%{}%",
task_key_substring.trim().to_ascii_lowercase()
));
}
}
}
#[async_trait]
impl BackgroundTaskReadRepository for MysqlBackgroundTaskRepository {
async fn find_run(
&self,
run_id: &str,
) -> Result<Option<StoredBackgroundTaskRun>, DataLayerError> {
let row = sqlx::query(&format!("{RUN_COLUMNS} WHERE id = ? LIMIT 1"))
.bind(run_id)
.fetch_optional(&self.pool)
.await
.map_sql_err()?;
row.as_ref().map(map_run_row).transpose()
}
async fn list_runs(
&self,
query: &BackgroundTaskListQuery,
) -> Result<StoredBackgroundTaskRunPage, DataLayerError> {
let limit = query.limit.max(1);
let mut count_builder =
QueryBuilder::<MySql>::new("SELECT COUNT(id) AS total FROM background_task_runs");
Self::apply_run_filter(&mut count_builder, query);
let total = count_builder
.build_query_scalar::<i64>()
.fetch_one(&self.pool)
.await
.map_sql_err()?;
let mut builder = QueryBuilder::<MySql>::new(RUN_COLUMNS);
Self::apply_run_filter(&mut builder, query);
builder
.push(" ORDER BY created_at_unix_secs DESC, updated_at_unix_secs DESC")
.push(" LIMIT ")
.push_bind(i64_from_usize(limit, "run limit")?)
.push(" OFFSET ")
.push_bind(i64_from_usize(query.offset, "run offset")?);
let rows = builder.build().fetch_all(&self.pool).await.map_sql_err()?;
let items = rows
.iter()
.map(map_run_row)
.collect::<Result<Vec<_>, _>>()?;
Ok(StoredBackgroundTaskRunPage {
items,
total: usize::try_from(total).unwrap_or_default(),
})
}
async fn list_events(
&self,
run_id: &str,
offset: usize,
limit: usize,
) -> Result<Vec<StoredBackgroundTaskEvent>, DataLayerError> {
let limit = limit.max(1);
let rows = sqlx::query(&format!(
"{EVENT_COLUMNS} WHERE run_id = ? ORDER BY created_at_unix_secs ASC, id ASC LIMIT ? OFFSET ?"
))
.bind(run_id)
.bind(i64_from_usize(limit, "event limit")?)
.bind(i64_from_usize(offset, "event offset")?)
.fetch_all(&self.pool)
.await
.map_sql_err()?;
rows.iter().map(map_event_row).collect()
}
async fn summarize_runs(&self) -> Result<BackgroundTaskSummary, DataLayerError> {
let total = sqlx::query_scalar::<_, i64>("SELECT COUNT(id) FROM background_task_runs")
.fetch_one(&self.pool)
.await
.map_sql_err()?;
let running_count = sqlx::query_scalar::<_, i64>(
"SELECT COUNT(id) FROM background_task_runs WHERE status = 'running'",
)
.fetch_one(&self.pool)
.await
.map_sql_err()?;
let status_rows = sqlx::query(
"SELECT status, COUNT(id) AS total FROM background_task_runs GROUP BY status",
)
.fetch_all(&self.pool)
.await
.map_sql_err()?;
let kind_rows =
sqlx::query("SELECT kind, COUNT(id) AS total FROM background_task_runs GROUP BY kind")
.fetch_all(&self.pool)
.await
.map_sql_err()?;
let mut by_status = std::collections::BTreeMap::new();
for row in status_rows {
let key: String = row.try_get("status").map_sql_err()?;
let count: i64 = row.try_get("total").map_sql_err()?;
by_status.insert(key, u64::try_from(count).unwrap_or_default());
}
let mut by_kind = std::collections::BTreeMap::new();
for row in kind_rows {
let key: String = row.try_get("kind").map_sql_err()?;
let count: i64 = row.try_get("total").map_sql_err()?;
by_kind.insert(key, u64::try_from(count).unwrap_or_default());
}
Ok(BackgroundTaskSummary {
total: u64::try_from(total).unwrap_or_default(),
running_count: u64::try_from(running_count).unwrap_or_default(),
by_status,
by_kind,
})
}
}
#[async_trait]
impl BackgroundTaskWriteRepository for MysqlBackgroundTaskRepository {
async fn upsert_run(
&self,
run: UpsertBackgroundTaskRun,
) -> Result<StoredBackgroundTaskRun, DataLayerError> {
run.validate()?;
sqlx::query(
r#"
INSERT INTO background_task_runs (
id,
task_key,
kind,
`trigger`,
status,
attempt,
max_attempts,
owner_instance,
progress_percent,
progress_message,
payload_json,
result_json,
error_message,
cancel_requested,
created_by,
created_at_unix_secs,
started_at_unix_secs,
finished_at_unix_secs,
updated_at_unix_secs
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
ON DUPLICATE KEY UPDATE
task_key = VALUES(task_key),
kind = VALUES(kind),
`trigger` = VALUES(`trigger`),
status = VALUES(status),
attempt = VALUES(attempt),
max_attempts = VALUES(max_attempts),
owner_instance = VALUES(owner_instance),
progress_percent = VALUES(progress_percent),
progress_message = VALUES(progress_message),
payload_json = VALUES(payload_json),
result_json = VALUES(result_json),
error_message = VALUES(error_message),
cancel_requested = VALUES(cancel_requested),
created_by = VALUES(created_by),
created_at_unix_secs = VALUES(created_at_unix_secs),
started_at_unix_secs = VALUES(started_at_unix_secs),
finished_at_unix_secs = VALUES(finished_at_unix_secs),
updated_at_unix_secs = VALUES(updated_at_unix_secs)
"#,
)
.bind(&run.id)
.bind(&run.task_key)
.bind(run.kind.as_database())
.bind(&run.trigger)
.bind(run.status.as_database())
.bind(i64::from(run.attempt))
.bind(i64::from(run.max_attempts))
.bind(run.owner_instance.as_deref())
.bind(i32::from(run.progress_percent))
.bind(run.progress_message.as_deref())
.bind(json_to_string(&run.payload_json, "payload_json")?)
.bind(json_to_string(&run.result_json, "result_json")?)
.bind(run.error_message.as_deref())
.bind(run.cancel_requested)
.bind(run.created_by.as_deref())
.bind(u64_to_i64(
run.created_at_unix_secs,
"created_at_unix_secs",
)?)
.bind(run.started_at_unix_secs.map(|value| value as i64))
.bind(run.finished_at_unix_secs.map(|value| value as i64))
.bind(u64_to_i64(
run.updated_at_unix_secs,
"updated_at_unix_secs",
)?)
.execute(&self.pool)
.await
.map_sql_err()?;
self.find_run(&run.id).await?.ok_or_else(|| {
DataLayerError::UnexpectedValue("background task run missing after upsert".to_string())
})
}
async fn request_cancel(
&self,
run_id: &str,
updated_at_unix_secs: u64,
) -> Result<bool, DataLayerError> {
let affected = sqlx::query(
"UPDATE background_task_runs SET cancel_requested = TRUE, updated_at_unix_secs = ? WHERE id = ?",
)
.bind(u64_to_i64(updated_at_unix_secs, "updated_at_unix_secs")?)
.bind(run_id)
.execute(&self.pool)
.await
.map_sql_err()?
.rows_affected();
Ok(affected > 0)
}
async fn upsert_event(
&self,
event: UpsertBackgroundTaskEvent,
) -> Result<StoredBackgroundTaskEvent, DataLayerError> {
event.validate()?;
sqlx::query(
r#"
INSERT INTO background_task_events (
id, run_id, event_type, message, payload_json, created_at_unix_secs
) VALUES (?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
run_id = VALUES(run_id),
event_type = VALUES(event_type),
message = VALUES(message),
payload_json = VALUES(payload_json),
created_at_unix_secs = VALUES(created_at_unix_secs)
"#,
)
.bind(&event.id)
.bind(&event.run_id)
.bind(&event.event_type)
.bind(&event.message)
.bind(json_to_string(&event.payload_json, "payload_json")?)
.bind(u64_to_i64(
event.created_at_unix_secs,
"created_at_unix_secs",
)?)
.execute(&self.pool)
.await
.map_sql_err()?;
let row = sqlx::query(&format!("{EVENT_COLUMNS} WHERE id = ? LIMIT 1"))
.bind(&event.id)
.fetch_one(&self.pool)
.await
.map_sql_err()?;
map_event_row(&row)
}
}
fn map_run_row(row: &MySqlRow) -> Result<StoredBackgroundTaskRun, DataLayerError> {
let kind: String = row.try_get("kind").map_sql_err()?;
let status: String = row.try_get("status").map_sql_err()?;
let attempt: i64 = row.try_get("attempt").map_sql_err()?;
let max_attempts: i64 = row.try_get("max_attempts").map_sql_err()?;
let progress_percent: i32 = row.try_get("progress_percent").map_sql_err()?;
let created_at_unix_secs: i64 = row.try_get("created_at_unix_secs").map_sql_err()?;
let started_at_unix_secs: Option<i64> = row.try_get("started_at_unix_secs").map_sql_err()?;
let finished_at_unix_secs: Option<i64> = row.try_get("finished_at_unix_secs").map_sql_err()?;
let updated_at_unix_secs: i64 = row.try_get("updated_at_unix_secs").map_sql_err()?;
Ok(StoredBackgroundTaskRun {
id: row.try_get("id").map_sql_err()?,
task_key: row.try_get("task_key").map_sql_err()?,
kind: BackgroundTaskKind::from_database(&kind)?,
trigger: row.try_get("trigger").map_sql_err()?,
status: BackgroundTaskStatus::from_database(&status)?,
attempt: u32::try_from(attempt).unwrap_or_default(),
max_attempts: u32::try_from(max_attempts).unwrap_or_default(),
owner_instance: row.try_get("owner_instance").map_sql_err()?,
progress_percent: u16::try_from(progress_percent).unwrap_or_default(),
progress_message: row.try_get("progress_message").map_sql_err()?,
payload_json: parse_optional_json(
row.try_get("payload_json").ok().flatten(),
"payload_json",
)?,
result_json: parse_optional_json(row.try_get("result_json").ok().flatten(), "result_json")?,
error_message: row.try_get("error_message").map_sql_err()?,
cancel_requested: row.try_get("cancel_requested").map_sql_err()?,
created_by: row.try_get("created_by").map_sql_err()?,
created_at_unix_secs: u64::try_from(created_at_unix_secs).unwrap_or_default(),
started_at_unix_secs: started_at_unix_secs.and_then(|value| u64::try_from(value).ok()),
finished_at_unix_secs: finished_at_unix_secs.and_then(|value| u64::try_from(value).ok()),
updated_at_unix_secs: u64::try_from(updated_at_unix_secs).unwrap_or_default(),
})
}
fn map_event_row(row: &MySqlRow) -> Result<StoredBackgroundTaskEvent, DataLayerError> {
let created_at_unix_secs: i64 = row.try_get("created_at_unix_secs").map_sql_err()?;
Ok(StoredBackgroundTaskEvent {
id: row.try_get("id").map_sql_err()?,
run_id: row.try_get("run_id").map_sql_err()?,
event_type: row.try_get("event_type").map_sql_err()?,
message: row.try_get("message").map_sql_err()?,
payload_json: parse_optional_json(
row.try_get("payload_json").ok().flatten(),
"payload_json",
)?,
created_at_unix_secs: u64::try_from(created_at_unix_secs).unwrap_or_default(),
})
}
fn i64_from_usize(value: usize, label: &str) -> Result<i64, DataLayerError> {
i64::try_from(value).map_err(|_| {
DataLayerError::UnexpectedValue(format!("background task {label} overflow: {value}"))
})
}
fn u64_to_i64(value: u64, label: &str) -> Result<i64, DataLayerError> {
i64::try_from(value).map_err(|_| {
DataLayerError::UnexpectedValue(format!("background task {label} overflow: {value}"))
})
}
fn json_to_string(
value: &Option<serde_json::Value>,
field_name: &str,
) -> Result<Option<String>, DataLayerError> {
value
.as_ref()
.map(|value| {
serde_json::to_string(value).map_err(|err| {
DataLayerError::UnexpectedValue(format!(
"background task {field_name} is unserializable: {err}"
))
})
})
.transpose()
}
fn parse_optional_json(
value: Option<String>,
field_name: &str,
) -> Result<Option<serde_json::Value>, DataLayerError> {
value
.filter(|value| !value.trim().is_empty())
.map(|value| {
serde_json::from_str(&value).map_err(|err| {
DataLayerError::UnexpectedValue(format!(
"background task {field_name} contains invalid JSON: {err}"
))
})
})
.transpose()
}

426
crates/aether-data/src/repository/background_tasks/postgres.rs Normal file
View File

@@ -0,0 +1,426 @@
use async_trait::async_trait;
use sqlx::{postgres::PgRow, PgPool, Postgres, QueryBuilder, Row};
use super::{
BackgroundTaskKind, BackgroundTaskListQuery, BackgroundTaskReadRepository,
BackgroundTaskStatus, BackgroundTaskSummary, BackgroundTaskWriteRepository,
StoredBackgroundTaskEvent, StoredBackgroundTaskRun, StoredBackgroundTaskRunPage,
UpsertBackgroundTaskEvent, UpsertBackgroundTaskRun,
};
use crate::error::SqlxResultExt;
use crate::DataLayerError;
const RUN_COLUMNS: &str = r#"
SELECT
id,
task_key,
kind,
"trigger",
status,
attempt,
max_attempts,
owner_instance,
progress_percent,
progress_message,
payload_json,
result_json,
error_message,
cancel_requested,
created_by,
created_at_unix_secs,
started_at_unix_secs,
finished_at_unix_secs,
updated_at_unix_secs
FROM background_task_runs
"#;
const EVENT_COLUMNS: &str = r#"
SELECT
id,
run_id,
event_type,
message,
payload_json,
created_at_unix_secs
FROM background_task_events
"#;
#[derive(Debug, Clone)]
pub struct SqlxBackgroundTaskRepository {
pool: PgPool,
}
impl SqlxBackgroundTaskRepository {
pub fn new(pool: PgPool) -> Self {
Self { pool }
}
fn apply_run_filter(
builder: &mut QueryBuilder<'_, Postgres>,
query: &BackgroundTaskListQuery,
include_where: bool,
) {
let mut has_where = include_where;
let mut push_where = |builder: &mut QueryBuilder<'_, Postgres>| {
if has_where {
builder.push(" AND ");
} else {
builder.push(" WHERE ");
has_where = true;
}
};
if let Some(kind) = query.kind {
push_where(builder);
builder.push("kind = ").push_bind(kind.as_database());
}
if let Some(status) = query.status {
push_where(builder);
builder.push("status = ").push_bind(status.as_database());
}
if let Some(trigger) = query.trigger.as_deref() {
push_where(builder);
builder
.push("\"trigger\" = ")
.push_bind(trigger.to_string());
}
if let Some(task_key_substring) = query.task_key_substring.as_deref() {
push_where(builder);
builder
.push("task_key ILIKE ")
.push_bind(format!("%{}%", task_key_substring.trim()));
}
}
}
#[async_trait]
impl BackgroundTaskReadRepository for SqlxBackgroundTaskRepository {
async fn find_run(
&self,
run_id: &str,
) -> Result<Option<StoredBackgroundTaskRun>, DataLayerError> {
let row = sqlx::query(&format!("{RUN_COLUMNS} WHERE id = $1 LIMIT 1"))
.bind(run_id)
.fetch_optional(&self.pool)
.await
.map_postgres_err()?;
row.as_ref().map(map_run_row).transpose()
}
async fn list_runs(
&self,
query: &BackgroundTaskListQuery,
) -> Result<StoredBackgroundTaskRunPage, DataLayerError> {
let limit = query.limit.max(1);
let mut count_builder =
QueryBuilder::<Postgres>::new("SELECT COUNT(id) AS total FROM background_task_runs");
Self::apply_run_filter(&mut count_builder, query, false);
let total = count_builder
.build_query_scalar::<i64>()
.fetch_one(&self.pool)
.await
.map_postgres_err()?;
let mut builder = QueryBuilder::<Postgres>::new(RUN_COLUMNS);
Self::apply_run_filter(&mut builder, query, false);
builder
.push(" ORDER BY created_at_unix_secs DESC, updated_at_unix_secs DESC")
.push(" LIMIT ")
.push_bind(i64_from_usize(limit, "background task run limit")?)
.push(" OFFSET ")
.push_bind(i64_from_usize(query.offset, "background task run offset")?);
let rows = builder
.build()
.fetch_all(&self.pool)
.await
.map_postgres_err()?;
let items = rows
.iter()
.map(map_run_row)
.collect::<Result<Vec<_>, _>>()?;
Ok(StoredBackgroundTaskRunPage {
items,
total: usize::try_from(total).unwrap_or_default(),
})
}
async fn list_events(
&self,
run_id: &str,
offset: usize,
limit: usize,
) -> Result<Vec<StoredBackgroundTaskEvent>, DataLayerError> {
let limit = limit.max(1);
let rows = sqlx::query(&format!(
"{EVENT_COLUMNS} WHERE run_id = $1 ORDER BY created_at_unix_secs ASC, id ASC LIMIT $2 OFFSET $3"
))
.bind(run_id)
.bind(i64_from_usize(limit, "background task event limit")?)
.bind(i64_from_usize(offset, "background task event offset")?)
.fetch_all(&self.pool)
.await
.map_postgres_err()?;
rows.iter().map(map_event_row).collect()
}
async fn summarize_runs(&self) -> Result<BackgroundTaskSummary, DataLayerError> {
let total = sqlx::query_scalar::<_, i64>("SELECT COUNT(id) FROM background_task_runs")
.fetch_one(&self.pool)
.await
.map_postgres_err()?;
let running_count = sqlx::query_scalar::<_, i64>(
"SELECT COUNT(id) FROM background_task_runs WHERE status = 'running'",
)
.fetch_one(&self.pool)
.await
.map_postgres_err()?;
let status_rows = sqlx::query(
"SELECT status, COUNT(id) AS total FROM background_task_runs GROUP BY status",
)
.fetch_all(&self.pool)
.await
.map_postgres_err()?;
let kind_rows =
sqlx::query("SELECT kind, COUNT(id) AS total FROM background_task_runs GROUP BY kind")
.fetch_all(&self.pool)
.await
.map_postgres_err()?;
let mut by_status = std::collections::BTreeMap::new();
for row in status_rows {
let key: String = row.try_get("status").map_postgres_err()?;
let count: i64 = row.try_get("total").map_postgres_err()?;
by_status.insert(key, u64::try_from(count).unwrap_or_default());
}
let mut by_kind = std::collections::BTreeMap::new();
for row in kind_rows {
let key: String = row.try_get("kind").map_postgres_err()?;
let count: i64 = row.try_get("total").map_postgres_err()?;
by_kind.insert(key, u64::try_from(count).unwrap_or_default());
}
Ok(BackgroundTaskSummary {
total: u64::try_from(total).unwrap_or_default(),
running_count: u64::try_from(running_count).unwrap_or_default(),
by_status,
by_kind,
})
}
}
#[async_trait]
impl BackgroundTaskWriteRepository for SqlxBackgroundTaskRepository {
async fn upsert_run(
&self,
run: UpsertBackgroundTaskRun,
) -> Result<StoredBackgroundTaskRun, DataLayerError> {
run.validate()?;
sqlx::query(
r#"
INSERT INTO background_task_runs (
id,
task_key,
kind,
"trigger",
status,
attempt,
max_attempts,
owner_instance,
progress_percent,
progress_message,
payload_json,
result_json,
error_message,
cancel_requested,
created_by,
created_at_unix_secs,
started_at_unix_secs,
finished_at_unix_secs,
updated_at_unix_secs
) VALUES (
$1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19
)
ON CONFLICT(id) DO UPDATE SET
task_key = EXCLUDED.task_key,
kind = EXCLUDED.kind,
"trigger" = EXCLUDED."trigger",
status = EXCLUDED.status,
attempt = EXCLUDED.attempt,
max_attempts = EXCLUDED.max_attempts,
owner_instance = EXCLUDED.owner_instance,
progress_percent = EXCLUDED.progress_percent,
progress_message = EXCLUDED.progress_message,
payload_json = EXCLUDED.payload_json,
result_json = EXCLUDED.result_json,
error_message = EXCLUDED.error_message,
cancel_requested = EXCLUDED.cancel_requested,
created_by = EXCLUDED.created_by,
created_at_unix_secs = EXCLUDED.created_at_unix_secs,
started_at_unix_secs = EXCLUDED.started_at_unix_secs,
finished_at_unix_secs = EXCLUDED.finished_at_unix_secs,
updated_at_unix_secs = EXCLUDED.updated_at_unix_secs
"#,
)
.bind(&run.id)
.bind(&run.task_key)
.bind(run.kind.as_database())
.bind(&run.trigger)
.bind(run.status.as_database())
.bind(u32_to_i32(run.attempt, "attempt")?)
.bind(u32_to_i32(run.max_attempts, "max_attempts")?)
.bind(run.owner_instance.as_deref())
.bind(i32::from(run.progress_percent))
.bind(run.progress_message.as_deref())
.bind(run.payload_json.clone())
.bind(run.result_json.clone())
.bind(run.error_message.as_deref())
.bind(run.cancel_requested)
.bind(run.created_by.as_deref())
.bind(u64_to_i64(
run.created_at_unix_secs,
"created_at_unix_secs",
)?)
.bind(run.started_at_unix_secs.map(|value| value as i64))
.bind(run.finished_at_unix_secs.map(|value| value as i64))
.bind(u64_to_i64(
run.updated_at_unix_secs,
"updated_at_unix_secs",
)?)
.execute(&self.pool)
.await
.map_postgres_err()?;
self.find_run(&run.id).await?.ok_or_else(|| {
DataLayerError::UnexpectedValue("background task run missing after upsert".to_string())
})
}
async fn request_cancel(
&self,
run_id: &str,
updated_at_unix_secs: u64,
) -> Result<bool, DataLayerError> {
let affected = sqlx::query(
"UPDATE background_task_runs SET cancel_requested = TRUE, updated_at_unix_secs = $2 WHERE id = $1",
)
.bind(run_id)
.bind(u64_to_i64(updated_at_unix_secs, "updated_at_unix_secs")?)
.execute(&self.pool)
.await
.map_postgres_err()?
.rows_affected();
Ok(affected > 0)
}
async fn upsert_event(
&self,
event: UpsertBackgroundTaskEvent,
) -> Result<StoredBackgroundTaskEvent, DataLayerError> {
event.validate()?;
sqlx::query(
r#"
INSERT INTO background_task_events (
id,
run_id,
event_type,
message,
payload_json,
created_at_unix_secs
) VALUES ($1,$2,$3,$4,$5,$6)
ON CONFLICT(id) DO UPDATE SET
run_id = EXCLUDED.run_id,
event_type = EXCLUDED.event_type,
message = EXCLUDED.message,
payload_json = EXCLUDED.payload_json,
created_at_unix_secs = EXCLUDED.created_at_unix_secs
"#,
)
.bind(&event.id)
.bind(&event.run_id)
.bind(&event.event_type)
.bind(&event.message)
.bind(event.payload_json.clone())
.bind(u64_to_i64(
event.created_at_unix_secs,
"created_at_unix_secs",
)?)
.execute(&self.pool)
.await
.map_postgres_err()?;
let row = sqlx::query(&format!("{EVENT_COLUMNS} WHERE id = $1 LIMIT 1"))
.bind(&event.id)
.fetch_one(&self.pool)
.await
.map_postgres_err()?;
map_event_row(&row)
}
}
fn map_run_row(row: &PgRow) -> Result<StoredBackgroundTaskRun, DataLayerError> {
let kind: String = row.try_get("kind").map_postgres_err()?;
let status: String = row.try_get("status").map_postgres_err()?;
let attempt: i32 = row.try_get("attempt").map_postgres_err()?;
let max_attempts: i32 = row.try_get("max_attempts").map_postgres_err()?;
let progress_percent: i32 = row.try_get("progress_percent").map_postgres_err()?;
let created_at_unix_secs: i64 = row.try_get("created_at_unix_secs").map_postgres_err()?;
let started_at_unix_secs: Option<i64> =
row.try_get("started_at_unix_secs").map_postgres_err()?;
let finished_at_unix_secs: Option<i64> =
row.try_get("finished_at_unix_secs").map_postgres_err()?;
let updated_at_unix_secs: i64 = row.try_get("updated_at_unix_secs").map_postgres_err()?;
Ok(StoredBackgroundTaskRun {
id: row.try_get("id").map_postgres_err()?,
task_key: row.try_get("task_key").map_postgres_err()?,
kind: BackgroundTaskKind::from_database(&kind)?,
trigger: row.try_get("trigger").map_postgres_err()?,
status: BackgroundTaskStatus::from_database(&status)?,
attempt: u32::try_from(attempt).unwrap_or_default(),
max_attempts: u32::try_from(max_attempts).unwrap_or_default(),
owner_instance: row.try_get("owner_instance").map_postgres_err()?,
progress_percent: u16::try_from(progress_percent).unwrap_or_default(),
progress_message: row.try_get("progress_message").map_postgres_err()?,
payload_json: row.try_get("payload_json").map_postgres_err()?,
result_json: row.try_get("result_json").map_postgres_err()?,
error_message: row.try_get("error_message").map_postgres_err()?,
cancel_requested: row.try_get("cancel_requested").map_postgres_err()?,
created_by: row.try_get("created_by").map_postgres_err()?,
created_at_unix_secs: u64::try_from(created_at_unix_secs).unwrap_or_default(),
started_at_unix_secs: started_at_unix_secs.and_then(|value| u64::try_from(value).ok()),
finished_at_unix_secs: finished_at_unix_secs.and_then(|value| u64::try_from(value).ok()),
updated_at_unix_secs: u64::try_from(updated_at_unix_secs).unwrap_or_default(),
})
}
fn map_event_row(row: &PgRow) -> Result<StoredBackgroundTaskEvent, DataLayerError> {
let created_at_unix_secs: i64 = row.try_get("created_at_unix_secs").map_postgres_err()?;
Ok(StoredBackgroundTaskEvent {
id: row.try_get("id").map_postgres_err()?,
run_id: row.try_get("run_id").map_postgres_err()?,
event_type: row.try_get("event_type").map_postgres_err()?,
message: row.try_get("message").map_postgres_err()?,
payload_json: row.try_get("payload_json").map_postgres_err()?,
created_at_unix_secs: u64::try_from(created_at_unix_secs).unwrap_or_default(),
})
}
fn i64_from_usize(value: usize, label: &str) -> Result<i64, DataLayerError> {
i64::try_from(value).map_err(|_| {
DataLayerError::UnexpectedValue(format!("background task {label} overflow: {value}"))
})
}
fn u64_to_i64(value: u64, label: &str) -> Result<i64, DataLayerError> {
i64::try_from(value).map_err(|_| {
DataLayerError::UnexpectedValue(format!("background task {label} overflow: {value}"))
})
}
fn u32_to_i32(value: u32, label: &str) -> Result<i32, DataLayerError> {
i32::try_from(value).map_err(|_| {
DataLayerError::UnexpectedValue(format!("background task {label} overflow: {value}"))
})
}

430
crates/aether-data/src/repository/background_tasks/sqlite.rs Normal file
View File

@@ -0,0 +1,430 @@
use async_trait::async_trait;
use sqlx::{sqlite::SqliteRow, QueryBuilder, Row, Sqlite};
use super::{
BackgroundTaskKind, BackgroundTaskListQuery, BackgroundTaskReadRepository,
BackgroundTaskStatus, BackgroundTaskSummary, BackgroundTaskWriteRepository,
StoredBackgroundTaskEvent, StoredBackgroundTaskRun, StoredBackgroundTaskRunPage,
UpsertBackgroundTaskEvent, UpsertBackgroundTaskRun,
};
use crate::driver::sqlite::SqlitePool;
use crate::error::SqlResultExt;
use crate::DataLayerError;
const RUN_COLUMNS: &str = r#"
SELECT
id,
task_key,
kind,
"trigger",
status,
attempt,
max_attempts,
owner_instance,
progress_percent,
progress_message,
payload_json,
result_json,
error_message,
cancel_requested,
created_by,
created_at_unix_secs,
started_at_unix_secs,
finished_at_unix_secs,
updated_at_unix_secs
FROM background_task_runs
"#;
const EVENT_COLUMNS: &str = r#"
SELECT
id,
run_id,
event_type,
message,
payload_json,
created_at_unix_secs
FROM background_task_events
"#;
#[derive(Debug, Clone)]
pub struct SqliteBackgroundTaskRepository {
pool: SqlitePool,
}
impl SqliteBackgroundTaskRepository {
pub fn new(pool: SqlitePool) -> Self {
Self { pool }
}
fn apply_run_filter(builder: &mut QueryBuilder<'_, Sqlite>, query: &BackgroundTaskListQuery) {
let mut has_where = false;
if let Some(kind) = query.kind {
if !has_where {
builder.push(" WHERE ");
has_where = true;
} else {
builder.push(" AND ");
}
builder.push("kind = ").push_bind(kind.as_database());
}
if let Some(status) = query.status {
if !has_where {
builder.push(" WHERE ");
has_where = true;
} else {
builder.push(" AND ");
}
builder.push("status = ").push_bind(status.as_database());
}
if let Some(trigger) = query.trigger.as_deref() {
if !has_where {
builder.push(" WHERE ");
has_where = true;
} else {
builder.push(" AND ");
}
builder
.push("\"trigger\" = ")
.push_bind(trigger.to_string());
}
if let Some(task_key_substring) = query.task_key_substring.as_deref() {
if !has_where {
builder.push(" WHERE ");
} else {
builder.push(" AND ");
}
builder.push("LOWER(task_key) LIKE ").push_bind(format!(
"%{}%",
task_key_substring.trim().to_ascii_lowercase()
));
}
}
}
#[async_trait]
impl BackgroundTaskReadRepository for SqliteBackgroundTaskRepository {
async fn find_run(
&self,
run_id: &str,
) -> Result<Option<StoredBackgroundTaskRun>, DataLayerError> {
let row = sqlx::query(&format!("{RUN_COLUMNS} WHERE id = ? LIMIT 1"))
.bind(run_id)
.fetch_optional(&self.pool)
.await
.map_sql_err()?;
row.as_ref().map(map_run_row).transpose()
}
async fn list_runs(
&self,
query: &BackgroundTaskListQuery,
) -> Result<StoredBackgroundTaskRunPage, DataLayerError> {
let limit = query.limit.max(1);
let mut count_builder =
QueryBuilder::<Sqlite>::new("SELECT COUNT(id) AS total FROM background_task_runs");
Self::apply_run_filter(&mut count_builder, query);
let total = count_builder
.build_query_scalar::<i64>()
.fetch_one(&self.pool)
.await
.map_sql_err()?;
let mut builder = QueryBuilder::<Sqlite>::new(RUN_COLUMNS);
Self::apply_run_filter(&mut builder, query);
builder
.push(" ORDER BY created_at_unix_secs DESC, updated_at_unix_secs DESC")
.push(" LIMIT ")
.push_bind(i64_from_usize(limit, "run limit")?)
.push(" OFFSET ")
.push_bind(i64_from_usize(query.offset, "run offset")?);
let rows = builder.build().fetch_all(&self.pool).await.map_sql_err()?;
let items = rows
.iter()
.map(map_run_row)
.collect::<Result<Vec<_>, _>>()?;
Ok(StoredBackgroundTaskRunPage {
items,
total: usize::try_from(total).unwrap_or_default(),
})
}
async fn list_events(
&self,
run_id: &str,
offset: usize,
limit: usize,
) -> Result<Vec<StoredBackgroundTaskEvent>, DataLayerError> {
let limit = limit.max(1);
let rows = sqlx::query(&format!(
"{EVENT_COLUMNS} WHERE run_id = ? ORDER BY created_at_unix_secs ASC, id ASC LIMIT ? OFFSET ?"
))
.bind(run_id)
.bind(i64_from_usize(limit, "event limit")?)
.bind(i64_from_usize(offset, "event offset")?)
.fetch_all(&self.pool)
.await
.map_sql_err()?;
rows.iter().map(map_event_row).collect()
}
async fn summarize_runs(&self) -> Result<BackgroundTaskSummary, DataLayerError> {
let total = sqlx::query_scalar::<_, i64>("SELECT COUNT(id) FROM background_task_runs")
.fetch_one(&self.pool)
.await
.map_sql_err()?;
let running_count = sqlx::query_scalar::<_, i64>(
"SELECT COUNT(id) FROM background_task_runs WHERE status = 'running'",
)
.fetch_one(&self.pool)
.await
.map_sql_err()?;
let status_rows = sqlx::query(
"SELECT status, COUNT(id) AS total FROM background_task_runs GROUP BY status",
)
.fetch_all(&self.pool)
.await
.map_sql_err()?;
let kind_rows =
sqlx::query("SELECT kind, COUNT(id) AS total FROM background_task_runs GROUP BY kind")
.fetch_all(&self.pool)
.await
.map_sql_err()?;
let mut by_status = std::collections::BTreeMap::new();
for row in status_rows {
let key: String = row.try_get("status").map_sql_err()?;
let count: i64 = row.try_get("total").map_sql_err()?;
by_status.insert(key, u64::try_from(count).unwrap_or_default());
}
let mut by_kind = std::collections::BTreeMap::new();
for row in kind_rows {
let key: String = row.try_get("kind").map_sql_err()?;
let count: i64 = row.try_get("total").map_sql_err()?;
by_kind.insert(key, u64::try_from(count).unwrap_or_default());
}
Ok(BackgroundTaskSummary {
total: u64::try_from(total).unwrap_or_default(),
running_count: u64::try_from(running_count).unwrap_or_default(),
by_status,
by_kind,
})
}
}
#[async_trait]
impl BackgroundTaskWriteRepository for SqliteBackgroundTaskRepository {
async fn upsert_run(
&self,
run: UpsertBackgroundTaskRun,
) -> Result<StoredBackgroundTaskRun, DataLayerError> {
run.validate()?;
sqlx::query(
r#"
INSERT INTO background_task_runs (
id,
task_key,
kind,
"trigger",
status,
attempt,
max_attempts,
owner_instance,
progress_percent,
progress_message,
payload_json,
result_json,
error_message,
cancel_requested,
created_by,
created_at_unix_secs,
started_at_unix_secs,
finished_at_unix_secs,
updated_at_unix_secs
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
ON CONFLICT(id) DO UPDATE SET
task_key = excluded.task_key,
kind = excluded.kind,
"trigger" = excluded."trigger",
status = excluded.status,
attempt = excluded.attempt,
max_attempts = excluded.max_attempts,
owner_instance = excluded.owner_instance,
progress_percent = excluded.progress_percent,
progress_message = excluded.progress_message,
payload_json = excluded.payload_json,
result_json = excluded.result_json,
error_message = excluded.error_message,
cancel_requested = excluded.cancel_requested,
created_by = excluded.created_by,
created_at_unix_secs = excluded.created_at_unix_secs,
started_at_unix_secs = excluded.started_at_unix_secs,
finished_at_unix_secs = excluded.finished_at_unix_secs,
updated_at_unix_secs = excluded.updated_at_unix_secs
"#,
)
.bind(&run.id)
.bind(&run.task_key)
.bind(run.kind.as_database())
.bind(&run.trigger)
.bind(run.status.as_database())
.bind(i64::from(run.attempt))
.bind(i64::from(run.max_attempts))
.bind(run.owner_instance.as_deref())
.bind(i32::from(run.progress_percent))
.bind(run.progress_message.as_deref())
.bind(run.payload_json.as_ref().map(serde_json::Value::to_string))
.bind(run.result_json.as_ref().map(serde_json::Value::to_string))
.bind(run.error_message.as_deref())
.bind(run.cancel_requested)
.bind(run.created_by.as_deref())
.bind(u64_to_i64(
run.created_at_unix_secs,
"created_at_unix_secs",
)?)
.bind(run.started_at_unix_secs.map(|value| value as i64))
.bind(run.finished_at_unix_secs.map(|value| value as i64))
.bind(u64_to_i64(
run.updated_at_unix_secs,
"updated_at_unix_secs",
)?)
.execute(&self.pool)
.await
.map_sql_err()?;
self.find_run(&run.id).await?.ok_or_else(|| {
DataLayerError::UnexpectedValue("background task run missing after upsert".to_string())
})
}
async fn request_cancel(
&self,
run_id: &str,
updated_at_unix_secs: u64,
) -> Result<bool, DataLayerError> {
let affected = sqlx::query(
"UPDATE background_task_runs SET cancel_requested = 1, updated_at_unix_secs = ? WHERE id = ?",
)
.bind(u64_to_i64(updated_at_unix_secs, "updated_at_unix_secs")?)
.bind(run_id)
.execute(&self.pool)
.await
.map_sql_err()?
.rows_affected();
Ok(affected > 0)
}
async fn upsert_event(
&self,
event: UpsertBackgroundTaskEvent,
) -> Result<StoredBackgroundTaskEvent, DataLayerError> {
event.validate()?;
sqlx::query(
r#"
INSERT INTO background_task_events (
id, run_id, event_type, message, payload_json, created_at_unix_secs
) VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
run_id = excluded.run_id,
event_type = excluded.event_type,
message = excluded.message,
payload_json = excluded.payload_json,
created_at_unix_secs = excluded.created_at_unix_secs
"#,
)
.bind(&event.id)
.bind(&event.run_id)
.bind(&event.event_type)
.bind(&event.message)
.bind(
event
.payload_json
.as_ref()
.map(serde_json::Value::to_string),
)
.bind(u64_to_i64(
event.created_at_unix_secs,
"created_at_unix_secs",
)?)
.execute(&self.pool)
.await
.map_sql_err()?;
let row = sqlx::query(&format!("{EVENT_COLUMNS} WHERE id = ? LIMIT 1"))
.bind(&event.id)
.fetch_one(&self.pool)
.await
.map_sql_err()?;
map_event_row(&row)
}
}
fn map_run_row(row: &SqliteRow) -> Result<StoredBackgroundTaskRun, DataLayerError> {
let kind: String = row.try_get("kind").map_sql_err()?;
let status: String = row.try_get("status").map_sql_err()?;
let attempt: i64 = row.try_get("attempt").map_sql_err()?;
let max_attempts: i64 = row.try_get("max_attempts").map_sql_err()?;
let progress_percent: i32 = row.try_get("progress_percent").map_sql_err()?;
let created_at_unix_secs: i64 = row.try_get("created_at_unix_secs").map_sql_err()?;
let started_at_unix_secs: Option<i64> = row.try_get("started_at_unix_secs").map_sql_err()?;
let finished_at_unix_secs: Option<i64> = row.try_get("finished_at_unix_secs").map_sql_err()?;
let updated_at_unix_secs: i64 = row.try_get("updated_at_unix_secs").map_sql_err()?;
Ok(StoredBackgroundTaskRun {
id: row.try_get("id").map_sql_err()?,
task_key: row.try_get("task_key").map_sql_err()?,
kind: BackgroundTaskKind::from_database(&kind)?,
trigger: row.try_get("trigger").map_sql_err()?,
status: BackgroundTaskStatus::from_database(&status)?,
attempt: u32::try_from(attempt).unwrap_or_default(),
max_attempts: u32::try_from(max_attempts).unwrap_or_default(),
owner_instance: row.try_get("owner_instance").map_sql_err()?,
progress_percent: u16::try_from(progress_percent).unwrap_or_default(),
progress_message: row.try_get("progress_message").map_sql_err()?,
payload_json: parse_optional_json(row.try_get("payload_json").map_sql_err()?)?,
result_json: parse_optional_json(row.try_get("result_json").map_sql_err()?)?,
error_message: row.try_get("error_message").map_sql_err()?,
cancel_requested: row.try_get("cancel_requested").map_sql_err()?,
created_by: row.try_get("created_by").map_sql_err()?,
created_at_unix_secs: u64::try_from(created_at_unix_secs).unwrap_or_default(),
started_at_unix_secs: started_at_unix_secs.and_then(|value| u64::try_from(value).ok()),
finished_at_unix_secs: finished_at_unix_secs.and_then(|value| u64::try_from(value).ok()),
updated_at_unix_secs: u64::try_from(updated_at_unix_secs).unwrap_or_default(),
})
}
fn map_event_row(row: &SqliteRow) -> Result<StoredBackgroundTaskEvent, DataLayerError> {
let created_at_unix_secs: i64 = row.try_get("created_at_unix_secs").map_sql_err()?;
Ok(StoredBackgroundTaskEvent {
id: row.try_get("id").map_sql_err()?,
run_id: row.try_get("run_id").map_sql_err()?,
event_type: row.try_get("event_type").map_sql_err()?,
message: row.try_get("message").map_sql_err()?,
payload_json: parse_optional_json(row.try_get("payload_json").map_sql_err()?)?,
created_at_unix_secs: u64::try_from(created_at_unix_secs).unwrap_or_default(),
})
}
fn parse_optional_json(value: Option<String>) -> Result<Option<serde_json::Value>, DataLayerError> {
value
.map(|raw| {
serde_json::from_str::<serde_json::Value>(&raw).map_err(|err| {
DataLayerError::UnexpectedValue(format!(
"invalid background task json payload: {err}"
))
})
})
.transpose()
}
fn i64_from_usize(value: usize, label: &str) -> Result<i64, DataLayerError> {
i64::try_from(value).map_err(|_| {
DataLayerError::UnexpectedValue(format!("background task {label} overflow: {value}"))
})
}
fn u64_to_i64(value: u64, label: &str) -> Result<i64, DataLayerError> {
i64::try_from(value).map_err(|_| {
DataLayerError::UnexpectedValue(format!("background task {label} overflow: {value}"))
})
}

1
crates/aether-data/src/repository/mod.rs
View File

@@ -8,6 +8,7 @@ pub mod announcements;
pub mod audit;
pub mod auth;
pub mod auth_modules;
pub mod background_tasks;
pub mod billing;
pub mod candidate_selection;
pub mod candidates;

15
crates/aether-task-runtime/Cargo.toml Normal file
View File

@@ -0,0 +1,15 @@
[package]
name = "aether-task-runtime"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
description = "Shared async task runtime primitives for Aether services"
[dependencies]
aether-runtime.workspace = true
serde.workspace = true
serde_json.workspace = true
tokio.workspace = true
tokio-util.workspace = true
tracing.workspace = true

227
crates/aether-task-runtime/src/lib.rs Normal file
View File

@@ -0,0 +1,227 @@
use std::future::Future;
use aether_runtime::task::spawn_named;
use tokio::task::{JoinHandle, JoinSet};
use tokio_util::sync::CancellationToken;
use tracing::warn;
#[derive(
Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize,
)]
pub enum TaskKind {
Scheduled,
Daemon,
OnDemand,
FireAndForget,
}
impl TaskKind {
pub const fn as_str(self) -> &'static str {
match self {
Self::Scheduled => "scheduled",
Self::Daemon => "daemon",
Self::OnDemand => "on_demand",
Self::FireAndForget => "fire_and_forget",
}
}
}
#[derive(
Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize,
)]
pub enum TaskStatus {
Queued,
Running,
Retrying,
Succeeded,
Failed,
Cancelled,
Skipped,
}
impl TaskStatus {
pub const fn as_str(self) -> &'static str {
match self {
Self::Queued => "queued",
Self::Running => "running",
Self::Retrying => "retrying",
Self::Succeeded => "succeeded",
Self::Failed => "failed",
Self::Cancelled => "cancelled",
Self::Skipped => "skipped",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct RetryPolicy {
pub max_attempts: u32,
}
impl Default for RetryPolicy {
fn default() -> Self {
Self { max_attempts: 1 }
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct TaskDefinition {
pub key: &'static str,
pub kind: TaskKind,
pub trigger: &'static str,
pub singleton: bool,
pub persist_history: bool,
pub retry_policy: RetryPolicy,
}
impl TaskDefinition {
pub const fn new(
key: &'static str,
kind: TaskKind,
trigger: &'static str,
singleton: bool,
persist_history: bool,
retry_policy: RetryPolicy,
) -> Self {
Self {
key,
kind,
trigger,
singleton,
persist_history,
retry_policy,
}
}
}
#[derive(Debug, Clone)]
pub struct TaskContext<TPayload = serde_json::Value> {
run_id: String,
task_key: String,
payload: Option<TPayload>,
cancellation_token: CancellationToken,
}
impl<TPayload> TaskContext<TPayload> {
pub fn new(
run_id: impl Into<String>,
task_key: impl Into<String>,
payload: Option<TPayload>,
cancellation_token: CancellationToken,
) -> Self {
Self {
run_id: run_id.into(),
task_key: task_key.into(),
payload,
cancellation_token,
}
}
pub fn run_id(&self) -> &str {
&self.run_id
}
pub fn task_key(&self) -> &str {
&self.task_key
}
pub fn payload(&self) -> Option<&TPayload> {
self.payload.as_ref()
}
pub fn cancellation_token(&self) -> CancellationToken {
self.cancellation_token.clone()
}
pub fn is_cancelled(&self) -> bool {
self.cancellation_token.is_cancelled()
}
pub async fn cancelled(&self) {
self.cancellation_token.cancelled().await;
}
}
#[derive(Debug)]
pub struct TaskSupervisor {
cancellation_token: CancellationToken,
join_set: JoinSet<()>,
supervised_task_count: usize,
}
impl TaskSupervisor {
pub fn new() -> Self {
Self {
cancellation_token: CancellationToken::new(),
join_set: JoinSet::new(),
supervised_task_count: 0,
}
}
pub fn cancellation_token(&self) -> CancellationToken {
self.cancellation_token.clone()
}
pub fn spawn_named<F>(&mut self, task_name: &'static str, future: F)
where
F: Future<Output = ()> + Send + 'static,
{
self.supervised_task_count = self.supervised_task_count.saturating_add(1);
let cancellation_token = self.cancellation_token.clone();
self.join_set.spawn(async move {
let mut handle = spawn_named(task_name, future);
tokio::select! {
_ = cancellation_token.cancelled() => {
handle.abort();
let _ = handle.await;
}
result = &mut handle => {
if let Err(error) = result {
warn!(task = task_name, error = ?error, "supervised task failed");
}
}
}
});
}
pub fn supervise_handle(&mut self, task_name: &'static str, mut handle: JoinHandle<()>) {
self.supervised_task_count = self.supervised_task_count.saturating_add(1);
let cancellation_token = self.cancellation_token.clone();
self.join_set.spawn(async move {
tokio::select! {
_ = cancellation_token.cancelled() => {
handle.abort();
let _ = handle.await;
}
result = &mut handle => {
if let Err(error) = result {
warn!(task = task_name, error = ?error, "supervised task failed");
}
}
}
});
}
pub fn is_empty(&self) -> bool {
self.supervised_task_count == 0
}
pub fn task_count(&self) -> usize {
self.supervised_task_count
}
pub fn cancel(&self) {
self.cancellation_token.cancel();
}
pub async fn shutdown(mut self) {
self.cancel();
while self.join_set.join_next().await.is_some() {}
}
}
impl Default for TaskSupervisor {
fn default() -> Self {
Self::new()
}
}

217
frontend/src/api/async-tasks.ts
View File

@@ -1,39 +1,78 @@
import apiClient from './client'
// 异步任务状态
export type AsyncTaskStatus = 'pending' | 'submitted' | 'queued' | 'processing' | 'completed' | 'failed' | 'cancelled'
export type AsyncTaskStatus =
| 'queued'
| 'running'
| 'retrying'
| 'succeeded'
| 'failed'
| 'cancelled'
| 'skipped'
| 'pending'
| 'submitted'
| 'processing'
| 'completed'
// 异步任务类型
export type AsyncTaskType = 'video'
export type AsyncTaskKind = 'scheduled' | 'daemon' | 'on_demand' | 'fire_and_forget'
export type AsyncTaskType = AsyncTaskKind | 'video'
// 异步任务列表项
export interface AsyncTaskItem {
id: string
external_task_id: string
user_id: string
username: string
task_type: AsyncTaskType
model: string
prompt: string
status: AsyncTaskStatus
progress_percent: number
progress_message: string | null
provider_id: string
provider_name: string
duration_seconds: number
resolution: string
aspect_ratio: string
video_url: string | null
error_code: string | null
error_message: string | null
poll_count: number
max_poll_count: number
created_at: string
completed_at: string | null
submitted_at: string | null
export interface AsyncTaskDefinition {
task_key: string
kind: AsyncTaskKind
trigger: string
max_attempts: number
singleton: boolean
persist_history: boolean
}
export interface AsyncTaskItem {
id: string
task_key?: string
kind?: AsyncTaskKind
trigger?: string
task_type?: AsyncTaskType
external_task_id?: string
user_id?: string
username?: string
model?: string
prompt?: string
status: AsyncTaskStatus
attempt?: number
max_attempts?: number
owner_instance?: string | null
progress_percent: number
progress_message: string | null
payload?: unknown
result?: unknown
error_message: string | null
cancel_requested?: boolean
created_by?: string | null
provider_id?: string
provider_name?: string
duration_seconds?: number
resolution?: string
aspect_ratio?: string
video_url?: string | null
error_code?: string | null
poll_count?: number
max_poll_count?: number
created_at: string
started_at?: string | null
updated_at?: string | null
finished_at?: string | null
completed_at?: string | null
submitted_at?: string | null
}
export interface AsyncTaskEvent {
id: string
run_id: string
event_type: string
message: string
payload: unknown
created_at: string
}
// 候选 Key 信息
export interface CandidateKeyInfo {
index: number
provider_id: string
@@ -47,7 +86,6 @@ export interface CandidateKeyInfo {
selected?: boolean
}
// 请求元数据
export interface AsyncTaskRequestMetadata {
candidate_keys: CandidateKeyInfo[]
selected_key_id: string
@@ -56,111 +94,118 @@ export interface AsyncTaskRequestMetadata {
user_agent: string
request_id: string
request_headers?: Record<string, string>
poll_raw_response?: unknown // 轮询完成时的原始响应
billing_snapshot?: unknown // 计费快照
poll_raw_response?: unknown
billing_snapshot?: unknown
}
// 异步任务详情
export interface AsyncTaskDetail extends AsyncTaskItem {
api_key_id: string
endpoint_id: string
key_id: string
client_api_format: string
provider_api_format: string
format_converted: boolean
original_request_body: unknown
converted_request_body: unknown
size: string | null
video_urls: string[] | null
thumbnail_url: string | null
video_size_bytes: number | null
video_duration_seconds: number | null // 实际视频时长（秒）
video_expires_at: string | null
stored_video_path: string | null
storage_provider: string | null
retry_count: number
max_retries: number
poll_interval_seconds: number
next_poll_at: string | null
updated_at: string | null
endpoint: {
api_key_id?: string
endpoint_id?: string
key_id?: string
client_api_format?: string
provider_api_format?: string
format_converted?: boolean
original_request_body?: unknown
converted_request_body?: unknown
size?: string | null
video_urls?: string[] | null
thumbnail_url?: string | null
video_size_bytes?: number | null
video_duration_seconds?: number | null
video_expires_at?: string | null
stored_video_path?: string | null
storage_provider?: string | null
retry_count?: number
max_retries?: number
poll_interval_seconds?: number
next_poll_at?: string | null
endpoint?: {
id: string
base_url: string
api_format: string
} | null
request_metadata: AsyncTaskRequestMetadata | null
request_metadata?: AsyncTaskRequestMetadata | null
}
// 异步任务列表响应
export interface AsyncTaskListResponse {
items: AsyncTaskItem[]
total: number
page: number
page_size: number
pages: number
definitions?: AsyncTaskDefinition[]
}
// 异步任务统计响应
export interface AsyncTaskStatsResponse {
total: number
by_status: Record<AsyncTaskStatus, number>
by_model: Record<string, number>
today_count: number
active_users?: number // 仅管理员
processing_count?: number // 仅管理员
running_count?: number
registered_tasks?: number
by_status: Partial<Record<AsyncTaskStatus, number>>
by_kind?: Partial<Record<AsyncTaskKind, number>>
by_model?: Record<string, number>
today_count?: number
active_users?: number
processing_count?: number
}
// 异步任务查询参数
export interface AsyncTaskQueryParams {
status?: AsyncTaskStatus
kind?: AsyncTaskKind
task_type?: AsyncTaskType
task_key?: string
trigger?: string
user_id?: string
model?: string
page?: number
page_size?: number
}
function normalizeStatus(status: AsyncTaskStatus): AsyncTaskStatus {
if (status === 'processing') return 'running'
if (status === 'submitted' || status === 'pending') return 'queued'
if (status === 'completed') return 'succeeded'
return status
}
export const asyncTasksApi = {
/**
* 获取异步任务列表
*/
async list(params: AsyncTaskQueryParams = {}): Promise<AsyncTaskListResponse> {
const searchParams = new URLSearchParams()
if (params.status) searchParams.append('status', params.status)
if (params.task_type) searchParams.append('task_type', params.task_type)
if (params.user_id) searchParams.append('user_id', params.user_id)
if (params.model) searchParams.append('model', params.model)
if (params.status) searchParams.append('status', normalizeStatus(params.status))
if (params.kind) searchParams.append('kind', params.kind)
if (params.task_type && params.task_type !== 'video') searchParams.append('kind', params.task_type)
if (params.task_key) searchParams.append('task_key', params.task_key)
if (params.trigger) searchParams.append('trigger', params.trigger)
if (params.page) searchParams.append('page', params.page.toString())
if (params.page_size) searchParams.append('page_size', params.page_size.toString())
const query = searchParams.toString()
// 后端 API 路径保持不变，前端抽象为异步任务
const url = query ? `/api/admin/video-tasks?${query}` : '/api/admin/video-tasks'
const url = query ? `/api/admin/tasks?${query}` : '/api/admin/tasks'
const response = await apiClient.get(url)
return response.data
},
/**
* 获取异步任务统计
*/
async getStats(): Promise<AsyncTaskStatsResponse> {
const response = await apiClient.get('/api/admin/video-tasks/stats')
const response = await apiClient.get('/api/admin/tasks/stats')
return response.data
},
/**
* 获取异步任务详情
*/
async getDetail(taskId: string): Promise<AsyncTaskDetail> {
const response = await apiClient.get(`/api/admin/video-tasks/${taskId}`)
const response = await apiClient.get(`/api/admin/tasks/${taskId}`)
return response.data
},
async getEvents(taskId: string): Promise<{ items: AsyncTaskEvent[] }> {
const response = await apiClient.get(`/api/admin/tasks/${taskId}/events`)
return response.data
},
/**
* 取消异步任务
*/
async cancel(taskId: string): Promise<{ id: string; status: string; message: string }> {
const response = await apiClient.post(`/api/admin/video-tasks/${taskId}/cancel`)
const response = await apiClient.post(`/api/admin/tasks/${taskId}/cancel`)
return response.data
},
async trigger(taskKey: string, payload: Record<string, unknown> = {}): Promise<{ run_id: string; status: string }> {
const response = await apiClient.post(`/api/admin/tasks/${taskKey}/trigger`, payload)
return response.data
},
}

138
frontend/src/views/admin/AsyncTasks.vue
View File

@@ -28,12 +28,12 @@
<div class="w-10 h-10 rounded-lg bg-blue-500/10 flex items-center justify-center">
<Loader2
class="w-5 h-5 text-blue-500"
:class="{ 'animate-spin': (stats?.processing_count ?? 0) > 0 }"
:class="{ 'animate-spin': runningCount > 0 }"
/>
</div>
<div>
<p class="text-2xl font-bold">
{{ stats?.processing_count ?? stats?.by_status?.processing ?? '-' }}
{{ runningCount || '-' }}
</p>
<p class="text-xs text-muted-foreground">
处理中
@@ -51,7 +51,7 @@
</div>
<div>
<p class="text-2xl font-bold">
{{ stats?.by_status?.completed ?? '-' }}
{{ stats?.by_status?.succeeded ?? '-' }}
</p>
<p class="text-xs text-muted-foreground">
已完成
@@ -69,10 +69,10 @@
</div>
<div>
<p class="text-2xl font-bold">
{{ stats?.today_count ?? '-' }}
{{ stats?.registered_tasks ?? '-' }}
</p>
<p class="text-xs text-muted-foreground">
今日任务
已注册
</p>
</div>
</div>
@@ -102,14 +102,17 @@
<SelectItem value="all">
全部状态
</SelectItem>
<SelectItem value="submitted">
已提交
<SelectItem value="queued">
排队中
</SelectItem>
<SelectItem value="processing">
处理中
<SelectItem value="running">
运行中
</SelectItem>
<SelectItem value="completed">
已完成
<SelectItem value="retrying">
重试中
</SelectItem>
<SelectItem value="succeeded">
成功
</SelectItem>
<SelectItem value="failed">
失败
@@ -117,13 +120,16 @@
<SelectItem value="cancelled">
已取消
</SelectItem>
<SelectItem value="skipped">
已跳过
</SelectItem>
</SelectContent>
</Select>
<!-- 模型筛选 -->
<Input
v-model="filterModel"
type="text"
placeholder="模型..."
placeholder="任务 Key..."
class="w-32 h-8 text-xs"
/>
<!-- 刷新按钮 -->
@@ -207,13 +213,13 @@
v-if="isVideoTask(task)"
class="w-4 h-4 text-muted-foreground shrink-0"
/>
<span class="font-medium text-sm truncate">{{ task.model }}</span>
<span class="font-medium text-sm truncate">{{ displayTaskName(task) }}</span>
</div>
<p
class="text-xs text-muted-foreground truncate max-w-[280px]"
:title="task.prompt"
:title="displayTaskDescription(task)"
>
{{ task.prompt }}
{{ displayTaskDescription(task) }}
</p>
</div>
</TableCell>
@@ -229,7 +235,7 @@
</div>
<div class="flex items-center gap-1.5 text-muted-foreground">
<Server class="w-3 h-3" />
<span class="truncate max-w-[100px]">{{ task.provider_name }}</span>
<span class="truncate max-w-[100px]">{{ displayTaskSource(task) }}</span>
</div>
</div>
</TableCell>
@@ -243,7 +249,7 @@
{{ getStatusLabel(task.status) }}
</Badge>
<div
v-if="task.progress_percent > 0 && task.status === 'processing'"
v-if="task.progress_percent > 0 && isRunningStatus(task.status)"
class="w-full"
>
<div class="flex items-center gap-2">
@@ -262,11 +268,11 @@
<TableCell>
<div class="text-xs space-y-0.5 text-muted-foreground">
<div
v-if="task.duration_seconds"
v-if="task.duration_seconds || task.attempt"
class="flex items-center gap-1"
>
<Timer class="w-3 h-3" />
<span>{{ task.duration_seconds }}s</span>
<span>{{ task.duration_seconds ? `${task.duration_seconds}s` : `${task.attempt}/${task.max_attempts ?? 1}` }}</span>
</div>
<div v-if="task.resolution">
{{ task.resolution }}
@@ -284,11 +290,11 @@
<span>{{ formatDate(task.created_at) }}</span>
</div>
<div
v-if="task.completed_at"
v-if="finishTime(task)"
class="flex items-center gap-1.5 text-green-600 dark:text-green-400"
>
<CheckCircle class="w-3 h-3" />
<span>{{ formatDate(task.completed_at) }}</span>
<span>{{ formatDate(finishTime(task)) }}</span>
</div>
</div>
</TableCell>
@@ -305,6 +311,7 @@
<Eye class="w-4 h-4" />
</Button>
<Button
v-if="isVideoTask(task)"
variant="ghost"
size="icon"
class="h-7 w-7"
@@ -337,7 +344,7 @@
v-if="isVideoTask(task)"
class="w-4 h-4 text-muted-foreground shrink-0"
/>
<span class="font-medium text-sm truncate">{{ task.model }}</span>
<span class="font-medium text-sm truncate">{{ displayTaskName(task) }}</span>
</div>
<Badge
:variant="getStatusVariant(task.status)"
@@ -349,7 +356,7 @@
<!-- 进度条（如果有） -->
<div
v-if="task.progress_percent > 0 && task.status === 'processing'"
v-if="task.progress_percent > 0 && isRunningStatus(task.status)"
class="space-y-1"
>
<div class="h-1.5 bg-muted rounded-full overflow-hidden">
@@ -365,7 +372,7 @@
<!-- Prompt -->
<p class="text-sm text-muted-foreground line-clamp-2">
{{ task.prompt }}
{{ displayTaskDescription(task) }}
</p>
<!-- 信息网格 -->
@@ -379,24 +386,25 @@
</div>
<div class="flex items-center gap-1.5 text-muted-foreground">
<Server class="w-3 h-3" />
<span class="truncate">{{ task.provider_name }}</span>
<span class="truncate">{{ displayTaskSource(task) }}</span>
</div>
<div class="flex items-center gap-1.5 text-muted-foreground">
<Clock class="w-3 h-3" />
<span>{{ formatDate(task.created_at) }}</span>
</div>
<div
v-if="task.duration_seconds"
v-if="task.duration_seconds || task.attempt"
class="flex items-center gap-1.5 text-muted-foreground"
>
<Timer class="w-3 h-3" />
<span>{{ task.duration_seconds }}s</span>
<span>{{ task.duration_seconds ? `${task.duration_seconds}s` : `${task.attempt}/${task.max_attempts ?? 1}` }}</span>
</div>
</div>
<!-- 操作按钮 -->
<div class="flex justify-end gap-2">
<Button
v-if="isVideoTask(task)"
variant="outline"
size="sm"
class="h-7 text-xs"
@@ -459,7 +467,7 @@
v-if="isVideoTask(selectedTask)"
class="w-3.5 h-3.5 mr-1"
/>
<span>{{ selectedTask.model }}</span>
<span>{{ displayTaskName(selectedTask) }}</span>
</div>
<Badge :variant="getStatusVariant(selectedTask.status)">
{{ getStatusLabel(selectedTask.status) }}
@@ -503,11 +511,11 @@
<span>用户: {{ selectedTask.username }}</span>
</template>
<span class="opacity-40">|</span>
<span>Provider: {{ selectedTask.provider_name }}</span>
<span>{{ displayTaskSource(selectedTask) }}</span>
</div>
<!-- 进度条 -->
<div
v-if="selectedTask.progress_percent > 0 && selectedTask.status === 'processing'"
v-if="selectedTask.progress_percent > 0 && isRunningStatus(selectedTask.status)"
class="mt-3"
>
<div class="flex items-center gap-3">
@@ -641,7 +649,7 @@
<!-- 任务完成但无视频 -->
<div
v-else-if="selectedTask.status === 'completed'"
v-else-if="isSucceededStatus(selectedTask.status) && isVideoTask(selectedTask)"
class="p-4 bg-amber-50 dark:bg-amber-900/20 rounded-lg border border-amber-200 dark:border-amber-800 text-center"
>
<Video class="w-8 h-8 mx-auto mb-2 text-amber-500" />
@@ -660,14 +668,14 @@
variant="outline"
size="sm"
class="h-6 px-2 text-xs"
@click="copyToClipboard(selectedTask.prompt)"
@click="copyToClipboard(displayTaskDescription(selectedTask))"
>
<Copy class="w-3 h-3 mr-1" />
复制
</Button>
</div>
<div class="p-3 bg-muted/50 rounded-lg border border-border/60 text-sm whitespace-pre-wrap break-words max-h-32 overflow-y-auto leading-relaxed">
{{ selectedTask.prompt }}
{{ displayTaskDescription(selectedTask) }}
</div>
</div>
@@ -738,7 +746,7 @@
轮询
</p>
<p class="text-sm font-medium">
{{ selectedTask.poll_count }} / {{ selectedTask.max_poll_count }}
{{ selectedTask.poll_count ?? selectedTask.attempt ?? 0 }} / {{ selectedTask.max_poll_count ?? selectedTask.max_attempts ?? 1 }}
</p>
</div>
<div class="p-3 bg-muted/30 rounded-lg">
@@ -746,7 +754,7 @@
重试
</p>
<p class="text-sm font-medium">
{{ selectedTask.retry_count }} / {{ selectedTask.max_retries }}
{{ selectedTask.retry_count ?? 0 }} / {{ selectedTask.max_retries ?? selectedTask.max_attempts ?? 1 }}
</p>
</div>
<div class="p-3 bg-muted/30 rounded-lg">
@@ -754,7 +762,7 @@
轮询间隔
</p>
<p class="text-sm font-medium">
{{ selectedTask.poll_interval_seconds }}s
{{ selectedTask.poll_interval_seconds ? `${selectedTask.poll_interval_seconds}s` : selectedTask.trigger ?? '-' }}
</p>
</div>
<div
@@ -780,13 +788,13 @@
<span>{{ formatTimeWithMs(selectedTask.created_at) }}</span>
<span class="time-arrow-container">
<span
v-if="selectedTask.completed_at"
v-if="finishTime(selectedTask)"
class="time-duration"
>+{{ calcDuration(selectedTask.created_at, selectedTask.completed_at) }}</span>
>+{{ calcDuration(selectedTask.created_at, finishTime(selectedTask) || selectedTask.created_at) }}</span>
<span class="time-arrow">→</span>
</span>
<template v-if="selectedTask.completed_at">
<span>{{ formatTimeWithMs(selectedTask.completed_at) }}</span>
<template v-if="finishTime(selectedTask)">
<span>{{ formatTimeWithMs(finishTime(selectedTask)) }}</span>
</template>
<span
v-else
@@ -917,10 +925,47 @@ let overviewRefreshInFlight = false
// 使用记录详情抽屉状态
const usageDetailOpen = ref(false)
const usageRequestId = ref<string | null>(null)
const runningCount = computed(() => {
return stats.value?.running_count
?? stats.value?.processing_count
?? stats.value?.by_status?.running
?? stats.value?.by_status?.processing
?? 0
})
// 判断是否为视频任务
function isVideoTask(task: AsyncTaskItem): boolean {
return task.task_type === 'video' || !!task.video_url || !!task.duration_seconds
return task.task_type === 'video' || !!task.video_url || !!task.duration_seconds || task.task_key === 'video.task.poller'
}
function displayTaskName(task: AsyncTaskItem | AsyncTaskDetail): string {
return task.model || task.task_key || task.id
}
function displayTaskDescription(task: AsyncTaskItem | AsyncTaskDetail): string {
if (task.prompt) return task.prompt
if (task.progress_message) return task.progress_message
if (task.error_message) return task.error_message
if (task.payload) return formatJson(task.payload)
return task.trigger || task.kind || '-'
}
function displayTaskSource(task: AsyncTaskItem | AsyncTaskDetail): string {
if (task.provider_name) return `Provider: ${task.provider_name}`
if (task.kind || task.trigger) return `${task.kind ?? 'task'} / ${task.trigger ?? '-'}`
return task.owner_instance || '-'
}
function finishTime(task: AsyncTaskItem | AsyncTaskDetail): string | null {
return task.finished_at || task.completed_at || null
}
function isRunningStatus(status: string): boolean {
return ['running', 'retrying', 'processing', 'submitted', 'pending', 'queued'].includes(status)
}
function isSucceededStatus(status: string): boolean {
return status === 'succeeded' || status === 'completed'
}
// 获取任务列表
@@ -929,7 +974,7 @@ async function fetchTasks() {
try {
const response = await asyncTasksApi.list({
status: filterStatus.value !== 'all' ? filterStatus.value as AsyncTaskStatus : undefined,
model: filterModel.value || undefined,
task_key: filterModel.value || undefined,
page: currentPage.value,
page_size: pageSize.value,
})
@@ -1085,6 +1130,7 @@ async function cancelTask(task: AsyncTaskItem | AsyncTaskDetail) {
// 状态相关
function getStatusVariant(status: string): 'default' | 'secondary' | 'destructive' | 'outline' {
switch (status) {
case 'succeeded':
case 'completed':
return 'default'
case 'failed':
@@ -1101,16 +1147,20 @@ function getStatusLabel(status: string): string {
pending: '待处理',
submitted: '已提交',
queued: '排队中',
running: '运行中',
retrying: '重试中',
processing: '处理中',
succeeded: '成功',
completed: '已完成',
failed: '失败',
cancelled: '已取消',
skipped: '已跳过',
}
return labels[status] || status
}
function canCancel(status: string): boolean {
return ['pending', 'submitted', 'queued', 'processing'].includes(status)
return ['pending', 'submitted', 'queued', 'processing', 'running', 'retrying'].includes(status)
}
// 格式化日期（简短格式，用于表格列表）
@@ -1229,7 +1279,7 @@ watch(filterModel, () => {
// 检查是否有进行中的任务
const hasProcessingTasks = computed(() => {
return tasks.value.some(t =>
['pending', 'submitted', 'queued', 'processing'].includes(t.status)
['pending', 'submitted', 'queued', 'processing', 'running', 'retrying'].includes(t.status)
)
})