i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-03 01:40:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add scoped manual usage cleanup

Browse Source
This commit is contained in:
HsungKayphoon
2026-05-16 14:53:10 +08:00
parent 75b7319465
commit 74a1e5ad7d
17 changed files with 1746 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
apps/aether-gateway/src/maintenance/runtime/cleanup_runs.rs
View File

@@ -489,7 +489,7 @@ fn request_body_cleanup_record(
}
}
async fn record_cleanup_run(
pub(crate) async fn record_admin_cleanup_run(
data: &GatewayDataState,
record: AdminCleanupRunRecord,
) -> Result<(), DataLayerError> {
@@ -509,6 +509,13 @@ async fn record_cleanup_run(
Ok(())
}
async fn record_cleanup_run(
data: &GatewayDataState,
record: AdminCleanupRunRecord,
) -> Result<(), DataLayerError> {
record_admin_cleanup_run(data, record).await
}
fn parse_cleanup_run_records(value: Value) -> Vec<AdminCleanupRunRecord> {
value
.as_array()

229
apps/aether-gateway/src/maintenance/runtime/runners.rs
View File

@@ -1,5 +1,6 @@
use aether_data_contracts::DataLayerError;
use serde_json::json;
use std::sync::Arc;
use std::time::Instant;
use tracing::{info, warn};
@@ -11,11 +12,12 @@ use super::{
cleanup_expired_gemini_file_mappings_once, cleanup_proxy_node_metrics_once,
cleanup_request_candidates_once, cleanup_stale_pending_requests_once,
cleanup_stale_proxy_nodes_once, collect_proxy_upgrade_rollout_probes, now_unix_secs,
perform_db_maintenance_once, perform_provider_checkin_once, perform_stats_aggregation_once,
perform_stats_hourly_aggregation_once, perform_usage_cleanup_once,
perform_usage_cleanup_once_with_override, perform_wallet_daily_usage_aggregation_once,
record_completed_cleanup_run, record_failed_cleanup_run, record_proxy_upgrade_traffic_success,
summarize_database_pool,
perform_db_maintenance_once, perform_manual_usage_cleanup_once, perform_provider_checkin_once,
perform_stats_aggregation_once, perform_stats_hourly_aggregation_once,
perform_usage_cleanup_once, perform_wallet_daily_usage_aggregation_once,
record_admin_cleanup_run, record_completed_cleanup_run, record_failed_cleanup_run,
record_proxy_upgrade_traffic_success, summarize_database_pool, AdminCleanupRunRecord,
ManualUsageCleanupOptions,
};
pub(super) async fn run_audit_cleanup_once(data: &GatewayDataState) -> Result<(), DataLayerError> {
@@ -299,15 +301,14 @@ pub(super) async fn run_usage_cleanup_once(data: &GatewayDataState) -> Result<()
Ok(())
}
pub(crate) async fn run_manual_usage_cleanup_once(
data: &GatewayDataState,
override_older_than_days: Option<u32>,
pub(crate) async fn start_manual_usage_cleanup_task(
data: Arc<GatewayDataState>,
options: ManualUsageCleanupOptions,
actor_user_id: Option<String>,
) -> Result<aether_data_contracts::repository::usage::UsageCleanupSummary, ManualUsageCleanupError>
{
) -> Result<AdminCleanupRunRecord, ManualUsageCleanupError> {
use super::{list_admin_cleanup_run_records, USAGE_CLEANUP_KIND};
let existing = list_admin_cleanup_run_records(data)
let existing = list_admin_cleanup_run_records(&data)
.await
.map_err(ManualUsageCleanupError::DataLayer)?;
if existing
@@ -318,10 +319,42 @@ pub(crate) async fn run_manual_usage_cleanup_once(
}
let started_at_unix_secs = now_unix_secs();
let record = AdminCleanupRunRecord {
id: uuid::Uuid::new_v4().to_string(),
kind: USAGE_CLEANUP_KIND.to_string(),
trigger: "manual".to_string(),
status: "processing".to_string(),
message: manual_usage_cleanup_start_message(options),
started_at_unix_secs,
completed_at_unix_secs: None,
duration_ms: None,
summary: manual_usage_cleanup_progress_summary(options, 0, None, actor_user_id.as_deref()),
error: None,
};
record_admin_cleanup_run(&data, record.clone())
.await
.map_err(ManualUsageCleanupError::DataLayer)?;
tokio::spawn(run_manual_usage_cleanup_task(
data,
record.clone(),
options,
actor_user_id,
));
Ok(record)
}
pub(crate) async fn run_manual_usage_cleanup_once(
data: &GatewayDataState,
options: ManualUsageCleanupOptions,
actor_user_id: Option<String>,
) -> Result<aether_data_contracts::repository::usage::UsageCleanupSummary, ManualUsageCleanupError>
{
use super::USAGE_CLEANUP_KIND;
let started_at = Instant::now();
let override_duration =
override_older_than_days.map(|days| chrono::Duration::days(i64::from(days)));
let summary = match perform_usage_cleanup_once_with_override(data, override_duration).await {
let started_at_unix_secs = now_unix_secs();
let summary = match perform_manual_usage_cleanup_once(data, options).await {
Ok(summary) => summary,
Err(err) => {
record_failed_cleanup_run(
@@ -336,17 +369,8 @@ pub(crate) async fn run_manual_usage_cleanup_once(
return Err(ManualUsageCleanupError::DataLayer(err));
}
};
let total = summary
.body_externalized
.saturating_add(summary.legacy_body_refs_migrated)
.saturating_add(summary.body_cleaned)
.saturating_add(summary.header_cleaned)
.saturating_add(summary.keys_cleaned)
.saturating_add(summary.records_deleted);
let message = match override_older_than_days {
Some(days) => format!("请求记录手动清理完成，清理 {days} 天前的记录，影响 {total} 项"),
None => format!("请求记录手动清理完成（按当前策略），影响 {total} 项"),
};
let total = usage_cleanup_total(summary);
let message = manual_usage_cleanup_completed_message(options, total);
record_completed_cleanup_run(
data,
USAGE_CLEANUP_KIND,
@@ -360,7 +384,9 @@ pub(crate) async fn run_manual_usage_cleanup_once(
"header_cleaned": summary.header_cleaned,
"keys_cleaned": summary.keys_cleaned,
"records_deleted": summary.records_deleted,
"requested_older_than_days": override_older_than_days,
"mode": options.mode.as_str(),
"requested_older_than_days": options.requested_older_than_days,
"targets": options.targets,
"actor_user_id": actor_user_id,
}),
message,
@@ -371,7 +397,8 @@ pub(crate) async fn run_manual_usage_cleanup_once(
log_type = "ops",
worker = "usage_cleanup",
trigger = "manual",
requested_older_than_days = override_older_than_days,
mode = options.mode.as_str(),
requested_older_than_days = options.requested_older_than_days,
actor_user_id = actor_user_id.as_deref(),
total_affected = total,
"gateway finished manual usage cleanup"
@@ -379,6 +406,154 @@ pub(crate) async fn run_manual_usage_cleanup_once(
Ok(summary)
}
async fn run_manual_usage_cleanup_task(
data: Arc<GatewayDataState>,
initial_record: AdminCleanupRunRecord,
options: ManualUsageCleanupOptions,
actor_user_id: Option<String>,
) {
let started_at = Instant::now();
match perform_manual_usage_cleanup_once(&data, options).await {
Ok(summary) => {
let total = usage_cleanup_total(summary);
let record = AdminCleanupRunRecord {
id: initial_record.id,
kind: initial_record.kind,
trigger: initial_record.trigger,
status: "completed".to_string(),
message: manual_usage_cleanup_completed_message(options, total),
started_at_unix_secs: initial_record.started_at_unix_secs,
completed_at_unix_secs: Some(now_unix_secs()),
duration_ms: Some(
started_at
.elapsed()
.as_millis()
.try_into()
.unwrap_or(u64::MAX),
),
summary: manual_usage_cleanup_progress_summary(
options,
100,
Some(summary),
actor_user_id.as_deref(),
),
error: None,
};
if let Err(err) = record_admin_cleanup_run(&data, record).await {
warn!(error = %err, "failed to record manual usage cleanup completion");
}
info!(
event_name = "usage_cleanup_manual_completed",
log_type = "ops",
worker = "usage_cleanup",
trigger = "manual",
mode = options.mode.as_str(),
requested_older_than_days = options.requested_older_than_days,
actor_user_id = actor_user_id.as_deref(),
total_affected = total,
"gateway finished manual usage cleanup task"
);
}
Err(err) => {
let record = AdminCleanupRunRecord {
id: initial_record.id,
kind: initial_record.kind,
trigger: initial_record.trigger,
status: "failed".to_string(),
message: "请求记录手动清理失败".to_string(),
started_at_unix_secs: initial_record.started_at_unix_secs,
completed_at_unix_secs: Some(now_unix_secs()),
duration_ms: Some(
started_at
.elapsed()
.as_millis()
.try_into()
.unwrap_or(u64::MAX),
),
summary: manual_usage_cleanup_progress_summary(
options,
100,
None,
actor_user_id.as_deref(),
),
error: Some(err.to_string()),
};
if let Err(record_err) = record_admin_cleanup_run(&data, record).await {
warn!(error = %record_err, "failed to record manual usage cleanup failure");
}
warn!(error = %err, "manual usage cleanup task failed");
}
}
}
fn usage_cleanup_total(
summary: aether_data_contracts::repository::usage::UsageCleanupSummary,
) -> usize {
summary
.body_externalized
.saturating_add(summary.legacy_body_refs_migrated)
.saturating_add(summary.body_cleaned)
.saturating_add(summary.header_cleaned)
.saturating_add(summary.keys_cleaned)
.saturating_add(summary.records_deleted)
}
fn manual_usage_cleanup_start_message(options: ManualUsageCleanupOptions) -> String {
match options.mode {
super::ManualUsageCleanupMode::BeforeNow => {
"请求记录手动清理已开始，清理当前时刻之前的已选请求体".to_string()
}
super::ManualUsageCleanupMode::OlderThanDays => format!(
"请求记录手动清理已开始，清理 {} 天前的已选内容",
options.requested_older_than_days.unwrap_or_default()
),
super::ManualUsageCleanupMode::Policy => {
"请求记录手动清理已开始，按当前策略清理已选内容".to_string()
}
}
}
fn manual_usage_cleanup_completed_message(
options: ManualUsageCleanupOptions,
total: usize,
) -> String {
match options.mode {
super::ManualUsageCleanupMode::BeforeNow => {
format!("请求记录手动清理完成，已清理当前时刻之前的已选请求体，影响 {total} 项")
}
super::ManualUsageCleanupMode::OlderThanDays => format!(
"请求记录手动清理完成，清理 {} 天前的已选内容，影响 {total} 项",
options.requested_older_than_days.unwrap_or_default()
),
super::ManualUsageCleanupMode::Policy => {
format!("请求记录手动清理完成（按当前策略），影响 {total} 项")
}
}
}
fn manual_usage_cleanup_progress_summary(
options: ManualUsageCleanupOptions,
progress_percent: u8,
summary: Option<aether_data_contracts::repository::usage::UsageCleanupSummary>,
actor_user_id: Option<&str>,
) -> serde_json::Value {
let summary = summary.unwrap_or_default();
json!({
"mode": options.mode.as_str(),
"requested_older_than_days": options.requested_older_than_days,
"targets": options.targets,
"progress_percent": progress_percent,
"body_externalized": summary.body_externalized,
"legacy_body_refs_migrated": summary.legacy_body_refs_migrated,
"body_cleaned": summary.body_cleaned,
"header_cleaned": summary.header_cleaned,
"keys_cleaned": summary.keys_cleaned,
"records_deleted": summary.records_deleted,
"total": usage_cleanup_total(summary),
"actor_user_id": actor_user_id,
})
}
#[derive(Debug)]
pub(crate) enum ManualUsageCleanupError {
AlreadyRunning,

35
apps/aether-gateway/src/maintenance/runtime/tests.rs
View File

@@ -28,12 +28,12 @@ use super::{
spawn_stats_hourly_aggregation_worker, spawn_usage_cleanup_worker,
spawn_wallet_daily_usage_aggregation_worker, start_proxy_upgrade_rollout,
stats_aggregation_target_day, stats_hourly_aggregation_target_hour, summarize_database_pool,
usage_cleanup_settings, usage_cleanup_window, usage_cleanup_window_with_override,
wallet_daily_usage_aggregation_target, AppState, DbMaintenanceRunSummary,
FailedPendingUsageRow, GatewayDataState, ProxyNodeMetricsCleanupSettings,
ProxyUpgradeRolloutProbeConfig, StalePendingUsageRow, UsageCleanupSettings, USAGE_CLEANUP_HOUR,
USAGE_CLEANUP_MINUTE, WALLET_DAILY_USAGE_AGGREGATION_HOUR,
WALLET_DAILY_USAGE_AGGREGATION_MINUTE,
usage_cleanup_settings, usage_cleanup_window, usage_cleanup_window_for_mode,
usage_cleanup_window_with_override, wallet_daily_usage_aggregation_target, AppState,
DbMaintenanceRunSummary, FailedPendingUsageRow, GatewayDataState, ManualUsageCleanupMode,
ProxyNodeMetricsCleanupSettings, ProxyUpgradeRolloutProbeConfig, StalePendingUsageRow,
UsageCleanupSettings, USAGE_CLEANUP_HOUR, USAGE_CLEANUP_MINUTE,
WALLET_DAILY_USAGE_AGGREGATION_HOUR, WALLET_DAILY_USAGE_AGGREGATION_MINUTE,
};
#[tokio::test]
@@ -986,6 +986,29 @@ fn usage_cleanup_window_with_override_is_always_non_aggressive() {
assert_eq!(passthrough, policy);
}
#[test]
fn usage_cleanup_before_now_window_uses_current_timestamp_only() {
let now_utc = "2026-03-18T03:00:00Z"
.parse::<DateTime<Utc>>()
.expect("timestamp should parse");
let settings = UsageCleanupSettings {
detail_retention_days: 7,
compressed_retention_days: 30,
header_retention_days: 90,
log_retention_days: 365,
batch_size: 123,
auto_delete_expired_keys: false,
};
let window =
usage_cleanup_window_for_mode(now_utc, settings, ManualUsageCleanupMode::BeforeNow, None);
assert_eq!(window.detail_cutoff, now_utc);
assert_eq!(window.compressed_cutoff, now_utc);
assert_eq!(window.header_cutoff, now_utc);
assert_eq!(window.log_cutoff, now_utc);
}
#[tokio::test]
async fn summarize_database_pool_uses_busy_connections_for_usage_rate() {
let data = GatewayDataState::from_config(crate::data::GatewayDataConfig::from_postgres_config(

146
apps/aether-gateway/src/maintenance/runtime/usage_cleanup.rs
View File

@@ -1,4 +1,6 @@
use aether_data_contracts::repository::usage::{UsageCleanupSummary, UsageCleanupWindow};
use aether_data_contracts::repository::usage::{
UsageCleanupExecutionMode, UsageCleanupSummary, UsageCleanupTargets, UsageCleanupWindow,
};
use aether_data_contracts::DataLayerError;
use chrono::Utc;
@@ -15,6 +17,8 @@ pub(crate) struct ManualUsageCleanupPreview {
pub compressed_cutoff: chrono::DateTime<Utc>,
pub header_cutoff: chrono::DateTime<Utc>,
pub log_cutoff: chrono::DateTime<Utc>,
pub mode: ManualUsageCleanupMode,
pub targets: UsageCleanupTargets,
pub requested_older_than_days: Option<u32>,
pub detail_count: u64,
pub compressed_count: u64,
@@ -22,49 +26,131 @@ pub(crate) struct ManualUsageCleanupPreview {
pub log_count: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum ManualUsageCleanupMode {
Policy,
OlderThanDays,
BeforeNow,
}
impl ManualUsageCleanupMode {
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::Policy => "policy",
Self::OlderThanDays => "older_than_days",
Self::BeforeNow => "before_now",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct ManualUsageCleanupOptions {
pub(crate) mode: ManualUsageCleanupMode,
pub(crate) requested_older_than_days: Option<u32>,
pub(crate) targets: UsageCleanupTargets,
}
impl ManualUsageCleanupOptions {
pub(crate) const fn policy() -> Self {
Self {
mode: ManualUsageCleanupMode::Policy,
requested_older_than_days: None,
targets: UsageCleanupTargets::all_policy_targets(),
}
}
}
pub(super) async fn perform_usage_cleanup_once(
data: &GatewayDataState,
) -> Result<UsageCleanupSummary, DataLayerError> {
perform_usage_cleanup_once_with_override(data, None).await
perform_usage_cleanup_once_with_override(data, None, true).await
}
pub(super) async fn perform_usage_cleanup_once_with_override(
data: &GatewayDataState,
override_older_than: Option<chrono::Duration>,
respect_auto_enabled: bool,
) -> Result<UsageCleanupSummary, DataLayerError> {
let options = ManualUsageCleanupOptions {
mode: if override_older_than.is_some() {
ManualUsageCleanupMode::OlderThanDays
} else {
ManualUsageCleanupMode::Policy
},
requested_older_than_days: None,
targets: UsageCleanupTargets::all_policy_targets(),
};
perform_usage_cleanup_once_with_options(
data,
options,
override_older_than,
respect_auto_enabled,
)
.await
}
pub(super) async fn perform_manual_usage_cleanup_once(
data: &GatewayDataState,
options: ManualUsageCleanupOptions,
) -> Result<UsageCleanupSummary, DataLayerError> {
let override_duration = options
.requested_older_than_days
.map(|days| chrono::Duration::days(i64::from(days)));
perform_usage_cleanup_once_with_options(data, options, override_duration, false).await
}
async fn perform_usage_cleanup_once_with_options(
data: &GatewayDataState,
options: ManualUsageCleanupOptions,
override_older_than: Option<chrono::Duration>,
respect_auto_enabled: bool,
) -> Result<UsageCleanupSummary, DataLayerError> {
if !data.has_usage_writer() {
return Ok(UsageCleanupSummary::default());
}
if override_older_than.is_none()
if respect_auto_enabled
&& override_older_than.is_none()
&& !system_config_bool(data, "enable_auto_cleanup", true).await?
{
return Ok(UsageCleanupSummary::default());
}
let window = compute_usage_cleanup_window(data, override_older_than).await?;
let window = compute_usage_cleanup_window(data, options.mode, override_older_than).await?;
let settings = usage_cleanup_settings(data).await?;
data.cleanup_usage(
&window,
settings.batch_size,
settings.auto_delete_expired_keys,
options.targets,
cleanup_execution_mode(options.mode),
)
.await
}
pub(crate) async fn preview_manual_usage_cleanup(
data: &GatewayDataState,
override_older_than_days: Option<u32>,
options: ManualUsageCleanupOptions,
) -> Result<ManualUsageCleanupPreview, DataLayerError> {
let override_duration =
override_older_than_days.map(|days| chrono::Duration::days(i64::from(days)));
let window = compute_usage_cleanup_window(data, override_duration).await?;
let counts = data.preview_usage_cleanup(&window).await?;
let override_duration = options
.requested_older_than_days
.map(|days| chrono::Duration::days(i64::from(days)));
let window = compute_usage_cleanup_window(data, options.mode, override_duration).await?;
let counts = data
.preview_usage_cleanup(
&window,
options.targets,
cleanup_execution_mode(options.mode),
)
.await?;
Ok(ManualUsageCleanupPreview {
detail_cutoff: window.detail_cutoff,
compressed_cutoff: window.compressed_cutoff,
header_cutoff: window.header_cutoff,
log_cutoff: window.log_cutoff,
requested_older_than_days: override_older_than_days,
mode: options.mode,
targets: options.targets,
requested_older_than_days: options.requested_older_than_days,
detail_count: counts.detail,
compressed_count: counts.compressed,
header_count: counts.header,
@@ -74,11 +160,43 @@ pub(crate) async fn preview_manual_usage_cleanup(
async fn compute_usage_cleanup_window(
data: &GatewayDataState,
mode: ManualUsageCleanupMode,
override_older_than: Option<chrono::Duration>,
) -> Result<UsageCleanupWindow, DataLayerError> {
let settings = usage_cleanup_settings(data).await?;
Ok(match override_older_than {
Some(duration) => usage_cleanup_window_with_override(Utc::now(), settings, Some(duration)),
None => usage_cleanup_window(Utc::now(), settings),
})
Ok(usage_cleanup_window_for_mode(
Utc::now(),
settings,
mode,
override_older_than,
))
}
pub(super) fn usage_cleanup_window_for_mode(
now_utc: chrono::DateTime<Utc>,
settings: super::UsageCleanupSettings,
mode: ManualUsageCleanupMode,
override_older_than: Option<chrono::Duration>,
) -> UsageCleanupWindow {
match mode {
ManualUsageCleanupMode::Policy => usage_cleanup_window(now_utc, settings),
ManualUsageCleanupMode::OlderThanDays => {
usage_cleanup_window_with_override(now_utc, settings, override_older_than)
}
ManualUsageCleanupMode::BeforeNow => UsageCleanupWindow {
detail_cutoff: now_utc,
compressed_cutoff: now_utc,
header_cutoff: now_utc,
log_cutoff: now_utc,
},
}
}
fn cleanup_execution_mode(mode: ManualUsageCleanupMode) -> UsageCleanupExecutionMode {
match mode {
ManualUsageCleanupMode::BeforeNow => UsageCleanupExecutionMode::BeforeNowBodyFields,
ManualUsageCleanupMode::Policy | ManualUsageCleanupMode::OlderThanDays => {
UsageCleanupExecutionMode::Policy
}
}
}