i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-01 17:00:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add scoped manual usage cleanup

Browse Source
This commit is contained in:
HsungKayphoon
2026-05-16 14:53:10 +08:00
parent 75b7319465
commit 74a1e5ad7d
17 changed files with 1746 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
apps/aether-gateway/src/data/state/runtime.rs
View File

@@ -37,7 +37,8 @@ use super::{
use aether_data_contracts::repository::usage::{
PendingUsageCleanupSummary, ProviderApiKeyWindowUsageRequest,
StoredProviderApiKeyWindowUsageSummary, StoredUsageDailySummary, UsageAuditListQuery,
UsageCleanupSummary, UsageCleanupWindow, UsageDailyHeatmapQuery,
UsageCleanupExecutionMode, UsageCleanupSummary, UsageCleanupTargets, UsageCleanupWindow,
UsageDailyHeatmapQuery,
};
use aether_runtime_state::RuntimeQueueStore;
use aether_video_tasks_core::read_data_backed_video_task_response;
@@ -980,11 +981,13 @@ impl GatewayDataState {
window: &UsageCleanupWindow,
batch_size: usize,
auto_delete_expired_keys: bool,
targets: UsageCleanupTargets,
mode: UsageCleanupExecutionMode,
) -> Result<UsageCleanupSummary, DataLayerError> {
match &self.usage_writer {
Some(repository) => {
repository
.cleanup_usage(window, batch_size, auto_delete_expired_keys)
.cleanup_usage(window, batch_size, auto_delete_expired_keys, targets, mode)
.await
}
None => Ok(UsageCleanupSummary::default()),
@@ -994,10 +997,16 @@ impl GatewayDataState {
pub(crate) async fn preview_usage_cleanup(
&self,
window: &UsageCleanupWindow,
targets: UsageCleanupTargets,
mode: UsageCleanupExecutionMode,
) -> Result<aether_data_contracts::repository::usage::UsageCleanupPreviewCounts, DataLayerError>
{
match &self.usage_writer {
Some(repository) => repository.preview_usage_cleanup(window).await,
Some(repository) => {
repository
.preview_usage_cleanup(window, targets, mode)
.await
}
None => {
Ok(aether_data_contracts::repository::usage::UsageCleanupPreviewCounts::default())
}

352
apps/aether-gateway/src/handlers/admin/system/core/system_routes.rs
View File

@@ -17,7 +17,9 @@ use crate::handlers::admin::system::shared::settings::{
build_admin_system_stats_payload, current_aether_version, fetch_latest_admin_system_release,
};
use crate::handlers::admin::system::shared::smtp::build_admin_smtp_test_payload;
use crate::maintenance::{ManualUsageCleanupMode, ManualUsageCleanupOptions};
use crate::GatewayError;
use aether_data_contracts::repository::usage::UsageCleanupTargets;
use axum::{
body::{Body, Bytes},
http,
@@ -26,6 +28,7 @@ use axum::{
};
use serde_json::json;
use std::time::Instant;
use url::form_urlencoded;
pub(super) async fn maybe_build_local_admin_core_system_response(
state: &AdminAppState<'_>,
@@ -263,7 +266,7 @@ pub(super) async fn maybe_build_local_admin_core_system_response(
&& request_path == "/api/admin/system/cleanup/usage/manual"
{
return Ok(Some(
build_manual_usage_cleanup_response(state, request_body).await?,
build_manual_usage_cleanup_response(state, request_context, request_body).await?,
));
}
@@ -625,50 +628,36 @@ async fn build_admin_system_cleanup_payload(
async fn build_manual_usage_cleanup_response(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
request_body: Option<&Bytes>,
) -> Result<Response<Body>, GatewayError> {
let older_than_days = match parse_manual_usage_cleanup_request(request_body) {
let options = match parse_manual_usage_cleanup_request(request_body) {
Ok(value) => value,
Err(response) => return Ok(response),
};
let actor_user_id = request_context
.decision()
.and_then(|decision| decision.admin_principal.as_ref())
.map(|principal| principal.user_id.clone());
match crate::maintenance::run_manual_usage_cleanup_once(
&state.app().data,
older_than_days,
None,
match crate::maintenance::start_manual_usage_cleanup_task(
std::sync::Arc::clone(&state.app().data),
options,
actor_user_id,
)
.await
{
Ok(summary) => {
let total = summary
.body_externalized
.saturating_add(summary.legacy_body_refs_migrated)
.saturating_add(summary.body_cleaned)
.saturating_add(summary.header_cleaned)
.saturating_add(summary.keys_cleaned)
.saturating_add(summary.records_deleted);
let message = match older_than_days {
Some(days) => {
format!("请求记录手动清理完成，清理 {days} 天前的记录，影响 {total} 项")
}
None => format!("请求记录手动清理完成（按当前策略），影响 {total} 项"),
};
Ok(task) => {
let payload = json!({
"message": message,
"requested_older_than_days": older_than_days,
"summary": {
"body_externalized": summary.body_externalized,
"legacy_body_refs_migrated": summary.legacy_body_refs_migrated,
"body_cleaned": summary.body_cleaned,
"header_cleaned": summary.header_cleaned,
"keys_cleaned": summary.keys_cleaned,
"records_deleted": summary.records_deleted,
},
"total_affected": total,
"message": task.message,
"mode": options.mode.as_str(),
"requested_older_than_days": options.requested_older_than_days,
"targets": options.targets,
"task": task,
});
Ok(attach_admin_audit_response(
Json(payload).into_response(),
"admin_system_usage_cleanup_completed",
"admin_system_usage_cleanup_started",
"manual_usage_cleanup",
"usage_cleanup",
"global",
@@ -696,12 +685,20 @@ async fn build_manual_usage_cleanup_preview_response(
Ok(value) => value,
Err(response) => return Ok(response),
};
let preview =
crate::maintenance::preview_manual_usage_cleanup(&state.app().data, older_than_days)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?;
let options = match parse_manual_usage_cleanup_query_options(
request_context.query_string(),
older_than_days,
) {
Ok(value) => value,
Err(response) => return Ok(response),
};
let preview = crate::maintenance::preview_manual_usage_cleanup(&state.app().data, options)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?;
Ok(Json(json!({
"mode": preview.mode.as_str(),
"requested_older_than_days": preview.requested_older_than_days,
"targets": preview.targets,
"effective_cutoffs": {
"detail": preview.detail_cutoff,
"compressed": preview.compressed_cutoff,
@@ -720,12 +717,12 @@ async fn build_manual_usage_cleanup_preview_response(
fn parse_manual_usage_cleanup_request(
request_body: Option<&Bytes>,
) -> Result<Option<u32>, Response<Body>> {
) -> Result<ManualUsageCleanupOptions, Response<Body>> {
let Some(body) = request_body else {
return Ok(None);
return Ok(ManualUsageCleanupOptions::policy());
};
if body.is_empty() {
return Ok(None);
return Ok(ManualUsageCleanupOptions::policy());
}
let parsed: serde_json::Value = match serde_json::from_slice(body) {
Ok(value) => value,
@@ -738,45 +735,208 @@ fn parse_manual_usage_cleanup_request(
}
};
let Some(object) = parsed.as_object() else {
return Ok(None);
return Err(bad_manual_cleanup_request("请求体必须为 JSON 对象"));
};
match object.get("older_than_days") {
None | Some(serde_json::Value::Null) => Ok(None),
Some(value) => value
.as_u64()
.and_then(|value| u32::try_from(value).ok())
.filter(|days| *days >= 1)
.map(Some)
.ok_or_else(|| {
(
http::StatusCode::BAD_REQUEST,
Json(json!({
"detail": "older_than_days 必须为正整数",
})),
)
.into_response()
}),
parse_manual_usage_cleanup_options(
object.get("mode").and_then(serde_json::Value::as_str),
object.get("older_than_days"),
object.get("targets"),
)
}
fn parse_manual_usage_cleanup_query_options(
query_string: Option<&str>,
older_than_days: Option<u32>,
) -> Result<ManualUsageCleanupOptions, Response<Body>> {
let mode = query_param(query_string, "mode");
let targets = query_param(query_string, "targets").map(serde_json::Value::String);
let older_value =
older_than_days.map(|days| serde_json::Value::Number(serde_json::Number::from(days)));
parse_manual_usage_cleanup_options(mode.as_deref(), older_value.as_ref(), targets.as_ref())
}
fn parse_manual_usage_cleanup_options(
raw_mode: Option<&str>,
older_than_days: Option<&serde_json::Value>,
targets_value: Option<&serde_json::Value>,
) -> Result<ManualUsageCleanupOptions, Response<Body>> {
let (requested_older_than_days, requested_before_now) =
parse_manual_cleanup_older_than_days(older_than_days)?;
let mode =
parse_manual_cleanup_mode(raw_mode, requested_older_than_days, requested_before_now)?;
if mode == ManualUsageCleanupMode::OlderThanDays && requested_older_than_days.is_none() {
return Err(bad_manual_cleanup_request(
"older_than_days 模式必须提供正整数天数",
));
}
if mode == ManualUsageCleanupMode::BeforeNow && requested_older_than_days.is_some() {
return Err(bad_manual_cleanup_request(
"before_now 模式不能同时提供 older_than_days",
));
}
if raw_mode.is_some() && requested_before_now && mode != ManualUsageCleanupMode::BeforeNow {
return Err(bad_manual_cleanup_request(
"older_than_days 为 0 时必须使用 before_now 模式",
));
}
if raw_mode.is_some()
&& mode == ManualUsageCleanupMode::Policy
&& requested_older_than_days.is_some()
{
return Err(bad_manual_cleanup_request(
"policy 模式不能同时提供 older_than_days",
));
}
let targets = parse_manual_cleanup_targets(targets_value, mode)?;
if !targets.any_selected() {
return Err(bad_manual_cleanup_request("至少选择一个清理范围"));
}
if mode == ManualUsageCleanupMode::BeforeNow
&& (targets.headers || targets.records || targets.expired_keys)
{
return Err(bad_manual_cleanup_request(
"清理当前时刻之前只允许选择详细请求体和压缩请求体",
));
}
Ok(ManualUsageCleanupOptions {
mode,
requested_older_than_days,
targets,
})
}
fn parse_manual_cleanup_mode(
raw_mode: Option<&str>,
requested_older_than_days: Option<u32>,
requested_before_now: bool,
) -> Result<ManualUsageCleanupMode, Response<Body>> {
let Some(raw) = raw_mode.map(str::trim).filter(|value| !value.is_empty()) else {
return Ok(if requested_before_now {
ManualUsageCleanupMode::BeforeNow
} else if requested_older_than_days.is_some() {
ManualUsageCleanupMode::OlderThanDays
} else {
ManualUsageCleanupMode::Policy
});
};
match raw {
"policy" => Ok(ManualUsageCleanupMode::Policy),
"older_than_days" => Ok(ManualUsageCleanupMode::OlderThanDays),
"before_now" => Ok(ManualUsageCleanupMode::BeforeNow),
_ => Err(bad_manual_cleanup_request(
"mode 必须为 policy、older_than_days 或 before_now",
)),
}
}
fn parse_older_than_days_query(query_string: Option<&str>) -> Result<Option<u32>, Response<Body>> {
let Some(query) = query_string.filter(|value| !value.is_empty()) else {
return Ok(None);
};
let value = query
.split('&')
.filter_map(|pair| pair.split_once('='))
.find_map(|(key, value)| {
if key == "older_than_days" && !value.is_empty() {
Some(value)
} else {
None
fn parse_manual_cleanup_older_than_days(
value: Option<&serde_json::Value>,
) -> Result<(Option<u32>, bool), Response<Body>> {
match value {
None | Some(serde_json::Value::Null) => Ok((None, false)),
Some(value) => {
let Some(raw) = value.as_u64() else {
return Err(bad_manual_cleanup_request("older_than_days 必须为非负整数"));
};
if raw == 0 {
return Ok((None, true));
}
let days = u32::try_from(raw)
.ok()
.filter(|days| *days >= 1)
.ok_or_else(|| bad_manual_cleanup_request("older_than_days 必须为正整数"))?;
Ok((Some(days), false))
}
}
}
fn parse_manual_cleanup_targets(
value: Option<&serde_json::Value>,
mode: ManualUsageCleanupMode,
) -> Result<UsageCleanupTargets, Response<Body>> {
let Some(value) = value else {
return Ok(match mode {
ManualUsageCleanupMode::BeforeNow => UsageCleanupTargets::body_targets(),
ManualUsageCleanupMode::Policy | ManualUsageCleanupMode::OlderThanDays => {
UsageCleanupTargets::all_policy_targets()
}
});
let Some(raw) = value else {
};
if value.is_null() {
return Ok(match mode {
ManualUsageCleanupMode::BeforeNow => UsageCleanupTargets::body_targets(),
ManualUsageCleanupMode::Policy | ManualUsageCleanupMode::OlderThanDays => {
UsageCleanupTargets::all_policy_targets()
}
});
}
let raw_targets = match value {
serde_json::Value::Array(items) => items
.iter()
.map(|item| {
item.as_str()
.map(str::to_string)
.ok_or_else(|| bad_manual_cleanup_request("targets 必须为字符串数组"))
})
.collect::<Result<Vec<_>, _>>()?,
serde_json::Value::String(raw) => raw
.split(',')
.map(str::trim)
.filter(|item| !item.is_empty())
.map(str::to_string)
.collect(),
_ => return Err(bad_manual_cleanup_request("targets 必须为字符串数组")),
};
let mut targets = UsageCleanupTargets {
detail_body: false,
compressed_body: false,
headers: false,
records: false,
expired_keys: false,
};
for raw in raw_targets {
match raw.as_str() {
"detail_body" | "detail" | "raw_body" => targets.detail_body = true,
"compressed_body" | "compressed" => targets.compressed_body = true,
"headers" | "header" => targets.headers = true,
"records" | "log" | "logs" => targets.records = true,
"expired_keys" => targets.expired_keys = true,
"all" => targets = UsageCleanupTargets::all_policy_targets(),
_ => {
return Err(bad_manual_cleanup_request(
"targets 只能包含 detail_body、compressed_body、headers、records",
))
}
}
}
Ok(targets)
}
fn bad_manual_cleanup_request(detail: impl Into<String>) -> Response<Body> {
(
http::StatusCode::BAD_REQUEST,
Json(json!({ "detail": detail.into() })),
)
.into_response()
}
fn query_param(query_string: Option<&str>, name: &str) -> Option<String> {
let query = query_string.filter(|value| !value.is_empty())?;
form_urlencoded::parse(query.as_bytes())
.find_map(|(key, value)| (key == name && !value.is_empty()).then(|| value.into_owned()))
}
fn parse_older_than_days_query(query_string: Option<&str>) -> Result<Option<u32>, Response<Body>> {
let Some(value) = query_param(query_string, "older_than_days") else {
return Ok(None);
};
raw.parse::<u32>()
value
.parse::<u32>()
.ok()
.filter(|days| *days >= 1)
.map(Some)
@@ -790,3 +950,55 @@ fn parse_older_than_days_query(query_string: Option<&str>) -> Result<Option<u32>
.into_response()
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn manual_cleanup_request_defaults_to_policy_targets() {
let options = parse_manual_usage_cleanup_request(None).expect("default request is valid");
assert_eq!(options.mode, ManualUsageCleanupMode::Policy);
assert_eq!(options.requested_older_than_days, None);
assert_eq!(options.targets, UsageCleanupTargets::all_policy_targets());
}
#[test]
fn manual_cleanup_request_treats_zero_days_as_before_now_body_only() {
let body = Bytes::from_static(br#"{"older_than_days":0}"#);
let options =
parse_manual_usage_cleanup_request(Some(&body)).expect("before-now request is valid");
assert_eq!(options.mode, ManualUsageCleanupMode::BeforeNow);
assert_eq!(options.requested_older_than_days, None);
assert_eq!(options.targets, UsageCleanupTargets::body_targets());
}
#[test]
fn manual_cleanup_request_rejects_before_now_headers() {
let body = Bytes::from_static(br#"{"mode":"before_now","targets":["headers"]}"#);
assert!(parse_manual_usage_cleanup_request(Some(&body)).is_err());
}
#[test]
fn manual_cleanup_preview_query_decodes_comma_separated_targets() {
let options = parse_manual_usage_cleanup_query_options(
Some("mode=before_now&targets=detail_body%2Ccompressed_body"),
None,
)
.expect("encoded targets query is valid");
assert_eq!(options.mode, ManualUsageCleanupMode::BeforeNow);
assert_eq!(options.targets, UsageCleanupTargets::body_targets());
}
#[test]
fn manual_cleanup_request_rejects_non_object_body() {
let body = Bytes::from_static(br#"[]"#);
assert!(parse_manual_usage_cleanup_request(Some(&body)).is_err());
}
}

7
apps/aether-gateway/src/maintenance/mod.rs
View File

@@ -21,9 +21,10 @@ pub(crate) use runtime::{
spawn_stats_aggregation_worker, spawn_stats_hourly_aggregation_worker,
spawn_usage_cleanup_worker, spawn_wallet_daily_usage_aggregation_worker,
start_admin_request_body_cleanup_task, start_admin_system_purge_task,
start_proxy_upgrade_rollout, AccountSelfCheckRunSummary, AdminCleanupRunRecord,
AdminCleanupTaskKind, AdminStatsRebuildSummary, AdminSystemCleanupSummary,
ManualUsageCleanupError, OAuthTokenRefreshRunSummary, PoolQuotaProbeRunSummary,
start_manual_usage_cleanup_task, start_proxy_upgrade_rollout, AccountSelfCheckRunSummary,
AdminCleanupRunRecord, AdminCleanupTaskKind, AdminStatsRebuildSummary,
AdminSystemCleanupSummary, ManualUsageCleanupError, ManualUsageCleanupMode,
ManualUsageCleanupOptions, OAuthTokenRefreshRunSummary, PoolQuotaProbeRunSummary,
PoolQuotaProbeWorkerConfig, ProviderCheckinRunSummary, ProxyUpgradeRolloutCancelSummary,
ProxyUpgradeRolloutConflictClearSummary, ProxyUpgradeRolloutNodeActionSummary,
ProxyUpgradeRolloutProbeConfig, ProxyUpgradeRolloutSkippedRestoreSummary,

14
apps/aether-gateway/src/maintenance/runtime.rs
View File

@@ -61,9 +61,9 @@ pub(crate) use aether_data_contracts::repository::usage::{
};
use audit_cleanup::*;
pub(crate) use cleanup_runs::{
list_admin_cleanup_run_records, record_completed_cleanup_run, record_failed_cleanup_run,
start_admin_request_body_cleanup_task, start_admin_system_purge_task, AdminCleanupRunRecord,
AdminCleanupTaskKind, USAGE_CLEANUP_KIND,
list_admin_cleanup_run_records, record_admin_cleanup_run, record_completed_cleanup_run,
record_failed_cleanup_run, start_admin_request_body_cleanup_task,
start_admin_system_purge_task, AdminCleanupRunRecord, AdminCleanupTaskKind, USAGE_CLEANUP_KIND,
};
use config::*;
use db_maintenance::*;
@@ -98,12 +98,16 @@ pub(crate) use proxy_upgrade_rollout::{
};
use request_candidate_cleanup::*;
use runners::*;
pub(crate) use runners::{run_manual_usage_cleanup_once, ManualUsageCleanupError};
pub(crate) use runners::{
run_manual_usage_cleanup_once, start_manual_usage_cleanup_task, ManualUsageCleanupError,
};
use schedule::*;
use stats_daily::*;
use stats_hourly::*;
pub(crate) use usage_cleanup::preview_manual_usage_cleanup;
use usage_cleanup::*;
pub(crate) use usage_cleanup::{
preview_manual_usage_cleanup, ManualUsageCleanupMode, ManualUsageCleanupOptions,
};
use wallet_daily_usage::*;
pub(crate) use workers::*;

9
apps/aether-gateway/src/maintenance/runtime/cleanup_runs.rs
View File

@@ -489,7 +489,7 @@ fn request_body_cleanup_record(
}
}
async fn record_cleanup_run(
pub(crate) async fn record_admin_cleanup_run(
data: &GatewayDataState,
record: AdminCleanupRunRecord,
) -> Result<(), DataLayerError> {
@@ -509,6 +509,13 @@ async fn record_cleanup_run(
Ok(())
}
async fn record_cleanup_run(
data: &GatewayDataState,
record: AdminCleanupRunRecord,
) -> Result<(), DataLayerError> {
record_admin_cleanup_run(data, record).await
}
fn parse_cleanup_run_records(value: Value) -> Vec<AdminCleanupRunRecord> {
value
.as_array()

229
apps/aether-gateway/src/maintenance/runtime/runners.rs
View File

@@ -1,5 +1,6 @@
use aether_data_contracts::DataLayerError;
use serde_json::json;
use std::sync::Arc;
use std::time::Instant;
use tracing::{info, warn};
@@ -11,11 +12,12 @@ use super::{
cleanup_expired_gemini_file_mappings_once, cleanup_proxy_node_metrics_once,
cleanup_request_candidates_once, cleanup_stale_pending_requests_once,
cleanup_stale_proxy_nodes_once, collect_proxy_upgrade_rollout_probes, now_unix_secs,
perform_db_maintenance_once, perform_provider_checkin_once, perform_stats_aggregation_once,
perform_stats_hourly_aggregation_once, perform_usage_cleanup_once,
perform_usage_cleanup_once_with_override, perform_wallet_daily_usage_aggregation_once,
record_completed_cleanup_run, record_failed_cleanup_run, record_proxy_upgrade_traffic_success,
summarize_database_pool,
perform_db_maintenance_once, perform_manual_usage_cleanup_once, perform_provider_checkin_once,
perform_stats_aggregation_once, perform_stats_hourly_aggregation_once,
perform_usage_cleanup_once, perform_wallet_daily_usage_aggregation_once,
record_admin_cleanup_run, record_completed_cleanup_run, record_failed_cleanup_run,
record_proxy_upgrade_traffic_success, summarize_database_pool, AdminCleanupRunRecord,
ManualUsageCleanupOptions,
};
pub(super) async fn run_audit_cleanup_once(data: &GatewayDataState) -> Result<(), DataLayerError> {
@@ -299,15 +301,14 @@ pub(super) async fn run_usage_cleanup_once(data: &GatewayDataState) -> Result<()
Ok(())
}
pub(crate) async fn run_manual_usage_cleanup_once(
data: &GatewayDataState,
override_older_than_days: Option<u32>,
pub(crate) async fn start_manual_usage_cleanup_task(
data: Arc<GatewayDataState>,
options: ManualUsageCleanupOptions,
actor_user_id: Option<String>,
) -> Result<aether_data_contracts::repository::usage::UsageCleanupSummary, ManualUsageCleanupError>
{
) -> Result<AdminCleanupRunRecord, ManualUsageCleanupError> {
use super::{list_admin_cleanup_run_records, USAGE_CLEANUP_KIND};
let existing = list_admin_cleanup_run_records(data)
let existing = list_admin_cleanup_run_records(&data)
.await
.map_err(ManualUsageCleanupError::DataLayer)?;
if existing
@@ -318,10 +319,42 @@ pub(crate) async fn run_manual_usage_cleanup_once(
}
let started_at_unix_secs = now_unix_secs();
let record = AdminCleanupRunRecord {
id: uuid::Uuid::new_v4().to_string(),
kind: USAGE_CLEANUP_KIND.to_string(),
trigger: "manual".to_string(),
status: "processing".to_string(),
message: manual_usage_cleanup_start_message(options),
started_at_unix_secs,
completed_at_unix_secs: None,
duration_ms: None,
summary: manual_usage_cleanup_progress_summary(options, 0, None, actor_user_id.as_deref()),
error: None,
};
record_admin_cleanup_run(&data, record.clone())
.await
.map_err(ManualUsageCleanupError::DataLayer)?;
tokio::spawn(run_manual_usage_cleanup_task(
data,
record.clone(),
options,
actor_user_id,
));
Ok(record)
}
pub(crate) async fn run_manual_usage_cleanup_once(
data: &GatewayDataState,
options: ManualUsageCleanupOptions,
actor_user_id: Option<String>,
) -> Result<aether_data_contracts::repository::usage::UsageCleanupSummary, ManualUsageCleanupError>
{
use super::USAGE_CLEANUP_KIND;
let started_at = Instant::now();
let override_duration =
override_older_than_days.map(|days| chrono::Duration::days(i64::from(days)));
let summary = match perform_usage_cleanup_once_with_override(data, override_duration).await {
let started_at_unix_secs = now_unix_secs();
let summary = match perform_manual_usage_cleanup_once(data, options).await {
Ok(summary) => summary,
Err(err) => {
record_failed_cleanup_run(
@@ -336,17 +369,8 @@ pub(crate) async fn run_manual_usage_cleanup_once(
return Err(ManualUsageCleanupError::DataLayer(err));
}
};
let total = summary
.body_externalized
.saturating_add(summary.legacy_body_refs_migrated)
.saturating_add(summary.body_cleaned)
.saturating_add(summary.header_cleaned)
.saturating_add(summary.keys_cleaned)
.saturating_add(summary.records_deleted);
let message = match override_older_than_days {
Some(days) => format!("请求记录手动清理完成，清理 {days} 天前的记录，影响 {total} 项"),
None => format!("请求记录手动清理完成（按当前策略），影响 {total} 项"),
};
let total = usage_cleanup_total(summary);
let message = manual_usage_cleanup_completed_message(options, total);
record_completed_cleanup_run(
data,
USAGE_CLEANUP_KIND,
@@ -360,7 +384,9 @@ pub(crate) async fn run_manual_usage_cleanup_once(
"header_cleaned": summary.header_cleaned,
"keys_cleaned": summary.keys_cleaned,
"records_deleted": summary.records_deleted,
"requested_older_than_days": override_older_than_days,
"mode": options.mode.as_str(),
"requested_older_than_days": options.requested_older_than_days,
"targets": options.targets,
"actor_user_id": actor_user_id,
}),
message,
@@ -371,7 +397,8 @@ pub(crate) async fn run_manual_usage_cleanup_once(
log_type = "ops",
worker = "usage_cleanup",
trigger = "manual",
requested_older_than_days = override_older_than_days,
mode = options.mode.as_str(),
requested_older_than_days = options.requested_older_than_days,
actor_user_id = actor_user_id.as_deref(),
total_affected = total,
"gateway finished manual usage cleanup"
@@ -379,6 +406,154 @@ pub(crate) async fn run_manual_usage_cleanup_once(
Ok(summary)
}
async fn run_manual_usage_cleanup_task(
data: Arc<GatewayDataState>,
initial_record: AdminCleanupRunRecord,
options: ManualUsageCleanupOptions,
actor_user_id: Option<String>,
) {
let started_at = Instant::now();
match perform_manual_usage_cleanup_once(&data, options).await {
Ok(summary) => {
let total = usage_cleanup_total(summary);
let record = AdminCleanupRunRecord {
id: initial_record.id,
kind: initial_record.kind,
trigger: initial_record.trigger,
status: "completed".to_string(),
message: manual_usage_cleanup_completed_message(options, total),
started_at_unix_secs: initial_record.started_at_unix_secs,
completed_at_unix_secs: Some(now_unix_secs()),
duration_ms: Some(
started_at
.elapsed()
.as_millis()
.try_into()
.unwrap_or(u64::MAX),
),
summary: manual_usage_cleanup_progress_summary(
options,
100,
Some(summary),
actor_user_id.as_deref(),
),
error: None,
};
if let Err(err) = record_admin_cleanup_run(&data, record).await {
warn!(error = %err, "failed to record manual usage cleanup completion");
}
info!(
event_name = "usage_cleanup_manual_completed",
log_type = "ops",
worker = "usage_cleanup",
trigger = "manual",
mode = options.mode.as_str(),
requested_older_than_days = options.requested_older_than_days,
actor_user_id = actor_user_id.as_deref(),
total_affected = total,
"gateway finished manual usage cleanup task"
);
}
Err(err) => {
let record = AdminCleanupRunRecord {
id: initial_record.id,
kind: initial_record.kind,
trigger: initial_record.trigger,
status: "failed".to_string(),
message: "请求记录手动清理失败".to_string(),
started_at_unix_secs: initial_record.started_at_unix_secs,
completed_at_unix_secs: Some(now_unix_secs()),
duration_ms: Some(
started_at
.elapsed()
.as_millis()
.try_into()
.unwrap_or(u64::MAX),
),
summary: manual_usage_cleanup_progress_summary(
options,
100,
None,
actor_user_id.as_deref(),
),
error: Some(err.to_string()),
};
if let Err(record_err) = record_admin_cleanup_run(&data, record).await {
warn!(error = %record_err, "failed to record manual usage cleanup failure");
}
warn!(error = %err, "manual usage cleanup task failed");
}
}
}
fn usage_cleanup_total(
summary: aether_data_contracts::repository::usage::UsageCleanupSummary,
) -> usize {
summary
.body_externalized
.saturating_add(summary.legacy_body_refs_migrated)
.saturating_add(summary.body_cleaned)
.saturating_add(summary.header_cleaned)
.saturating_add(summary.keys_cleaned)
.saturating_add(summary.records_deleted)
}
fn manual_usage_cleanup_start_message(options: ManualUsageCleanupOptions) -> String {
match options.mode {
super::ManualUsageCleanupMode::BeforeNow => {
"请求记录手动清理已开始，清理当前时刻之前的已选请求体".to_string()
}
super::ManualUsageCleanupMode::OlderThanDays => format!(
"请求记录手动清理已开始，清理 {} 天前的已选内容",
options.requested_older_than_days.unwrap_or_default()
),
super::ManualUsageCleanupMode::Policy => {
"请求记录手动清理已开始，按当前策略清理已选内容".to_string()
}
}
}
fn manual_usage_cleanup_completed_message(
options: ManualUsageCleanupOptions,
total: usize,
) -> String {
match options.mode {
super::ManualUsageCleanupMode::BeforeNow => {
format!("请求记录手动清理完成，已清理当前时刻之前的已选请求体，影响 {total} 项")
}
super::ManualUsageCleanupMode::OlderThanDays => format!(
"请求记录手动清理完成，清理 {} 天前的已选内容，影响 {total} 项",
options.requested_older_than_days.unwrap_or_default()
),
super::ManualUsageCleanupMode::Policy => {
format!("请求记录手动清理完成（按当前策略），影响 {total} 项")
}
}
}
fn manual_usage_cleanup_progress_summary(
options: ManualUsageCleanupOptions,
progress_percent: u8,
summary: Option<aether_data_contracts::repository::usage::UsageCleanupSummary>,
actor_user_id: Option<&str>,
) -> serde_json::Value {
let summary = summary.unwrap_or_default();
json!({
"mode": options.mode.as_str(),
"requested_older_than_days": options.requested_older_than_days,
"targets": options.targets,
"progress_percent": progress_percent,
"body_externalized": summary.body_externalized,
"legacy_body_refs_migrated": summary.legacy_body_refs_migrated,
"body_cleaned": summary.body_cleaned,
"header_cleaned": summary.header_cleaned,
"keys_cleaned": summary.keys_cleaned,
"records_deleted": summary.records_deleted,
"total": usage_cleanup_total(summary),
"actor_user_id": actor_user_id,
})
}
#[derive(Debug)]
pub(crate) enum ManualUsageCleanupError {
AlreadyRunning,

35
apps/aether-gateway/src/maintenance/runtime/tests.rs
View File

@@ -28,12 +28,12 @@ use super::{
spawn_stats_hourly_aggregation_worker, spawn_usage_cleanup_worker,
spawn_wallet_daily_usage_aggregation_worker, start_proxy_upgrade_rollout,
stats_aggregation_target_day, stats_hourly_aggregation_target_hour, summarize_database_pool,
usage_cleanup_settings, usage_cleanup_window, usage_cleanup_window_with_override,
wallet_daily_usage_aggregation_target, AppState, DbMaintenanceRunSummary,
FailedPendingUsageRow, GatewayDataState, ProxyNodeMetricsCleanupSettings,
ProxyUpgradeRolloutProbeConfig, StalePendingUsageRow, UsageCleanupSettings, USAGE_CLEANUP_HOUR,
USAGE_CLEANUP_MINUTE, WALLET_DAILY_USAGE_AGGREGATION_HOUR,
WALLET_DAILY_USAGE_AGGREGATION_MINUTE,
usage_cleanup_settings, usage_cleanup_window, usage_cleanup_window_for_mode,
usage_cleanup_window_with_override, wallet_daily_usage_aggregation_target, AppState,
DbMaintenanceRunSummary, FailedPendingUsageRow, GatewayDataState, ManualUsageCleanupMode,
ProxyNodeMetricsCleanupSettings, ProxyUpgradeRolloutProbeConfig, StalePendingUsageRow,
UsageCleanupSettings, USAGE_CLEANUP_HOUR, USAGE_CLEANUP_MINUTE,
WALLET_DAILY_USAGE_AGGREGATION_HOUR, WALLET_DAILY_USAGE_AGGREGATION_MINUTE,
};
#[tokio::test]
@@ -986,6 +986,29 @@ fn usage_cleanup_window_with_override_is_always_non_aggressive() {
assert_eq!(passthrough, policy);
}
#[test]
fn usage_cleanup_before_now_window_uses_current_timestamp_only() {
let now_utc = "2026-03-18T03:00:00Z"
.parse::<DateTime<Utc>>()
.expect("timestamp should parse");
let settings = UsageCleanupSettings {
detail_retention_days: 7,
compressed_retention_days: 30,
header_retention_days: 90,
log_retention_days: 365,
batch_size: 123,
auto_delete_expired_keys: false,
};
let window =
usage_cleanup_window_for_mode(now_utc, settings, ManualUsageCleanupMode::BeforeNow, None);
assert_eq!(window.detail_cutoff, now_utc);
assert_eq!(window.compressed_cutoff, now_utc);
assert_eq!(window.header_cutoff, now_utc);
assert_eq!(window.log_cutoff, now_utc);
}
#[tokio::test]
async fn summarize_database_pool_uses_busy_connections_for_usage_rate() {
let data = GatewayDataState::from_config(crate::data::GatewayDataConfig::from_postgres_config(

146
apps/aether-gateway/src/maintenance/runtime/usage_cleanup.rs
View File

@@ -1,4 +1,6 @@
use aether_data_contracts::repository::usage::{UsageCleanupSummary, UsageCleanupWindow};
use aether_data_contracts::repository::usage::{
UsageCleanupExecutionMode, UsageCleanupSummary, UsageCleanupTargets, UsageCleanupWindow,
};
use aether_data_contracts::DataLayerError;
use chrono::Utc;
@@ -15,6 +17,8 @@ pub(crate) struct ManualUsageCleanupPreview {
pub compressed_cutoff: chrono::DateTime<Utc>,
pub header_cutoff: chrono::DateTime<Utc>,
pub log_cutoff: chrono::DateTime<Utc>,
pub mode: ManualUsageCleanupMode,
pub targets: UsageCleanupTargets,
pub requested_older_than_days: Option<u32>,
pub detail_count: u64,
pub compressed_count: u64,
@@ -22,49 +26,131 @@ pub(crate) struct ManualUsageCleanupPreview {
pub log_count: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum ManualUsageCleanupMode {
Policy,
OlderThanDays,
BeforeNow,
}
impl ManualUsageCleanupMode {
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::Policy => "policy",
Self::OlderThanDays => "older_than_days",
Self::BeforeNow => "before_now",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct ManualUsageCleanupOptions {
pub(crate) mode: ManualUsageCleanupMode,
pub(crate) requested_older_than_days: Option<u32>,
pub(crate) targets: UsageCleanupTargets,
}
impl ManualUsageCleanupOptions {
pub(crate) const fn policy() -> Self {
Self {
mode: ManualUsageCleanupMode::Policy,
requested_older_than_days: None,
targets: UsageCleanupTargets::all_policy_targets(),
}
}
}
pub(super) async fn perform_usage_cleanup_once(
data: &GatewayDataState,
) -> Result<UsageCleanupSummary, DataLayerError> {
perform_usage_cleanup_once_with_override(data, None).await
perform_usage_cleanup_once_with_override(data, None, true).await
}
pub(super) async fn perform_usage_cleanup_once_with_override(
data: &GatewayDataState,
override_older_than: Option<chrono::Duration>,
respect_auto_enabled: bool,
) -> Result<UsageCleanupSummary, DataLayerError> {
let options = ManualUsageCleanupOptions {
mode: if override_older_than.is_some() {
ManualUsageCleanupMode::OlderThanDays
} else {
ManualUsageCleanupMode::Policy
},
requested_older_than_days: None,
targets: UsageCleanupTargets::all_policy_targets(),
};
perform_usage_cleanup_once_with_options(
data,
options,
override_older_than,
respect_auto_enabled,
)
.await
}
pub(super) async fn perform_manual_usage_cleanup_once(
data: &GatewayDataState,
options: ManualUsageCleanupOptions,
) -> Result<UsageCleanupSummary, DataLayerError> {
let override_duration = options
.requested_older_than_days
.map(|days| chrono::Duration::days(i64::from(days)));
perform_usage_cleanup_once_with_options(data, options, override_duration, false).await
}
async fn perform_usage_cleanup_once_with_options(
data: &GatewayDataState,
options: ManualUsageCleanupOptions,
override_older_than: Option<chrono::Duration>,
respect_auto_enabled: bool,
) -> Result<UsageCleanupSummary, DataLayerError> {
if !data.has_usage_writer() {
return Ok(UsageCleanupSummary::default());
}
if override_older_than.is_none()
if respect_auto_enabled
&& override_older_than.is_none()
&& !system_config_bool(data, "enable_auto_cleanup", true).await?
{
return Ok(UsageCleanupSummary::default());
}
let window = compute_usage_cleanup_window(data, override_older_than).await?;
let window = compute_usage_cleanup_window(data, options.mode, override_older_than).await?;
let settings = usage_cleanup_settings(data).await?;
data.cleanup_usage(
&window,
settings.batch_size,
settings.auto_delete_expired_keys,
options.targets,
cleanup_execution_mode(options.mode),
)
.await
}
pub(crate) async fn preview_manual_usage_cleanup(
data: &GatewayDataState,
override_older_than_days: Option<u32>,
options: ManualUsageCleanupOptions,
) -> Result<ManualUsageCleanupPreview, DataLayerError> {
let override_duration =
override_older_than_days.map(|days| chrono::Duration::days(i64::from(days)));
let window = compute_usage_cleanup_window(data, override_duration).await?;
let counts = data.preview_usage_cleanup(&window).await?;
let override_duration = options
.requested_older_than_days
.map(|days| chrono::Duration::days(i64::from(days)));
let window = compute_usage_cleanup_window(data, options.mode, override_duration).await?;
let counts = data
.preview_usage_cleanup(
&window,
options.targets,
cleanup_execution_mode(options.mode),
)
.await?;
Ok(ManualUsageCleanupPreview {
detail_cutoff: window.detail_cutoff,
compressed_cutoff: window.compressed_cutoff,
header_cutoff: window.header_cutoff,
log_cutoff: window.log_cutoff,
requested_older_than_days: override_older_than_days,
mode: options.mode,
targets: options.targets,
requested_older_than_days: options.requested_older_than_days,
detail_count: counts.detail,
compressed_count: counts.compressed,
header_count: counts.header,
@@ -74,11 +160,43 @@ pub(crate) async fn preview_manual_usage_cleanup(
async fn compute_usage_cleanup_window(
data: &GatewayDataState,
mode: ManualUsageCleanupMode,
override_older_than: Option<chrono::Duration>,
) -> Result<UsageCleanupWindow, DataLayerError> {
let settings = usage_cleanup_settings(data).await?;
Ok(match override_older_than {
Some(duration) => usage_cleanup_window_with_override(Utc::now(), settings, Some(duration)),
None => usage_cleanup_window(Utc::now(), settings),
})
Ok(usage_cleanup_window_for_mode(
Utc::now(),
settings,
mode,
override_older_than,
))
}
pub(super) fn usage_cleanup_window_for_mode(
now_utc: chrono::DateTime<Utc>,
settings: super::UsageCleanupSettings,
mode: ManualUsageCleanupMode,
override_older_than: Option<chrono::Duration>,
) -> UsageCleanupWindow {
match mode {
ManualUsageCleanupMode::Policy => usage_cleanup_window(now_utc, settings),
ManualUsageCleanupMode::OlderThanDays => {
usage_cleanup_window_with_override(now_utc, settings, override_older_than)
}
ManualUsageCleanupMode::BeforeNow => UsageCleanupWindow {
detail_cutoff: now_utc,
compressed_cutoff: now_utc,
header_cutoff: now_utc,
log_cutoff: now_utc,
},
}
}
fn cleanup_execution_mode(mode: ManualUsageCleanupMode) -> UsageCleanupExecutionMode {
match mode {
ManualUsageCleanupMode::BeforeNow => UsageCleanupExecutionMode::BeforeNowBodyFields,
ManualUsageCleanupMode::Policy | ManualUsageCleanupMode::OlderThanDays => {
UsageCleanupExecutionMode::Policy
}
}
}