i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 17:30:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add scoped manual usage cleanup

Browse Source
This commit is contained in:
HsungKayphoon
2026-05-16 14:53:10 +08:00
parent 75b7319465
commit 74a1e5ad7d
17 changed files with 1746 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
apps/aether-gateway/src/data/state/runtime.rs
View File

@@ -37,7 +37,8 @@ use super::{
use aether_data_contracts::repository::usage::{ use aether_data_contracts::repository::usage::{
PendingUsageCleanupSummary, ProviderApiKeyWindowUsageRequest, PendingUsageCleanupSummary, ProviderApiKeyWindowUsageRequest,
StoredProviderApiKeyWindowUsageSummary, StoredUsageDailySummary, UsageAuditListQuery, StoredProviderApiKeyWindowUsageSummary, StoredUsageDailySummary, UsageAuditListQuery,
UsageCleanupSummary, UsageCleanupWindow, UsageDailyHeatmapQuery, UsageCleanupExecutionMode, UsageCleanupSummary, UsageCleanupTargets, UsageCleanupWindow,
UsageDailyHeatmapQuery,
}; };
use aether_runtime_state::RuntimeQueueStore; use aether_runtime_state::RuntimeQueueStore;
use aether_video_tasks_core::read_data_backed_video_task_response; use aether_video_tasks_core::read_data_backed_video_task_response;
@@ -980,11 +981,13 @@ impl GatewayDataState {
window: &UsageCleanupWindow, window: &UsageCleanupWindow,
batch_size: usize, batch_size: usize,
auto_delete_expired_keys: bool, auto_delete_expired_keys: bool,
targets: UsageCleanupTargets,
mode: UsageCleanupExecutionMode,
) -> Result<UsageCleanupSummary, DataLayerError> { ) -> Result<UsageCleanupSummary, DataLayerError> {
match &self.usage_writer { match &self.usage_writer {
Some(repository) => { Some(repository) => {
repository repository
.cleanup_usage(window, batch_size, auto_delete_expired_keys) .cleanup_usage(window, batch_size, auto_delete_expired_keys, targets, mode)
.await .await
} }
None => Ok(UsageCleanupSummary::default()), None => Ok(UsageCleanupSummary::default()),
@@ -994,10 +997,16 @@ impl GatewayDataState {
pub(crate) async fn preview_usage_cleanup( pub(crate) async fn preview_usage_cleanup(
&self, &self,
window: &UsageCleanupWindow, window: &UsageCleanupWindow,
targets: UsageCleanupTargets,
mode: UsageCleanupExecutionMode,
) -> Result<aether_data_contracts::repository::usage::UsageCleanupPreviewCounts, DataLayerError> ) -> Result<aether_data_contracts::repository::usage::UsageCleanupPreviewCounts, DataLayerError>
{ {
match &self.usage_writer { match &self.usage_writer {
Some(repository) => repository.preview_usage_cleanup(window).await, Some(repository) => {
repository
.preview_usage_cleanup(window, targets, mode)
.await
}
None => { None => {
Ok(aether_data_contracts::repository::usage::UsageCleanupPreviewCounts::default()) Ok(aether_data_contracts::repository::usage::UsageCleanupPreviewCounts::default())
} }

352
apps/aether-gateway/src/handlers/admin/system/core/system_routes.rs
View File

@@ -17,7 +17,9 @@ use crate::handlers::admin::system::shared::settings::{
build_admin_system_stats_payload, current_aether_version, fetch_latest_admin_system_release, build_admin_system_stats_payload, current_aether_version, fetch_latest_admin_system_release,
}; };
use crate::handlers::admin::system::shared::smtp::build_admin_smtp_test_payload; use crate::handlers::admin::system::shared::smtp::build_admin_smtp_test_payload;
use crate::maintenance::{ManualUsageCleanupMode, ManualUsageCleanupOptions};
use crate::GatewayError; use crate::GatewayError;
use aether_data_contracts::repository::usage::UsageCleanupTargets;
use axum::{ use axum::{
body::{Body, Bytes}, body::{Body, Bytes},
http, http,
@@ -26,6 +28,7 @@ use axum::{
}; };
use serde_json::json; use serde_json::json;
use std::time::Instant; use std::time::Instant;
use url::form_urlencoded;
pub(super) async fn maybe_build_local_admin_core_system_response( pub(super) async fn maybe_build_local_admin_core_system_response(
state: &AdminAppState<'_>, state: &AdminAppState<'_>,
@@ -263,7 +266,7 @@ pub(super) async fn maybe_build_local_admin_core_system_response(
&& request_path == "/api/admin/system/cleanup/usage/manual" && request_path == "/api/admin/system/cleanup/usage/manual"
{ {
return Ok(Some( return Ok(Some(
build_manual_usage_cleanup_response(state, request_body).await?, build_manual_usage_cleanup_response(state, request_context, request_body).await?,
)); ));
} }
@@ -625,50 +628,36 @@ async fn build_admin_system_cleanup_payload(
async fn build_manual_usage_cleanup_response( async fn build_manual_usage_cleanup_response(
state: &AdminAppState<'_>, state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
request_body: Option<&Bytes>, request_body: Option<&Bytes>,
) -> Result<Response<Body>, GatewayError> { ) -> Result<Response<Body>, GatewayError> {
let older_than_days = match parse_manual_usage_cleanup_request(request_body) { let options = match parse_manual_usage_cleanup_request(request_body) {
Ok(value) => value, Ok(value) => value,
Err(response) => return Ok(response), Err(response) => return Ok(response),
}; };
let actor_user_id = request_context
.decision()
.and_then(|decision| decision.admin_principal.as_ref())
.map(|principal| principal.user_id.clone());
match crate::maintenance::run_manual_usage_cleanup_once( match crate::maintenance::start_manual_usage_cleanup_task(
&state.app().data, std::sync::Arc::clone(&state.app().data),
older_than_days, options,
None, actor_user_id,
) )
.await .await
{ {
Ok(summary) => { Ok(task) => {
let total = summary
.body_externalized
.saturating_add(summary.legacy_body_refs_migrated)
.saturating_add(summary.body_cleaned)
.saturating_add(summary.header_cleaned)
.saturating_add(summary.keys_cleaned)
.saturating_add(summary.records_deleted);
let message = match older_than_days {
Some(days) => {
format!("请求记录手动清理完成，清理 {days} 天前的记录，影响 {total} 项")
}
None => format!("请求记录手动清理完成（按当前策略），影响 {total} 项"),
};
let payload = json!({ let payload = json!({
"message": message, "message": task.message,
"requested_older_than_days": older_than_days, "mode": options.mode.as_str(),
"summary": { "requested_older_than_days": options.requested_older_than_days,
"body_externalized": summary.body_externalized, "targets": options.targets,
"legacy_body_refs_migrated": summary.legacy_body_refs_migrated, "task": task,
"body_cleaned": summary.body_cleaned,
"header_cleaned": summary.header_cleaned,
"keys_cleaned": summary.keys_cleaned,
"records_deleted": summary.records_deleted,
},
"total_affected": total,
}); });
Ok(attach_admin_audit_response( Ok(attach_admin_audit_response(
Json(payload).into_response(), Json(payload).into_response(),
"admin_system_usage_cleanup_completed", "admin_system_usage_cleanup_started",
"manual_usage_cleanup", "manual_usage_cleanup",
"usage_cleanup", "usage_cleanup",
"global", "global",
@@ -696,12 +685,20 @@ async fn build_manual_usage_cleanup_preview_response(
Ok(value) => value, Ok(value) => value,
Err(response) => return Ok(response), Err(response) => return Ok(response),
}; };
let preview = let options = match parse_manual_usage_cleanup_query_options(
crate::maintenance::preview_manual_usage_cleanup(&state.app().data, older_than_days) request_context.query_string(),
.await older_than_days,
.map_err(|err| GatewayError::Internal(err.to_string()))?; ) {
Ok(value) => value,
Err(response) => return Ok(response),
};
let preview = crate::maintenance::preview_manual_usage_cleanup(&state.app().data, options)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?;
Ok(Json(json!({ Ok(Json(json!({
"mode": preview.mode.as_str(),
"requested_older_than_days": preview.requested_older_than_days, "requested_older_than_days": preview.requested_older_than_days,
"targets": preview.targets,
"effective_cutoffs": { "effective_cutoffs": {
"detail": preview.detail_cutoff, "detail": preview.detail_cutoff,
"compressed": preview.compressed_cutoff, "compressed": preview.compressed_cutoff,
@@ -720,12 +717,12 @@ async fn build_manual_usage_cleanup_preview_response(
fn parse_manual_usage_cleanup_request( fn parse_manual_usage_cleanup_request(
request_body: Option<&Bytes>, request_body: Option<&Bytes>,
) -> Result<Option<u32>, Response<Body>> { ) -> Result<ManualUsageCleanupOptions, Response<Body>> {
let Some(body) = request_body else { let Some(body) = request_body else {
return Ok(None); return Ok(ManualUsageCleanupOptions::policy());
}; };
if body.is_empty() { if body.is_empty() {
return Ok(None); return Ok(ManualUsageCleanupOptions::policy());
} }
let parsed: serde_json::Value = match serde_json::from_slice(body) { let parsed: serde_json::Value = match serde_json::from_slice(body) {
Ok(value) => value, Ok(value) => value,
@@ -738,45 +735,208 @@ fn parse_manual_usage_cleanup_request(
} }
}; };
let Some(object) = parsed.as_object() else { let Some(object) = parsed.as_object() else {
return Ok(None); return Err(bad_manual_cleanup_request("请求体必须为 JSON 对象"));
}; };
match object.get("older_than_days") { parse_manual_usage_cleanup_options(
None | Some(serde_json::Value::Null) => Ok(None), object.get("mode").and_then(serde_json::Value::as_str),
Some(value) => value object.get("older_than_days"),
.as_u64() object.get("targets"),
.and_then(|value| u32::try_from(value).ok()) )
.filter(|days| *days >= 1) }
.map(Some)
.ok_or_else(|| { fn parse_manual_usage_cleanup_query_options(
( query_string: Option<&str>,
http::StatusCode::BAD_REQUEST, older_than_days: Option<u32>,
Json(json!({ ) -> Result<ManualUsageCleanupOptions, Response<Body>> {
"detail": "older_than_days 必须为正整数", let mode = query_param(query_string, "mode");
})), let targets = query_param(query_string, "targets").map(serde_json::Value::String);
) let older_value =
.into_response() older_than_days.map(|days| serde_json::Value::Number(serde_json::Number::from(days)));
}), parse_manual_usage_cleanup_options(mode.as_deref(), older_value.as_ref(), targets.as_ref())
}
fn parse_manual_usage_cleanup_options(
raw_mode: Option<&str>,
older_than_days: Option<&serde_json::Value>,
targets_value: Option<&serde_json::Value>,
) -> Result<ManualUsageCleanupOptions, Response<Body>> {
let (requested_older_than_days, requested_before_now) =
parse_manual_cleanup_older_than_days(older_than_days)?;
let mode =
parse_manual_cleanup_mode(raw_mode, requested_older_than_days, requested_before_now)?;
if mode == ManualUsageCleanupMode::OlderThanDays && requested_older_than_days.is_none() {
return Err(bad_manual_cleanup_request(
"older_than_days 模式必须提供正整数天数",
));
}
if mode == ManualUsageCleanupMode::BeforeNow && requested_older_than_days.is_some() {
return Err(bad_manual_cleanup_request(
"before_now 模式不能同时提供 older_than_days",
));
}
if raw_mode.is_some() && requested_before_now && mode != ManualUsageCleanupMode::BeforeNow {
return Err(bad_manual_cleanup_request(
"older_than_days 为 0 时必须使用 before_now 模式",
));
}
if raw_mode.is_some()
&& mode == ManualUsageCleanupMode::Policy
&& requested_older_than_days.is_some()
{
return Err(bad_manual_cleanup_request(
"policy 模式不能同时提供 older_than_days",
));
}
let targets = parse_manual_cleanup_targets(targets_value, mode)?;
if !targets.any_selected() {
return Err(bad_manual_cleanup_request("至少选择一个清理范围"));
}
if mode == ManualUsageCleanupMode::BeforeNow
&& (targets.headers || targets.records || targets.expired_keys)
{
return Err(bad_manual_cleanup_request(
"清理当前时刻之前只允许选择详细请求体和压缩请求体",
));
}
Ok(ManualUsageCleanupOptions {
mode,
requested_older_than_days,
targets,
})
}
fn parse_manual_cleanup_mode(
raw_mode: Option<&str>,
requested_older_than_days: Option<u32>,
requested_before_now: bool,
) -> Result<ManualUsageCleanupMode, Response<Body>> {
let Some(raw) = raw_mode.map(str::trim).filter(|value| !value.is_empty()) else {
return Ok(if requested_before_now {
ManualUsageCleanupMode::BeforeNow
} else if requested_older_than_days.is_some() {
ManualUsageCleanupMode::OlderThanDays
} else {
ManualUsageCleanupMode::Policy
});
};
match raw {
"policy" => Ok(ManualUsageCleanupMode::Policy),
"older_than_days" => Ok(ManualUsageCleanupMode::OlderThanDays),
"before_now" => Ok(ManualUsageCleanupMode::BeforeNow),
_ => Err(bad_manual_cleanup_request(
"mode 必须为 policy、older_than_days 或 before_now",
)),
} }
} }
fn parse_older_than_days_query(query_string: Option<&str>) -> Result<Option<u32>, Response<Body>> { fn parse_manual_cleanup_older_than_days(
let Some(query) = query_string.filter(|value| !value.is_empty()) else { value: Option<&serde_json::Value>,
return Ok(None); ) -> Result<(Option<u32>, bool), Response<Body>> {
}; match value {
let value = query None | Some(serde_json::Value::Null) => Ok((None, false)),
.split('&') Some(value) => {
.filter_map(|pair| pair.split_once('=')) let Some(raw) = value.as_u64() else {
.find_map(|(key, value)| { return Err(bad_manual_cleanup_request("older_than_days 必须为非负整数"));
if key == "older_than_days" && !value.is_empty() { };
Some(value) if raw == 0 {
} else { return Ok((None, true));
None }
let days = u32::try_from(raw)
.ok()
.filter(|days| *days >= 1)
.ok_or_else(|| bad_manual_cleanup_request("older_than_days 必须为正整数"))?;
Ok((Some(days), false))
}
}
}
fn parse_manual_cleanup_targets(
value: Option<&serde_json::Value>,
mode: ManualUsageCleanupMode,
) -> Result<UsageCleanupTargets, Response<Body>> {
let Some(value) = value else {
return Ok(match mode {
ManualUsageCleanupMode::BeforeNow => UsageCleanupTargets::body_targets(),
ManualUsageCleanupMode::Policy | ManualUsageCleanupMode::OlderThanDays => {
UsageCleanupTargets::all_policy_targets()
} }
}); });
let Some(raw) = value else { };
if value.is_null() {
return Ok(match mode {
ManualUsageCleanupMode::BeforeNow => UsageCleanupTargets::body_targets(),
ManualUsageCleanupMode::Policy | ManualUsageCleanupMode::OlderThanDays => {
UsageCleanupTargets::all_policy_targets()
}
});
}
let raw_targets = match value {
serde_json::Value::Array(items) => items
.iter()
.map(|item| {
item.as_str()
.map(str::to_string)
.ok_or_else(|| bad_manual_cleanup_request("targets 必须为字符串数组"))
})
.collect::<Result<Vec<_>, _>>()?,
serde_json::Value::String(raw) => raw
.split(',')
.map(str::trim)
.filter(|item| !item.is_empty())
.map(str::to_string)
.collect(),
_ => return Err(bad_manual_cleanup_request("targets 必须为字符串数组")),
};
let mut targets = UsageCleanupTargets {
detail_body: false,
compressed_body: false,
headers: false,
records: false,
expired_keys: false,
};
for raw in raw_targets {
match raw.as_str() {
"detail_body" | "detail" | "raw_body" => targets.detail_body = true,
"compressed_body" | "compressed" => targets.compressed_body = true,
"headers" | "header" => targets.headers = true,
"records" | "log" | "logs" => targets.records = true,
"expired_keys" => targets.expired_keys = true,
"all" => targets = UsageCleanupTargets::all_policy_targets(),
_ => {
return Err(bad_manual_cleanup_request(
"targets 只能包含 detail_body、compressed_body、headers、records",
))
}
}
}
Ok(targets)
}
fn bad_manual_cleanup_request(detail: impl Into<String>) -> Response<Body> {
(
http::StatusCode::BAD_REQUEST,
Json(json!({ "detail": detail.into() })),
)
.into_response()
}
fn query_param(query_string: Option<&str>, name: &str) -> Option<String> {
let query = query_string.filter(|value| !value.is_empty())?;
form_urlencoded::parse(query.as_bytes())
.find_map(|(key, value)| (key == name && !value.is_empty()).then(|| value.into_owned()))
}
fn parse_older_than_days_query(query_string: Option<&str>) -> Result<Option<u32>, Response<Body>> {
let Some(value) = query_param(query_string, "older_than_days") else {
return Ok(None); return Ok(None);
}; };
raw.parse::<u32>() value
.parse::<u32>()
.ok() .ok()
.filter(|days| *days >= 1) .filter(|days| *days >= 1)
.map(Some) .map(Some)
@@ -790,3 +950,55 @@ fn parse_older_than_days_query(query_string: Option<&str>) -> Result<Option<u32>
.into_response() .into_response()
}) })
} }
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn manual_cleanup_request_defaults_to_policy_targets() {
let options = parse_manual_usage_cleanup_request(None).expect("default request is valid");
assert_eq!(options.mode, ManualUsageCleanupMode::Policy);
assert_eq!(options.requested_older_than_days, None);
assert_eq!(options.targets, UsageCleanupTargets::all_policy_targets());
}
#[test]
fn manual_cleanup_request_treats_zero_days_as_before_now_body_only() {
let body = Bytes::from_static(br#"{"older_than_days":0}"#);
let options =
parse_manual_usage_cleanup_request(Some(&body)).expect("before-now request is valid");
assert_eq!(options.mode, ManualUsageCleanupMode::BeforeNow);
assert_eq!(options.requested_older_than_days, None);
assert_eq!(options.targets, UsageCleanupTargets::body_targets());
}
#[test]
fn manual_cleanup_request_rejects_before_now_headers() {
let body = Bytes::from_static(br#"{"mode":"before_now","targets":["headers"]}"#);
assert!(parse_manual_usage_cleanup_request(Some(&body)).is_err());
}
#[test]
fn manual_cleanup_preview_query_decodes_comma_separated_targets() {
let options = parse_manual_usage_cleanup_query_options(
Some("mode=before_now&targets=detail_body%2Ccompressed_body"),
None,
)
.expect("encoded targets query is valid");
assert_eq!(options.mode, ManualUsageCleanupMode::BeforeNow);
assert_eq!(options.targets, UsageCleanupTargets::body_targets());
}
#[test]
fn manual_cleanup_request_rejects_non_object_body() {
let body = Bytes::from_static(br#"[]"#);
assert!(parse_manual_usage_cleanup_request(Some(&body)).is_err());
}
}

7
apps/aether-gateway/src/maintenance/mod.rs
View File

@@ -21,9 +21,10 @@ pub(crate) use runtime::{
spawn_stats_aggregation_worker, spawn_stats_hourly_aggregation_worker, spawn_stats_aggregation_worker, spawn_stats_hourly_aggregation_worker,
spawn_usage_cleanup_worker, spawn_wallet_daily_usage_aggregation_worker, spawn_usage_cleanup_worker, spawn_wallet_daily_usage_aggregation_worker,
start_admin_request_body_cleanup_task, start_admin_system_purge_task, start_admin_request_body_cleanup_task, start_admin_system_purge_task,
start_proxy_upgrade_rollout, AccountSelfCheckRunSummary, AdminCleanupRunRecord, start_manual_usage_cleanup_task, start_proxy_upgrade_rollout, AccountSelfCheckRunSummary,
AdminCleanupTaskKind, AdminStatsRebuildSummary, AdminSystemCleanupSummary, AdminCleanupRunRecord, AdminCleanupTaskKind, AdminStatsRebuildSummary,
ManualUsageCleanupError, OAuthTokenRefreshRunSummary, PoolQuotaProbeRunSummary, AdminSystemCleanupSummary, ManualUsageCleanupError, ManualUsageCleanupMode,
ManualUsageCleanupOptions, OAuthTokenRefreshRunSummary, PoolQuotaProbeRunSummary,
PoolQuotaProbeWorkerConfig, ProviderCheckinRunSummary, ProxyUpgradeRolloutCancelSummary, PoolQuotaProbeWorkerConfig, ProviderCheckinRunSummary, ProxyUpgradeRolloutCancelSummary,
ProxyUpgradeRolloutConflictClearSummary, ProxyUpgradeRolloutNodeActionSummary, ProxyUpgradeRolloutConflictClearSummary, ProxyUpgradeRolloutNodeActionSummary,
ProxyUpgradeRolloutProbeConfig, ProxyUpgradeRolloutSkippedRestoreSummary, ProxyUpgradeRolloutProbeConfig, ProxyUpgradeRolloutSkippedRestoreSummary,

14
apps/aether-gateway/src/maintenance/runtime.rs
View File

@@ -61,9 +61,9 @@ pub(crate) use aether_data_contracts::repository::usage::{
}; };
use audit_cleanup::*; use audit_cleanup::*;
pub(crate) use cleanup_runs::{ pub(crate) use cleanup_runs::{
list_admin_cleanup_run_records, record_completed_cleanup_run, record_failed_cleanup_run, list_admin_cleanup_run_records, record_admin_cleanup_run, record_completed_cleanup_run,
start_admin_request_body_cleanup_task, start_admin_system_purge_task, AdminCleanupRunRecord, record_failed_cleanup_run, start_admin_request_body_cleanup_task,
AdminCleanupTaskKind, USAGE_CLEANUP_KIND, start_admin_system_purge_task, AdminCleanupRunRecord, AdminCleanupTaskKind, USAGE_CLEANUP_KIND,
}; };
use config::*; use config::*;
use db_maintenance::*; use db_maintenance::*;
@@ -98,12 +98,16 @@ pub(crate) use proxy_upgrade_rollout::{
}; };
use request_candidate_cleanup::*; use request_candidate_cleanup::*;
use runners::*; use runners::*;
pub(crate) use runners::{run_manual_usage_cleanup_once, ManualUsageCleanupError}; pub(crate) use runners::{
run_manual_usage_cleanup_once, start_manual_usage_cleanup_task, ManualUsageCleanupError,
};
use schedule::*; use schedule::*;
use stats_daily::*; use stats_daily::*;
use stats_hourly::*; use stats_hourly::*;
pub(crate) use usage_cleanup::preview_manual_usage_cleanup;
use usage_cleanup::*; use usage_cleanup::*;
pub(crate) use usage_cleanup::{
preview_manual_usage_cleanup, ManualUsageCleanupMode, ManualUsageCleanupOptions,
};
use wallet_daily_usage::*; use wallet_daily_usage::*;
pub(crate) use workers::*; pub(crate) use workers::*;

9
apps/aether-gateway/src/maintenance/runtime/cleanup_runs.rs
View File

@@ -489,7 +489,7 @@ fn request_body_cleanup_record(
} }
} }
async fn record_cleanup_run( pub(crate) async fn record_admin_cleanup_run(
data: &GatewayDataState, data: &GatewayDataState,
record: AdminCleanupRunRecord, record: AdminCleanupRunRecord,
) -> Result<(), DataLayerError> { ) -> Result<(), DataLayerError> {
@@ -509,6 +509,13 @@ async fn record_cleanup_run(
Ok(()) Ok(())
} }
async fn record_cleanup_run(
data: &GatewayDataState,
record: AdminCleanupRunRecord,
) -> Result<(), DataLayerError> {
record_admin_cleanup_run(data, record).await
}
fn parse_cleanup_run_records(value: Value) -> Vec<AdminCleanupRunRecord> { fn parse_cleanup_run_records(value: Value) -> Vec<AdminCleanupRunRecord> {
value value
.as_array() .as_array()

229
apps/aether-gateway/src/maintenance/runtime/runners.rs
View File

@@ -1,5 +1,6 @@
use aether_data_contracts::DataLayerError; use aether_data_contracts::DataLayerError;
use serde_json::json; use serde_json::json;
use std::sync::Arc;
use std::time::Instant; use std::time::Instant;
use tracing::{info, warn}; use tracing::{info, warn};
@@ -11,11 +12,12 @@ use super::{
cleanup_expired_gemini_file_mappings_once, cleanup_proxy_node_metrics_once, cleanup_expired_gemini_file_mappings_once, cleanup_proxy_node_metrics_once,
cleanup_request_candidates_once, cleanup_stale_pending_requests_once, cleanup_request_candidates_once, cleanup_stale_pending_requests_once,
cleanup_stale_proxy_nodes_once, collect_proxy_upgrade_rollout_probes, now_unix_secs, cleanup_stale_proxy_nodes_once, collect_proxy_upgrade_rollout_probes, now_unix_secs,
perform_db_maintenance_once, perform_provider_checkin_once, perform_stats_aggregation_once, perform_db_maintenance_once, perform_manual_usage_cleanup_once, perform_provider_checkin_once,
perform_stats_hourly_aggregation_once, perform_usage_cleanup_once, perform_stats_aggregation_once, perform_stats_hourly_aggregation_once,
perform_usage_cleanup_once_with_override, perform_wallet_daily_usage_aggregation_once, perform_usage_cleanup_once, perform_wallet_daily_usage_aggregation_once,
record_completed_cleanup_run, record_failed_cleanup_run, record_proxy_upgrade_traffic_success, record_admin_cleanup_run, record_completed_cleanup_run, record_failed_cleanup_run,
summarize_database_pool, record_proxy_upgrade_traffic_success, summarize_database_pool, AdminCleanupRunRecord,
ManualUsageCleanupOptions,
}; };
pub(super) async fn run_audit_cleanup_once(data: &GatewayDataState) -> Result<(), DataLayerError> { pub(super) async fn run_audit_cleanup_once(data: &GatewayDataState) -> Result<(), DataLayerError> {
@@ -299,15 +301,14 @@ pub(super) async fn run_usage_cleanup_once(data: &GatewayDataState) -> Result<()
Ok(()) Ok(())
} }
pub(crate) async fn run_manual_usage_cleanup_once( pub(crate) async fn start_manual_usage_cleanup_task(
data: &GatewayDataState, data: Arc<GatewayDataState>,
override_older_than_days: Option<u32>, options: ManualUsageCleanupOptions,
actor_user_id: Option<String>, actor_user_id: Option<String>,
) -> Result<aether_data_contracts::repository::usage::UsageCleanupSummary, ManualUsageCleanupError> ) -> Result<AdminCleanupRunRecord, ManualUsageCleanupError> {
{
use super::{list_admin_cleanup_run_records, USAGE_CLEANUP_KIND}; use super::{list_admin_cleanup_run_records, USAGE_CLEANUP_KIND};
let existing = list_admin_cleanup_run_records(data) let existing = list_admin_cleanup_run_records(&data)
.await .await
.map_err(ManualUsageCleanupError::DataLayer)?; .map_err(ManualUsageCleanupError::DataLayer)?;
if existing if existing
@@ -318,10 +319,42 @@ pub(crate) async fn run_manual_usage_cleanup_once(
} }
let started_at_unix_secs = now_unix_secs(); let started_at_unix_secs = now_unix_secs();
let record = AdminCleanupRunRecord {
id: uuid::Uuid::new_v4().to_string(),
kind: USAGE_CLEANUP_KIND.to_string(),
trigger: "manual".to_string(),
status: "processing".to_string(),
message: manual_usage_cleanup_start_message(options),
started_at_unix_secs,
completed_at_unix_secs: None,
duration_ms: None,
summary: manual_usage_cleanup_progress_summary(options, 0, None, actor_user_id.as_deref()),
error: None,
};
record_admin_cleanup_run(&data, record.clone())
.await
.map_err(ManualUsageCleanupError::DataLayer)?;
tokio::spawn(run_manual_usage_cleanup_task(
data,
record.clone(),
options,
actor_user_id,
));
Ok(record)
}
pub(crate) async fn run_manual_usage_cleanup_once(
data: &GatewayDataState,
options: ManualUsageCleanupOptions,
actor_user_id: Option<String>,
) -> Result<aether_data_contracts::repository::usage::UsageCleanupSummary, ManualUsageCleanupError>
{
use super::USAGE_CLEANUP_KIND;
let started_at = Instant::now(); let started_at = Instant::now();
let override_duration = let started_at_unix_secs = now_unix_secs();
override_older_than_days.map(|days| chrono::Duration::days(i64::from(days))); let summary = match perform_manual_usage_cleanup_once(data, options).await {
let summary = match perform_usage_cleanup_once_with_override(data, override_duration).await {
Ok(summary) => summary, Ok(summary) => summary,
Err(err) => { Err(err) => {
record_failed_cleanup_run( record_failed_cleanup_run(
@@ -336,17 +369,8 @@ pub(crate) async fn run_manual_usage_cleanup_once(
return Err(ManualUsageCleanupError::DataLayer(err)); return Err(ManualUsageCleanupError::DataLayer(err));
} }
}; };
let total = summary let total = usage_cleanup_total(summary);
.body_externalized let message = manual_usage_cleanup_completed_message(options, total);
.saturating_add(summary.legacy_body_refs_migrated)
.saturating_add(summary.body_cleaned)
.saturating_add(summary.header_cleaned)
.saturating_add(summary.keys_cleaned)
.saturating_add(summary.records_deleted);
let message = match override_older_than_days {
Some(days) => format!("请求记录手动清理完成，清理 {days} 天前的记录，影响 {total} 项"),
None => format!("请求记录手动清理完成（按当前策略），影响 {total} 项"),
};
record_completed_cleanup_run( record_completed_cleanup_run(
data, data,
USAGE_CLEANUP_KIND, USAGE_CLEANUP_KIND,
@@ -360,7 +384,9 @@ pub(crate) async fn run_manual_usage_cleanup_once(
"header_cleaned": summary.header_cleaned, "header_cleaned": summary.header_cleaned,
"keys_cleaned": summary.keys_cleaned, "keys_cleaned": summary.keys_cleaned,
"records_deleted": summary.records_deleted, "records_deleted": summary.records_deleted,
"requested_older_than_days": override_older_than_days, "mode": options.mode.as_str(),
"requested_older_than_days": options.requested_older_than_days,
"targets": options.targets,
"actor_user_id": actor_user_id, "actor_user_id": actor_user_id,
}), }),
message, message,
@@ -371,7 +397,8 @@ pub(crate) async fn run_manual_usage_cleanup_once(
log_type = "ops", log_type = "ops",
worker = "usage_cleanup", worker = "usage_cleanup",
trigger = "manual", trigger = "manual",
requested_older_than_days = override_older_than_days, mode = options.mode.as_str(),
requested_older_than_days = options.requested_older_than_days,
actor_user_id = actor_user_id.as_deref(), actor_user_id = actor_user_id.as_deref(),
total_affected = total, total_affected = total,
"gateway finished manual usage cleanup" "gateway finished manual usage cleanup"
@@ -379,6 +406,154 @@ pub(crate) async fn run_manual_usage_cleanup_once(
Ok(summary) Ok(summary)
} }
async fn run_manual_usage_cleanup_task(
data: Arc<GatewayDataState>,
initial_record: AdminCleanupRunRecord,
options: ManualUsageCleanupOptions,
actor_user_id: Option<String>,
) {
let started_at = Instant::now();
match perform_manual_usage_cleanup_once(&data, options).await {
Ok(summary) => {
let total = usage_cleanup_total(summary);
let record = AdminCleanupRunRecord {
id: initial_record.id,
kind: initial_record.kind,
trigger: initial_record.trigger,
status: "completed".to_string(),
message: manual_usage_cleanup_completed_message(options, total),
started_at_unix_secs: initial_record.started_at_unix_secs,
completed_at_unix_secs: Some(now_unix_secs()),
duration_ms: Some(
started_at
.elapsed()
.as_millis()
.try_into()
.unwrap_or(u64::MAX),
),
summary: manual_usage_cleanup_progress_summary(
options,
100,
Some(summary),
actor_user_id.as_deref(),
),
error: None,
};
if let Err(err) = record_admin_cleanup_run(&data, record).await {
warn!(error = %err, "failed to record manual usage cleanup completion");
}
info!(
event_name = "usage_cleanup_manual_completed",
log_type = "ops",
worker = "usage_cleanup",
trigger = "manual",
mode = options.mode.as_str(),
requested_older_than_days = options.requested_older_than_days,
actor_user_id = actor_user_id.as_deref(),
total_affected = total,
"gateway finished manual usage cleanup task"
);
}
Err(err) => {
let record = AdminCleanupRunRecord {
id: initial_record.id,
kind: initial_record.kind,
trigger: initial_record.trigger,
status: "failed".to_string(),
message: "请求记录手动清理失败".to_string(),
started_at_unix_secs: initial_record.started_at_unix_secs,
completed_at_unix_secs: Some(now_unix_secs()),
duration_ms: Some(
started_at
.elapsed()
.as_millis()
.try_into()
.unwrap_or(u64::MAX),
),
summary: manual_usage_cleanup_progress_summary(
options,
100,
None,
actor_user_id.as_deref(),
),
error: Some(err.to_string()),
};
if let Err(record_err) = record_admin_cleanup_run(&data, record).await {
warn!(error = %record_err, "failed to record manual usage cleanup failure");
}
warn!(error = %err, "manual usage cleanup task failed");
}
}
}
fn usage_cleanup_total(
summary: aether_data_contracts::repository::usage::UsageCleanupSummary,
) -> usize {
summary
.body_externalized
.saturating_add(summary.legacy_body_refs_migrated)
.saturating_add(summary.body_cleaned)
.saturating_add(summary.header_cleaned)
.saturating_add(summary.keys_cleaned)
.saturating_add(summary.records_deleted)
}
fn manual_usage_cleanup_start_message(options: ManualUsageCleanupOptions) -> String {
match options.mode {
super::ManualUsageCleanupMode::BeforeNow => {
"请求记录手动清理已开始，清理当前时刻之前的已选请求体".to_string()
}
super::ManualUsageCleanupMode::OlderThanDays => format!(
"请求记录手动清理已开始，清理 {} 天前的已选内容",
options.requested_older_than_days.unwrap_or_default()
),
super::ManualUsageCleanupMode::Policy => {
"请求记录手动清理已开始，按当前策略清理已选内容".to_string()
}
}
}
fn manual_usage_cleanup_completed_message(
options: ManualUsageCleanupOptions,
total: usize,
) -> String {
match options.mode {
super::ManualUsageCleanupMode::BeforeNow => {
format!("请求记录手动清理完成，已清理当前时刻之前的已选请求体，影响 {total} 项")
}
super::ManualUsageCleanupMode::OlderThanDays => format!(
"请求记录手动清理完成，清理 {} 天前的已选内容，影响 {total} 项",
options.requested_older_than_days.unwrap_or_default()
),
super::ManualUsageCleanupMode::Policy => {
format!("请求记录手动清理完成（按当前策略），影响 {total} 项")
}
}
}
fn manual_usage_cleanup_progress_summary(
options: ManualUsageCleanupOptions,
progress_percent: u8,
summary: Option<aether_data_contracts::repository::usage::UsageCleanupSummary>,
actor_user_id: Option<&str>,
) -> serde_json::Value {
let summary = summary.unwrap_or_default();
json!({
"mode": options.mode.as_str(),
"requested_older_than_days": options.requested_older_than_days,
"targets": options.targets,
"progress_percent": progress_percent,
"body_externalized": summary.body_externalized,
"legacy_body_refs_migrated": summary.legacy_body_refs_migrated,
"body_cleaned": summary.body_cleaned,
"header_cleaned": summary.header_cleaned,
"keys_cleaned": summary.keys_cleaned,
"records_deleted": summary.records_deleted,
"total": usage_cleanup_total(summary),
"actor_user_id": actor_user_id,
})
}
#[derive(Debug)] #[derive(Debug)]
pub(crate) enum ManualUsageCleanupError { pub(crate) enum ManualUsageCleanupError {
AlreadyRunning, AlreadyRunning,

35
apps/aether-gateway/src/maintenance/runtime/tests.rs
View File

@@ -28,12 +28,12 @@ use super::{
spawn_stats_hourly_aggregation_worker, spawn_usage_cleanup_worker, spawn_stats_hourly_aggregation_worker, spawn_usage_cleanup_worker,
spawn_wallet_daily_usage_aggregation_worker, start_proxy_upgrade_rollout, spawn_wallet_daily_usage_aggregation_worker, start_proxy_upgrade_rollout,
stats_aggregation_target_day, stats_hourly_aggregation_target_hour, summarize_database_pool, stats_aggregation_target_day, stats_hourly_aggregation_target_hour, summarize_database_pool,
usage_cleanup_settings, usage_cleanup_window, usage_cleanup_window_with_override, usage_cleanup_settings, usage_cleanup_window, usage_cleanup_window_for_mode,
wallet_daily_usage_aggregation_target, AppState, DbMaintenanceRunSummary, usage_cleanup_window_with_override, wallet_daily_usage_aggregation_target, AppState,
FailedPendingUsageRow, GatewayDataState, ProxyNodeMetricsCleanupSettings, DbMaintenanceRunSummary, FailedPendingUsageRow, GatewayDataState, ManualUsageCleanupMode,
ProxyUpgradeRolloutProbeConfig, StalePendingUsageRow, UsageCleanupSettings, USAGE_CLEANUP_HOUR, ProxyNodeMetricsCleanupSettings, ProxyUpgradeRolloutProbeConfig, StalePendingUsageRow,
USAGE_CLEANUP_MINUTE, WALLET_DAILY_USAGE_AGGREGATION_HOUR, UsageCleanupSettings, USAGE_CLEANUP_HOUR, USAGE_CLEANUP_MINUTE,
WALLET_DAILY_USAGE_AGGREGATION_MINUTE, WALLET_DAILY_USAGE_AGGREGATION_HOUR, WALLET_DAILY_USAGE_AGGREGATION_MINUTE,
}; };
#[tokio::test] #[tokio::test]
@@ -986,6 +986,29 @@ fn usage_cleanup_window_with_override_is_always_non_aggressive() {
assert_eq!(passthrough, policy); assert_eq!(passthrough, policy);
} }
#[test]
fn usage_cleanup_before_now_window_uses_current_timestamp_only() {
let now_utc = "2026-03-18T03:00:00Z"
.parse::<DateTime<Utc>>()
.expect("timestamp should parse");
let settings = UsageCleanupSettings {
detail_retention_days: 7,
compressed_retention_days: 30,
header_retention_days: 90,
log_retention_days: 365,
batch_size: 123,
auto_delete_expired_keys: false,
};
let window =
usage_cleanup_window_for_mode(now_utc, settings, ManualUsageCleanupMode::BeforeNow, None);
assert_eq!(window.detail_cutoff, now_utc);
assert_eq!(window.compressed_cutoff, now_utc);
assert_eq!(window.header_cutoff, now_utc);
assert_eq!(window.log_cutoff, now_utc);
}
#[tokio::test] #[tokio::test]
async fn summarize_database_pool_uses_busy_connections_for_usage_rate() { async fn summarize_database_pool_uses_busy_connections_for_usage_rate() {
let data = GatewayDataState::from_config(crate::data::GatewayDataConfig::from_postgres_config( let data = GatewayDataState::from_config(crate::data::GatewayDataConfig::from_postgres_config(

146
apps/aether-gateway/src/maintenance/runtime/usage_cleanup.rs
View File

@@ -1,4 +1,6 @@
use aether_data_contracts::repository::usage::{UsageCleanupSummary, UsageCleanupWindow}; use aether_data_contracts::repository::usage::{
UsageCleanupExecutionMode, UsageCleanupSummary, UsageCleanupTargets, UsageCleanupWindow,
};
use aether_data_contracts::DataLayerError; use aether_data_contracts::DataLayerError;
use chrono::Utc; use chrono::Utc;
@@ -15,6 +17,8 @@ pub(crate) struct ManualUsageCleanupPreview {
pub compressed_cutoff: chrono::DateTime<Utc>, pub compressed_cutoff: chrono::DateTime<Utc>,
pub header_cutoff: chrono::DateTime<Utc>, pub header_cutoff: chrono::DateTime<Utc>,
pub log_cutoff: chrono::DateTime<Utc>, pub log_cutoff: chrono::DateTime<Utc>,
pub mode: ManualUsageCleanupMode,
pub targets: UsageCleanupTargets,
pub requested_older_than_days: Option<u32>, pub requested_older_than_days: Option<u32>,
pub detail_count: u64, pub detail_count: u64,
pub compressed_count: u64, pub compressed_count: u64,
@@ -22,49 +26,131 @@ pub(crate) struct ManualUsageCleanupPreview {
pub log_count: u64, pub log_count: u64,
} }
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum ManualUsageCleanupMode {
Policy,
OlderThanDays,
BeforeNow,
}
impl ManualUsageCleanupMode {
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::Policy => "policy",
Self::OlderThanDays => "older_than_days",
Self::BeforeNow => "before_now",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct ManualUsageCleanupOptions {
pub(crate) mode: ManualUsageCleanupMode,
pub(crate) requested_older_than_days: Option<u32>,
pub(crate) targets: UsageCleanupTargets,
}
impl ManualUsageCleanupOptions {
pub(crate) const fn policy() -> Self {
Self {
mode: ManualUsageCleanupMode::Policy,
requested_older_than_days: None,
targets: UsageCleanupTargets::all_policy_targets(),
}
}
}
pub(super) async fn perform_usage_cleanup_once( pub(super) async fn perform_usage_cleanup_once(
data: &GatewayDataState, data: &GatewayDataState,
) -> Result<UsageCleanupSummary, DataLayerError> { ) -> Result<UsageCleanupSummary, DataLayerError> {
perform_usage_cleanup_once_with_override(data, None).await perform_usage_cleanup_once_with_override(data, None, true).await
} }
pub(super) async fn perform_usage_cleanup_once_with_override( pub(super) async fn perform_usage_cleanup_once_with_override(
data: &GatewayDataState, data: &GatewayDataState,
override_older_than: Option<chrono::Duration>, override_older_than: Option<chrono::Duration>,
respect_auto_enabled: bool,
) -> Result<UsageCleanupSummary, DataLayerError> {
let options = ManualUsageCleanupOptions {
mode: if override_older_than.is_some() {
ManualUsageCleanupMode::OlderThanDays
} else {
ManualUsageCleanupMode::Policy
},
requested_older_than_days: None,
targets: UsageCleanupTargets::all_policy_targets(),
};
perform_usage_cleanup_once_with_options(
data,
options,
override_older_than,
respect_auto_enabled,
)
.await
}
pub(super) async fn perform_manual_usage_cleanup_once(
data: &GatewayDataState,
options: ManualUsageCleanupOptions,
) -> Result<UsageCleanupSummary, DataLayerError> {
let override_duration = options
.requested_older_than_days
.map(|days| chrono::Duration::days(i64::from(days)));
perform_usage_cleanup_once_with_options(data, options, override_duration, false).await
}
async fn perform_usage_cleanup_once_with_options(
data: &GatewayDataState,
options: ManualUsageCleanupOptions,
override_older_than: Option<chrono::Duration>,
respect_auto_enabled: bool,
) -> Result<UsageCleanupSummary, DataLayerError> { ) -> Result<UsageCleanupSummary, DataLayerError> {
if !data.has_usage_writer() { if !data.has_usage_writer() {
return Ok(UsageCleanupSummary::default()); return Ok(UsageCleanupSummary::default());
} }
if override_older_than.is_none() if respect_auto_enabled
&& override_older_than.is_none()
&& !system_config_bool(data, "enable_auto_cleanup", true).await? && !system_config_bool(data, "enable_auto_cleanup", true).await?
{ {
return Ok(UsageCleanupSummary::default()); return Ok(UsageCleanupSummary::default());
} }
let window = compute_usage_cleanup_window(data, override_older_than).await?; let window = compute_usage_cleanup_window(data, options.mode, override_older_than).await?;
let settings = usage_cleanup_settings(data).await?; let settings = usage_cleanup_settings(data).await?;
data.cleanup_usage( data.cleanup_usage(
&window, &window,
settings.batch_size, settings.batch_size,
settings.auto_delete_expired_keys, settings.auto_delete_expired_keys,
options.targets,
cleanup_execution_mode(options.mode),
) )
.await .await
} }
pub(crate) async fn preview_manual_usage_cleanup( pub(crate) async fn preview_manual_usage_cleanup(
data: &GatewayDataState, data: &GatewayDataState,
override_older_than_days: Option<u32>, options: ManualUsageCleanupOptions,
) -> Result<ManualUsageCleanupPreview, DataLayerError> { ) -> Result<ManualUsageCleanupPreview, DataLayerError> {
let override_duration = let override_duration = options
override_older_than_days.map(|days| chrono::Duration::days(i64::from(days))); .requested_older_than_days
let window = compute_usage_cleanup_window(data, override_duration).await?; .map(|days| chrono::Duration::days(i64::from(days)));
let counts = data.preview_usage_cleanup(&window).await?; let window = compute_usage_cleanup_window(data, options.mode, override_duration).await?;
let counts = data
.preview_usage_cleanup(
&window,
options.targets,
cleanup_execution_mode(options.mode),
)
.await?;
Ok(ManualUsageCleanupPreview { Ok(ManualUsageCleanupPreview {
detail_cutoff: window.detail_cutoff, detail_cutoff: window.detail_cutoff,
compressed_cutoff: window.compressed_cutoff, compressed_cutoff: window.compressed_cutoff,
header_cutoff: window.header_cutoff, header_cutoff: window.header_cutoff,
log_cutoff: window.log_cutoff, log_cutoff: window.log_cutoff,
requested_older_than_days: override_older_than_days, mode: options.mode,
targets: options.targets,
requested_older_than_days: options.requested_older_than_days,
detail_count: counts.detail, detail_count: counts.detail,
compressed_count: counts.compressed, compressed_count: counts.compressed,
header_count: counts.header, header_count: counts.header,
@@ -74,11 +160,43 @@ pub(crate) async fn preview_manual_usage_cleanup(
async fn compute_usage_cleanup_window( async fn compute_usage_cleanup_window(
data: &GatewayDataState, data: &GatewayDataState,
mode: ManualUsageCleanupMode,
override_older_than: Option<chrono::Duration>, override_older_than: Option<chrono::Duration>,
) -> Result<UsageCleanupWindow, DataLayerError> { ) -> Result<UsageCleanupWindow, DataLayerError> {
let settings = usage_cleanup_settings(data).await?; let settings = usage_cleanup_settings(data).await?;
Ok(match override_older_than { Ok(usage_cleanup_window_for_mode(
Some(duration) => usage_cleanup_window_with_override(Utc::now(), settings, Some(duration)), Utc::now(),
None => usage_cleanup_window(Utc::now(), settings), settings,
}) mode,
override_older_than,
))
}
pub(super) fn usage_cleanup_window_for_mode(
now_utc: chrono::DateTime<Utc>,
settings: super::UsageCleanupSettings,
mode: ManualUsageCleanupMode,
override_older_than: Option<chrono::Duration>,
) -> UsageCleanupWindow {
match mode {
ManualUsageCleanupMode::Policy => usage_cleanup_window(now_utc, settings),
ManualUsageCleanupMode::OlderThanDays => {
usage_cleanup_window_with_override(now_utc, settings, override_older_than)
}
ManualUsageCleanupMode::BeforeNow => UsageCleanupWindow {
detail_cutoff: now_utc,
compressed_cutoff: now_utc,
header_cutoff: now_utc,
log_cutoff: now_utc,
},
}
}
fn cleanup_execution_mode(mode: ManualUsageCleanupMode) -> UsageCleanupExecutionMode {
match mode {
ManualUsageCleanupMode::BeforeNow => UsageCleanupExecutionMode::BeforeNowBodyFields,
ManualUsageCleanupMode::Policy | ManualUsageCleanupMode::OlderThanDays => {
UsageCleanupExecutionMode::Policy
}
}
} }

14
crates/aether-data-contracts/src/repository/usage/mod.rs
View File

@@ -18,12 +18,12 @@ pub use types::{
UsageAuditListQuery, UsageAuditSummaryQuery, UsageBodyCaptureResult, UsageBodyCaptureState, UsageAuditListQuery, UsageAuditSummaryQuery, UsageBodyCaptureResult, UsageBodyCaptureState,
UsageBodyCaptureStorage, UsageBodyField, UsageBreakdownGroupBy, UsageBreakdownSummaryQuery, UsageBodyCaptureStorage, UsageBodyField, UsageBreakdownGroupBy, UsageBreakdownSummaryQuery,
UsageCacheAffinityHitSummaryQuery, UsageCacheAffinityIntervalGroupBy, UsageCacheAffinityHitSummaryQuery, UsageCacheAffinityIntervalGroupBy,
UsageCacheAffinityIntervalQuery, UsageCacheHitSummaryQuery, UsageCleanupPreviewCounts, UsageCacheAffinityIntervalQuery, UsageCacheHitSummaryQuery, UsageCleanupExecutionMode,
UsageCleanupSummary, UsageCleanupWindow, UsageCostSavingsSummaryQuery, UsageDailyHeatmapQuery, UsageCleanupPreviewCounts, UsageCleanupSummary, UsageCleanupTargets, UsageCleanupWindow,
UsageDashboardDailyBreakdownQuery, UsageDashboardProviderCountsQuery, UsageCostSavingsSummaryQuery, UsageDailyHeatmapQuery, UsageDashboardDailyBreakdownQuery,
UsageDashboardSummaryQuery, UsageErrorDistributionQuery, UsageLeaderboardGroupBy, UsageDashboardProviderCountsQuery, UsageDashboardSummaryQuery, UsageErrorDistributionQuery,
UsageLeaderboardQuery, UsageMonitoringErrorCountQuery, UsageMonitoringErrorListQuery, UsageLeaderboardGroupBy, UsageLeaderboardQuery, UsageMonitoringErrorCountQuery,
UsagePerformancePercentilesQuery, UsageProviderPerformanceQuery, UsageReadRepository, UsageMonitoringErrorListQuery, UsagePerformancePercentilesQuery, UsageProviderPerformanceQuery,
UsageRepository, UsageSettledCostSummaryQuery, UsageTimeSeriesGranularity, UsageReadRepository, UsageRepository, UsageSettledCostSummaryQuery, UsageTimeSeriesGranularity,
UsageTimeSeriesQuery, UsageWriteRepository, UsageTimeSeriesQuery, UsageWriteRepository,
}; };

60
crates/aether-data-contracts/src/repository/usage/types.rs
View File

@@ -1688,16 +1688,20 @@ pub trait UsageWriteRepository: Send + Sync {
window: &UsageCleanupWindow, window: &UsageCleanupWindow,
batch_size: usize, batch_size: usize,
auto_delete_expired_keys: bool, auto_delete_expired_keys: bool,
targets: UsageCleanupTargets,
mode: UsageCleanupExecutionMode,
) -> Result<UsageCleanupSummary, crate::DataLayerError> { ) -> Result<UsageCleanupSummary, crate::DataLayerError> {
let _ = (window, batch_size, auto_delete_expired_keys); let _ = (window, batch_size, auto_delete_expired_keys, targets, mode);
Ok(UsageCleanupSummary::default()) Ok(UsageCleanupSummary::default())
} }
async fn preview_usage_cleanup( async fn preview_usage_cleanup(
&self, &self,
window: &UsageCleanupWindow, window: &UsageCleanupWindow,
targets: UsageCleanupTargets,
mode: UsageCleanupExecutionMode,
) -> Result<UsageCleanupPreviewCounts, crate::DataLayerError> { ) -> Result<UsageCleanupPreviewCounts, crate::DataLayerError> {
let _ = window; let _ = (window, targets, mode);
Ok(UsageCleanupPreviewCounts::default()) Ok(UsageCleanupPreviewCounts::default())
} }
} }
@@ -1730,6 +1734,58 @@ pub struct UsageCleanupWindow {
pub log_cutoff: DateTime<Utc>, pub log_cutoff: DateTime<Utc>,
} }
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct UsageCleanupTargets {
pub detail_body: bool,
pub compressed_body: bool,
pub headers: bool,
pub records: bool,
pub expired_keys: bool,
}
impl UsageCleanupTargets {
pub const fn all_policy_targets() -> Self {
Self {
detail_body: true,
compressed_body: true,
headers: true,
records: true,
expired_keys: true,
}
}
pub const fn body_targets() -> Self {
Self {
detail_body: true,
compressed_body: true,
headers: false,
records: false,
expired_keys: false,
}
}
pub const fn any_selected(self) -> bool {
self.detail_body
|| self.compressed_body
|| self.headers
|| self.records
|| self.expired_keys
}
}
impl Default for UsageCleanupTargets {
fn default() -> Self {
Self::all_policy_targets()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)]
pub enum UsageCleanupExecutionMode {
#[default]
Policy,
BeforeNowBodyFields,
}
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct UsageCleanupPreviewCounts { pub struct UsageCleanupPreviewCounts {
pub detail: u64, pub detail: u64,

599
crates/aether-data/src/repository/usage/postgres/cleanup.rs
View File

@@ -1,8 +1,8 @@
use std::io::Write; use std::io::Write;
use aether_data_contracts::repository::usage::{ use aether_data_contracts::repository::usage::{
parse_usage_body_ref, usage_body_ref, UsageBodyField, UsageCleanupPreviewCounts, parse_usage_body_ref, usage_body_ref, UsageBodyField, UsageCleanupExecutionMode,
UsageCleanupSummary, UsageCleanupWindow, UsageCleanupPreviewCounts, UsageCleanupSummary, UsageCleanupTargets, UsageCleanupWindow,
}; };
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use flate2::{write::GzEncoder, Compression}; use flate2::{write::GzEncoder, Compression};
@@ -129,6 +129,64 @@ WHERE created_at < $1
ORDER BY created_at ASC, id ASC ORDER BY created_at ASC, id ASC
LIMIT $3 LIMIT $3
"#; "#;
const SELECT_USAGE_RAW_BODY_BATCH_SQL: &str = r#"
SELECT id, request_id
FROM usage
WHERE created_at < $1
AND (
request_body IS NOT NULL
OR response_body IS NOT NULL
OR provider_request_body IS NOT NULL
OR client_response_body IS NOT NULL
)
ORDER BY created_at ASC, id ASC
LIMIT $2
"#;
const CLEAR_USAGE_RAW_BODY_FIELDS_SQL: &str = r#"
UPDATE usage
SET request_body = NULL,
response_body = NULL,
provider_request_body = NULL,
client_response_body = NULL
WHERE id = ANY($1)
"#;
const SELECT_USAGE_COMPRESSED_BODY_BATCH_SQL: &str = r#"
SELECT id, request_id
FROM usage
WHERE created_at < $1
AND (
request_body_compressed IS NOT NULL
OR response_body_compressed IS NOT NULL
OR provider_request_body_compressed IS NOT NULL
OR client_response_body_compressed IS NOT NULL
OR EXISTS (
SELECT 1
FROM usage_body_blobs
WHERE usage_body_blobs.request_id = usage.request_id
)
OR EXISTS (
SELECT 1
FROM usage_http_audits
WHERE usage_http_audits.request_id = usage.request_id
AND (
usage_http_audits.request_body_ref IS NOT NULL
OR usage_http_audits.provider_request_body_ref IS NOT NULL
OR usage_http_audits.response_body_ref IS NOT NULL
OR usage_http_audits.client_response_body_ref IS NOT NULL
)
)
)
ORDER BY created_at ASC, id ASC
LIMIT $2
"#;
const CLEAR_USAGE_COMPRESSED_BODY_FIELDS_SQL: &str = r#"
UPDATE usage
SET request_body_compressed = NULL,
response_body_compressed = NULL,
provider_request_body_compressed = NULL,
client_response_body_compressed = NULL
WHERE id = ANY($1)
"#;
const CLEAR_USAGE_BODY_FIELDS_SQL: &str = r#" const CLEAR_USAGE_BODY_FIELDS_SQL: &str = r#"
UPDATE usage UPDATE usage
SET request_body = NULL, SET request_body = NULL,
@@ -419,49 +477,99 @@ impl SqlxUsageReadRepository {
window: &UsageCleanupWindow, window: &UsageCleanupWindow,
batch_size: usize, batch_size: usize,
auto_delete_expired_keys: bool, auto_delete_expired_keys: bool,
targets: UsageCleanupTargets,
mode: UsageCleanupExecutionMode,
) -> Result<UsageCleanupSummary, DataLayerError> { ) -> Result<UsageCleanupSummary, DataLayerError> {
if batch_size == 0 { if batch_size == 0 || !targets.any_selected() {
return Ok(UsageCleanupSummary::default()); return Ok(UsageCleanupSummary::default());
} }
if mode == UsageCleanupExecutionMode::BeforeNowBodyFields {
let body_externalized = if targets.detail_body {
cleanup_usage_raw_body_fields(&self.pool, window.detail_cutoff, batch_size).await?
} else {
0
};
let body_cleaned = if targets.compressed_body {
cleanup_usage_compressed_body_fields(
&self.pool,
window.compressed_cutoff,
batch_size,
)
.await?
} else {
0
};
return Ok(UsageCleanupSummary {
body_externalized,
legacy_body_refs_migrated: 0,
body_cleaned,
header_cleaned: 0,
keys_cleaned: 0,
records_deleted: 0,
});
}
let records_deleted = let records_deleted = if targets.records {
delete_old_usage_records(&self.pool, window.log_cutoff, batch_size).await?; delete_old_usage_records(&self.pool, window.log_cutoff, batch_size).await?
let header_cleaned = cleanup_usage_header_fields( } else {
&self.pool, 0
window.header_cutoff, };
batch_size, let header_cleaned = if targets.headers {
Some(window.log_cutoff), cleanup_usage_header_fields(
) &self.pool,
.await?; window.header_cutoff,
let legacy_body_refs_migrated = migrate_legacy_usage_body_ref_metadata( batch_size,
&self.pool, targets.records.then_some(window.log_cutoff),
window.detail_cutoff, )
batch_size, .await?
Some(window.compressed_cutoff), } else {
) 0
.await?; };
let body_cleaned = cleanup_usage_stale_body_fields( let body_cleaned = if targets.compressed_body {
&self.pool, cleanup_usage_stale_body_fields(
window.compressed_cutoff, &self.pool,
batch_size, window.compressed_cutoff,
Some(window.log_cutoff), batch_size,
) targets.records.then_some(window.log_cutoff),
.await?; )
let body_externalized = compress_usage_body_fields( .await?
&self.pool, } else {
window.detail_cutoff, 0
batch_size, };
Some(window.compressed_cutoff), let detail_body_newer_than = detail_body_newer_than(window, targets);
) let legacy_body_refs_migrated = if targets.detail_body {
.await?; migrate_legacy_usage_body_ref_metadata(
let keys_cleaned = &self.pool,
window.detail_cutoff,
batch_size,
detail_body_newer_than,
)
.await?
} else {
0
};
let body_externalized = if targets.detail_body {
compress_usage_body_fields(
&self.pool,
window.detail_cutoff,
batch_size,
detail_body_newer_than,
)
.await?
} else {
0
};
let keys_cleaned = if targets.expired_keys {
match cleanup_expired_api_keys(&self.pool, auto_delete_expired_keys).await { match cleanup_expired_api_keys(&self.pool, auto_delete_expired_keys).await {
Ok(count) => count, Ok(count) => count,
Err(err) => { Err(err) => {
warn!(error = %err, "usage cleanup expired api key sweep failed"); warn!(error = %err, "usage cleanup expired api key sweep failed");
0 0
} }
}; }
} else {
0
};
Ok(UsageCleanupSummary { Ok(UsageCleanupSummary {
body_externalized, body_externalized,
@@ -477,45 +585,400 @@ impl SqlxUsageReadRepository {
pub async fn preview_usage_cleanup_impl( pub async fn preview_usage_cleanup_impl(
pool: &PostgresPool, pool: &PostgresPool,
window: &UsageCleanupWindow, window: &UsageCleanupWindow,
targets: UsageCleanupTargets,
mode: UsageCleanupExecutionMode,
) -> Result<UsageCleanupPreviewCounts, DataLayerError> { ) -> Result<UsageCleanupPreviewCounts, DataLayerError> {
let detail: i64 = sqlx::query_scalar( if mode == UsageCleanupExecutionMode::BeforeNowBodyFields {
"SELECT COUNT(*)::bigint FROM usage WHERE created_at < $1 AND created_at >= $2", let detail = if targets.detail_body {
) count_usage_raw_body_candidates(pool, window.detail_cutoff).await?
.bind(window.detail_cutoff) } else {
.bind(window.compressed_cutoff) 0
.fetch_one(pool) };
.await let compressed = if targets.compressed_body {
.map_err(postgres_error)?; count_usage_compressed_body_candidates(pool, window.compressed_cutoff).await?
let compressed: i64 = sqlx::query_scalar( } else {
"SELECT COUNT(*)::bigint FROM usage WHERE created_at < $1 AND created_at >= $2", 0
) };
.bind(window.compressed_cutoff) return Ok(UsageCleanupPreviewCounts {
.bind(window.log_cutoff) detail,
.fetch_one(pool) compressed,
.await header: 0,
.map_err(postgres_error)?; log: 0,
let header: i64 = sqlx::query_scalar( });
"SELECT COUNT(*)::bigint FROM usage WHERE created_at < $1 AND created_at >= $2", }
)
.bind(window.header_cutoff) let detail = if targets.detail_body {
.bind(window.log_cutoff) count_usage_detail_body_candidates(
.fetch_one(pool) pool,
.await window.detail_cutoff,
.map_err(postgres_error)?; detail_body_newer_than(window, targets),
let log: i64 = sqlx::query_scalar("SELECT COUNT(*)::bigint FROM usage WHERE created_at < $1") )
.bind(window.log_cutoff) .await?
.fetch_one(pool) } else {
.await 0
.map_err(postgres_error)?; };
let compressed = if targets.compressed_body {
count_usage_stale_body_candidates(
pool,
window.compressed_cutoff,
targets.records.then_some(window.log_cutoff),
)
.await?
} else {
0
};
let header = if targets.headers {
count_usage_header_candidates(
pool,
window.header_cutoff,
targets.records.then_some(window.log_cutoff),
)
.await?
} else {
0
};
let log = if targets.records {
let log: i64 =
sqlx::query_scalar("SELECT COUNT(*)::bigint FROM usage WHERE created_at < $1")
.bind(window.log_cutoff)
.fetch_one(pool)
.await
.map_err(postgres_error)?;
u64::try_from(log).unwrap_or(0)
} else {
0
};
Ok(UsageCleanupPreviewCounts { Ok(UsageCleanupPreviewCounts {
detail: u64::try_from(detail).unwrap_or(0), detail,
compressed: u64::try_from(compressed).unwrap_or(0), compressed,
header: u64::try_from(header).unwrap_or(0), header,
log: u64::try_from(log).unwrap_or(0), log,
}) })
} }
async fn cleanup_usage_raw_body_fields(
pool: &PostgresPool,
cutoff_time: DateTime<Utc>,
batch_size: usize,
) -> Result<usize, DataLayerError> {
let mut total_cleaned = 0usize;
loop {
let rows = fetch_usage_body_cleanup_rows(
pool,
SELECT_USAGE_RAW_BODY_BATCH_SQL,
cutoff_time,
batch_size,
)
.await?;
if rows.is_empty() {
break;
}
let ids = rows.iter().map(|row| row.id.clone()).collect::<Vec<_>>();
let cleaned = sqlx::query(CLEAR_USAGE_RAW_BODY_FIELDS_SQL)
.bind(ids)
.execute(pool)
.await
.map_err(postgres_error)?
.rows_affected();
let cleaned = usize::try_from(cleaned).unwrap_or(usize::MAX);
total_cleaned += cleaned;
if cleaned == 0 || cleaned < batch_size {
break;
}
}
Ok(total_cleaned)
}
async fn cleanup_usage_compressed_body_fields(
pool: &PostgresPool,
cutoff_time: DateTime<Utc>,
batch_size: usize,
) -> Result<usize, DataLayerError> {
let mut total_cleaned = 0usize;
loop {
let rows = fetch_usage_body_cleanup_rows(
pool,
SELECT_USAGE_COMPRESSED_BODY_BATCH_SQL,
cutoff_time,
batch_size,
)
.await?;
if rows.is_empty() {
break;
}
let ids = rows.iter().map(|row| row.id.clone()).collect::<Vec<_>>();
let request_ids = rows
.iter()
.map(|row| row.request_id.clone())
.collect::<Vec<_>>();
let cleaned = sqlx::query(CLEAR_USAGE_COMPRESSED_BODY_FIELDS_SQL)
.bind(ids)
.execute(pool)
.await
.map_err(postgres_error)?
.rows_affected();
sqlx::query(DELETE_USAGE_BODY_BLOBS_SQL)
.bind(&request_ids)
.execute(pool)
.await
.map_err(postgres_error)?;
sqlx::query(CLEAR_USAGE_HTTP_AUDIT_BODY_REFS_SQL)
.bind(&request_ids)
.execute(pool)
.await
.map_err(postgres_error)?;
sqlx::query(DELETE_EMPTY_USAGE_HTTP_AUDITS_SQL)
.bind(request_ids)
.execute(pool)
.await
.map_err(postgres_error)?;
let cleaned = usize::try_from(cleaned).unwrap_or(usize::MAX);
total_cleaned += cleaned;
if cleaned == 0 || cleaned < batch_size {
break;
}
}
Ok(total_cleaned)
}
async fn fetch_usage_body_cleanup_rows(
pool: &PostgresPool,
sql: &str,
cutoff_time: DateTime<Utc>,
batch_size: usize,
) -> Result<Vec<UsageBodyCleanupRow>, DataLayerError> {
let rows = sqlx::query(sql)
.bind(cutoff_time)
.bind(i64::try_from(batch_size).unwrap_or(i64::MAX))
.fetch_all(pool)
.await
.map_err(postgres_error)?
.into_iter()
.map(|row| {
Ok(UsageBodyCleanupRow {
id: row.try_get::<String, _>("id").map_err(postgres_error)?,
request_id: row
.try_get::<String, _>("request_id")
.map_err(postgres_error)?,
})
})
.collect::<Result<Vec<_>, DataLayerError>>()?;
Ok(rows)
}
fn detail_body_newer_than(
window: &UsageCleanupWindow,
targets: UsageCleanupTargets,
) -> Option<DateTime<Utc>> {
[
targets.compressed_body.then_some(window.compressed_cutoff),
targets.records.then_some(window.log_cutoff),
]
.into_iter()
.flatten()
.max()
}
async fn count_usage_raw_body_candidates(
pool: &PostgresPool,
cutoff_time: DateTime<Utc>,
) -> Result<u64, DataLayerError> {
let count: i64 = sqlx::query_scalar(
r#"
SELECT COUNT(*)::bigint
FROM usage
WHERE created_at < $1
AND (
request_body IS NOT NULL
OR response_body IS NOT NULL
OR provider_request_body IS NOT NULL
OR client_response_body IS NOT NULL
)
"#,
)
.bind(cutoff_time)
.fetch_one(pool)
.await
.map_err(postgres_error)?;
Ok(u64::try_from(count).unwrap_or(0))
}
async fn count_usage_compressed_body_candidates(
pool: &PostgresPool,
cutoff_time: DateTime<Utc>,
) -> Result<u64, DataLayerError> {
let count: i64 = sqlx::query_scalar(
r#"
SELECT COUNT(*)::bigint
FROM usage
WHERE created_at < $1
AND (
request_body_compressed IS NOT NULL
OR response_body_compressed IS NOT NULL
OR provider_request_body_compressed IS NOT NULL
OR client_response_body_compressed IS NOT NULL
OR EXISTS (
SELECT 1
FROM usage_body_blobs
WHERE usage_body_blobs.request_id = usage.request_id
)
OR EXISTS (
SELECT 1
FROM usage_http_audits
WHERE usage_http_audits.request_id = usage.request_id
AND (
usage_http_audits.request_body_ref IS NOT NULL
OR usage_http_audits.provider_request_body_ref IS NOT NULL
OR usage_http_audits.response_body_ref IS NOT NULL
OR usage_http_audits.client_response_body_ref IS NOT NULL
)
)
)
"#,
)
.bind(cutoff_time)
.fetch_one(pool)
.await
.map_err(postgres_error)?;
Ok(u64::try_from(count).unwrap_or(0))
}
async fn count_usage_detail_body_candidates(
pool: &PostgresPool,
cutoff_time: DateTime<Utc>,
newer_than: Option<DateTime<Utc>>,
) -> Result<u64, DataLayerError> {
if matches!(newer_than, Some(value) if value >= cutoff_time) {
return Ok(0);
}
let count: i64 = sqlx::query_scalar(
r#"
SELECT COUNT(*)::bigint
FROM usage
WHERE created_at < $1
AND ($2::timestamptz IS NULL OR created_at >= $2)
AND (
request_body IS NOT NULL
OR request_body_compressed IS NOT NULL
OR response_body IS NOT NULL
OR response_body_compressed IS NOT NULL
OR provider_request_body IS NOT NULL
OR provider_request_body_compressed IS NOT NULL
OR client_response_body IS NOT NULL
OR client_response_body_compressed IS NOT NULL
OR (
request_metadata IS NOT NULL
AND (
request_metadata::jsonb ? 'request_body_ref'
OR request_metadata::jsonb ? 'provider_request_body_ref'
OR request_metadata::jsonb ? 'response_body_ref'
OR request_metadata::jsonb ? 'client_response_body_ref'
)
)
)
"#,
)
.bind(cutoff_time)
.bind(newer_than)
.fetch_one(pool)
.await
.map_err(postgres_error)?;
Ok(u64::try_from(count).unwrap_or(0))
}
async fn count_usage_stale_body_candidates(
pool: &PostgresPool,
cutoff_time: DateTime<Utc>,
newer_than: Option<DateTime<Utc>>,
) -> Result<u64, DataLayerError> {
if matches!(newer_than, Some(value) if value >= cutoff_time) {
return Ok(0);
}
let count: i64 = sqlx::query_scalar(
r#"
SELECT COUNT(*)::bigint
FROM usage
WHERE created_at < $1
AND ($2::timestamptz IS NULL OR created_at >= $2)
AND (
request_body IS NOT NULL
OR response_body IS NOT NULL
OR provider_request_body IS NOT NULL
OR client_response_body IS NOT NULL
OR request_body_compressed IS NOT NULL
OR response_body_compressed IS NOT NULL
OR provider_request_body_compressed IS NOT NULL
OR client_response_body_compressed IS NOT NULL
OR EXISTS (
SELECT 1
FROM usage_body_blobs
WHERE usage_body_blobs.request_id = usage.request_id
)
OR EXISTS (
SELECT 1
FROM usage_http_audits
WHERE usage_http_audits.request_id = usage.request_id
AND (
usage_http_audits.request_body_ref IS NOT NULL
OR usage_http_audits.provider_request_body_ref IS NOT NULL
OR usage_http_audits.response_body_ref IS NOT NULL
OR usage_http_audits.client_response_body_ref IS NOT NULL
)
)
)
"#,
)
.bind(cutoff_time)
.bind(newer_than)
.fetch_one(pool)
.await
.map_err(postgres_error)?;
Ok(u64::try_from(count).unwrap_or(0))
}
async fn count_usage_header_candidates(
pool: &PostgresPool,
cutoff_time: DateTime<Utc>,
newer_than: Option<DateTime<Utc>>,
) -> Result<u64, DataLayerError> {
if matches!(newer_than, Some(value) if value >= cutoff_time) {
return Ok(0);
}
let count: i64 = sqlx::query_scalar(
r#"
SELECT COUNT(*)::bigint
FROM usage
WHERE created_at < $1
AND ($2::timestamptz IS NULL OR created_at >= $2)
AND (
request_headers IS NOT NULL
OR response_headers IS NOT NULL
OR provider_request_headers IS NOT NULL
OR client_response_headers IS NOT NULL
OR EXISTS (
SELECT 1
FROM usage_http_audits
WHERE usage_http_audits.request_id = usage.request_id
AND (
usage_http_audits.request_headers IS NOT NULL
OR usage_http_audits.response_headers IS NOT NULL
OR usage_http_audits.provider_request_headers IS NOT NULL
OR usage_http_audits.client_response_headers IS NOT NULL
)
)
)
"#,
)
.bind(cutoff_time)
.bind(newer_than)
.fetch_one(pool)
.await
.map_err(postgres_error)?;
Ok(u64::try_from(count).unwrap_or(0))
}
async fn delete_old_usage_records( async fn delete_old_usage_records(
pool: &PostgresPool, pool: &PostgresPool,
cutoff_time: DateTime<Utc>, cutoff_time: DateTime<Utc>,

33
crates/aether-data/src/repository/usage/postgres/mod.rs
View File

@@ -11,12 +11,13 @@ use aether_data_contracts::repository::usage::{
UsageAuditAggregationQuery, UsageAuditKeywordSearchQuery, UsageAuditSummaryQuery, UsageAuditAggregationQuery, UsageAuditKeywordSearchQuery, UsageAuditSummaryQuery,
UsageBodyCaptureState, UsageBodyField, UsageBreakdownGroupBy, UsageBreakdownSummaryQuery, UsageBodyCaptureState, UsageBodyField, UsageBreakdownGroupBy, UsageBreakdownSummaryQuery,
UsageCacheAffinityHitSummaryQuery, UsageCacheAffinityIntervalGroupBy, UsageCacheAffinityHitSummaryQuery, UsageCacheAffinityIntervalGroupBy,
UsageCacheAffinityIntervalQuery, UsageCacheHitSummaryQuery, UsageCleanupSummary, UsageCacheAffinityIntervalQuery, UsageCacheHitSummaryQuery, UsageCleanupExecutionMode,
UsageCleanupWindow, UsageCostSavingsSummaryQuery, UsageDashboardDailyBreakdownQuery, UsageCleanupSummary, UsageCleanupTargets, UsageCleanupWindow, UsageCostSavingsSummaryQuery,
UsageDashboardProviderCountsQuery, UsageDashboardSummaryQuery, UsageErrorDistributionQuery, UsageDashboardDailyBreakdownQuery, UsageDashboardProviderCountsQuery,
UsageLeaderboardGroupBy, UsageLeaderboardQuery, UsageMonitoringErrorCountQuery, UsageDashboardSummaryQuery, UsageErrorDistributionQuery, UsageLeaderboardGroupBy,
UsageMonitoringErrorListQuery, UsagePerformancePercentilesQuery, UsageProviderPerformanceQuery, UsageLeaderboardQuery, UsageMonitoringErrorCountQuery, UsageMonitoringErrorListQuery,
UsageSettledCostSummaryQuery, UsageTimeSeriesGranularity, UsageTimeSeriesQuery, UsagePerformancePercentilesQuery, UsageProviderPerformanceQuery, UsageSettledCostSummaryQuery,
UsageTimeSeriesGranularity, UsageTimeSeriesQuery,
}; };
use async_trait::async_trait; use async_trait::async_trait;
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
@@ -8218,17 +8219,31 @@ impl UsageWriteRepository for SqlxUsageReadRepository {
window: &UsageCleanupWindow, window: &UsageCleanupWindow,
batch_size: usize, batch_size: usize,
auto_delete_expired_keys: bool, auto_delete_expired_keys: bool,
targets: UsageCleanupTargets,
mode: UsageCleanupExecutionMode,
) -> Result<UsageCleanupSummary, DataLayerError> { ) -> Result<UsageCleanupSummary, DataLayerError> {
Self::cleanup_usage(self, window, batch_size, auto_delete_expired_keys).await Self::cleanup_usage(
self,
window,
batch_size,
auto_delete_expired_keys,
targets,
mode,
)
.await
} }
async fn preview_usage_cleanup( async fn preview_usage_cleanup(
&self, &self,
window: &UsageCleanupWindow, window: &UsageCleanupWindow,
targets: UsageCleanupTargets,
mode: UsageCleanupExecutionMode,
) -> Result<aether_data_contracts::repository::usage::UsageCleanupPreviewCounts, DataLayerError> ) -> Result<aether_data_contracts::repository::usage::UsageCleanupPreviewCounts, DataLayerError>
{ {
crate::repository::usage::postgres::cleanup::preview_usage_cleanup_impl(&self.pool, window) crate::repository::usage::postgres::cleanup::preview_usage_cleanup_impl(
.await &self.pool, window, targets, mode,
)
.await
} }
} }

50
frontend/src/api/admin.ts
View File

@@ -310,15 +310,35 @@ export interface ManualUsageCleanupSummary {
records_deleted: number records_deleted: number
} }
export interface ManualUsageCleanupResponse { export type ManualUsageCleanupMode = 'policy' | 'older_than_days' | 'before_now'
export type ManualUsageCleanupTarget = 'detail_body' | 'compressed_body' | 'headers' | 'records'
export interface ManualUsageCleanupTargets {
detail_body: boolean
compressed_body: boolean
headers: boolean
records: boolean
expired_keys: boolean
}
export interface ManualUsageCleanupRequest {
mode?: ManualUsageCleanupMode
older_than_days?: number
targets?: ManualUsageCleanupTarget[]
}
export interface ManualUsageCleanupTaskResponse {
message: string message: string
mode: ManualUsageCleanupMode
requested_older_than_days: number | null requested_older_than_days: number | null
summary: ManualUsageCleanupSummary targets: ManualUsageCleanupTargets
total_affected: number task: CleanupRunRecord
} }
export interface ManualUsageCleanupPreview { export interface ManualUsageCleanupPreview {
mode: ManualUsageCleanupMode
requested_older_than_days: number | null requested_older_than_days: number | null
targets: ManualUsageCleanupTargets
effective_cutoffs: { effective_cutoffs: {
detail: string detail: string
compressed: string compressed: string
@@ -1218,14 +1238,20 @@ export const adminApi = {
}, },
async runManualUsageCleanup( async runManualUsageCleanup(
params: { older_than_days?: number } = {} params: ManualUsageCleanupRequest = {}
): Promise<ManualUsageCleanupResponse | ManualUsageCleanupConflict> { ): Promise<ManualUsageCleanupTaskResponse | ManualUsageCleanupConflict> {
const body: Record<string, number> = {} const body: ManualUsageCleanupRequest = {}
if (params.mode) {
body.mode = params.mode
}
if (typeof params.older_than_days === 'number') { if (typeof params.older_than_days === 'number') {
body.older_than_days = params.older_than_days body.older_than_days = params.older_than_days
} }
if (params.targets?.length) {
body.targets = params.targets
}
try { try {
const response = await apiClient.post<ManualUsageCleanupResponse>( const response = await apiClient.post<ManualUsageCleanupTaskResponse>(
'/api/admin/system/cleanup/usage/manual', '/api/admin/system/cleanup/usage/manual',
body body
) )
@@ -1240,12 +1266,18 @@ export const adminApi = {
}, },
async previewManualUsageCleanup( async previewManualUsageCleanup(
params: { older_than_days?: number } = {} params: ManualUsageCleanupRequest = {}
): Promise<ManualUsageCleanupPreview> { ): Promise<ManualUsageCleanupPreview> {
const query: Record<string, number> = {} const query: Record<string, string | number> = {}
if (params.mode) {
query.mode = params.mode
}
if (typeof params.older_than_days === 'number') { if (typeof params.older_than_days === 'number') {
query.older_than_days = params.older_than_days query.older_than_days = params.older_than_days
} }
if (params.targets?.length) {
query.targets = params.targets.join(',')
}
const response = await apiClient.get<ManualUsageCleanupPreview>( const response = await apiClient.get<ManualUsageCleanupPreview>(
'/api/admin/system/cleanup/usage/preview', '/api/admin/system/cleanup/usage/preview',
{ params: query } { params: query }

62
frontend/src/views/admin/system-settings/CleanupPolicySection.vue
View File

@@ -288,7 +288,8 @@
<ManualCleanupConfirmDialog <ManualCleanupConfirmDialog
:open="manualCleanupDialogOpen" :open="manualCleanupDialogOpen"
@update:open="manualCleanupDialogOpen = $event" @update:open="manualCleanupDialogOpen = $event"
@confirm="handleManualCleanupConfirm" @running-change="manualCleanupRunning = $event"
@completed="handleManualCleanupCompleted"
/> />
<div <div
@@ -402,7 +403,7 @@
<script setup lang="ts"> <script setup lang="ts">
import { onBeforeUnmount, onMounted, ref } from 'vue' import { onBeforeUnmount, onMounted, ref } from 'vue'
import { RefreshCw, Trash2 } from 'lucide-vue-next' import { RefreshCw, Trash2 } from 'lucide-vue-next'
import { adminApi, type CleanupRunRecord, type ManualUsageCleanupResponse } from '@/api/admin' import { adminApi, type CleanupRunRecord } from '@/api/admin'
import Button from '@/components/ui/button.vue' import Button from '@/components/ui/button.vue'
import Input from '@/components/ui/input.vue' import Input from '@/components/ui/input.vue'
import Label from '@/components/ui/label.vue' import Label from '@/components/ui/label.vue'
@@ -410,7 +411,6 @@ import Switch from '@/components/ui/switch.vue'
import { CardSection } from '@/components/layout' import { CardSection } from '@/components/layout'
import ManualCleanupConfirmDialog from './ManualCleanupConfirmDialog.vue' import ManualCleanupConfirmDialog from './ManualCleanupConfirmDialog.vue'
import { useToast } from '@/composables/useToast' import { useToast } from '@/composables/useToast'
import { parseApiError } from '@/utils/errorParser'
defineProps<{ defineProps<{
enableAutoCleanup: boolean enableAutoCleanup: boolean
@@ -459,52 +459,18 @@ function openManualCleanupDialog() {
manualCleanupDialogOpen.value = true manualCleanupDialogOpen.value = true
} }
async function handleManualCleanupConfirm(olderThanDays: number | undefined) { function handleManualCleanupCompleted(task: CleanupRunRecord) {
manualCleanupRunning.value = true manualCleanupRunning.value = false
try { manualCleanupResult.value = {
const response = await adminApi.runManualUsageCleanup( title: task.message,
typeof olderThanDays === 'number' ? { older_than_days: olderThanDays } : {}, description: cleanupSummaryText(task.summary),
)
if ('detail' in response && response.detail === 'usage_cleanup_already_running') {
manualCleanupResult.value = {
title: '已有一次清理正在进行中',
description: response.message,
}
toast.warning(response.message)
} else {
const completed = response as ManualUsageCleanupResponse
manualCleanupResult.value = {
title: completed.message,
description: summarizeManualCleanup(completed),
}
toast.success(completed.message)
}
manualCleanupDialogOpen.value = false
} catch (error) {
const message = parseApiError(error).message
manualCleanupResult.value = {
title: '请求记录清理失败',
description: message,
}
toast.error(message)
} finally {
manualCleanupRunning.value = false
void loadCleanupRuns()
} }
} if (task.status === 'failed') {
toast.error(task.error || task.message)
function summarizeManualCleanup(response: ManualUsageCleanupResponse): string { } else {
const { summary } = response toast.success(task.message)
const parts: string[] = []
if (summary.records_deleted > 0) parts.push(`删除整条记录 ${summary.records_deleted}`)
if (summary.body_externalized > 0) parts.push(`压缩 body ${summary.body_externalized}`)
if (summary.body_cleaned > 0) parts.push(`清除过期 body ${summary.body_cleaned}`)
if (summary.header_cleaned > 0) parts.push(`清除 headers ${summary.header_cleaned}`)
if (summary.legacy_body_refs_migrated > 0) {
parts.push(`迁移遗留引用 ${summary.legacy_body_refs_migrated}`)
} }
if (summary.keys_cleaned > 0) parts.push(`回收 Key ${summary.keys_cleaned}`) void loadCleanupRuns()
return parts.length > 0 ? parts.join(' / ') : '无数据变更'
} }
async function loadCleanupRuns() { async function loadCleanupRuns() {
@@ -568,7 +534,7 @@ function cleanupSummaryText(summary: Record<string, unknown>): string {
function summaryLabel(key: string): string { function summaryLabel(key: string): string {
const labels: Record<string, string> = { const labels: Record<string, string> = {
body_externalized: '压缩', body_externalized: '详细体',
legacy_body_refs_migrated: '迁移', legacy_body_refs_migrated: '迁移',
body_cleaned: '清体', body_cleaned: '清体',
header_cleaned: '清头', header_cleaned: '清头',

360
frontend/src/views/admin/system-settings/ManualCleanupConfirmDialog.vue
View File

@@ -3,30 +3,86 @@
:open="open" :open="open"
size="lg" size="lg"
title="立即清理请求记录" title="立即清理请求记录"
description="按现有分级保留策略主动清理请求记录，可选指定清理更早时间的数据。操作不可逆。" description="默认按当前分级保留策略执行，也可以选择指定范围。操作不可逆。"
:persistent="submitting" :persistent="isLocked"
@update:open="handleOpenChange" @update:open="handleOpenChange"
> >
<div class="px-4 sm:px-6 py-4 space-y-4"> <div class="px-4 sm:px-6 py-4 space-y-4">
<div> <div>
<Label class="block text-sm font-medium">
清理方式
</Label>
<div class="mt-2 grid grid-cols-1 sm:grid-cols-3 gap-2">
<button
v-for="item in modeOptions"
:key="item.value"
type="button"
class="rounded-md border px-3 py-2 text-left text-sm transition-colors"
:class="mode === item.value ? 'border-primary bg-primary/10 text-primary' : 'border-border bg-card hover:bg-muted/60'"
:disabled="isLocked"
@click="setMode(item.value)"
>
<span class="font-medium">{{ item.label }}</span>
<span class="mt-1 block text-xs text-muted-foreground">{{ item.description }}</span>
</button>
</div>
</div>
<div v-if="mode === 'older_than_days'">
<Label <Label
for="manual-cleanup-older-than-days" for="manual-cleanup-older-than-days"
class="block text-sm font-medium" class="block text-sm font-medium"
> >
清理 N 天前的记录（可选） 清理 N 天前的记录
</Label> </Label>
<Input <Input
id="manual-cleanup-older-than-days" id="manual-cleanup-older-than-days"
:model-value="olderThanDays ?? ''" :model-value="olderThanDays ?? ''"
type="number" type="number"
min="1" min="1"
placeholder="留空代表按当前保留策略" placeholder="例如 30"
class="mt-1" class="mt-1"
:disabled="submitting" :disabled="isLocked"
@update:model-value="handleDaysChange" @update:model-value="handleDaysChange"
/> />
<p class="mt-1 text-xs text-muted-foreground"> <p class="mt-1 text-xs text-muted-foreground">
留空代表按当前保留策略清理。填入数字代表清理 N 天前的记录；该值只能比策略更宽松，不会删除更新的数据。 该值会与当前策略取更保守的时间点，不会清理比策略更新的数据。
</p>
</div>
<div>
<Label class="block text-sm font-medium">
清理范围
</Label>
<div class="mt-2 grid grid-cols-1 sm:grid-cols-2 gap-2">
<label
v-for="target in targetOptions"
:key="target.value"
class="flex min-h-16 items-start gap-3 rounded-md border border-border bg-card px-3 py-2"
>
<Checkbox
class="mt-0.5"
:checked="selectedTargets.includes(target.value)"
:disabled="isLocked"
@update:checked="toggleTarget(target.value, $event)"
/>
<span>
<span class="block text-sm font-medium">{{ target.label }}</span>
<span class="block text-xs text-muted-foreground">{{ target.description }}</span>
</span>
</label>
</div>
<p
v-if="mode === 'before_now'"
class="mt-2 text-xs text-amber-600"
>
当前时刻之前模式只允许清理详细请求体和压缩请求体，不会清请求头或整条记录。
</p>
<p
v-if="targetError"
class="mt-2 text-xs text-destructive"
>
{{ targetError }}
</p> </p>
</div> </div>
@@ -39,7 +95,7 @@
v-if="!previewLoading" v-if="!previewLoading"
type="button" type="button"
class="text-xs text-muted-foreground hover:text-foreground" class="text-xs text-muted-foreground hover:text-foreground"
:disabled="submitting" :disabled="isLocked"
@click="loadPreview" @click="loadPreview"
> >
刷新预估 刷新预估
@@ -86,6 +142,45 @@
</div> </div>
</div> </div>
<div
v-if="activeTask || taskError"
class="rounded-md border border-border bg-card px-4 py-3"
>
<div class="flex items-center justify-between gap-3">
<div class="min-w-0">
<div class="text-sm font-medium">
{{ activeTask?.message || '请求记录清理失败' }}
</div>
<div class="mt-1 text-xs text-muted-foreground">
{{ activeTask ? cleanupStatusLabel(activeTask.status) : taskError }}
</div>
</div>
<span
v-if="activeTask"
:class="cleanupStatusClass(activeTask.status)"
class="shrink-0 text-xs"
>
{{ cleanupStatusLabel(activeTask.status) }}
</span>
</div>
<div
v-if="activeTask"
class="mt-3 h-2 overflow-hidden rounded-full bg-muted"
>
<div
class="h-full rounded-full bg-primary transition-all"
:class="{ 'animate-pulse': activeTask.status === 'processing' }"
:style="{ width: `${taskProgressPercent}%` }"
/>
</div>
<div
v-if="activeTask"
class="mt-2 text-xs text-muted-foreground"
>
{{ cleanupSummaryText(activeTask.summary) }}
</div>
</div>
<div> <div>
<Label <Label
for="manual-cleanup-confirm-phrase" for="manual-cleanup-confirm-phrase"
@@ -99,47 +194,59 @@
class="mt-1" class="mt-1"
autocomplete="off" autocomplete="off"
:placeholder="confirmPhrase" :placeholder="confirmPhrase"
:disabled="submitting" :disabled="isLocked || isFinished"
@update:model-value="typedPhrase = String($event)" @update:model-value="typedPhrase = String($event)"
@keydown.enter.prevent="maybeSubmitOnEnter" @keydown.enter.prevent="maybeSubmitOnEnter"
/> />
<p class="mt-1 text-xs text-muted-foreground"> <p class="mt-1 text-xs text-muted-foreground">
确认操作后会立刻执行清理，且不可撤销。 确认后会在当前弹窗中显示执行状态，完成前不能关闭。
</p> </p>
</div> </div>
</div> </div>
<template #footer> <template #footer>
<Button <Button
v-if="!isFinished"
variant="destructive" variant="destructive"
:disabled="!canSubmit" :disabled="!canSubmit"
@click="handleConfirm" @click="handleConfirm"
> >
{{ submitting ? '清理中…' : '确认清理' }} {{ isLocked ? '清理中…' : '确认清理' }}
</Button> </Button>
<Button <Button
variant="outline" variant="outline"
:disabled="submitting" :disabled="isLocked"
@click="handleCancel" @click="handleCancel"
> >
取消 {{ isFinished ? '关闭' : '取消' }}
</Button> </Button>
</template> </template>
</Dialog> </Dialog>
</template> </template>
<script setup lang="ts"> <script setup lang="ts">
import { computed, ref, watch } from 'vue' import { computed, onBeforeUnmount, ref, watch } from 'vue'
import { Dialog } from '@/components/ui' import { Dialog } from '@/components/ui'
import Button from '@/components/ui/button.vue' import Button from '@/components/ui/button.vue'
import Checkbox from '@/components/ui/checkbox.vue'
import Input from '@/components/ui/input.vue' import Input from '@/components/ui/input.vue'
import Label from '@/components/ui/label.vue' import Label from '@/components/ui/label.vue'
import { adminApi, type ManualUsageCleanupPreview } from '@/api/admin' import {
adminApi,
type CleanupRunRecord,
type ManualUsageCleanupPreview,
type ManualUsageCleanupRequest,
} from '@/api/admin'
import { parseApiError } from '@/utils/errorParser' import { parseApiError } from '@/utils/errorParser'
import { import {
MANUAL_USAGE_CLEANUP_CONFIRM_PHRASE, MANUAL_USAGE_CLEANUP_CONFIRM_PHRASE,
allowedTargetsForMode,
defaultManualCleanupTargets,
isConfirmPhraseMatched, isConfirmPhraseMatched,
normalizeManualCleanupTargets,
normalizeOlderThanDaysInput, normalizeOlderThanDaysInput,
type ManualCleanupMode,
type ManualCleanupTarget,
} from './manualCleanupForm' } from './manualCleanupForm'
const props = defineProps<{ const props = defineProps<{
@@ -148,50 +255,124 @@ const props = defineProps<{
const emit = defineEmits<{ const emit = defineEmits<{
'update:open': [value: boolean] 'update:open': [value: boolean]
confirm: [olderThanDays: number | undefined] 'running-change': [value: boolean]
completed: [task: CleanupRunRecord]
}>() }>()
const confirmPhrase = MANUAL_USAGE_CLEANUP_CONFIRM_PHRASE const confirmPhrase = MANUAL_USAGE_CLEANUP_CONFIRM_PHRASE
const mode = ref<ManualCleanupMode>('policy')
const olderThanDays = ref<number | null>(null) const olderThanDays = ref<number | null>(null)
const selectedTargets = ref<ManualCleanupTarget[]>(defaultManualCleanupTargets('policy'))
const targetsTouched = ref(false)
const typedPhrase = ref('') const typedPhrase = ref('')
const preview = ref<ManualUsageCleanupPreview | null>(null) const preview = ref<ManualUsageCleanupPreview | null>(null)
const previewLoading = ref(false) const previewLoading = ref(false)
const previewError = ref<string | null>(null) const previewError = ref<string | null>(null)
const submitting = ref(false) const submitting = ref(false)
const activeTask = ref<CleanupRunRecord | null>(null)
const taskError = ref<string | null>(null)
let previewDebounceTimer: ReturnType<typeof setTimeout> | null = null let previewDebounceTimer: ReturnType<typeof setTimeout> | null = null
let previewSeq = 0 let previewSeq = 0
let taskPollTimer: ReturnType<typeof window.setInterval> | null = null
const normalizedPhrase = computed(() => typedPhrase.value) const modeOptions: Array<{ value: ManualCleanupMode; label: string; description: string }> = [
{ value: 'policy', label: '按当前策略', description: '沿用页面上配置的保留天数' },
{ value: 'older_than_days', label: '指定天数前', description: '在策略内取更保守时间点' },
{ value: 'before_now', label: '当前时刻之前', description: '只清已选请求体内容' },
]
const targetLabels: Record<ManualCleanupTarget, { label: string; description: string }> = {
detail_body: { label: '详细请求体', description: '把详细 body 移入压缩/外置存储' },
compressed_body: { label: '压缩请求体', description: '删除已压缩或外置的 body 内容' },
headers: { label: '请求头', description: '清空请求/响应 headers 字段' },
records: { label: '整条记录', description: '删除超过记录保留期的 usage 行' },
}
const targetOptions = computed(() =>
allowedTargetsForMode(mode.value).map(value => ({
value,
...targetLabels[value],
}))
)
const normalizedTargets = computed(() =>
normalizeManualCleanupTargets(mode.value, selectedTargets.value)
)
const targetError = computed(() => {
if (normalizedTargets.value.length === 0) return '至少选择一个清理范围'
return null
})
const currentTaskRunning = computed(() => activeTask.value?.status === 'processing')
const isLocked = computed(() => submitting.value || currentTaskRunning.value)
const isFinished = computed(() =>
activeTask.value?.status === 'completed' || activeTask.value?.status === 'failed'
)
const canSubmit = computed( const canSubmit = computed(
() => () =>
!submitting.value && !isLocked.value &&
!isFinished.value &&
!previewLoading.value && !previewLoading.value &&
isConfirmPhraseMatched(normalizedPhrase.value), !targetError.value &&
modeIsValid.value &&
isConfirmPhraseMatched(typedPhrase.value),
) )
const modeIsValid = computed(() => mode.value !== 'older_than_days' || olderThanDays.value !== null)
const taskProgressPercent = computed(() => {
const task = activeTask.value
if (!task) return 0
if (task.status === 'completed') return 100
if (task.status === 'failed') return 100
const raw = task.summary?.progress_percent
if (typeof raw === 'number' && Number.isFinite(raw) && raw > 0) {
return Math.max(1, Math.min(99, Math.round(raw)))
}
return 12
})
watch( watch(
() => props.open, () => props.open,
(isOpen) => { (isOpen) => {
if (isOpen) { if (isOpen) {
resetForm() resetForm()
void loadPreview() void loadPreview()
} else if (previewDebounceTimer) { } else {
clearTimeout(previewDebounceTimer) clearPreviewTimer()
previewDebounceTimer = null stopTaskPolling()
} }
}, },
) )
function resetForm() { function resetForm() {
mode.value = 'policy'
olderThanDays.value = null olderThanDays.value = null
selectedTargets.value = defaultManualCleanupTargets('policy')
targetsTouched.value = false
typedPhrase.value = '' typedPhrase.value = ''
preview.value = null preview.value = null
previewError.value = null previewError.value = null
previewLoading.value = false previewLoading.value = false
submitting.value = false submitting.value = false
activeTask.value = null
taskError.value = null
emit('running-change', false)
}
function setMode(nextMode: ManualCleanupMode) {
if (isLocked.value || mode.value === nextMode) return
mode.value = nextMode
olderThanDays.value = null
selectedTargets.value = defaultManualCleanupTargets(nextMode)
targetsTouched.value = false
activeTask.value = null
taskError.value = null
schedulePreview()
} }
function handleDaysChange(value: string | number) { function handleDaysChange(value: string | number) {
@@ -199,33 +380,59 @@ function handleDaysChange(value: string | number) {
schedulePreview() schedulePreview()
} }
function schedulePreview() { function toggleTarget(target: ManualCleanupTarget, checked: boolean) {
if (previewDebounceTimer) { targetsTouched.value = true
clearTimeout(previewDebounceTimer) activeTask.value = null
taskError.value = null
const current = new Set(selectedTargets.value)
if (checked) {
current.add(target)
} else {
current.delete(target)
} }
selectedTargets.value = normalizeManualCleanupTargets(mode.value, Array.from(current))
schedulePreview()
}
function buildRequest(): ManualUsageCleanupRequest {
const request: ManualUsageCleanupRequest = { mode: mode.value }
if (mode.value === 'older_than_days' && olderThanDays.value !== null) {
request.older_than_days = olderThanDays.value
}
if (targetsTouched.value) {
request.targets = normalizedTargets.value
}
return request
}
function schedulePreview() {
clearPreviewTimer()
previewDebounceTimer = setTimeout(() => { previewDebounceTimer = setTimeout(() => {
previewDebounceTimer = null previewDebounceTimer = null
void loadPreview() void loadPreview()
}, 300) }, 300)
} }
function clearPreviewTimer() {
if (previewDebounceTimer) {
clearTimeout(previewDebounceTimer)
previewDebounceTimer = null
}
}
async function loadPreview() { async function loadPreview() {
const seq = ++previewSeq const seq = ++previewSeq
previewLoading.value = true previewLoading.value = true
previewError.value = null previewError.value = null
try { try {
const params: { older_than_days?: number } = {} const result = await adminApi.previewManualUsageCleanup(buildRequest())
if (olderThanDays.value !== null) {
params.older_than_days = olderThanDays.value
}
const result = await adminApi.previewManualUsageCleanup(params)
if (seq === previewSeq) { if (seq === previewSeq) {
preview.value = result preview.value = result
} }
} catch (error) { } catch (error) {
if (seq === previewSeq) { if (seq === previewSeq) {
preview.value = null preview.value = null
previewError.value = parseApiError(error).message previewError.value = parseApiError(error)
} }
} finally { } finally {
if (seq === previewSeq) { if (seq === previewSeq) {
@@ -235,14 +442,14 @@ async function loadPreview() {
} }
function handleOpenChange(value: boolean) { function handleOpenChange(value: boolean) {
if (!value && submitting.value) { if (!value && isLocked.value) {
return return
} }
emit('update:open', value) emit('update:open', value)
} }
function handleCancel() { function handleCancel() {
if (submitting.value) return if (isLocked.value) return
emit('update:open', false) emit('update:open', false)
} }
@@ -255,14 +462,101 @@ function maybeSubmitOnEnter() {
async function handleConfirm() { async function handleConfirm() {
if (!canSubmit.value) return if (!canSubmit.value) return
submitting.value = true submitting.value = true
taskError.value = null
try { try {
emit('confirm', olderThanDays.value ?? undefined) const response = await adminApi.runManualUsageCleanup(buildRequest())
if ('detail' in response && response.detail === 'usage_cleanup_already_running') {
taskError.value = response.message
emit('running-change', false)
return
}
activeTask.value = response.task
emit('running-change', response.task.status === 'processing')
if (response.task.status === 'processing') {
startTaskPolling(response.task.id)
} else {
emit('completed', response.task)
}
} catch (error) {
taskError.value = parseApiError(error)
emit('running-change', false)
} finally { } finally {
submitting.value = false submitting.value = false
} }
} }
function startTaskPolling(taskId: string) {
stopTaskPolling()
void pollTask(taskId)
taskPollTimer = window.setInterval(() => {
void pollTask(taskId)
}, 1_500)
}
function stopTaskPolling() {
if (taskPollTimer) {
window.clearInterval(taskPollTimer)
taskPollTimer = null
}
}
async function pollTask(taskId: string) {
try {
const response = await adminApi.getCleanupRuns()
const task = response.items.find(item => item.id === taskId)
if (!task) return
activeTask.value = task
const running = task.status === 'processing'
emit('running-change', running)
if (!running) {
stopTaskPolling()
emit('completed', task)
void loadPreview()
}
} catch (error) {
taskError.value = parseApiError(error)
}
}
function cleanupStatusLabel(status: string): string {
if (status === 'processing') return '执行中'
if (status === 'failed') return '失败'
return '完成'
}
function cleanupStatusClass(status: string): string {
if (status === 'processing') return 'text-amber-500'
if (status === 'failed') return 'text-destructive'
return 'text-emerald-500'
}
function cleanupSummaryText(summary: Record<string, unknown>): string {
const total = typeof summary.total === 'number' ? summary.total : null
if (total !== null && total > 0) return `影响 ${total} 项`
const entries = Object.entries(summary)
.filter(([key, value]) => key !== 'progress_percent' && typeof value === 'number' && value > 0)
.map(([key, value]) => `${summaryLabel(key)} ${value}`)
return entries.length > 0 ? entries.join(' / ') : '等待后台返回结果'
}
function summaryLabel(key: string): string {
const labels: Record<string, string> = {
body_externalized: '详细体',
legacy_body_refs_migrated: '迁移',
body_cleaned: '清体',
header_cleaned: '清头',
keys_cleaned: 'Key',
records_deleted: '删记录',
}
return labels[key] || key
}
function formatCount(value: number): string { function formatCount(value: number): string {
return value.toLocaleString() return value.toLocaleString()
} }
onBeforeUnmount(() => {
clearPreviewTimer()
stopTaskPolling()
})
</script> </script>

33
frontend/src/views/admin/system-settings/__tests__/manualCleanupForm.spec.ts
View File

@@ -1,8 +1,11 @@
import { describe, expect, it } from 'vitest' import { describe, expect, it } from 'vitest'
import { import {
allowedTargetsForMode,
defaultManualCleanupTargets,
MANUAL_USAGE_CLEANUP_CONFIRM_PHRASE, MANUAL_USAGE_CLEANUP_CONFIRM_PHRASE,
isConfirmPhraseMatched, isConfirmPhraseMatched,
normalizeManualCleanupTargets,
normalizeConfirmPhraseInput, normalizeConfirmPhraseInput,
normalizeOlderThanDaysInput, normalizeOlderThanDaysInput,
} from '../manualCleanupForm' } from '../manualCleanupForm'
@@ -61,4 +64,34 @@ describe('manualCleanupForm', () => {
expect(normalizeOlderThanDaysInput('abc')).toBeNull() expect(normalizeOlderThanDaysInput('abc')).toBeNull()
}) })
}) })
describe('cleanup targets', () => {
it('keeps all ranges available for policy cleanup', () => {
expect(allowedTargetsForMode('policy')).toEqual([
'detail_body',
'compressed_body',
'headers',
'records',
])
expect(defaultManualCleanupTargets('older_than_days')).toEqual([
'detail_body',
'compressed_body',
'headers',
'records',
])
})
it('limits before-now cleanup to body targets', () => {
expect(allowedTargetsForMode('before_now')).toEqual([
'detail_body',
'compressed_body',
])
expect(normalizeManualCleanupTargets('before_now', [
'detail_body',
'headers',
'compressed_body',
'records',
])).toEqual(['detail_body', 'compressed_body'])
})
})
}) })

33
frontend/src/views/admin/system-settings/manualCleanupForm.ts
View File

@@ -1,5 +1,20 @@
export const MANUAL_USAGE_CLEANUP_CONFIRM_PHRASE = '确认清理' export const MANUAL_USAGE_CLEANUP_CONFIRM_PHRASE = '确认清理'
export type ManualCleanupMode = 'policy' | 'older_than_days' | 'before_now'
export type ManualCleanupTarget = 'detail_body' | 'compressed_body' | 'headers' | 'records'
export const MANUAL_CLEANUP_TARGETS: ManualCleanupTarget[] = [
'detail_body',
'compressed_body',
'headers',
'records',
]
export const BEFORE_NOW_ALLOWED_TARGETS: ManualCleanupTarget[] = [
'detail_body',
'compressed_body',
]
export function normalizeConfirmPhraseInput(raw: string): string { export function normalizeConfirmPhraseInput(raw: string): string {
return raw.replace(/\r?\n/g, '').trim() return raw.replace(/\r?\n/g, '').trim()
} }
@@ -14,3 +29,21 @@ export function normalizeOlderThanDaysInput(raw: string | number | null | undefi
if (!Number.isFinite(parsed) || parsed <= 0) return null if (!Number.isFinite(parsed) || parsed <= 0) return null
return Math.floor(parsed) return Math.floor(parsed)
} }
export function allowedTargetsForMode(mode: ManualCleanupMode): ManualCleanupTarget[] {
return mode === 'before_now' ? BEFORE_NOW_ALLOWED_TARGETS : MANUAL_CLEANUP_TARGETS
}
export function normalizeManualCleanupTargets(
mode: ManualCleanupMode,
targets: ManualCleanupTarget[],
): ManualCleanupTarget[] {
const allowed = new Set(allowedTargetsForMode(mode))
return targets.filter((target, index) =>
allowed.has(target) && targets.indexOf(target) === index
)
}
export function defaultManualCleanupTargets(mode: ManualCleanupMode): ManualCleanupTarget[] {
return allowedTargetsForMode(mode)
}