feat: add scoped manual usage cleanup

This commit is contained in:
HsungKayphoon
2026-05-16 14:53:10 +08:00
parent 75b7319465
commit 74a1e5ad7d
17 changed files with 1746 additions and 305 deletions

View File

@@ -37,7 +37,8 @@ use super::{
use aether_data_contracts::repository::usage::{
PendingUsageCleanupSummary, ProviderApiKeyWindowUsageRequest,
StoredProviderApiKeyWindowUsageSummary, StoredUsageDailySummary, UsageAuditListQuery,
UsageCleanupSummary, UsageCleanupWindow, UsageDailyHeatmapQuery,
UsageCleanupExecutionMode, UsageCleanupSummary, UsageCleanupTargets, UsageCleanupWindow,
UsageDailyHeatmapQuery,
};
use aether_runtime_state::RuntimeQueueStore;
use aether_video_tasks_core::read_data_backed_video_task_response;
@@ -980,11 +981,13 @@ impl GatewayDataState {
window: &UsageCleanupWindow,
batch_size: usize,
auto_delete_expired_keys: bool,
targets: UsageCleanupTargets,
mode: UsageCleanupExecutionMode,
) -> Result<UsageCleanupSummary, DataLayerError> {
match &self.usage_writer {
Some(repository) => {
repository
.cleanup_usage(window, batch_size, auto_delete_expired_keys)
.cleanup_usage(window, batch_size, auto_delete_expired_keys, targets, mode)
.await
}
None => Ok(UsageCleanupSummary::default()),
@@ -994,10 +997,16 @@ impl GatewayDataState {
pub(crate) async fn preview_usage_cleanup(
&self,
window: &UsageCleanupWindow,
targets: UsageCleanupTargets,
mode: UsageCleanupExecutionMode,
) -> Result<aether_data_contracts::repository::usage::UsageCleanupPreviewCounts, DataLayerError>
{
match &self.usage_writer {
Some(repository) => repository.preview_usage_cleanup(window).await,
Some(repository) => {
repository
.preview_usage_cleanup(window, targets, mode)
.await
}
None => {
Ok(aether_data_contracts::repository::usage::UsageCleanupPreviewCounts::default())
}

View File

@@ -17,7 +17,9 @@ use crate::handlers::admin::system::shared::settings::{
build_admin_system_stats_payload, current_aether_version, fetch_latest_admin_system_release,
};
use crate::handlers::admin::system::shared::smtp::build_admin_smtp_test_payload;
use crate::maintenance::{ManualUsageCleanupMode, ManualUsageCleanupOptions};
use crate::GatewayError;
use aether_data_contracts::repository::usage::UsageCleanupTargets;
use axum::{
body::{Body, Bytes},
http,
@@ -26,6 +28,7 @@ use axum::{
};
use serde_json::json;
use std::time::Instant;
use url::form_urlencoded;
pub(super) async fn maybe_build_local_admin_core_system_response(
state: &AdminAppState<'_>,
@@ -263,7 +266,7 @@ pub(super) async fn maybe_build_local_admin_core_system_response(
&& request_path == "/api/admin/system/cleanup/usage/manual"
{
return Ok(Some(
build_manual_usage_cleanup_response(state, request_body).await?,
build_manual_usage_cleanup_response(state, request_context, request_body).await?,
));
}
@@ -625,50 +628,36 @@ async fn build_admin_system_cleanup_payload(
async fn build_manual_usage_cleanup_response(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
request_body: Option<&Bytes>,
) -> Result<Response<Body>, GatewayError> {
let older_than_days = match parse_manual_usage_cleanup_request(request_body) {
let options = match parse_manual_usage_cleanup_request(request_body) {
Ok(value) => value,
Err(response) => return Ok(response),
};
let actor_user_id = request_context
.decision()
.and_then(|decision| decision.admin_principal.as_ref())
.map(|principal| principal.user_id.clone());
match crate::maintenance::run_manual_usage_cleanup_once(
&state.app().data,
older_than_days,
None,
match crate::maintenance::start_manual_usage_cleanup_task(
std::sync::Arc::clone(&state.app().data),
options,
actor_user_id,
)
.await
{
Ok(summary) => {
let total = summary
.body_externalized
.saturating_add(summary.legacy_body_refs_migrated)
.saturating_add(summary.body_cleaned)
.saturating_add(summary.header_cleaned)
.saturating_add(summary.keys_cleaned)
.saturating_add(summary.records_deleted);
let message = match older_than_days {
Some(days) => {
format!("请求记录手动清理完成,清理 {days} 天前的记录,影响 {total}")
}
None => format!("请求记录手动清理完成(按当前策略),影响 {total}"),
};
Ok(task) => {
let payload = json!({
"message": message,
"requested_older_than_days": older_than_days,
"summary": {
"body_externalized": summary.body_externalized,
"legacy_body_refs_migrated": summary.legacy_body_refs_migrated,
"body_cleaned": summary.body_cleaned,
"header_cleaned": summary.header_cleaned,
"keys_cleaned": summary.keys_cleaned,
"records_deleted": summary.records_deleted,
},
"total_affected": total,
"message": task.message,
"mode": options.mode.as_str(),
"requested_older_than_days": options.requested_older_than_days,
"targets": options.targets,
"task": task,
});
Ok(attach_admin_audit_response(
Json(payload).into_response(),
"admin_system_usage_cleanup_completed",
"admin_system_usage_cleanup_started",
"manual_usage_cleanup",
"usage_cleanup",
"global",
@@ -696,12 +685,20 @@ async fn build_manual_usage_cleanup_preview_response(
Ok(value) => value,
Err(response) => return Ok(response),
};
let preview =
crate::maintenance::preview_manual_usage_cleanup(&state.app().data, older_than_days)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?;
let options = match parse_manual_usage_cleanup_query_options(
request_context.query_string(),
older_than_days,
) {
Ok(value) => value,
Err(response) => return Ok(response),
};
let preview = crate::maintenance::preview_manual_usage_cleanup(&state.app().data, options)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?;
Ok(Json(json!({
"mode": preview.mode.as_str(),
"requested_older_than_days": preview.requested_older_than_days,
"targets": preview.targets,
"effective_cutoffs": {
"detail": preview.detail_cutoff,
"compressed": preview.compressed_cutoff,
@@ -720,12 +717,12 @@ async fn build_manual_usage_cleanup_preview_response(
fn parse_manual_usage_cleanup_request(
request_body: Option<&Bytes>,
) -> Result<Option<u32>, Response<Body>> {
) -> Result<ManualUsageCleanupOptions, Response<Body>> {
let Some(body) = request_body else {
return Ok(None);
return Ok(ManualUsageCleanupOptions::policy());
};
if body.is_empty() {
return Ok(None);
return Ok(ManualUsageCleanupOptions::policy());
}
let parsed: serde_json::Value = match serde_json::from_slice(body) {
Ok(value) => value,
@@ -738,45 +735,208 @@ fn parse_manual_usage_cleanup_request(
}
};
let Some(object) = parsed.as_object() else {
return Ok(None);
return Err(bad_manual_cleanup_request("请求体必须为 JSON 对象"));
};
match object.get("older_than_days") {
None | Some(serde_json::Value::Null) => Ok(None),
Some(value) => value
.as_u64()
.and_then(|value| u32::try_from(value).ok())
.filter(|days| *days >= 1)
.map(Some)
.ok_or_else(|| {
(
http::StatusCode::BAD_REQUEST,
Json(json!({
"detail": "older_than_days 必须为正整数",
})),
)
.into_response()
}),
parse_manual_usage_cleanup_options(
object.get("mode").and_then(serde_json::Value::as_str),
object.get("older_than_days"),
object.get("targets"),
)
}
fn parse_manual_usage_cleanup_query_options(
query_string: Option<&str>,
older_than_days: Option<u32>,
) -> Result<ManualUsageCleanupOptions, Response<Body>> {
let mode = query_param(query_string, "mode");
let targets = query_param(query_string, "targets").map(serde_json::Value::String);
let older_value =
older_than_days.map(|days| serde_json::Value::Number(serde_json::Number::from(days)));
parse_manual_usage_cleanup_options(mode.as_deref(), older_value.as_ref(), targets.as_ref())
}
fn parse_manual_usage_cleanup_options(
raw_mode: Option<&str>,
older_than_days: Option<&serde_json::Value>,
targets_value: Option<&serde_json::Value>,
) -> Result<ManualUsageCleanupOptions, Response<Body>> {
let (requested_older_than_days, requested_before_now) =
parse_manual_cleanup_older_than_days(older_than_days)?;
let mode =
parse_manual_cleanup_mode(raw_mode, requested_older_than_days, requested_before_now)?;
if mode == ManualUsageCleanupMode::OlderThanDays && requested_older_than_days.is_none() {
return Err(bad_manual_cleanup_request(
"older_than_days 模式必须提供正整数天数",
));
}
if mode == ManualUsageCleanupMode::BeforeNow && requested_older_than_days.is_some() {
return Err(bad_manual_cleanup_request(
"before_now 模式不能同时提供 older_than_days",
));
}
if raw_mode.is_some() && requested_before_now && mode != ManualUsageCleanupMode::BeforeNow {
return Err(bad_manual_cleanup_request(
"older_than_days 为 0 时必须使用 before_now 模式",
));
}
if raw_mode.is_some()
&& mode == ManualUsageCleanupMode::Policy
&& requested_older_than_days.is_some()
{
return Err(bad_manual_cleanup_request(
"policy 模式不能同时提供 older_than_days",
));
}
let targets = parse_manual_cleanup_targets(targets_value, mode)?;
if !targets.any_selected() {
return Err(bad_manual_cleanup_request("至少选择一个清理范围"));
}
if mode == ManualUsageCleanupMode::BeforeNow
&& (targets.headers || targets.records || targets.expired_keys)
{
return Err(bad_manual_cleanup_request(
"清理当前时刻之前只允许选择详细请求体和压缩请求体",
));
}
Ok(ManualUsageCleanupOptions {
mode,
requested_older_than_days,
targets,
})
}
fn parse_manual_cleanup_mode(
raw_mode: Option<&str>,
requested_older_than_days: Option<u32>,
requested_before_now: bool,
) -> Result<ManualUsageCleanupMode, Response<Body>> {
let Some(raw) = raw_mode.map(str::trim).filter(|value| !value.is_empty()) else {
return Ok(if requested_before_now {
ManualUsageCleanupMode::BeforeNow
} else if requested_older_than_days.is_some() {
ManualUsageCleanupMode::OlderThanDays
} else {
ManualUsageCleanupMode::Policy
});
};
match raw {
"policy" => Ok(ManualUsageCleanupMode::Policy),
"older_than_days" => Ok(ManualUsageCleanupMode::OlderThanDays),
"before_now" => Ok(ManualUsageCleanupMode::BeforeNow),
_ => Err(bad_manual_cleanup_request(
"mode 必须为 policy、older_than_days 或 before_now",
)),
}
}
fn parse_older_than_days_query(query_string: Option<&str>) -> Result<Option<u32>, Response<Body>> {
let Some(query) = query_string.filter(|value| !value.is_empty()) else {
return Ok(None);
};
let value = query
.split('&')
.filter_map(|pair| pair.split_once('='))
.find_map(|(key, value)| {
if key == "older_than_days" && !value.is_empty() {
Some(value)
} else {
None
fn parse_manual_cleanup_older_than_days(
value: Option<&serde_json::Value>,
) -> Result<(Option<u32>, bool), Response<Body>> {
match value {
None | Some(serde_json::Value::Null) => Ok((None, false)),
Some(value) => {
let Some(raw) = value.as_u64() else {
return Err(bad_manual_cleanup_request("older_than_days 必须为非负整数"));
};
if raw == 0 {
return Ok((None, true));
}
let days = u32::try_from(raw)
.ok()
.filter(|days| *days >= 1)
.ok_or_else(|| bad_manual_cleanup_request("older_than_days 必须为正整数"))?;
Ok((Some(days), false))
}
}
}
fn parse_manual_cleanup_targets(
value: Option<&serde_json::Value>,
mode: ManualUsageCleanupMode,
) -> Result<UsageCleanupTargets, Response<Body>> {
let Some(value) = value else {
return Ok(match mode {
ManualUsageCleanupMode::BeforeNow => UsageCleanupTargets::body_targets(),
ManualUsageCleanupMode::Policy | ManualUsageCleanupMode::OlderThanDays => {
UsageCleanupTargets::all_policy_targets()
}
});
let Some(raw) = value else {
};
if value.is_null() {
return Ok(match mode {
ManualUsageCleanupMode::BeforeNow => UsageCleanupTargets::body_targets(),
ManualUsageCleanupMode::Policy | ManualUsageCleanupMode::OlderThanDays => {
UsageCleanupTargets::all_policy_targets()
}
});
}
let raw_targets = match value {
serde_json::Value::Array(items) => items
.iter()
.map(|item| {
item.as_str()
.map(str::to_string)
.ok_or_else(|| bad_manual_cleanup_request("targets 必须为字符串数组"))
})
.collect::<Result<Vec<_>, _>>()?,
serde_json::Value::String(raw) => raw
.split(',')
.map(str::trim)
.filter(|item| !item.is_empty())
.map(str::to_string)
.collect(),
_ => return Err(bad_manual_cleanup_request("targets 必须为字符串数组")),
};
let mut targets = UsageCleanupTargets {
detail_body: false,
compressed_body: false,
headers: false,
records: false,
expired_keys: false,
};
for raw in raw_targets {
match raw.as_str() {
"detail_body" | "detail" | "raw_body" => targets.detail_body = true,
"compressed_body" | "compressed" => targets.compressed_body = true,
"headers" | "header" => targets.headers = true,
"records" | "log" | "logs" => targets.records = true,
"expired_keys" => targets.expired_keys = true,
"all" => targets = UsageCleanupTargets::all_policy_targets(),
_ => {
return Err(bad_manual_cleanup_request(
"targets 只能包含 detail_body、compressed_body、headers、records",
))
}
}
}
Ok(targets)
}
fn bad_manual_cleanup_request(detail: impl Into<String>) -> Response<Body> {
(
http::StatusCode::BAD_REQUEST,
Json(json!({ "detail": detail.into() })),
)
.into_response()
}
fn query_param(query_string: Option<&str>, name: &str) -> Option<String> {
let query = query_string.filter(|value| !value.is_empty())?;
form_urlencoded::parse(query.as_bytes())
.find_map(|(key, value)| (key == name && !value.is_empty()).then(|| value.into_owned()))
}
fn parse_older_than_days_query(query_string: Option<&str>) -> Result<Option<u32>, Response<Body>> {
let Some(value) = query_param(query_string, "older_than_days") else {
return Ok(None);
};
raw.parse::<u32>()
value
.parse::<u32>()
.ok()
.filter(|days| *days >= 1)
.map(Some)
@@ -790,3 +950,55 @@ fn parse_older_than_days_query(query_string: Option<&str>) -> Result<Option<u32>
.into_response()
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn manual_cleanup_request_defaults_to_policy_targets() {
let options = parse_manual_usage_cleanup_request(None).expect("default request is valid");
assert_eq!(options.mode, ManualUsageCleanupMode::Policy);
assert_eq!(options.requested_older_than_days, None);
assert_eq!(options.targets, UsageCleanupTargets::all_policy_targets());
}
#[test]
fn manual_cleanup_request_treats_zero_days_as_before_now_body_only() {
let body = Bytes::from_static(br#"{"older_than_days":0}"#);
let options =
parse_manual_usage_cleanup_request(Some(&body)).expect("before-now request is valid");
assert_eq!(options.mode, ManualUsageCleanupMode::BeforeNow);
assert_eq!(options.requested_older_than_days, None);
assert_eq!(options.targets, UsageCleanupTargets::body_targets());
}
#[test]
fn manual_cleanup_request_rejects_before_now_headers() {
let body = Bytes::from_static(br#"{"mode":"before_now","targets":["headers"]}"#);
assert!(parse_manual_usage_cleanup_request(Some(&body)).is_err());
}
#[test]
fn manual_cleanup_preview_query_decodes_comma_separated_targets() {
let options = parse_manual_usage_cleanup_query_options(
Some("mode=before_now&targets=detail_body%2Ccompressed_body"),
None,
)
.expect("encoded targets query is valid");
assert_eq!(options.mode, ManualUsageCleanupMode::BeforeNow);
assert_eq!(options.targets, UsageCleanupTargets::body_targets());
}
#[test]
fn manual_cleanup_request_rejects_non_object_body() {
let body = Bytes::from_static(br#"[]"#);
assert!(parse_manual_usage_cleanup_request(Some(&body)).is_err());
}
}

View File

@@ -21,9 +21,10 @@ pub(crate) use runtime::{
spawn_stats_aggregation_worker, spawn_stats_hourly_aggregation_worker,
spawn_usage_cleanup_worker, spawn_wallet_daily_usage_aggregation_worker,
start_admin_request_body_cleanup_task, start_admin_system_purge_task,
start_proxy_upgrade_rollout, AccountSelfCheckRunSummary, AdminCleanupRunRecord,
AdminCleanupTaskKind, AdminStatsRebuildSummary, AdminSystemCleanupSummary,
ManualUsageCleanupError, OAuthTokenRefreshRunSummary, PoolQuotaProbeRunSummary,
start_manual_usage_cleanup_task, start_proxy_upgrade_rollout, AccountSelfCheckRunSummary,
AdminCleanupRunRecord, AdminCleanupTaskKind, AdminStatsRebuildSummary,
AdminSystemCleanupSummary, ManualUsageCleanupError, ManualUsageCleanupMode,
ManualUsageCleanupOptions, OAuthTokenRefreshRunSummary, PoolQuotaProbeRunSummary,
PoolQuotaProbeWorkerConfig, ProviderCheckinRunSummary, ProxyUpgradeRolloutCancelSummary,
ProxyUpgradeRolloutConflictClearSummary, ProxyUpgradeRolloutNodeActionSummary,
ProxyUpgradeRolloutProbeConfig, ProxyUpgradeRolloutSkippedRestoreSummary,

View File

@@ -61,9 +61,9 @@ pub(crate) use aether_data_contracts::repository::usage::{
};
use audit_cleanup::*;
pub(crate) use cleanup_runs::{
list_admin_cleanup_run_records, record_completed_cleanup_run, record_failed_cleanup_run,
start_admin_request_body_cleanup_task, start_admin_system_purge_task, AdminCleanupRunRecord,
AdminCleanupTaskKind, USAGE_CLEANUP_KIND,
list_admin_cleanup_run_records, record_admin_cleanup_run, record_completed_cleanup_run,
record_failed_cleanup_run, start_admin_request_body_cleanup_task,
start_admin_system_purge_task, AdminCleanupRunRecord, AdminCleanupTaskKind, USAGE_CLEANUP_KIND,
};
use config::*;
use db_maintenance::*;
@@ -98,12 +98,16 @@ pub(crate) use proxy_upgrade_rollout::{
};
use request_candidate_cleanup::*;
use runners::*;
pub(crate) use runners::{run_manual_usage_cleanup_once, ManualUsageCleanupError};
pub(crate) use runners::{
run_manual_usage_cleanup_once, start_manual_usage_cleanup_task, ManualUsageCleanupError,
};
use schedule::*;
use stats_daily::*;
use stats_hourly::*;
pub(crate) use usage_cleanup::preview_manual_usage_cleanup;
use usage_cleanup::*;
pub(crate) use usage_cleanup::{
preview_manual_usage_cleanup, ManualUsageCleanupMode, ManualUsageCleanupOptions,
};
use wallet_daily_usage::*;
pub(crate) use workers::*;

View File

@@ -489,7 +489,7 @@ fn request_body_cleanup_record(
}
}
async fn record_cleanup_run(
pub(crate) async fn record_admin_cleanup_run(
data: &GatewayDataState,
record: AdminCleanupRunRecord,
) -> Result<(), DataLayerError> {
@@ -509,6 +509,13 @@ async fn record_cleanup_run(
Ok(())
}
async fn record_cleanup_run(
data: &GatewayDataState,
record: AdminCleanupRunRecord,
) -> Result<(), DataLayerError> {
record_admin_cleanup_run(data, record).await
}
fn parse_cleanup_run_records(value: Value) -> Vec<AdminCleanupRunRecord> {
value
.as_array()

View File

@@ -1,5 +1,6 @@
use aether_data_contracts::DataLayerError;
use serde_json::json;
use std::sync::Arc;
use std::time::Instant;
use tracing::{info, warn};
@@ -11,11 +12,12 @@ use super::{
cleanup_expired_gemini_file_mappings_once, cleanup_proxy_node_metrics_once,
cleanup_request_candidates_once, cleanup_stale_pending_requests_once,
cleanup_stale_proxy_nodes_once, collect_proxy_upgrade_rollout_probes, now_unix_secs,
perform_db_maintenance_once, perform_provider_checkin_once, perform_stats_aggregation_once,
perform_stats_hourly_aggregation_once, perform_usage_cleanup_once,
perform_usage_cleanup_once_with_override, perform_wallet_daily_usage_aggregation_once,
record_completed_cleanup_run, record_failed_cleanup_run, record_proxy_upgrade_traffic_success,
summarize_database_pool,
perform_db_maintenance_once, perform_manual_usage_cleanup_once, perform_provider_checkin_once,
perform_stats_aggregation_once, perform_stats_hourly_aggregation_once,
perform_usage_cleanup_once, perform_wallet_daily_usage_aggregation_once,
record_admin_cleanup_run, record_completed_cleanup_run, record_failed_cleanup_run,
record_proxy_upgrade_traffic_success, summarize_database_pool, AdminCleanupRunRecord,
ManualUsageCleanupOptions,
};
pub(super) async fn run_audit_cleanup_once(data: &GatewayDataState) -> Result<(), DataLayerError> {
@@ -299,15 +301,14 @@ pub(super) async fn run_usage_cleanup_once(data: &GatewayDataState) -> Result<()
Ok(())
}
pub(crate) async fn run_manual_usage_cleanup_once(
data: &GatewayDataState,
override_older_than_days: Option<u32>,
pub(crate) async fn start_manual_usage_cleanup_task(
data: Arc<GatewayDataState>,
options: ManualUsageCleanupOptions,
actor_user_id: Option<String>,
) -> Result<aether_data_contracts::repository::usage::UsageCleanupSummary, ManualUsageCleanupError>
{
) -> Result<AdminCleanupRunRecord, ManualUsageCleanupError> {
use super::{list_admin_cleanup_run_records, USAGE_CLEANUP_KIND};
let existing = list_admin_cleanup_run_records(data)
let existing = list_admin_cleanup_run_records(&data)
.await
.map_err(ManualUsageCleanupError::DataLayer)?;
if existing
@@ -318,10 +319,42 @@ pub(crate) async fn run_manual_usage_cleanup_once(
}
let started_at_unix_secs = now_unix_secs();
let record = AdminCleanupRunRecord {
id: uuid::Uuid::new_v4().to_string(),
kind: USAGE_CLEANUP_KIND.to_string(),
trigger: "manual".to_string(),
status: "processing".to_string(),
message: manual_usage_cleanup_start_message(options),
started_at_unix_secs,
completed_at_unix_secs: None,
duration_ms: None,
summary: manual_usage_cleanup_progress_summary(options, 0, None, actor_user_id.as_deref()),
error: None,
};
record_admin_cleanup_run(&data, record.clone())
.await
.map_err(ManualUsageCleanupError::DataLayer)?;
tokio::spawn(run_manual_usage_cleanup_task(
data,
record.clone(),
options,
actor_user_id,
));
Ok(record)
}
pub(crate) async fn run_manual_usage_cleanup_once(
data: &GatewayDataState,
options: ManualUsageCleanupOptions,
actor_user_id: Option<String>,
) -> Result<aether_data_contracts::repository::usage::UsageCleanupSummary, ManualUsageCleanupError>
{
use super::USAGE_CLEANUP_KIND;
let started_at = Instant::now();
let override_duration =
override_older_than_days.map(|days| chrono::Duration::days(i64::from(days)));
let summary = match perform_usage_cleanup_once_with_override(data, override_duration).await {
let started_at_unix_secs = now_unix_secs();
let summary = match perform_manual_usage_cleanup_once(data, options).await {
Ok(summary) => summary,
Err(err) => {
record_failed_cleanup_run(
@@ -336,17 +369,8 @@ pub(crate) async fn run_manual_usage_cleanup_once(
return Err(ManualUsageCleanupError::DataLayer(err));
}
};
let total = summary
.body_externalized
.saturating_add(summary.legacy_body_refs_migrated)
.saturating_add(summary.body_cleaned)
.saturating_add(summary.header_cleaned)
.saturating_add(summary.keys_cleaned)
.saturating_add(summary.records_deleted);
let message = match override_older_than_days {
Some(days) => format!("请求记录手动清理完成,清理 {days} 天前的记录,影响 {total}"),
None => format!("请求记录手动清理完成(按当前策略),影响 {total}"),
};
let total = usage_cleanup_total(summary);
let message = manual_usage_cleanup_completed_message(options, total);
record_completed_cleanup_run(
data,
USAGE_CLEANUP_KIND,
@@ -360,7 +384,9 @@ pub(crate) async fn run_manual_usage_cleanup_once(
"header_cleaned": summary.header_cleaned,
"keys_cleaned": summary.keys_cleaned,
"records_deleted": summary.records_deleted,
"requested_older_than_days": override_older_than_days,
"mode": options.mode.as_str(),
"requested_older_than_days": options.requested_older_than_days,
"targets": options.targets,
"actor_user_id": actor_user_id,
}),
message,
@@ -371,7 +397,8 @@ pub(crate) async fn run_manual_usage_cleanup_once(
log_type = "ops",
worker = "usage_cleanup",
trigger = "manual",
requested_older_than_days = override_older_than_days,
mode = options.mode.as_str(),
requested_older_than_days = options.requested_older_than_days,
actor_user_id = actor_user_id.as_deref(),
total_affected = total,
"gateway finished manual usage cleanup"
@@ -379,6 +406,154 @@ pub(crate) async fn run_manual_usage_cleanup_once(
Ok(summary)
}
async fn run_manual_usage_cleanup_task(
data: Arc<GatewayDataState>,
initial_record: AdminCleanupRunRecord,
options: ManualUsageCleanupOptions,
actor_user_id: Option<String>,
) {
let started_at = Instant::now();
match perform_manual_usage_cleanup_once(&data, options).await {
Ok(summary) => {
let total = usage_cleanup_total(summary);
let record = AdminCleanupRunRecord {
id: initial_record.id,
kind: initial_record.kind,
trigger: initial_record.trigger,
status: "completed".to_string(),
message: manual_usage_cleanup_completed_message(options, total),
started_at_unix_secs: initial_record.started_at_unix_secs,
completed_at_unix_secs: Some(now_unix_secs()),
duration_ms: Some(
started_at
.elapsed()
.as_millis()
.try_into()
.unwrap_or(u64::MAX),
),
summary: manual_usage_cleanup_progress_summary(
options,
100,
Some(summary),
actor_user_id.as_deref(),
),
error: None,
};
if let Err(err) = record_admin_cleanup_run(&data, record).await {
warn!(error = %err, "failed to record manual usage cleanup completion");
}
info!(
event_name = "usage_cleanup_manual_completed",
log_type = "ops",
worker = "usage_cleanup",
trigger = "manual",
mode = options.mode.as_str(),
requested_older_than_days = options.requested_older_than_days,
actor_user_id = actor_user_id.as_deref(),
total_affected = total,
"gateway finished manual usage cleanup task"
);
}
Err(err) => {
let record = AdminCleanupRunRecord {
id: initial_record.id,
kind: initial_record.kind,
trigger: initial_record.trigger,
status: "failed".to_string(),
message: "请求记录手动清理失败".to_string(),
started_at_unix_secs: initial_record.started_at_unix_secs,
completed_at_unix_secs: Some(now_unix_secs()),
duration_ms: Some(
started_at
.elapsed()
.as_millis()
.try_into()
.unwrap_or(u64::MAX),
),
summary: manual_usage_cleanup_progress_summary(
options,
100,
None,
actor_user_id.as_deref(),
),
error: Some(err.to_string()),
};
if let Err(record_err) = record_admin_cleanup_run(&data, record).await {
warn!(error = %record_err, "failed to record manual usage cleanup failure");
}
warn!(error = %err, "manual usage cleanup task failed");
}
}
}
fn usage_cleanup_total(
summary: aether_data_contracts::repository::usage::UsageCleanupSummary,
) -> usize {
summary
.body_externalized
.saturating_add(summary.legacy_body_refs_migrated)
.saturating_add(summary.body_cleaned)
.saturating_add(summary.header_cleaned)
.saturating_add(summary.keys_cleaned)
.saturating_add(summary.records_deleted)
}
fn manual_usage_cleanup_start_message(options: ManualUsageCleanupOptions) -> String {
match options.mode {
super::ManualUsageCleanupMode::BeforeNow => {
"请求记录手动清理已开始,清理当前时刻之前的已选请求体".to_string()
}
super::ManualUsageCleanupMode::OlderThanDays => format!(
"请求记录手动清理已开始,清理 {} 天前的已选内容",
options.requested_older_than_days.unwrap_or_default()
),
super::ManualUsageCleanupMode::Policy => {
"请求记录手动清理已开始,按当前策略清理已选内容".to_string()
}
}
}
fn manual_usage_cleanup_completed_message(
options: ManualUsageCleanupOptions,
total: usize,
) -> String {
match options.mode {
super::ManualUsageCleanupMode::BeforeNow => {
format!("请求记录手动清理完成,已清理当前时刻之前的已选请求体,影响 {total}")
}
super::ManualUsageCleanupMode::OlderThanDays => format!(
"请求记录手动清理完成,清理 {} 天前的已选内容,影响 {total}",
options.requested_older_than_days.unwrap_or_default()
),
super::ManualUsageCleanupMode::Policy => {
format!("请求记录手动清理完成(按当前策略),影响 {total}")
}
}
}
fn manual_usage_cleanup_progress_summary(
options: ManualUsageCleanupOptions,
progress_percent: u8,
summary: Option<aether_data_contracts::repository::usage::UsageCleanupSummary>,
actor_user_id: Option<&str>,
) -> serde_json::Value {
let summary = summary.unwrap_or_default();
json!({
"mode": options.mode.as_str(),
"requested_older_than_days": options.requested_older_than_days,
"targets": options.targets,
"progress_percent": progress_percent,
"body_externalized": summary.body_externalized,
"legacy_body_refs_migrated": summary.legacy_body_refs_migrated,
"body_cleaned": summary.body_cleaned,
"header_cleaned": summary.header_cleaned,
"keys_cleaned": summary.keys_cleaned,
"records_deleted": summary.records_deleted,
"total": usage_cleanup_total(summary),
"actor_user_id": actor_user_id,
})
}
#[derive(Debug)]
pub(crate) enum ManualUsageCleanupError {
AlreadyRunning,

View File

@@ -28,12 +28,12 @@ use super::{
spawn_stats_hourly_aggregation_worker, spawn_usage_cleanup_worker,
spawn_wallet_daily_usage_aggregation_worker, start_proxy_upgrade_rollout,
stats_aggregation_target_day, stats_hourly_aggregation_target_hour, summarize_database_pool,
usage_cleanup_settings, usage_cleanup_window, usage_cleanup_window_with_override,
wallet_daily_usage_aggregation_target, AppState, DbMaintenanceRunSummary,
FailedPendingUsageRow, GatewayDataState, ProxyNodeMetricsCleanupSettings,
ProxyUpgradeRolloutProbeConfig, StalePendingUsageRow, UsageCleanupSettings, USAGE_CLEANUP_HOUR,
USAGE_CLEANUP_MINUTE, WALLET_DAILY_USAGE_AGGREGATION_HOUR,
WALLET_DAILY_USAGE_AGGREGATION_MINUTE,
usage_cleanup_settings, usage_cleanup_window, usage_cleanup_window_for_mode,
usage_cleanup_window_with_override, wallet_daily_usage_aggregation_target, AppState,
DbMaintenanceRunSummary, FailedPendingUsageRow, GatewayDataState, ManualUsageCleanupMode,
ProxyNodeMetricsCleanupSettings, ProxyUpgradeRolloutProbeConfig, StalePendingUsageRow,
UsageCleanupSettings, USAGE_CLEANUP_HOUR, USAGE_CLEANUP_MINUTE,
WALLET_DAILY_USAGE_AGGREGATION_HOUR, WALLET_DAILY_USAGE_AGGREGATION_MINUTE,
};
#[tokio::test]
@@ -986,6 +986,29 @@ fn usage_cleanup_window_with_override_is_always_non_aggressive() {
assert_eq!(passthrough, policy);
}
#[test]
fn usage_cleanup_before_now_window_uses_current_timestamp_only() {
let now_utc = "2026-03-18T03:00:00Z"
.parse::<DateTime<Utc>>()
.expect("timestamp should parse");
let settings = UsageCleanupSettings {
detail_retention_days: 7,
compressed_retention_days: 30,
header_retention_days: 90,
log_retention_days: 365,
batch_size: 123,
auto_delete_expired_keys: false,
};
let window =
usage_cleanup_window_for_mode(now_utc, settings, ManualUsageCleanupMode::BeforeNow, None);
assert_eq!(window.detail_cutoff, now_utc);
assert_eq!(window.compressed_cutoff, now_utc);
assert_eq!(window.header_cutoff, now_utc);
assert_eq!(window.log_cutoff, now_utc);
}
#[tokio::test]
async fn summarize_database_pool_uses_busy_connections_for_usage_rate() {
let data = GatewayDataState::from_config(crate::data::GatewayDataConfig::from_postgres_config(

View File

@@ -1,4 +1,6 @@
use aether_data_contracts::repository::usage::{UsageCleanupSummary, UsageCleanupWindow};
use aether_data_contracts::repository::usage::{
UsageCleanupExecutionMode, UsageCleanupSummary, UsageCleanupTargets, UsageCleanupWindow,
};
use aether_data_contracts::DataLayerError;
use chrono::Utc;
@@ -15,6 +17,8 @@ pub(crate) struct ManualUsageCleanupPreview {
pub compressed_cutoff: chrono::DateTime<Utc>,
pub header_cutoff: chrono::DateTime<Utc>,
pub log_cutoff: chrono::DateTime<Utc>,
pub mode: ManualUsageCleanupMode,
pub targets: UsageCleanupTargets,
pub requested_older_than_days: Option<u32>,
pub detail_count: u64,
pub compressed_count: u64,
@@ -22,49 +26,131 @@ pub(crate) struct ManualUsageCleanupPreview {
pub log_count: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum ManualUsageCleanupMode {
Policy,
OlderThanDays,
BeforeNow,
}
impl ManualUsageCleanupMode {
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::Policy => "policy",
Self::OlderThanDays => "older_than_days",
Self::BeforeNow => "before_now",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct ManualUsageCleanupOptions {
pub(crate) mode: ManualUsageCleanupMode,
pub(crate) requested_older_than_days: Option<u32>,
pub(crate) targets: UsageCleanupTargets,
}
impl ManualUsageCleanupOptions {
pub(crate) const fn policy() -> Self {
Self {
mode: ManualUsageCleanupMode::Policy,
requested_older_than_days: None,
targets: UsageCleanupTargets::all_policy_targets(),
}
}
}
pub(super) async fn perform_usage_cleanup_once(
data: &GatewayDataState,
) -> Result<UsageCleanupSummary, DataLayerError> {
perform_usage_cleanup_once_with_override(data, None).await
perform_usage_cleanup_once_with_override(data, None, true).await
}
pub(super) async fn perform_usage_cleanup_once_with_override(
data: &GatewayDataState,
override_older_than: Option<chrono::Duration>,
respect_auto_enabled: bool,
) -> Result<UsageCleanupSummary, DataLayerError> {
let options = ManualUsageCleanupOptions {
mode: if override_older_than.is_some() {
ManualUsageCleanupMode::OlderThanDays
} else {
ManualUsageCleanupMode::Policy
},
requested_older_than_days: None,
targets: UsageCleanupTargets::all_policy_targets(),
};
perform_usage_cleanup_once_with_options(
data,
options,
override_older_than,
respect_auto_enabled,
)
.await
}
pub(super) async fn perform_manual_usage_cleanup_once(
data: &GatewayDataState,
options: ManualUsageCleanupOptions,
) -> Result<UsageCleanupSummary, DataLayerError> {
let override_duration = options
.requested_older_than_days
.map(|days| chrono::Duration::days(i64::from(days)));
perform_usage_cleanup_once_with_options(data, options, override_duration, false).await
}
async fn perform_usage_cleanup_once_with_options(
data: &GatewayDataState,
options: ManualUsageCleanupOptions,
override_older_than: Option<chrono::Duration>,
respect_auto_enabled: bool,
) -> Result<UsageCleanupSummary, DataLayerError> {
if !data.has_usage_writer() {
return Ok(UsageCleanupSummary::default());
}
if override_older_than.is_none()
if respect_auto_enabled
&& override_older_than.is_none()
&& !system_config_bool(data, "enable_auto_cleanup", true).await?
{
return Ok(UsageCleanupSummary::default());
}
let window = compute_usage_cleanup_window(data, override_older_than).await?;
let window = compute_usage_cleanup_window(data, options.mode, override_older_than).await?;
let settings = usage_cleanup_settings(data).await?;
data.cleanup_usage(
&window,
settings.batch_size,
settings.auto_delete_expired_keys,
options.targets,
cleanup_execution_mode(options.mode),
)
.await
}
pub(crate) async fn preview_manual_usage_cleanup(
data: &GatewayDataState,
override_older_than_days: Option<u32>,
options: ManualUsageCleanupOptions,
) -> Result<ManualUsageCleanupPreview, DataLayerError> {
let override_duration =
override_older_than_days.map(|days| chrono::Duration::days(i64::from(days)));
let window = compute_usage_cleanup_window(data, override_duration).await?;
let counts = data.preview_usage_cleanup(&window).await?;
let override_duration = options
.requested_older_than_days
.map(|days| chrono::Duration::days(i64::from(days)));
let window = compute_usage_cleanup_window(data, options.mode, override_duration).await?;
let counts = data
.preview_usage_cleanup(
&window,
options.targets,
cleanup_execution_mode(options.mode),
)
.await?;
Ok(ManualUsageCleanupPreview {
detail_cutoff: window.detail_cutoff,
compressed_cutoff: window.compressed_cutoff,
header_cutoff: window.header_cutoff,
log_cutoff: window.log_cutoff,
requested_older_than_days: override_older_than_days,
mode: options.mode,
targets: options.targets,
requested_older_than_days: options.requested_older_than_days,
detail_count: counts.detail,
compressed_count: counts.compressed,
header_count: counts.header,
@@ -74,11 +160,43 @@ pub(crate) async fn preview_manual_usage_cleanup(
async fn compute_usage_cleanup_window(
data: &GatewayDataState,
mode: ManualUsageCleanupMode,
override_older_than: Option<chrono::Duration>,
) -> Result<UsageCleanupWindow, DataLayerError> {
let settings = usage_cleanup_settings(data).await?;
Ok(match override_older_than {
Some(duration) => usage_cleanup_window_with_override(Utc::now(), settings, Some(duration)),
None => usage_cleanup_window(Utc::now(), settings),
})
Ok(usage_cleanup_window_for_mode(
Utc::now(),
settings,
mode,
override_older_than,
))
}
pub(super) fn usage_cleanup_window_for_mode(
now_utc: chrono::DateTime<Utc>,
settings: super::UsageCleanupSettings,
mode: ManualUsageCleanupMode,
override_older_than: Option<chrono::Duration>,
) -> UsageCleanupWindow {
match mode {
ManualUsageCleanupMode::Policy => usage_cleanup_window(now_utc, settings),
ManualUsageCleanupMode::OlderThanDays => {
usage_cleanup_window_with_override(now_utc, settings, override_older_than)
}
ManualUsageCleanupMode::BeforeNow => UsageCleanupWindow {
detail_cutoff: now_utc,
compressed_cutoff: now_utc,
header_cutoff: now_utc,
log_cutoff: now_utc,
},
}
}
fn cleanup_execution_mode(mode: ManualUsageCleanupMode) -> UsageCleanupExecutionMode {
match mode {
ManualUsageCleanupMode::BeforeNow => UsageCleanupExecutionMode::BeforeNowBodyFields,
ManualUsageCleanupMode::Policy | ManualUsageCleanupMode::OlderThanDays => {
UsageCleanupExecutionMode::Policy
}
}
}

View File

@@ -18,12 +18,12 @@ pub use types::{
UsageAuditListQuery, UsageAuditSummaryQuery, UsageBodyCaptureResult, UsageBodyCaptureState,
UsageBodyCaptureStorage, UsageBodyField, UsageBreakdownGroupBy, UsageBreakdownSummaryQuery,
UsageCacheAffinityHitSummaryQuery, UsageCacheAffinityIntervalGroupBy,
UsageCacheAffinityIntervalQuery, UsageCacheHitSummaryQuery, UsageCleanupPreviewCounts,
UsageCleanupSummary, UsageCleanupWindow, UsageCostSavingsSummaryQuery, UsageDailyHeatmapQuery,
UsageDashboardDailyBreakdownQuery, UsageDashboardProviderCountsQuery,
UsageDashboardSummaryQuery, UsageErrorDistributionQuery, UsageLeaderboardGroupBy,
UsageLeaderboardQuery, UsageMonitoringErrorCountQuery, UsageMonitoringErrorListQuery,
UsagePerformancePercentilesQuery, UsageProviderPerformanceQuery, UsageReadRepository,
UsageRepository, UsageSettledCostSummaryQuery, UsageTimeSeriesGranularity,
UsageCacheAffinityIntervalQuery, UsageCacheHitSummaryQuery, UsageCleanupExecutionMode,
UsageCleanupPreviewCounts, UsageCleanupSummary, UsageCleanupTargets, UsageCleanupWindow,
UsageCostSavingsSummaryQuery, UsageDailyHeatmapQuery, UsageDashboardDailyBreakdownQuery,
UsageDashboardProviderCountsQuery, UsageDashboardSummaryQuery, UsageErrorDistributionQuery,
UsageLeaderboardGroupBy, UsageLeaderboardQuery, UsageMonitoringErrorCountQuery,
UsageMonitoringErrorListQuery, UsagePerformancePercentilesQuery, UsageProviderPerformanceQuery,
UsageReadRepository, UsageRepository, UsageSettledCostSummaryQuery, UsageTimeSeriesGranularity,
UsageTimeSeriesQuery, UsageWriteRepository,
};

View File

@@ -1688,16 +1688,20 @@ pub trait UsageWriteRepository: Send + Sync {
window: &UsageCleanupWindow,
batch_size: usize,
auto_delete_expired_keys: bool,
targets: UsageCleanupTargets,
mode: UsageCleanupExecutionMode,
) -> Result<UsageCleanupSummary, crate::DataLayerError> {
let _ = (window, batch_size, auto_delete_expired_keys);
let _ = (window, batch_size, auto_delete_expired_keys, targets, mode);
Ok(UsageCleanupSummary::default())
}
async fn preview_usage_cleanup(
&self,
window: &UsageCleanupWindow,
targets: UsageCleanupTargets,
mode: UsageCleanupExecutionMode,
) -> Result<UsageCleanupPreviewCounts, crate::DataLayerError> {
let _ = window;
let _ = (window, targets, mode);
Ok(UsageCleanupPreviewCounts::default())
}
}
@@ -1730,6 +1734,58 @@ pub struct UsageCleanupWindow {
pub log_cutoff: DateTime<Utc>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct UsageCleanupTargets {
pub detail_body: bool,
pub compressed_body: bool,
pub headers: bool,
pub records: bool,
pub expired_keys: bool,
}
impl UsageCleanupTargets {
pub const fn all_policy_targets() -> Self {
Self {
detail_body: true,
compressed_body: true,
headers: true,
records: true,
expired_keys: true,
}
}
pub const fn body_targets() -> Self {
Self {
detail_body: true,
compressed_body: true,
headers: false,
records: false,
expired_keys: false,
}
}
pub const fn any_selected(self) -> bool {
self.detail_body
|| self.compressed_body
|| self.headers
|| self.records
|| self.expired_keys
}
}
impl Default for UsageCleanupTargets {
fn default() -> Self {
Self::all_policy_targets()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)]
pub enum UsageCleanupExecutionMode {
#[default]
Policy,
BeforeNowBodyFields,
}
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct UsageCleanupPreviewCounts {
pub detail: u64,

View File

@@ -1,8 +1,8 @@
use std::io::Write;
use aether_data_contracts::repository::usage::{
parse_usage_body_ref, usage_body_ref, UsageBodyField, UsageCleanupPreviewCounts,
UsageCleanupSummary, UsageCleanupWindow,
parse_usage_body_ref, usage_body_ref, UsageBodyField, UsageCleanupExecutionMode,
UsageCleanupPreviewCounts, UsageCleanupSummary, UsageCleanupTargets, UsageCleanupWindow,
};
use chrono::{DateTime, Utc};
use flate2::{write::GzEncoder, Compression};
@@ -129,6 +129,64 @@ WHERE created_at < $1
ORDER BY created_at ASC, id ASC
LIMIT $3
"#;
const SELECT_USAGE_RAW_BODY_BATCH_SQL: &str = r#"
SELECT id, request_id
FROM usage
WHERE created_at < $1
AND (
request_body IS NOT NULL
OR response_body IS NOT NULL
OR provider_request_body IS NOT NULL
OR client_response_body IS NOT NULL
)
ORDER BY created_at ASC, id ASC
LIMIT $2
"#;
const CLEAR_USAGE_RAW_BODY_FIELDS_SQL: &str = r#"
UPDATE usage
SET request_body = NULL,
response_body = NULL,
provider_request_body = NULL,
client_response_body = NULL
WHERE id = ANY($1)
"#;
const SELECT_USAGE_COMPRESSED_BODY_BATCH_SQL: &str = r#"
SELECT id, request_id
FROM usage
WHERE created_at < $1
AND (
request_body_compressed IS NOT NULL
OR response_body_compressed IS NOT NULL
OR provider_request_body_compressed IS NOT NULL
OR client_response_body_compressed IS NOT NULL
OR EXISTS (
SELECT 1
FROM usage_body_blobs
WHERE usage_body_blobs.request_id = usage.request_id
)
OR EXISTS (
SELECT 1
FROM usage_http_audits
WHERE usage_http_audits.request_id = usage.request_id
AND (
usage_http_audits.request_body_ref IS NOT NULL
OR usage_http_audits.provider_request_body_ref IS NOT NULL
OR usage_http_audits.response_body_ref IS NOT NULL
OR usage_http_audits.client_response_body_ref IS NOT NULL
)
)
)
ORDER BY created_at ASC, id ASC
LIMIT $2
"#;
const CLEAR_USAGE_COMPRESSED_BODY_FIELDS_SQL: &str = r#"
UPDATE usage
SET request_body_compressed = NULL,
response_body_compressed = NULL,
provider_request_body_compressed = NULL,
client_response_body_compressed = NULL
WHERE id = ANY($1)
"#;
const CLEAR_USAGE_BODY_FIELDS_SQL: &str = r#"
UPDATE usage
SET request_body = NULL,
@@ -419,49 +477,99 @@ impl SqlxUsageReadRepository {
window: &UsageCleanupWindow,
batch_size: usize,
auto_delete_expired_keys: bool,
targets: UsageCleanupTargets,
mode: UsageCleanupExecutionMode,
) -> Result<UsageCleanupSummary, DataLayerError> {
if batch_size == 0 {
if batch_size == 0 || !targets.any_selected() {
return Ok(UsageCleanupSummary::default());
}
if mode == UsageCleanupExecutionMode::BeforeNowBodyFields {
let body_externalized = if targets.detail_body {
cleanup_usage_raw_body_fields(&self.pool, window.detail_cutoff, batch_size).await?
} else {
0
};
let body_cleaned = if targets.compressed_body {
cleanup_usage_compressed_body_fields(
&self.pool,
window.compressed_cutoff,
batch_size,
)
.await?
} else {
0
};
return Ok(UsageCleanupSummary {
body_externalized,
legacy_body_refs_migrated: 0,
body_cleaned,
header_cleaned: 0,
keys_cleaned: 0,
records_deleted: 0,
});
}
let records_deleted =
delete_old_usage_records(&self.pool, window.log_cutoff, batch_size).await?;
let header_cleaned = cleanup_usage_header_fields(
&self.pool,
window.header_cutoff,
batch_size,
Some(window.log_cutoff),
)
.await?;
let legacy_body_refs_migrated = migrate_legacy_usage_body_ref_metadata(
&self.pool,
window.detail_cutoff,
batch_size,
Some(window.compressed_cutoff),
)
.await?;
let body_cleaned = cleanup_usage_stale_body_fields(
&self.pool,
window.compressed_cutoff,
batch_size,
Some(window.log_cutoff),
)
.await?;
let body_externalized = compress_usage_body_fields(
&self.pool,
window.detail_cutoff,
batch_size,
Some(window.compressed_cutoff),
)
.await?;
let keys_cleaned =
let records_deleted = if targets.records {
delete_old_usage_records(&self.pool, window.log_cutoff, batch_size).await?
} else {
0
};
let header_cleaned = if targets.headers {
cleanup_usage_header_fields(
&self.pool,
window.header_cutoff,
batch_size,
targets.records.then_some(window.log_cutoff),
)
.await?
} else {
0
};
let body_cleaned = if targets.compressed_body {
cleanup_usage_stale_body_fields(
&self.pool,
window.compressed_cutoff,
batch_size,
targets.records.then_some(window.log_cutoff),
)
.await?
} else {
0
};
let detail_body_newer_than = detail_body_newer_than(window, targets);
let legacy_body_refs_migrated = if targets.detail_body {
migrate_legacy_usage_body_ref_metadata(
&self.pool,
window.detail_cutoff,
batch_size,
detail_body_newer_than,
)
.await?
} else {
0
};
let body_externalized = if targets.detail_body {
compress_usage_body_fields(
&self.pool,
window.detail_cutoff,
batch_size,
detail_body_newer_than,
)
.await?
} else {
0
};
let keys_cleaned = if targets.expired_keys {
match cleanup_expired_api_keys(&self.pool, auto_delete_expired_keys).await {
Ok(count) => count,
Err(err) => {
warn!(error = %err, "usage cleanup expired api key sweep failed");
0
}
};
}
} else {
0
};
Ok(UsageCleanupSummary {
body_externalized,
@@ -477,45 +585,400 @@ impl SqlxUsageReadRepository {
pub async fn preview_usage_cleanup_impl(
pool: &PostgresPool,
window: &UsageCleanupWindow,
targets: UsageCleanupTargets,
mode: UsageCleanupExecutionMode,
) -> Result<UsageCleanupPreviewCounts, DataLayerError> {
let detail: i64 = sqlx::query_scalar(
"SELECT COUNT(*)::bigint FROM usage WHERE created_at < $1 AND created_at >= $2",
)
.bind(window.detail_cutoff)
.bind(window.compressed_cutoff)
.fetch_one(pool)
.await
.map_err(postgres_error)?;
let compressed: i64 = sqlx::query_scalar(
"SELECT COUNT(*)::bigint FROM usage WHERE created_at < $1 AND created_at >= $2",
)
.bind(window.compressed_cutoff)
.bind(window.log_cutoff)
.fetch_one(pool)
.await
.map_err(postgres_error)?;
let header: i64 = sqlx::query_scalar(
"SELECT COUNT(*)::bigint FROM usage WHERE created_at < $1 AND created_at >= $2",
)
.bind(window.header_cutoff)
.bind(window.log_cutoff)
.fetch_one(pool)
.await
.map_err(postgres_error)?;
let log: i64 = sqlx::query_scalar("SELECT COUNT(*)::bigint FROM usage WHERE created_at < $1")
.bind(window.log_cutoff)
.fetch_one(pool)
.await
.map_err(postgres_error)?;
if mode == UsageCleanupExecutionMode::BeforeNowBodyFields {
let detail = if targets.detail_body {
count_usage_raw_body_candidates(pool, window.detail_cutoff).await?
} else {
0
};
let compressed = if targets.compressed_body {
count_usage_compressed_body_candidates(pool, window.compressed_cutoff).await?
} else {
0
};
return Ok(UsageCleanupPreviewCounts {
detail,
compressed,
header: 0,
log: 0,
});
}
let detail = if targets.detail_body {
count_usage_detail_body_candidates(
pool,
window.detail_cutoff,
detail_body_newer_than(window, targets),
)
.await?
} else {
0
};
let compressed = if targets.compressed_body {
count_usage_stale_body_candidates(
pool,
window.compressed_cutoff,
targets.records.then_some(window.log_cutoff),
)
.await?
} else {
0
};
let header = if targets.headers {
count_usage_header_candidates(
pool,
window.header_cutoff,
targets.records.then_some(window.log_cutoff),
)
.await?
} else {
0
};
let log = if targets.records {
let log: i64 =
sqlx::query_scalar("SELECT COUNT(*)::bigint FROM usage WHERE created_at < $1")
.bind(window.log_cutoff)
.fetch_one(pool)
.await
.map_err(postgres_error)?;
u64::try_from(log).unwrap_or(0)
} else {
0
};
Ok(UsageCleanupPreviewCounts {
detail: u64::try_from(detail).unwrap_or(0),
compressed: u64::try_from(compressed).unwrap_or(0),
header: u64::try_from(header).unwrap_or(0),
log: u64::try_from(log).unwrap_or(0),
detail,
compressed,
header,
log,
})
}
async fn cleanup_usage_raw_body_fields(
pool: &PostgresPool,
cutoff_time: DateTime<Utc>,
batch_size: usize,
) -> Result<usize, DataLayerError> {
let mut total_cleaned = 0usize;
loop {
let rows = fetch_usage_body_cleanup_rows(
pool,
SELECT_USAGE_RAW_BODY_BATCH_SQL,
cutoff_time,
batch_size,
)
.await?;
if rows.is_empty() {
break;
}
let ids = rows.iter().map(|row| row.id.clone()).collect::<Vec<_>>();
let cleaned = sqlx::query(CLEAR_USAGE_RAW_BODY_FIELDS_SQL)
.bind(ids)
.execute(pool)
.await
.map_err(postgres_error)?
.rows_affected();
let cleaned = usize::try_from(cleaned).unwrap_or(usize::MAX);
total_cleaned += cleaned;
if cleaned == 0 || cleaned < batch_size {
break;
}
}
Ok(total_cleaned)
}
async fn cleanup_usage_compressed_body_fields(
pool: &PostgresPool,
cutoff_time: DateTime<Utc>,
batch_size: usize,
) -> Result<usize, DataLayerError> {
let mut total_cleaned = 0usize;
loop {
let rows = fetch_usage_body_cleanup_rows(
pool,
SELECT_USAGE_COMPRESSED_BODY_BATCH_SQL,
cutoff_time,
batch_size,
)
.await?;
if rows.is_empty() {
break;
}
let ids = rows.iter().map(|row| row.id.clone()).collect::<Vec<_>>();
let request_ids = rows
.iter()
.map(|row| row.request_id.clone())
.collect::<Vec<_>>();
let cleaned = sqlx::query(CLEAR_USAGE_COMPRESSED_BODY_FIELDS_SQL)
.bind(ids)
.execute(pool)
.await
.map_err(postgres_error)?
.rows_affected();
sqlx::query(DELETE_USAGE_BODY_BLOBS_SQL)
.bind(&request_ids)
.execute(pool)
.await
.map_err(postgres_error)?;
sqlx::query(CLEAR_USAGE_HTTP_AUDIT_BODY_REFS_SQL)
.bind(&request_ids)
.execute(pool)
.await
.map_err(postgres_error)?;
sqlx::query(DELETE_EMPTY_USAGE_HTTP_AUDITS_SQL)
.bind(request_ids)
.execute(pool)
.await
.map_err(postgres_error)?;
let cleaned = usize::try_from(cleaned).unwrap_or(usize::MAX);
total_cleaned += cleaned;
if cleaned == 0 || cleaned < batch_size {
break;
}
}
Ok(total_cleaned)
}
async fn fetch_usage_body_cleanup_rows(
pool: &PostgresPool,
sql: &str,
cutoff_time: DateTime<Utc>,
batch_size: usize,
) -> Result<Vec<UsageBodyCleanupRow>, DataLayerError> {
let rows = sqlx::query(sql)
.bind(cutoff_time)
.bind(i64::try_from(batch_size).unwrap_or(i64::MAX))
.fetch_all(pool)
.await
.map_err(postgres_error)?
.into_iter()
.map(|row| {
Ok(UsageBodyCleanupRow {
id: row.try_get::<String, _>("id").map_err(postgres_error)?,
request_id: row
.try_get::<String, _>("request_id")
.map_err(postgres_error)?,
})
})
.collect::<Result<Vec<_>, DataLayerError>>()?;
Ok(rows)
}
fn detail_body_newer_than(
window: &UsageCleanupWindow,
targets: UsageCleanupTargets,
) -> Option<DateTime<Utc>> {
[
targets.compressed_body.then_some(window.compressed_cutoff),
targets.records.then_some(window.log_cutoff),
]
.into_iter()
.flatten()
.max()
}
async fn count_usage_raw_body_candidates(
pool: &PostgresPool,
cutoff_time: DateTime<Utc>,
) -> Result<u64, DataLayerError> {
let count: i64 = sqlx::query_scalar(
r#"
SELECT COUNT(*)::bigint
FROM usage
WHERE created_at < $1
AND (
request_body IS NOT NULL
OR response_body IS NOT NULL
OR provider_request_body IS NOT NULL
OR client_response_body IS NOT NULL
)
"#,
)
.bind(cutoff_time)
.fetch_one(pool)
.await
.map_err(postgres_error)?;
Ok(u64::try_from(count).unwrap_or(0))
}
async fn count_usage_compressed_body_candidates(
pool: &PostgresPool,
cutoff_time: DateTime<Utc>,
) -> Result<u64, DataLayerError> {
let count: i64 = sqlx::query_scalar(
r#"
SELECT COUNT(*)::bigint
FROM usage
WHERE created_at < $1
AND (
request_body_compressed IS NOT NULL
OR response_body_compressed IS NOT NULL
OR provider_request_body_compressed IS NOT NULL
OR client_response_body_compressed IS NOT NULL
OR EXISTS (
SELECT 1
FROM usage_body_blobs
WHERE usage_body_blobs.request_id = usage.request_id
)
OR EXISTS (
SELECT 1
FROM usage_http_audits
WHERE usage_http_audits.request_id = usage.request_id
AND (
usage_http_audits.request_body_ref IS NOT NULL
OR usage_http_audits.provider_request_body_ref IS NOT NULL
OR usage_http_audits.response_body_ref IS NOT NULL
OR usage_http_audits.client_response_body_ref IS NOT NULL
)
)
)
"#,
)
.bind(cutoff_time)
.fetch_one(pool)
.await
.map_err(postgres_error)?;
Ok(u64::try_from(count).unwrap_or(0))
}
async fn count_usage_detail_body_candidates(
pool: &PostgresPool,
cutoff_time: DateTime<Utc>,
newer_than: Option<DateTime<Utc>>,
) -> Result<u64, DataLayerError> {
if matches!(newer_than, Some(value) if value >= cutoff_time) {
return Ok(0);
}
let count: i64 = sqlx::query_scalar(
r#"
SELECT COUNT(*)::bigint
FROM usage
WHERE created_at < $1
AND ($2::timestamptz IS NULL OR created_at >= $2)
AND (
request_body IS NOT NULL
OR request_body_compressed IS NOT NULL
OR response_body IS NOT NULL
OR response_body_compressed IS NOT NULL
OR provider_request_body IS NOT NULL
OR provider_request_body_compressed IS NOT NULL
OR client_response_body IS NOT NULL
OR client_response_body_compressed IS NOT NULL
OR (
request_metadata IS NOT NULL
AND (
request_metadata::jsonb ? 'request_body_ref'
OR request_metadata::jsonb ? 'provider_request_body_ref'
OR request_metadata::jsonb ? 'response_body_ref'
OR request_metadata::jsonb ? 'client_response_body_ref'
)
)
)
"#,
)
.bind(cutoff_time)
.bind(newer_than)
.fetch_one(pool)
.await
.map_err(postgres_error)?;
Ok(u64::try_from(count).unwrap_or(0))
}
async fn count_usage_stale_body_candidates(
pool: &PostgresPool,
cutoff_time: DateTime<Utc>,
newer_than: Option<DateTime<Utc>>,
) -> Result<u64, DataLayerError> {
if matches!(newer_than, Some(value) if value >= cutoff_time) {
return Ok(0);
}
let count: i64 = sqlx::query_scalar(
r#"
SELECT COUNT(*)::bigint
FROM usage
WHERE created_at < $1
AND ($2::timestamptz IS NULL OR created_at >= $2)
AND (
request_body IS NOT NULL
OR response_body IS NOT NULL
OR provider_request_body IS NOT NULL
OR client_response_body IS NOT NULL
OR request_body_compressed IS NOT NULL
OR response_body_compressed IS NOT NULL
OR provider_request_body_compressed IS NOT NULL
OR client_response_body_compressed IS NOT NULL
OR EXISTS (
SELECT 1
FROM usage_body_blobs
WHERE usage_body_blobs.request_id = usage.request_id
)
OR EXISTS (
SELECT 1
FROM usage_http_audits
WHERE usage_http_audits.request_id = usage.request_id
AND (
usage_http_audits.request_body_ref IS NOT NULL
OR usage_http_audits.provider_request_body_ref IS NOT NULL
OR usage_http_audits.response_body_ref IS NOT NULL
OR usage_http_audits.client_response_body_ref IS NOT NULL
)
)
)
"#,
)
.bind(cutoff_time)
.bind(newer_than)
.fetch_one(pool)
.await
.map_err(postgres_error)?;
Ok(u64::try_from(count).unwrap_or(0))
}
async fn count_usage_header_candidates(
pool: &PostgresPool,
cutoff_time: DateTime<Utc>,
newer_than: Option<DateTime<Utc>>,
) -> Result<u64, DataLayerError> {
if matches!(newer_than, Some(value) if value >= cutoff_time) {
return Ok(0);
}
let count: i64 = sqlx::query_scalar(
r#"
SELECT COUNT(*)::bigint
FROM usage
WHERE created_at < $1
AND ($2::timestamptz IS NULL OR created_at >= $2)
AND (
request_headers IS NOT NULL
OR response_headers IS NOT NULL
OR provider_request_headers IS NOT NULL
OR client_response_headers IS NOT NULL
OR EXISTS (
SELECT 1
FROM usage_http_audits
WHERE usage_http_audits.request_id = usage.request_id
AND (
usage_http_audits.request_headers IS NOT NULL
OR usage_http_audits.response_headers IS NOT NULL
OR usage_http_audits.provider_request_headers IS NOT NULL
OR usage_http_audits.client_response_headers IS NOT NULL
)
)
)
"#,
)
.bind(cutoff_time)
.bind(newer_than)
.fetch_one(pool)
.await
.map_err(postgres_error)?;
Ok(u64::try_from(count).unwrap_or(0))
}
async fn delete_old_usage_records(
pool: &PostgresPool,
cutoff_time: DateTime<Utc>,

View File

@@ -11,12 +11,13 @@ use aether_data_contracts::repository::usage::{
UsageAuditAggregationQuery, UsageAuditKeywordSearchQuery, UsageAuditSummaryQuery,
UsageBodyCaptureState, UsageBodyField, UsageBreakdownGroupBy, UsageBreakdownSummaryQuery,
UsageCacheAffinityHitSummaryQuery, UsageCacheAffinityIntervalGroupBy,
UsageCacheAffinityIntervalQuery, UsageCacheHitSummaryQuery, UsageCleanupSummary,
UsageCleanupWindow, UsageCostSavingsSummaryQuery, UsageDashboardDailyBreakdownQuery,
UsageDashboardProviderCountsQuery, UsageDashboardSummaryQuery, UsageErrorDistributionQuery,
UsageLeaderboardGroupBy, UsageLeaderboardQuery, UsageMonitoringErrorCountQuery,
UsageMonitoringErrorListQuery, UsagePerformancePercentilesQuery, UsageProviderPerformanceQuery,
UsageSettledCostSummaryQuery, UsageTimeSeriesGranularity, UsageTimeSeriesQuery,
UsageCacheAffinityIntervalQuery, UsageCacheHitSummaryQuery, UsageCleanupExecutionMode,
UsageCleanupSummary, UsageCleanupTargets, UsageCleanupWindow, UsageCostSavingsSummaryQuery,
UsageDashboardDailyBreakdownQuery, UsageDashboardProviderCountsQuery,
UsageDashboardSummaryQuery, UsageErrorDistributionQuery, UsageLeaderboardGroupBy,
UsageLeaderboardQuery, UsageMonitoringErrorCountQuery, UsageMonitoringErrorListQuery,
UsagePerformancePercentilesQuery, UsageProviderPerformanceQuery, UsageSettledCostSummaryQuery,
UsageTimeSeriesGranularity, UsageTimeSeriesQuery,
};
use async_trait::async_trait;
use chrono::{DateTime, Utc};
@@ -8218,17 +8219,31 @@ impl UsageWriteRepository for SqlxUsageReadRepository {
window: &UsageCleanupWindow,
batch_size: usize,
auto_delete_expired_keys: bool,
targets: UsageCleanupTargets,
mode: UsageCleanupExecutionMode,
) -> Result<UsageCleanupSummary, DataLayerError> {
Self::cleanup_usage(self, window, batch_size, auto_delete_expired_keys).await
Self::cleanup_usage(
self,
window,
batch_size,
auto_delete_expired_keys,
targets,
mode,
)
.await
}
async fn preview_usage_cleanup(
&self,
window: &UsageCleanupWindow,
targets: UsageCleanupTargets,
mode: UsageCleanupExecutionMode,
) -> Result<aether_data_contracts::repository::usage::UsageCleanupPreviewCounts, DataLayerError>
{
crate::repository::usage::postgres::cleanup::preview_usage_cleanup_impl(&self.pool, window)
.await
crate::repository::usage::postgres::cleanup::preview_usage_cleanup_impl(
&self.pool, window, targets, mode,
)
.await
}
}

View File

@@ -310,15 +310,35 @@ export interface ManualUsageCleanupSummary {
records_deleted: number
}
export interface ManualUsageCleanupResponse {
export type ManualUsageCleanupMode = 'policy' | 'older_than_days' | 'before_now'
export type ManualUsageCleanupTarget = 'detail_body' | 'compressed_body' | 'headers' | 'records'
export interface ManualUsageCleanupTargets {
detail_body: boolean
compressed_body: boolean
headers: boolean
records: boolean
expired_keys: boolean
}
export interface ManualUsageCleanupRequest {
mode?: ManualUsageCleanupMode
older_than_days?: number
targets?: ManualUsageCleanupTarget[]
}
export interface ManualUsageCleanupTaskResponse {
message: string
mode: ManualUsageCleanupMode
requested_older_than_days: number | null
summary: ManualUsageCleanupSummary
total_affected: number
targets: ManualUsageCleanupTargets
task: CleanupRunRecord
}
export interface ManualUsageCleanupPreview {
mode: ManualUsageCleanupMode
requested_older_than_days: number | null
targets: ManualUsageCleanupTargets
effective_cutoffs: {
detail: string
compressed: string
@@ -1218,14 +1238,20 @@ export const adminApi = {
},
async runManualUsageCleanup(
params: { older_than_days?: number } = {}
): Promise<ManualUsageCleanupResponse | ManualUsageCleanupConflict> {
const body: Record<string, number> = {}
params: ManualUsageCleanupRequest = {}
): Promise<ManualUsageCleanupTaskResponse | ManualUsageCleanupConflict> {
const body: ManualUsageCleanupRequest = {}
if (params.mode) {
body.mode = params.mode
}
if (typeof params.older_than_days === 'number') {
body.older_than_days = params.older_than_days
}
if (params.targets?.length) {
body.targets = params.targets
}
try {
const response = await apiClient.post<ManualUsageCleanupResponse>(
const response = await apiClient.post<ManualUsageCleanupTaskResponse>(
'/api/admin/system/cleanup/usage/manual',
body
)
@@ -1240,12 +1266,18 @@ export const adminApi = {
},
async previewManualUsageCleanup(
params: { older_than_days?: number } = {}
params: ManualUsageCleanupRequest = {}
): Promise<ManualUsageCleanupPreview> {
const query: Record<string, number> = {}
const query: Record<string, string | number> = {}
if (params.mode) {
query.mode = params.mode
}
if (typeof params.older_than_days === 'number') {
query.older_than_days = params.older_than_days
}
if (params.targets?.length) {
query.targets = params.targets.join(',')
}
const response = await apiClient.get<ManualUsageCleanupPreview>(
'/api/admin/system/cleanup/usage/preview',
{ params: query }

View File

@@ -288,7 +288,8 @@
<ManualCleanupConfirmDialog
:open="manualCleanupDialogOpen"
@update:open="manualCleanupDialogOpen = $event"
@confirm="handleManualCleanupConfirm"
@running-change="manualCleanupRunning = $event"
@completed="handleManualCleanupCompleted"
/>
<div
@@ -402,7 +403,7 @@
<script setup lang="ts">
import { onBeforeUnmount, onMounted, ref } from 'vue'
import { RefreshCw, Trash2 } from 'lucide-vue-next'
import { adminApi, type CleanupRunRecord, type ManualUsageCleanupResponse } from '@/api/admin'
import { adminApi, type CleanupRunRecord } from '@/api/admin'
import Button from '@/components/ui/button.vue'
import Input from '@/components/ui/input.vue'
import Label from '@/components/ui/label.vue'
@@ -410,7 +411,6 @@ import Switch from '@/components/ui/switch.vue'
import { CardSection } from '@/components/layout'
import ManualCleanupConfirmDialog from './ManualCleanupConfirmDialog.vue'
import { useToast } from '@/composables/useToast'
import { parseApiError } from '@/utils/errorParser'
defineProps<{
enableAutoCleanup: boolean
@@ -459,52 +459,18 @@ function openManualCleanupDialog() {
manualCleanupDialogOpen.value = true
}
async function handleManualCleanupConfirm(olderThanDays: number | undefined) {
manualCleanupRunning.value = true
try {
const response = await adminApi.runManualUsageCleanup(
typeof olderThanDays === 'number' ? { older_than_days: olderThanDays } : {},
)
if ('detail' in response && response.detail === 'usage_cleanup_already_running') {
manualCleanupResult.value = {
title: '已有一次清理正在进行中',
description: response.message,
}
toast.warning(response.message)
} else {
const completed = response as ManualUsageCleanupResponse
manualCleanupResult.value = {
title: completed.message,
description: summarizeManualCleanup(completed),
}
toast.success(completed.message)
}
manualCleanupDialogOpen.value = false
} catch (error) {
const message = parseApiError(error).message
manualCleanupResult.value = {
title: '请求记录清理失败',
description: message,
}
toast.error(message)
} finally {
manualCleanupRunning.value = false
void loadCleanupRuns()
function handleManualCleanupCompleted(task: CleanupRunRecord) {
manualCleanupRunning.value = false
manualCleanupResult.value = {
title: task.message,
description: cleanupSummaryText(task.summary),
}
}
function summarizeManualCleanup(response: ManualUsageCleanupResponse): string {
const { summary } = response
const parts: string[] = []
if (summary.records_deleted > 0) parts.push(`删除整条记录 ${summary.records_deleted}`)
if (summary.body_externalized > 0) parts.push(`压缩 body ${summary.body_externalized}`)
if (summary.body_cleaned > 0) parts.push(`清除过期 body ${summary.body_cleaned}`)
if (summary.header_cleaned > 0) parts.push(`清除 headers ${summary.header_cleaned}`)
if (summary.legacy_body_refs_migrated > 0) {
parts.push(`迁移遗留引用 ${summary.legacy_body_refs_migrated}`)
if (task.status === 'failed') {
toast.error(task.error || task.message)
} else {
toast.success(task.message)
}
if (summary.keys_cleaned > 0) parts.push(`回收 Key ${summary.keys_cleaned}`)
return parts.length > 0 ? parts.join(' / ') : '无数据变更'
void loadCleanupRuns()
}
async function loadCleanupRuns() {
@@ -568,7 +534,7 @@ function cleanupSummaryText(summary: Record<string, unknown>): string {
function summaryLabel(key: string): string {
const labels: Record<string, string> = {
body_externalized: '压缩',
body_externalized: '详细体',
legacy_body_refs_migrated: '迁移',
body_cleaned: '清体',
header_cleaned: '清头',

View File

@@ -3,30 +3,86 @@
:open="open"
size="lg"
title="立即清理请求记录"
description="按现有分级保留策略主动清理请求记录,可选指定清理更早时间的数据。操作不可逆。"
:persistent="submitting"
description="默认按当前分级保留策略执行,也可以选择指定范围。操作不可逆。"
:persistent="isLocked"
@update:open="handleOpenChange"
>
<div class="px-4 sm:px-6 py-4 space-y-4">
<div>
<Label class="block text-sm font-medium">
清理方式
</Label>
<div class="mt-2 grid grid-cols-1 sm:grid-cols-3 gap-2">
<button
v-for="item in modeOptions"
:key="item.value"
type="button"
class="rounded-md border px-3 py-2 text-left text-sm transition-colors"
:class="mode === item.value ? 'border-primary bg-primary/10 text-primary' : 'border-border bg-card hover:bg-muted/60'"
:disabled="isLocked"
@click="setMode(item.value)"
>
<span class="font-medium">{{ item.label }}</span>
<span class="mt-1 block text-xs text-muted-foreground">{{ item.description }}</span>
</button>
</div>
</div>
<div v-if="mode === 'older_than_days'">
<Label
for="manual-cleanup-older-than-days"
class="block text-sm font-medium"
>
清理 N 天前的记录可选
清理 N 天前的记录
</Label>
<Input
id="manual-cleanup-older-than-days"
:model-value="olderThanDays ?? ''"
type="number"
min="1"
placeholder="留空代表按当前保留策略"
placeholder="例如 30"
class="mt-1"
:disabled="submitting"
:disabled="isLocked"
@update:model-value="handleDaysChange"
/>
<p class="mt-1 text-xs text-muted-foreground">
留空代表按当前保留策略清理填入数字代表清理 N 天前的记录该值只能比策略更宽松不会删除更新的数据
该值会与当前策略取更保守的时间点不会清理比策略更新的数据
</p>
</div>
<div>
<Label class="block text-sm font-medium">
清理范围
</Label>
<div class="mt-2 grid grid-cols-1 sm:grid-cols-2 gap-2">
<label
v-for="target in targetOptions"
:key="target.value"
class="flex min-h-16 items-start gap-3 rounded-md border border-border bg-card px-3 py-2"
>
<Checkbox
class="mt-0.5"
:checked="selectedTargets.includes(target.value)"
:disabled="isLocked"
@update:checked="toggleTarget(target.value, $event)"
/>
<span>
<span class="block text-sm font-medium">{{ target.label }}</span>
<span class="block text-xs text-muted-foreground">{{ target.description }}</span>
</span>
</label>
</div>
<p
v-if="mode === 'before_now'"
class="mt-2 text-xs text-amber-600"
>
当前时刻之前模式只允许清理详细请求体和压缩请求体不会清请求头或整条记录
</p>
<p
v-if="targetError"
class="mt-2 text-xs text-destructive"
>
{{ targetError }}
</p>
</div>
@@ -39,7 +95,7 @@
v-if="!previewLoading"
type="button"
class="text-xs text-muted-foreground hover:text-foreground"
:disabled="submitting"
:disabled="isLocked"
@click="loadPreview"
>
刷新预估
@@ -86,6 +142,45 @@
</div>
</div>
<div
v-if="activeTask || taskError"
class="rounded-md border border-border bg-card px-4 py-3"
>
<div class="flex items-center justify-between gap-3">
<div class="min-w-0">
<div class="text-sm font-medium">
{{ activeTask?.message || '请求记录清理失败' }}
</div>
<div class="mt-1 text-xs text-muted-foreground">
{{ activeTask ? cleanupStatusLabel(activeTask.status) : taskError }}
</div>
</div>
<span
v-if="activeTask"
:class="cleanupStatusClass(activeTask.status)"
class="shrink-0 text-xs"
>
{{ cleanupStatusLabel(activeTask.status) }}
</span>
</div>
<div
v-if="activeTask"
class="mt-3 h-2 overflow-hidden rounded-full bg-muted"
>
<div
class="h-full rounded-full bg-primary transition-all"
:class="{ 'animate-pulse': activeTask.status === 'processing' }"
:style="{ width: `${taskProgressPercent}%` }"
/>
</div>
<div
v-if="activeTask"
class="mt-2 text-xs text-muted-foreground"
>
{{ cleanupSummaryText(activeTask.summary) }}
</div>
</div>
<div>
<Label
for="manual-cleanup-confirm-phrase"
@@ -99,47 +194,59 @@
class="mt-1"
autocomplete="off"
:placeholder="confirmPhrase"
:disabled="submitting"
:disabled="isLocked || isFinished"
@update:model-value="typedPhrase = String($event)"
@keydown.enter.prevent="maybeSubmitOnEnter"
/>
<p class="mt-1 text-xs text-muted-foreground">
确认操作后会立刻执行清理且不可撤销
确认后会在当前弹窗中显示执行状态完成前不能关闭
</p>
</div>
</div>
<template #footer>
<Button
v-if="!isFinished"
variant="destructive"
:disabled="!canSubmit"
@click="handleConfirm"
>
{{ submitting ? '清理中…' : '确认清理' }}
{{ isLocked ? '清理中…' : '确认清理' }}
</Button>
<Button
variant="outline"
:disabled="submitting"
:disabled="isLocked"
@click="handleCancel"
>
取消
{{ isFinished ? '关闭' : '取消' }}
</Button>
</template>
</Dialog>
</template>
<script setup lang="ts">
import { computed, ref, watch } from 'vue'
import { computed, onBeforeUnmount, ref, watch } from 'vue'
import { Dialog } from '@/components/ui'
import Button from '@/components/ui/button.vue'
import Checkbox from '@/components/ui/checkbox.vue'
import Input from '@/components/ui/input.vue'
import Label from '@/components/ui/label.vue'
import { adminApi, type ManualUsageCleanupPreview } from '@/api/admin'
import {
adminApi,
type CleanupRunRecord,
type ManualUsageCleanupPreview,
type ManualUsageCleanupRequest,
} from '@/api/admin'
import { parseApiError } from '@/utils/errorParser'
import {
MANUAL_USAGE_CLEANUP_CONFIRM_PHRASE,
allowedTargetsForMode,
defaultManualCleanupTargets,
isConfirmPhraseMatched,
normalizeManualCleanupTargets,
normalizeOlderThanDaysInput,
type ManualCleanupMode,
type ManualCleanupTarget,
} from './manualCleanupForm'
const props = defineProps<{
@@ -148,50 +255,124 @@ const props = defineProps<{
const emit = defineEmits<{
'update:open': [value: boolean]
confirm: [olderThanDays: number | undefined]
'running-change': [value: boolean]
completed: [task: CleanupRunRecord]
}>()
const confirmPhrase = MANUAL_USAGE_CLEANUP_CONFIRM_PHRASE
const mode = ref<ManualCleanupMode>('policy')
const olderThanDays = ref<number | null>(null)
const selectedTargets = ref<ManualCleanupTarget[]>(defaultManualCleanupTargets('policy'))
const targetsTouched = ref(false)
const typedPhrase = ref('')
const preview = ref<ManualUsageCleanupPreview | null>(null)
const previewLoading = ref(false)
const previewError = ref<string | null>(null)
const submitting = ref(false)
const activeTask = ref<CleanupRunRecord | null>(null)
const taskError = ref<string | null>(null)
let previewDebounceTimer: ReturnType<typeof setTimeout> | null = null
let previewSeq = 0
let taskPollTimer: ReturnType<typeof window.setInterval> | null = null
const normalizedPhrase = computed(() => typedPhrase.value)
const modeOptions: Array<{ value: ManualCleanupMode; label: string; description: string }> = [
{ value: 'policy', label: '按当前策略', description: '沿用页面上配置的保留天数' },
{ value: 'older_than_days', label: '指定天数前', description: '在策略内取更保守时间点' },
{ value: 'before_now', label: '当前时刻之前', description: '只清已选请求体内容' },
]
const targetLabels: Record<ManualCleanupTarget, { label: string; description: string }> = {
detail_body: { label: '详细请求体', description: '把详细 body 移入压缩/外置存储' },
compressed_body: { label: '压缩请求体', description: '删除已压缩或外置的 body 内容' },
headers: { label: '请求头', description: '清空请求/响应 headers 字段' },
records: { label: '整条记录', description: '删除超过记录保留期的 usage 行' },
}
const targetOptions = computed(() =>
allowedTargetsForMode(mode.value).map(value => ({
value,
...targetLabels[value],
}))
)
const normalizedTargets = computed(() =>
normalizeManualCleanupTargets(mode.value, selectedTargets.value)
)
const targetError = computed(() => {
if (normalizedTargets.value.length === 0) return '至少选择一个清理范围'
return null
})
const currentTaskRunning = computed(() => activeTask.value?.status === 'processing')
const isLocked = computed(() => submitting.value || currentTaskRunning.value)
const isFinished = computed(() =>
activeTask.value?.status === 'completed' || activeTask.value?.status === 'failed'
)
const canSubmit = computed(
() =>
!submitting.value &&
!isLocked.value &&
!isFinished.value &&
!previewLoading.value &&
isConfirmPhraseMatched(normalizedPhrase.value),
!targetError.value &&
modeIsValid.value &&
isConfirmPhraseMatched(typedPhrase.value),
)
const modeIsValid = computed(() => mode.value !== 'older_than_days' || olderThanDays.value !== null)
const taskProgressPercent = computed(() => {
const task = activeTask.value
if (!task) return 0
if (task.status === 'completed') return 100
if (task.status === 'failed') return 100
const raw = task.summary?.progress_percent
if (typeof raw === 'number' && Number.isFinite(raw) && raw > 0) {
return Math.max(1, Math.min(99, Math.round(raw)))
}
return 12
})
watch(
() => props.open,
(isOpen) => {
if (isOpen) {
resetForm()
void loadPreview()
} else if (previewDebounceTimer) {
clearTimeout(previewDebounceTimer)
previewDebounceTimer = null
} else {
clearPreviewTimer()
stopTaskPolling()
}
},
)
function resetForm() {
mode.value = 'policy'
olderThanDays.value = null
selectedTargets.value = defaultManualCleanupTargets('policy')
targetsTouched.value = false
typedPhrase.value = ''
preview.value = null
previewError.value = null
previewLoading.value = false
submitting.value = false
activeTask.value = null
taskError.value = null
emit('running-change', false)
}
function setMode(nextMode: ManualCleanupMode) {
if (isLocked.value || mode.value === nextMode) return
mode.value = nextMode
olderThanDays.value = null
selectedTargets.value = defaultManualCleanupTargets(nextMode)
targetsTouched.value = false
activeTask.value = null
taskError.value = null
schedulePreview()
}
function handleDaysChange(value: string | number) {
@@ -199,33 +380,59 @@ function handleDaysChange(value: string | number) {
schedulePreview()
}
function schedulePreview() {
if (previewDebounceTimer) {
clearTimeout(previewDebounceTimer)
function toggleTarget(target: ManualCleanupTarget, checked: boolean) {
targetsTouched.value = true
activeTask.value = null
taskError.value = null
const current = new Set(selectedTargets.value)
if (checked) {
current.add(target)
} else {
current.delete(target)
}
selectedTargets.value = normalizeManualCleanupTargets(mode.value, Array.from(current))
schedulePreview()
}
function buildRequest(): ManualUsageCleanupRequest {
const request: ManualUsageCleanupRequest = { mode: mode.value }
if (mode.value === 'older_than_days' && olderThanDays.value !== null) {
request.older_than_days = olderThanDays.value
}
if (targetsTouched.value) {
request.targets = normalizedTargets.value
}
return request
}
function schedulePreview() {
clearPreviewTimer()
previewDebounceTimer = setTimeout(() => {
previewDebounceTimer = null
void loadPreview()
}, 300)
}
function clearPreviewTimer() {
if (previewDebounceTimer) {
clearTimeout(previewDebounceTimer)
previewDebounceTimer = null
}
}
async function loadPreview() {
const seq = ++previewSeq
previewLoading.value = true
previewError.value = null
try {
const params: { older_than_days?: number } = {}
if (olderThanDays.value !== null) {
params.older_than_days = olderThanDays.value
}
const result = await adminApi.previewManualUsageCleanup(params)
const result = await adminApi.previewManualUsageCleanup(buildRequest())
if (seq === previewSeq) {
preview.value = result
}
} catch (error) {
if (seq === previewSeq) {
preview.value = null
previewError.value = parseApiError(error).message
previewError.value = parseApiError(error)
}
} finally {
if (seq === previewSeq) {
@@ -235,14 +442,14 @@ async function loadPreview() {
}
function handleOpenChange(value: boolean) {
if (!value && submitting.value) {
if (!value && isLocked.value) {
return
}
emit('update:open', value)
}
function handleCancel() {
if (submitting.value) return
if (isLocked.value) return
emit('update:open', false)
}
@@ -255,14 +462,101 @@ function maybeSubmitOnEnter() {
async function handleConfirm() {
if (!canSubmit.value) return
submitting.value = true
taskError.value = null
try {
emit('confirm', olderThanDays.value ?? undefined)
const response = await adminApi.runManualUsageCleanup(buildRequest())
if ('detail' in response && response.detail === 'usage_cleanup_already_running') {
taskError.value = response.message
emit('running-change', false)
return
}
activeTask.value = response.task
emit('running-change', response.task.status === 'processing')
if (response.task.status === 'processing') {
startTaskPolling(response.task.id)
} else {
emit('completed', response.task)
}
} catch (error) {
taskError.value = parseApiError(error)
emit('running-change', false)
} finally {
submitting.value = false
}
}
function startTaskPolling(taskId: string) {
stopTaskPolling()
void pollTask(taskId)
taskPollTimer = window.setInterval(() => {
void pollTask(taskId)
}, 1_500)
}
function stopTaskPolling() {
if (taskPollTimer) {
window.clearInterval(taskPollTimer)
taskPollTimer = null
}
}
async function pollTask(taskId: string) {
try {
const response = await adminApi.getCleanupRuns()
const task = response.items.find(item => item.id === taskId)
if (!task) return
activeTask.value = task
const running = task.status === 'processing'
emit('running-change', running)
if (!running) {
stopTaskPolling()
emit('completed', task)
void loadPreview()
}
} catch (error) {
taskError.value = parseApiError(error)
}
}
function cleanupStatusLabel(status: string): string {
if (status === 'processing') return '执行中'
if (status === 'failed') return '失败'
return '完成'
}
function cleanupStatusClass(status: string): string {
if (status === 'processing') return 'text-amber-500'
if (status === 'failed') return 'text-destructive'
return 'text-emerald-500'
}
function cleanupSummaryText(summary: Record<string, unknown>): string {
const total = typeof summary.total === 'number' ? summary.total : null
if (total !== null && total > 0) return `影响 ${total}`
const entries = Object.entries(summary)
.filter(([key, value]) => key !== 'progress_percent' && typeof value === 'number' && value > 0)
.map(([key, value]) => `${summaryLabel(key)} ${value}`)
return entries.length > 0 ? entries.join(' / ') : '等待后台返回结果'
}
function summaryLabel(key: string): string {
const labels: Record<string, string> = {
body_externalized: '详细体',
legacy_body_refs_migrated: '迁移',
body_cleaned: '清体',
header_cleaned: '清头',
keys_cleaned: 'Key',
records_deleted: '删记录',
}
return labels[key] || key
}
function formatCount(value: number): string {
return value.toLocaleString()
}
onBeforeUnmount(() => {
clearPreviewTimer()
stopTaskPolling()
})
</script>

View File

@@ -1,8 +1,11 @@
import { describe, expect, it } from 'vitest'
import {
allowedTargetsForMode,
defaultManualCleanupTargets,
MANUAL_USAGE_CLEANUP_CONFIRM_PHRASE,
isConfirmPhraseMatched,
normalizeManualCleanupTargets,
normalizeConfirmPhraseInput,
normalizeOlderThanDaysInput,
} from '../manualCleanupForm'
@@ -61,4 +64,34 @@ describe('manualCleanupForm', () => {
expect(normalizeOlderThanDaysInput('abc')).toBeNull()
})
})
describe('cleanup targets', () => {
it('keeps all ranges available for policy cleanup', () => {
expect(allowedTargetsForMode('policy')).toEqual([
'detail_body',
'compressed_body',
'headers',
'records',
])
expect(defaultManualCleanupTargets('older_than_days')).toEqual([
'detail_body',
'compressed_body',
'headers',
'records',
])
})
it('limits before-now cleanup to body targets', () => {
expect(allowedTargetsForMode('before_now')).toEqual([
'detail_body',
'compressed_body',
])
expect(normalizeManualCleanupTargets('before_now', [
'detail_body',
'headers',
'compressed_body',
'records',
])).toEqual(['detail_body', 'compressed_body'])
})
})
})

View File

@@ -1,5 +1,20 @@
export const MANUAL_USAGE_CLEANUP_CONFIRM_PHRASE = '确认清理'
export type ManualCleanupMode = 'policy' | 'older_than_days' | 'before_now'
export type ManualCleanupTarget = 'detail_body' | 'compressed_body' | 'headers' | 'records'
export const MANUAL_CLEANUP_TARGETS: ManualCleanupTarget[] = [
'detail_body',
'compressed_body',
'headers',
'records',
]
export const BEFORE_NOW_ALLOWED_TARGETS: ManualCleanupTarget[] = [
'detail_body',
'compressed_body',
]
export function normalizeConfirmPhraseInput(raw: string): string {
return raw.replace(/\r?\n/g, '').trim()
}
@@ -14,3 +29,21 @@ export function normalizeOlderThanDaysInput(raw: string | number | null | undefi
if (!Number.isFinite(parsed) || parsed <= 0) return null
return Math.floor(parsed)
}
export function allowedTargetsForMode(mode: ManualCleanupMode): ManualCleanupTarget[] {
return mode === 'before_now' ? BEFORE_NOW_ALLOWED_TARGETS : MANUAL_CLEANUP_TARGETS
}
export function normalizeManualCleanupTargets(
mode: ManualCleanupMode,
targets: ManualCleanupTarget[],
): ManualCleanupTarget[] {
const allowed = new Set(allowedTargetsForMode(mode))
return targets.filter((target, index) =>
allowed.has(target) && targets.indexOf(target) === index
)
}
export function defaultManualCleanupTargets(mode: ManualCleanupMode): ManualCleanupTarget[] {
return allowedTargetsForMode(mode)
}