fix(admin): run destructive purges as cleanup tasks

This commit is contained in:
fawney19
2026-05-10 00:41:21 +08:00
parent 391c2fbe5d
commit a81053e6ff
9 changed files with 682 additions and 126 deletions

View File

@@ -18,7 +18,6 @@ use crate::handlers::admin::system::shared::settings::{
};
use crate::handlers::admin::system::shared::smtp::build_admin_smtp_test_payload;
use crate::GatewayError;
use aether_data::repository::system::AdminSystemPurgeTarget;
use axum::{
body::{Body, Bytes},
http,
@@ -205,34 +204,21 @@ pub(super) async fn maybe_build_local_admin_core_system_response(
return Ok(Some(Json(json!({ "items": records })).into_response()));
}
if decision.route_kind.as_deref() == Some("purge_request_bodies_task")
&& request_method == http::Method::POST
&& request_path == "/api/admin/system/purge/request-bodies/task"
if let Some((task_kind, action, object_type, object_id)) =
admin_system_purge_task_for_route_kind(decision.route_kind.as_deref())
{
let task =
crate::maintenance::start_admin_request_body_cleanup_task(state.cloned_app()).await?;
if request_method != http::Method::POST {
return Ok(None);
}
let task = crate::maintenance::start_admin_system_purge_task(state.cloned_app(), task_kind)
.await?;
return Ok(Some(attach_admin_audit_response(
Json(json!({
"message": task.message.clone(),
"task": task,
}))
.into_response(),
"admin_system_request_body_cleanup_task_started",
"purge_request_bodies_async",
"request_bodies",
"all",
)));
}
if let Some((target, action, object_type, object_id)) =
admin_system_purge_target_for_route_kind(decision.route_kind.as_deref())
{
if request_method != http::Method::POST {
return Ok(None);
}
return Ok(Some(attach_admin_audit_response(
Json(build_admin_system_purge_payload(state, target).await?).into_response(),
"admin_system_data_purged",
"admin_system_purge_task_started",
action,
object_type,
object_id,
@@ -465,97 +451,55 @@ pub(super) async fn maybe_build_local_admin_core_system_response(
Ok(None)
}
fn admin_system_purge_target_for_route_kind(
fn admin_system_purge_task_for_route_kind(
route_kind: Option<&str>,
) -> Option<(
AdminSystemPurgeTarget,
crate::maintenance::AdminCleanupTaskKind,
&'static str,
&'static str,
&'static str,
)> {
match route_kind {
Some("purge_config") => Some((
AdminSystemPurgeTarget::Config,
"purge_system_config",
crate::maintenance::AdminCleanupTaskKind::Config,
"purge_system_config_async",
"system_config",
"global",
)),
Some("purge_users") => Some((
AdminSystemPurgeTarget::Users,
"purge_non_admin_users",
crate::maintenance::AdminCleanupTaskKind::Users,
"purge_non_admin_users_async",
"users",
"non_admin",
)),
Some("purge_usage") => Some((
AdminSystemPurgeTarget::Usage,
"purge_usage_records",
crate::maintenance::AdminCleanupTaskKind::Usage,
"purge_usage_records_async",
"usage",
"all",
)),
Some("purge_audit_logs") => Some((
AdminSystemPurgeTarget::AuditLogs,
"purge_audit_logs",
crate::maintenance::AdminCleanupTaskKind::AuditLogs,
"purge_audit_logs_async",
"audit_logs",
"all",
)),
Some("purge_request_bodies") => Some((
AdminSystemPurgeTarget::RequestBodies,
"purge_request_bodies",
Some("purge_request_bodies") | Some("purge_request_bodies_task") => Some((
crate::maintenance::AdminCleanupTaskKind::RequestBodies,
"purge_request_bodies_async",
"request_bodies",
"all",
)),
Some("purge_stats") => Some((AdminSystemPurgeTarget::Stats, "purge_stats", "stats", "all")),
Some("purge_stats") => Some((
crate::maintenance::AdminCleanupTaskKind::Stats,
"purge_stats_async",
"stats",
"all",
)),
_ => None,
}
}
async fn build_admin_system_purge_payload(
state: &AdminAppState<'_>,
target: AdminSystemPurgeTarget,
) -> Result<serde_json::Value, GatewayError> {
let summary = state.purge_admin_system_data(target).await?;
let total = summary.total();
let affected = summary.affected.clone();
if target == AdminSystemPurgeTarget::Stats {
let rebuild = state.rebuild_admin_stats_once().await?;
let message = if rebuild.capped {
format!(
"统计聚合已清空,已重建 {} 个小时桶和 {} 个日桶,仍有历史统计待后台任务继续重建",
rebuild.hourly_buckets, rebuild.daily_buckets
)
} else {
format!(
"统计聚合已清空并重建,删除 {} 行,重建 {} 个小时桶和 {} 个日桶",
total, rebuild.hourly_buckets, rebuild.daily_buckets
)
};
return Ok(json!({
"message": message,
"deleted": affected,
"rebuilt": {
"hourly_buckets": rebuild.hourly_buckets,
"daily_buckets": rebuild.daily_buckets,
"capped": rebuild.capped,
},
}));
}
let (message, count_key) = match target {
AdminSystemPurgeTarget::Config => ("系统配置已清空", "deleted"),
AdminSystemPurgeTarget::Users => ("非管理员用户已清空", "deleted"),
AdminSystemPurgeTarget::Usage => ("使用记录已清空", "deleted"),
AdminSystemPurgeTarget::AuditLogs => ("审计日志已清空", "deleted"),
AdminSystemPurgeTarget::RequestBodies => ("请求/响应体已清空", "cleaned"),
AdminSystemPurgeTarget::Stats => unreachable!("stats handled above"),
};
Ok(json!({
"message": format!("{message},影响 {} 行", total),
count_key: affected,
}))
}
async fn build_admin_system_cleanup_payload(
state: &AdminAppState<'_>,
) -> Result<serde_json::Value, GatewayError> {

View File

@@ -53,7 +53,8 @@ pub(crate) use aether_data_contracts::repository::usage::{
use audit_cleanup::*;
pub(crate) use cleanup_runs::{
list_admin_cleanup_run_records, record_completed_cleanup_run, record_failed_cleanup_run,
start_admin_request_body_cleanup_task, AdminCleanupRunRecord,
start_admin_request_body_cleanup_task, start_admin_system_purge_task, AdminCleanupRunRecord,
AdminCleanupTaskKind,
};
use config::*;
use db_maintenance::*;

View File

@@ -1,6 +1,6 @@
use std::time::Instant;
use aether_data::repository::system::AdminSystemPurgeSummary;
use aether_data::repository::system::{AdminSystemPurgeSummary, AdminSystemPurgeTarget};
use aether_data_contracts::DataLayerError;
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
@@ -29,6 +29,65 @@ pub(crate) struct AdminCleanupRunRecord {
pub(crate) error: Option<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum AdminCleanupTaskKind {
Config,
Users,
RequestBodies,
Usage,
AuditLogs,
Stats,
}
impl AdminCleanupTaskKind {
fn record_kind(self) -> &'static str {
match self {
Self::Config => "config_purge",
Self::Users => "users_purge",
Self::RequestBodies => "request_bodies",
Self::Usage => "usage_purge",
Self::AuditLogs => "audit_logs_purge",
Self::Stats => "stats_purge",
}
}
fn target(self) -> Option<AdminSystemPurgeTarget> {
match self {
Self::RequestBodies => None,
Self::Config => Some(AdminSystemPurgeTarget::Config),
Self::Users => Some(AdminSystemPurgeTarget::Users),
Self::Usage => Some(AdminSystemPurgeTarget::Usage),
Self::AuditLogs => Some(AdminSystemPurgeTarget::AuditLogs),
Self::Stats => Some(AdminSystemPurgeTarget::Stats),
}
}
fn start_message(self, batch_size: Option<usize>) -> String {
match self {
Self::Config => "系统配置后台清空已开始".to_string(),
Self::Users => "非管理员用户后台清空已开始".to_string(),
Self::RequestBodies => format!(
"请求/响应体后台清理已开始,每批 {}",
batch_size.unwrap_or(1)
),
Self::Usage => "使用记录后台清空已开始".to_string(),
Self::AuditLogs => "审计日志后台清空已开始".to_string(),
Self::Stats => "统计聚合后台清空和重建已开始".to_string(),
}
}
fn failure_message(self) -> &'static str {
match self {
Self::Config => "系统配置后台清空失败",
Self::Users => "非管理员用户后台清空失败",
Self::RequestBodies => "请求/响应体后台清理失败",
Self::Usage => "使用记录后台清空失败",
Self::AuditLogs => "审计日志后台清空失败",
Self::Stats => "统计聚合后台清空或重建失败",
}
}
}
pub(crate) async fn list_admin_cleanup_run_records(
data: &GatewayDataState,
) -> Result<Vec<AdminCleanupRunRecord>, DataLayerError> {
@@ -43,6 +102,13 @@ pub(crate) async fn list_admin_cleanup_run_records(
pub(crate) async fn start_admin_request_body_cleanup_task(
app: AppState,
) -> Result<AdminCleanupRunRecord, GatewayError> {
start_admin_system_purge_task(app, AdminCleanupTaskKind::RequestBodies).await
}
pub(crate) async fn start_admin_system_purge_task(
app: AppState,
kind: AdminCleanupTaskKind,
) -> Result<AdminCleanupRunRecord, GatewayError> {
let data = app.data.clone();
let records = list_admin_cleanup_run_records(&data)
@@ -50,42 +116,55 @@ pub(crate) async fn start_admin_request_body_cleanup_task(
.map_err(|err| GatewayError::Internal(err.to_string()))?;
if let Some(existing) = records
.iter()
.find(|record| record.kind == "request_bodies" && record.status == "processing")
.find(|record| record.kind == kind.record_kind() && record.status == "processing")
.cloned()
{
return Ok(existing);
}
let batch_size = system_config_usize(&data, "cleanup_batch_size", 1_000)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?
.max(1);
let batch_size = if kind == AdminCleanupTaskKind::RequestBodies {
Some(
system_config_usize(&data, "cleanup_batch_size", 1_000)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?
.max(1),
)
} else {
None
};
let started_at = now_unix_secs();
let record = AdminCleanupRunRecord {
id: uuid::Uuid::new_v4().to_string(),
kind: "request_bodies".to_string(),
kind: kind.record_kind().to_string(),
trigger: "manual".to_string(),
status: "processing".to_string(),
message: format!("请求/响应体后台清理已开始,每批 {batch_size} 条"),
message: kind.start_message(batch_size),
started_at_unix_secs: started_at,
completed_at_unix_secs: None,
duration_ms: None,
summary: json!({
"batch_size": batch_size,
"batches": 0,
"cleaned": {},
}),
summary: initial_task_summary(kind, batch_size),
error: None,
};
record_cleanup_run(&data, record.clone())
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?;
tokio::spawn(run_request_body_cleanup_task(
data,
record.clone(),
batch_size,
));
match kind {
AdminCleanupTaskKind::RequestBodies => {
tokio::spawn(run_request_body_cleanup_task(
data,
record.clone(),
batch_size.unwrap_or(1),
));
}
AdminCleanupTaskKind::Usage
| AdminCleanupTaskKind::AuditLogs
| AdminCleanupTaskKind::Config
| AdminCleanupTaskKind::Users
| AdminCleanupTaskKind::Stats => {
tokio::spawn(run_admin_system_purge_task(app, record.clone(), kind));
}
}
Ok(record)
}
@@ -235,6 +314,149 @@ async fn run_request_body_cleanup_task(
);
}
async fn run_admin_system_purge_task(
app: AppState,
initial_record: AdminCleanupRunRecord,
kind: AdminCleanupTaskKind,
) {
let started_at = Instant::now();
let data = app.data.clone();
match run_admin_system_purge_task_once(&app, kind).await {
Ok((summary, message)) => {
let completed = cleanup_task_record(
&initial_record,
"completed",
message,
summary,
Some(started_at),
None,
);
if let Err(err) = record_cleanup_run(&data, completed).await {
warn!(error = %err, "failed to record admin system purge task completion");
}
info!(
event_name = "admin_system_purge_task_completed",
log_type = "ops",
worker = "admin_system_purge_task",
kind = ?kind,
"gateway finished admin system purge task"
);
}
Err(err) => {
let failed = cleanup_task_record(
&initial_record,
"failed",
kind.failure_message().to_string(),
json!({}),
Some(started_at),
Some(format!("{err:?}")),
);
if let Err(record_err) = record_cleanup_run(&data, failed).await {
warn!(error = %record_err, "failed to record admin system purge task failure");
}
warn!(error = ?err, kind = ?kind, "admin system purge task failed");
}
}
}
async fn run_admin_system_purge_task_once(
app: &AppState,
kind: AdminCleanupTaskKind,
) -> Result<(Value, String), GatewayError> {
let target = kind
.target()
.expect("non-request-body cleanup task should map to purge target");
let purge = app.purge_admin_system_data(target).await?;
let deleted_total = purge.total();
let affected = purge.affected.clone();
if kind == AdminCleanupTaskKind::Stats {
let rebuild = app.rebuild_admin_stats_once().await?;
let message = if rebuild.capped {
format!(
"统计聚合后台清空完成,已重建 {} 个小时桶和 {} 个日桶,仍有历史统计待后台任务继续重建",
rebuild.hourly_buckets, rebuild.daily_buckets
)
} else {
format!(
"统计聚合后台清空并重建完成,删除 {} 行,重建 {} 个小时桶和 {} 个日桶",
deleted_total, rebuild.hourly_buckets, rebuild.daily_buckets
)
};
return Ok((
json!({
"deleted": affected,
"rebuilt": {
"hourly_buckets": rebuild.hourly_buckets,
"daily_buckets": rebuild.daily_buckets,
"capped": rebuild.capped,
},
"total": deleted_total,
}),
message,
));
}
let message = match kind {
AdminCleanupTaskKind::Config => format!("系统配置后台清空完成,影响 {deleted_total}"),
AdminCleanupTaskKind::Users => format!("非管理员用户后台清空完成,影响 {deleted_total}"),
AdminCleanupTaskKind::Usage => format!("使用记录后台清空完成,影响 {deleted_total}"),
AdminCleanupTaskKind::AuditLogs => format!("审计日志后台清空完成,影响 {deleted_total}"),
AdminCleanupTaskKind::RequestBodies | AdminCleanupTaskKind::Stats => unreachable!(),
};
Ok((
json!({
"deleted": affected,
"total": deleted_total,
}),
message,
))
}
fn initial_task_summary(kind: AdminCleanupTaskKind, batch_size: Option<usize>) -> Value {
match kind {
AdminCleanupTaskKind::RequestBodies => json!({
"batch_size": batch_size.unwrap_or(1),
"batches": 0,
"cleaned": {},
}),
AdminCleanupTaskKind::Stats => json!({
"deleted": {},
"rebuilt": {},
}),
AdminCleanupTaskKind::Config
| AdminCleanupTaskKind::Users
| AdminCleanupTaskKind::Usage
| AdminCleanupTaskKind::AuditLogs => json!({
"deleted": {},
}),
}
}
fn cleanup_task_record(
initial: &AdminCleanupRunRecord,
status: &str,
message: String,
summary: Value,
started_at: Option<Instant>,
error: Option<String>,
) -> AdminCleanupRunRecord {
let completed = matches!(status, "completed" | "failed");
AdminCleanupRunRecord {
id: initial.id.clone(),
kind: initial.kind.clone(),
trigger: initial.trigger.clone(),
status: status.to_string(),
message,
started_at_unix_secs: initial.started_at_unix_secs,
completed_at_unix_secs: completed.then(now_unix_secs),
duration_ms: started_at
.map(|value| value.elapsed().as_millis().try_into().unwrap_or(u64::MAX)),
summary,
error,
}
}
fn request_body_cleanup_record(
initial: &AdminCleanupRunRecord,
status: &str,

View File

@@ -10,7 +10,7 @@ use super::{
advance_proxy_upgrade_rollout_once, cleanup_audit_logs_once,
cleanup_expired_gemini_file_mappings_once, cleanup_proxy_node_metrics_once,
cleanup_request_candidates_once, cleanup_stale_pending_requests_once,
cleanup_stale_proxy_nodes_once, collect_proxy_upgrade_rollout_probes,
cleanup_stale_proxy_nodes_once, collect_proxy_upgrade_rollout_probes, now_unix_secs,
perform_db_maintenance_once, perform_provider_checkin_once, perform_stats_aggregation_once,
perform_stats_hourly_aggregation_once, perform_usage_cleanup_once,
perform_wallet_daily_usage_aggregation_once, record_completed_cleanup_run,