mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 01:10:23 +08:00
fix(admin): run destructive purges as cleanup tasks
This commit is contained in:
@@ -18,7 +18,6 @@ use crate::handlers::admin::system::shared::settings::{
|
||||
};
|
||||
use crate::handlers::admin::system::shared::smtp::build_admin_smtp_test_payload;
|
||||
use crate::GatewayError;
|
||||
use aether_data::repository::system::AdminSystemPurgeTarget;
|
||||
use axum::{
|
||||
body::{Body, Bytes},
|
||||
http,
|
||||
@@ -205,34 +204,21 @@ pub(super) async fn maybe_build_local_admin_core_system_response(
|
||||
return Ok(Some(Json(json!({ "items": records })).into_response()));
|
||||
}
|
||||
|
||||
if decision.route_kind.as_deref() == Some("purge_request_bodies_task")
|
||||
&& request_method == http::Method::POST
|
||||
&& request_path == "/api/admin/system/purge/request-bodies/task"
|
||||
if let Some((task_kind, action, object_type, object_id)) =
|
||||
admin_system_purge_task_for_route_kind(decision.route_kind.as_deref())
|
||||
{
|
||||
let task =
|
||||
crate::maintenance::start_admin_request_body_cleanup_task(state.cloned_app()).await?;
|
||||
if request_method != http::Method::POST {
|
||||
return Ok(None);
|
||||
}
|
||||
let task = crate::maintenance::start_admin_system_purge_task(state.cloned_app(), task_kind)
|
||||
.await?;
|
||||
return Ok(Some(attach_admin_audit_response(
|
||||
Json(json!({
|
||||
"message": task.message.clone(),
|
||||
"task": task,
|
||||
}))
|
||||
.into_response(),
|
||||
"admin_system_request_body_cleanup_task_started",
|
||||
"purge_request_bodies_async",
|
||||
"request_bodies",
|
||||
"all",
|
||||
)));
|
||||
}
|
||||
|
||||
if let Some((target, action, object_type, object_id)) =
|
||||
admin_system_purge_target_for_route_kind(decision.route_kind.as_deref())
|
||||
{
|
||||
if request_method != http::Method::POST {
|
||||
return Ok(None);
|
||||
}
|
||||
return Ok(Some(attach_admin_audit_response(
|
||||
Json(build_admin_system_purge_payload(state, target).await?).into_response(),
|
||||
"admin_system_data_purged",
|
||||
"admin_system_purge_task_started",
|
||||
action,
|
||||
object_type,
|
||||
object_id,
|
||||
@@ -465,97 +451,55 @@ pub(super) async fn maybe_build_local_admin_core_system_response(
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn admin_system_purge_target_for_route_kind(
|
||||
fn admin_system_purge_task_for_route_kind(
|
||||
route_kind: Option<&str>,
|
||||
) -> Option<(
|
||||
AdminSystemPurgeTarget,
|
||||
crate::maintenance::AdminCleanupTaskKind,
|
||||
&'static str,
|
||||
&'static str,
|
||||
&'static str,
|
||||
)> {
|
||||
match route_kind {
|
||||
Some("purge_config") => Some((
|
||||
AdminSystemPurgeTarget::Config,
|
||||
"purge_system_config",
|
||||
crate::maintenance::AdminCleanupTaskKind::Config,
|
||||
"purge_system_config_async",
|
||||
"system_config",
|
||||
"global",
|
||||
)),
|
||||
Some("purge_users") => Some((
|
||||
AdminSystemPurgeTarget::Users,
|
||||
"purge_non_admin_users",
|
||||
crate::maintenance::AdminCleanupTaskKind::Users,
|
||||
"purge_non_admin_users_async",
|
||||
"users",
|
||||
"non_admin",
|
||||
)),
|
||||
Some("purge_usage") => Some((
|
||||
AdminSystemPurgeTarget::Usage,
|
||||
"purge_usage_records",
|
||||
crate::maintenance::AdminCleanupTaskKind::Usage,
|
||||
"purge_usage_records_async",
|
||||
"usage",
|
||||
"all",
|
||||
)),
|
||||
Some("purge_audit_logs") => Some((
|
||||
AdminSystemPurgeTarget::AuditLogs,
|
||||
"purge_audit_logs",
|
||||
crate::maintenance::AdminCleanupTaskKind::AuditLogs,
|
||||
"purge_audit_logs_async",
|
||||
"audit_logs",
|
||||
"all",
|
||||
)),
|
||||
Some("purge_request_bodies") => Some((
|
||||
AdminSystemPurgeTarget::RequestBodies,
|
||||
"purge_request_bodies",
|
||||
Some("purge_request_bodies") | Some("purge_request_bodies_task") => Some((
|
||||
crate::maintenance::AdminCleanupTaskKind::RequestBodies,
|
||||
"purge_request_bodies_async",
|
||||
"request_bodies",
|
||||
"all",
|
||||
)),
|
||||
Some("purge_stats") => Some((AdminSystemPurgeTarget::Stats, "purge_stats", "stats", "all")),
|
||||
Some("purge_stats") => Some((
|
||||
crate::maintenance::AdminCleanupTaskKind::Stats,
|
||||
"purge_stats_async",
|
||||
"stats",
|
||||
"all",
|
||||
)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
async fn build_admin_system_purge_payload(
|
||||
state: &AdminAppState<'_>,
|
||||
target: AdminSystemPurgeTarget,
|
||||
) -> Result<serde_json::Value, GatewayError> {
|
||||
let summary = state.purge_admin_system_data(target).await?;
|
||||
let total = summary.total();
|
||||
let affected = summary.affected.clone();
|
||||
|
||||
if target == AdminSystemPurgeTarget::Stats {
|
||||
let rebuild = state.rebuild_admin_stats_once().await?;
|
||||
let message = if rebuild.capped {
|
||||
format!(
|
||||
"统计聚合已清空,已重建 {} 个小时桶和 {} 个日桶,仍有历史统计待后台任务继续重建",
|
||||
rebuild.hourly_buckets, rebuild.daily_buckets
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
"统计聚合已清空并重建,删除 {} 行,重建 {} 个小时桶和 {} 个日桶",
|
||||
total, rebuild.hourly_buckets, rebuild.daily_buckets
|
||||
)
|
||||
};
|
||||
return Ok(json!({
|
||||
"message": message,
|
||||
"deleted": affected,
|
||||
"rebuilt": {
|
||||
"hourly_buckets": rebuild.hourly_buckets,
|
||||
"daily_buckets": rebuild.daily_buckets,
|
||||
"capped": rebuild.capped,
|
||||
},
|
||||
}));
|
||||
}
|
||||
|
||||
let (message, count_key) = match target {
|
||||
AdminSystemPurgeTarget::Config => ("系统配置已清空", "deleted"),
|
||||
AdminSystemPurgeTarget::Users => ("非管理员用户已清空", "deleted"),
|
||||
AdminSystemPurgeTarget::Usage => ("使用记录已清空", "deleted"),
|
||||
AdminSystemPurgeTarget::AuditLogs => ("审计日志已清空", "deleted"),
|
||||
AdminSystemPurgeTarget::RequestBodies => ("请求/响应体已清空", "cleaned"),
|
||||
AdminSystemPurgeTarget::Stats => unreachable!("stats handled above"),
|
||||
};
|
||||
|
||||
Ok(json!({
|
||||
"message": format!("{message},影响 {} 行", total),
|
||||
count_key: affected,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn build_admin_system_cleanup_payload(
|
||||
state: &AdminAppState<'_>,
|
||||
) -> Result<serde_json::Value, GatewayError> {
|
||||
|
||||
@@ -53,7 +53,8 @@ pub(crate) use aether_data_contracts::repository::usage::{
|
||||
use audit_cleanup::*;
|
||||
pub(crate) use cleanup_runs::{
|
||||
list_admin_cleanup_run_records, record_completed_cleanup_run, record_failed_cleanup_run,
|
||||
start_admin_request_body_cleanup_task, AdminCleanupRunRecord,
|
||||
start_admin_request_body_cleanup_task, start_admin_system_purge_task, AdminCleanupRunRecord,
|
||||
AdminCleanupTaskKind,
|
||||
};
|
||||
use config::*;
|
||||
use db_maintenance::*;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use std::time::Instant;
|
||||
|
||||
use aether_data::repository::system::AdminSystemPurgeSummary;
|
||||
use aether_data::repository::system::{AdminSystemPurgeSummary, AdminSystemPurgeTarget};
|
||||
use aether_data_contracts::DataLayerError;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{json, Value};
|
||||
@@ -29,6 +29,65 @@ pub(crate) struct AdminCleanupRunRecord {
|
||||
pub(crate) error: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum AdminCleanupTaskKind {
|
||||
Config,
|
||||
Users,
|
||||
RequestBodies,
|
||||
Usage,
|
||||
AuditLogs,
|
||||
Stats,
|
||||
}
|
||||
|
||||
impl AdminCleanupTaskKind {
|
||||
fn record_kind(self) -> &'static str {
|
||||
match self {
|
||||
Self::Config => "config_purge",
|
||||
Self::Users => "users_purge",
|
||||
Self::RequestBodies => "request_bodies",
|
||||
Self::Usage => "usage_purge",
|
||||
Self::AuditLogs => "audit_logs_purge",
|
||||
Self::Stats => "stats_purge",
|
||||
}
|
||||
}
|
||||
|
||||
fn target(self) -> Option<AdminSystemPurgeTarget> {
|
||||
match self {
|
||||
Self::RequestBodies => None,
|
||||
Self::Config => Some(AdminSystemPurgeTarget::Config),
|
||||
Self::Users => Some(AdminSystemPurgeTarget::Users),
|
||||
Self::Usage => Some(AdminSystemPurgeTarget::Usage),
|
||||
Self::AuditLogs => Some(AdminSystemPurgeTarget::AuditLogs),
|
||||
Self::Stats => Some(AdminSystemPurgeTarget::Stats),
|
||||
}
|
||||
}
|
||||
|
||||
fn start_message(self, batch_size: Option<usize>) -> String {
|
||||
match self {
|
||||
Self::Config => "系统配置后台清空已开始".to_string(),
|
||||
Self::Users => "非管理员用户后台清空已开始".to_string(),
|
||||
Self::RequestBodies => format!(
|
||||
"请求/响应体后台清理已开始,每批 {} 条",
|
||||
batch_size.unwrap_or(1)
|
||||
),
|
||||
Self::Usage => "使用记录后台清空已开始".to_string(),
|
||||
Self::AuditLogs => "审计日志后台清空已开始".to_string(),
|
||||
Self::Stats => "统计聚合后台清空和重建已开始".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn failure_message(self) -> &'static str {
|
||||
match self {
|
||||
Self::Config => "系统配置后台清空失败",
|
||||
Self::Users => "非管理员用户后台清空失败",
|
||||
Self::RequestBodies => "请求/响应体后台清理失败",
|
||||
Self::Usage => "使用记录后台清空失败",
|
||||
Self::AuditLogs => "审计日志后台清空失败",
|
||||
Self::Stats => "统计聚合后台清空或重建失败",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn list_admin_cleanup_run_records(
|
||||
data: &GatewayDataState,
|
||||
) -> Result<Vec<AdminCleanupRunRecord>, DataLayerError> {
|
||||
@@ -43,6 +102,13 @@ pub(crate) async fn list_admin_cleanup_run_records(
|
||||
|
||||
pub(crate) async fn start_admin_request_body_cleanup_task(
|
||||
app: AppState,
|
||||
) -> Result<AdminCleanupRunRecord, GatewayError> {
|
||||
start_admin_system_purge_task(app, AdminCleanupTaskKind::RequestBodies).await
|
||||
}
|
||||
|
||||
pub(crate) async fn start_admin_system_purge_task(
|
||||
app: AppState,
|
||||
kind: AdminCleanupTaskKind,
|
||||
) -> Result<AdminCleanupRunRecord, GatewayError> {
|
||||
let data = app.data.clone();
|
||||
let records = list_admin_cleanup_run_records(&data)
|
||||
@@ -50,42 +116,55 @@ pub(crate) async fn start_admin_request_body_cleanup_task(
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
if let Some(existing) = records
|
||||
.iter()
|
||||
.find(|record| record.kind == "request_bodies" && record.status == "processing")
|
||||
.find(|record| record.kind == kind.record_kind() && record.status == "processing")
|
||||
.cloned()
|
||||
{
|
||||
return Ok(existing);
|
||||
}
|
||||
|
||||
let batch_size = system_config_usize(&data, "cleanup_batch_size", 1_000)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||
.max(1);
|
||||
let batch_size = if kind == AdminCleanupTaskKind::RequestBodies {
|
||||
Some(
|
||||
system_config_usize(&data, "cleanup_batch_size", 1_000)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||
.max(1),
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let started_at = now_unix_secs();
|
||||
let record = AdminCleanupRunRecord {
|
||||
id: uuid::Uuid::new_v4().to_string(),
|
||||
kind: "request_bodies".to_string(),
|
||||
kind: kind.record_kind().to_string(),
|
||||
trigger: "manual".to_string(),
|
||||
status: "processing".to_string(),
|
||||
message: format!("请求/响应体后台清理已开始,每批 {batch_size} 条"),
|
||||
message: kind.start_message(batch_size),
|
||||
started_at_unix_secs: started_at,
|
||||
completed_at_unix_secs: None,
|
||||
duration_ms: None,
|
||||
summary: json!({
|
||||
"batch_size": batch_size,
|
||||
"batches": 0,
|
||||
"cleaned": {},
|
||||
}),
|
||||
summary: initial_task_summary(kind, batch_size),
|
||||
error: None,
|
||||
};
|
||||
record_cleanup_run(&data, record.clone())
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
|
||||
tokio::spawn(run_request_body_cleanup_task(
|
||||
data,
|
||||
record.clone(),
|
||||
batch_size,
|
||||
));
|
||||
match kind {
|
||||
AdminCleanupTaskKind::RequestBodies => {
|
||||
tokio::spawn(run_request_body_cleanup_task(
|
||||
data,
|
||||
record.clone(),
|
||||
batch_size.unwrap_or(1),
|
||||
));
|
||||
}
|
||||
AdminCleanupTaskKind::Usage
|
||||
| AdminCleanupTaskKind::AuditLogs
|
||||
| AdminCleanupTaskKind::Config
|
||||
| AdminCleanupTaskKind::Users
|
||||
| AdminCleanupTaskKind::Stats => {
|
||||
tokio::spawn(run_admin_system_purge_task(app, record.clone(), kind));
|
||||
}
|
||||
}
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
@@ -235,6 +314,149 @@ async fn run_request_body_cleanup_task(
|
||||
);
|
||||
}
|
||||
|
||||
async fn run_admin_system_purge_task(
|
||||
app: AppState,
|
||||
initial_record: AdminCleanupRunRecord,
|
||||
kind: AdminCleanupTaskKind,
|
||||
) {
|
||||
let started_at = Instant::now();
|
||||
let data = app.data.clone();
|
||||
match run_admin_system_purge_task_once(&app, kind).await {
|
||||
Ok((summary, message)) => {
|
||||
let completed = cleanup_task_record(
|
||||
&initial_record,
|
||||
"completed",
|
||||
message,
|
||||
summary,
|
||||
Some(started_at),
|
||||
None,
|
||||
);
|
||||
if let Err(err) = record_cleanup_run(&data, completed).await {
|
||||
warn!(error = %err, "failed to record admin system purge task completion");
|
||||
}
|
||||
info!(
|
||||
event_name = "admin_system_purge_task_completed",
|
||||
log_type = "ops",
|
||||
worker = "admin_system_purge_task",
|
||||
kind = ?kind,
|
||||
"gateway finished admin system purge task"
|
||||
);
|
||||
}
|
||||
Err(err) => {
|
||||
let failed = cleanup_task_record(
|
||||
&initial_record,
|
||||
"failed",
|
||||
kind.failure_message().to_string(),
|
||||
json!({}),
|
||||
Some(started_at),
|
||||
Some(format!("{err:?}")),
|
||||
);
|
||||
if let Err(record_err) = record_cleanup_run(&data, failed).await {
|
||||
warn!(error = %record_err, "failed to record admin system purge task failure");
|
||||
}
|
||||
warn!(error = ?err, kind = ?kind, "admin system purge task failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_admin_system_purge_task_once(
|
||||
app: &AppState,
|
||||
kind: AdminCleanupTaskKind,
|
||||
) -> Result<(Value, String), GatewayError> {
|
||||
let target = kind
|
||||
.target()
|
||||
.expect("non-request-body cleanup task should map to purge target");
|
||||
let purge = app.purge_admin_system_data(target).await?;
|
||||
let deleted_total = purge.total();
|
||||
let affected = purge.affected.clone();
|
||||
|
||||
if kind == AdminCleanupTaskKind::Stats {
|
||||
let rebuild = app.rebuild_admin_stats_once().await?;
|
||||
let message = if rebuild.capped {
|
||||
format!(
|
||||
"统计聚合后台清空完成,已重建 {} 个小时桶和 {} 个日桶,仍有历史统计待后台任务继续重建",
|
||||
rebuild.hourly_buckets, rebuild.daily_buckets
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
"统计聚合后台清空并重建完成,删除 {} 行,重建 {} 个小时桶和 {} 个日桶",
|
||||
deleted_total, rebuild.hourly_buckets, rebuild.daily_buckets
|
||||
)
|
||||
};
|
||||
return Ok((
|
||||
json!({
|
||||
"deleted": affected,
|
||||
"rebuilt": {
|
||||
"hourly_buckets": rebuild.hourly_buckets,
|
||||
"daily_buckets": rebuild.daily_buckets,
|
||||
"capped": rebuild.capped,
|
||||
},
|
||||
"total": deleted_total,
|
||||
}),
|
||||
message,
|
||||
));
|
||||
}
|
||||
|
||||
let message = match kind {
|
||||
AdminCleanupTaskKind::Config => format!("系统配置后台清空完成,影响 {deleted_total} 行"),
|
||||
AdminCleanupTaskKind::Users => format!("非管理员用户后台清空完成,影响 {deleted_total} 行"),
|
||||
AdminCleanupTaskKind::Usage => format!("使用记录后台清空完成,影响 {deleted_total} 行"),
|
||||
AdminCleanupTaskKind::AuditLogs => format!("审计日志后台清空完成,影响 {deleted_total} 行"),
|
||||
AdminCleanupTaskKind::RequestBodies | AdminCleanupTaskKind::Stats => unreachable!(),
|
||||
};
|
||||
Ok((
|
||||
json!({
|
||||
"deleted": affected,
|
||||
"total": deleted_total,
|
||||
}),
|
||||
message,
|
||||
))
|
||||
}
|
||||
|
||||
fn initial_task_summary(kind: AdminCleanupTaskKind, batch_size: Option<usize>) -> Value {
|
||||
match kind {
|
||||
AdminCleanupTaskKind::RequestBodies => json!({
|
||||
"batch_size": batch_size.unwrap_or(1),
|
||||
"batches": 0,
|
||||
"cleaned": {},
|
||||
}),
|
||||
AdminCleanupTaskKind::Stats => json!({
|
||||
"deleted": {},
|
||||
"rebuilt": {},
|
||||
}),
|
||||
AdminCleanupTaskKind::Config
|
||||
| AdminCleanupTaskKind::Users
|
||||
| AdminCleanupTaskKind::Usage
|
||||
| AdminCleanupTaskKind::AuditLogs => json!({
|
||||
"deleted": {},
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn cleanup_task_record(
|
||||
initial: &AdminCleanupRunRecord,
|
||||
status: &str,
|
||||
message: String,
|
||||
summary: Value,
|
||||
started_at: Option<Instant>,
|
||||
error: Option<String>,
|
||||
) -> AdminCleanupRunRecord {
|
||||
let completed = matches!(status, "completed" | "failed");
|
||||
AdminCleanupRunRecord {
|
||||
id: initial.id.clone(),
|
||||
kind: initial.kind.clone(),
|
||||
trigger: initial.trigger.clone(),
|
||||
status: status.to_string(),
|
||||
message,
|
||||
started_at_unix_secs: initial.started_at_unix_secs,
|
||||
completed_at_unix_secs: completed.then(now_unix_secs),
|
||||
duration_ms: started_at
|
||||
.map(|value| value.elapsed().as_millis().try_into().unwrap_or(u64::MAX)),
|
||||
summary,
|
||||
error,
|
||||
}
|
||||
}
|
||||
|
||||
fn request_body_cleanup_record(
|
||||
initial: &AdminCleanupRunRecord,
|
||||
status: &str,
|
||||
|
||||
@@ -10,7 +10,7 @@ use super::{
|
||||
advance_proxy_upgrade_rollout_once, cleanup_audit_logs_once,
|
||||
cleanup_expired_gemini_file_mappings_once, cleanup_proxy_node_metrics_once,
|
||||
cleanup_request_candidates_once, cleanup_stale_pending_requests_once,
|
||||
cleanup_stale_proxy_nodes_once, collect_proxy_upgrade_rollout_probes,
|
||||
cleanup_stale_proxy_nodes_once, collect_proxy_upgrade_rollout_probes, now_unix_secs,
|
||||
perform_db_maintenance_once, perform_provider_checkin_once, perform_stats_aggregation_once,
|
||||
perform_stats_hourly_aggregation_once, perform_usage_cleanup_once,
|
||||
perform_wallet_daily_usage_aggregation_once, record_completed_cleanup_run,
|
||||
|
||||
Reference in New Issue
Block a user