mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-01 17:00:21 +08:00
fix(admin): run destructive purges as cleanup tasks
This commit is contained in:
@@ -18,7 +18,6 @@ use crate::handlers::admin::system::shared::settings::{
|
|||||||
};
|
};
|
||||||
use crate::handlers::admin::system::shared::smtp::build_admin_smtp_test_payload;
|
use crate::handlers::admin::system::shared::smtp::build_admin_smtp_test_payload;
|
||||||
use crate::GatewayError;
|
use crate::GatewayError;
|
||||||
use aether_data::repository::system::AdminSystemPurgeTarget;
|
|
||||||
use axum::{
|
use axum::{
|
||||||
body::{Body, Bytes},
|
body::{Body, Bytes},
|
||||||
http,
|
http,
|
||||||
@@ -205,34 +204,21 @@ pub(super) async fn maybe_build_local_admin_core_system_response(
|
|||||||
return Ok(Some(Json(json!({ "items": records })).into_response()));
|
return Ok(Some(Json(json!({ "items": records })).into_response()));
|
||||||
}
|
}
|
||||||
|
|
||||||
if decision.route_kind.as_deref() == Some("purge_request_bodies_task")
|
if let Some((task_kind, action, object_type, object_id)) =
|
||||||
&& request_method == http::Method::POST
|
admin_system_purge_task_for_route_kind(decision.route_kind.as_deref())
|
||||||
&& request_path == "/api/admin/system/purge/request-bodies/task"
|
|
||||||
{
|
{
|
||||||
let task =
|
if request_method != http::Method::POST {
|
||||||
crate::maintenance::start_admin_request_body_cleanup_task(state.cloned_app()).await?;
|
return Ok(None);
|
||||||
|
}
|
||||||
|
let task = crate::maintenance::start_admin_system_purge_task(state.cloned_app(), task_kind)
|
||||||
|
.await?;
|
||||||
return Ok(Some(attach_admin_audit_response(
|
return Ok(Some(attach_admin_audit_response(
|
||||||
Json(json!({
|
Json(json!({
|
||||||
"message": task.message.clone(),
|
"message": task.message.clone(),
|
||||||
"task": task,
|
"task": task,
|
||||||
}))
|
}))
|
||||||
.into_response(),
|
.into_response(),
|
||||||
"admin_system_request_body_cleanup_task_started",
|
"admin_system_purge_task_started",
|
||||||
"purge_request_bodies_async",
|
|
||||||
"request_bodies",
|
|
||||||
"all",
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some((target, action, object_type, object_id)) =
|
|
||||||
admin_system_purge_target_for_route_kind(decision.route_kind.as_deref())
|
|
||||||
{
|
|
||||||
if request_method != http::Method::POST {
|
|
||||||
return Ok(None);
|
|
||||||
}
|
|
||||||
return Ok(Some(attach_admin_audit_response(
|
|
||||||
Json(build_admin_system_purge_payload(state, target).await?).into_response(),
|
|
||||||
"admin_system_data_purged",
|
|
||||||
action,
|
action,
|
||||||
object_type,
|
object_type,
|
||||||
object_id,
|
object_id,
|
||||||
@@ -465,97 +451,55 @@ pub(super) async fn maybe_build_local_admin_core_system_response(
|
|||||||
Ok(None)
|
Ok(None)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn admin_system_purge_target_for_route_kind(
|
fn admin_system_purge_task_for_route_kind(
|
||||||
route_kind: Option<&str>,
|
route_kind: Option<&str>,
|
||||||
) -> Option<(
|
) -> Option<(
|
||||||
AdminSystemPurgeTarget,
|
crate::maintenance::AdminCleanupTaskKind,
|
||||||
&'static str,
|
&'static str,
|
||||||
&'static str,
|
&'static str,
|
||||||
&'static str,
|
&'static str,
|
||||||
)> {
|
)> {
|
||||||
match route_kind {
|
match route_kind {
|
||||||
Some("purge_config") => Some((
|
Some("purge_config") => Some((
|
||||||
AdminSystemPurgeTarget::Config,
|
crate::maintenance::AdminCleanupTaskKind::Config,
|
||||||
"purge_system_config",
|
"purge_system_config_async",
|
||||||
"system_config",
|
"system_config",
|
||||||
"global",
|
"global",
|
||||||
)),
|
)),
|
||||||
Some("purge_users") => Some((
|
Some("purge_users") => Some((
|
||||||
AdminSystemPurgeTarget::Users,
|
crate::maintenance::AdminCleanupTaskKind::Users,
|
||||||
"purge_non_admin_users",
|
"purge_non_admin_users_async",
|
||||||
"users",
|
"users",
|
||||||
"non_admin",
|
"non_admin",
|
||||||
)),
|
)),
|
||||||
Some("purge_usage") => Some((
|
Some("purge_usage") => Some((
|
||||||
AdminSystemPurgeTarget::Usage,
|
crate::maintenance::AdminCleanupTaskKind::Usage,
|
||||||
"purge_usage_records",
|
"purge_usage_records_async",
|
||||||
"usage",
|
"usage",
|
||||||
"all",
|
"all",
|
||||||
)),
|
)),
|
||||||
Some("purge_audit_logs") => Some((
|
Some("purge_audit_logs") => Some((
|
||||||
AdminSystemPurgeTarget::AuditLogs,
|
crate::maintenance::AdminCleanupTaskKind::AuditLogs,
|
||||||
"purge_audit_logs",
|
"purge_audit_logs_async",
|
||||||
"audit_logs",
|
"audit_logs",
|
||||||
"all",
|
"all",
|
||||||
)),
|
)),
|
||||||
Some("purge_request_bodies") => Some((
|
Some("purge_request_bodies") | Some("purge_request_bodies_task") => Some((
|
||||||
AdminSystemPurgeTarget::RequestBodies,
|
crate::maintenance::AdminCleanupTaskKind::RequestBodies,
|
||||||
"purge_request_bodies",
|
"purge_request_bodies_async",
|
||||||
"request_bodies",
|
"request_bodies",
|
||||||
"all",
|
"all",
|
||||||
)),
|
)),
|
||||||
Some("purge_stats") => Some((AdminSystemPurgeTarget::Stats, "purge_stats", "stats", "all")),
|
Some("purge_stats") => Some((
|
||||||
|
crate::maintenance::AdminCleanupTaskKind::Stats,
|
||||||
|
"purge_stats_async",
|
||||||
|
"stats",
|
||||||
|
"all",
|
||||||
|
)),
|
||||||
_ => None,
|
_ => None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn build_admin_system_purge_payload(
|
|
||||||
state: &AdminAppState<'_>,
|
|
||||||
target: AdminSystemPurgeTarget,
|
|
||||||
) -> Result<serde_json::Value, GatewayError> {
|
|
||||||
let summary = state.purge_admin_system_data(target).await?;
|
|
||||||
let total = summary.total();
|
|
||||||
let affected = summary.affected.clone();
|
|
||||||
|
|
||||||
if target == AdminSystemPurgeTarget::Stats {
|
|
||||||
let rebuild = state.rebuild_admin_stats_once().await?;
|
|
||||||
let message = if rebuild.capped {
|
|
||||||
format!(
|
|
||||||
"统计聚合已清空,已重建 {} 个小时桶和 {} 个日桶,仍有历史统计待后台任务继续重建",
|
|
||||||
rebuild.hourly_buckets, rebuild.daily_buckets
|
|
||||||
)
|
|
||||||
} else {
|
|
||||||
format!(
|
|
||||||
"统计聚合已清空并重建,删除 {} 行,重建 {} 个小时桶和 {} 个日桶",
|
|
||||||
total, rebuild.hourly_buckets, rebuild.daily_buckets
|
|
||||||
)
|
|
||||||
};
|
|
||||||
return Ok(json!({
|
|
||||||
"message": message,
|
|
||||||
"deleted": affected,
|
|
||||||
"rebuilt": {
|
|
||||||
"hourly_buckets": rebuild.hourly_buckets,
|
|
||||||
"daily_buckets": rebuild.daily_buckets,
|
|
||||||
"capped": rebuild.capped,
|
|
||||||
},
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
let (message, count_key) = match target {
|
|
||||||
AdminSystemPurgeTarget::Config => ("系统配置已清空", "deleted"),
|
|
||||||
AdminSystemPurgeTarget::Users => ("非管理员用户已清空", "deleted"),
|
|
||||||
AdminSystemPurgeTarget::Usage => ("使用记录已清空", "deleted"),
|
|
||||||
AdminSystemPurgeTarget::AuditLogs => ("审计日志已清空", "deleted"),
|
|
||||||
AdminSystemPurgeTarget::RequestBodies => ("请求/响应体已清空", "cleaned"),
|
|
||||||
AdminSystemPurgeTarget::Stats => unreachable!("stats handled above"),
|
|
||||||
};
|
|
||||||
|
|
||||||
Ok(json!({
|
|
||||||
"message": format!("{message},影响 {} 行", total),
|
|
||||||
count_key: affected,
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn build_admin_system_cleanup_payload(
|
async fn build_admin_system_cleanup_payload(
|
||||||
state: &AdminAppState<'_>,
|
state: &AdminAppState<'_>,
|
||||||
) -> Result<serde_json::Value, GatewayError> {
|
) -> Result<serde_json::Value, GatewayError> {
|
||||||
|
|||||||
@@ -53,7 +53,8 @@ pub(crate) use aether_data_contracts::repository::usage::{
|
|||||||
use audit_cleanup::*;
|
use audit_cleanup::*;
|
||||||
pub(crate) use cleanup_runs::{
|
pub(crate) use cleanup_runs::{
|
||||||
list_admin_cleanup_run_records, record_completed_cleanup_run, record_failed_cleanup_run,
|
list_admin_cleanup_run_records, record_completed_cleanup_run, record_failed_cleanup_run,
|
||||||
start_admin_request_body_cleanup_task, AdminCleanupRunRecord,
|
start_admin_request_body_cleanup_task, start_admin_system_purge_task, AdminCleanupRunRecord,
|
||||||
|
AdminCleanupTaskKind,
|
||||||
};
|
};
|
||||||
use config::*;
|
use config::*;
|
||||||
use db_maintenance::*;
|
use db_maintenance::*;
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
use std::time::Instant;
|
use std::time::Instant;
|
||||||
|
|
||||||
use aether_data::repository::system::AdminSystemPurgeSummary;
|
use aether_data::repository::system::{AdminSystemPurgeSummary, AdminSystemPurgeTarget};
|
||||||
use aether_data_contracts::DataLayerError;
|
use aether_data_contracts::DataLayerError;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use serde_json::{json, Value};
|
use serde_json::{json, Value};
|
||||||
@@ -29,6 +29,65 @@ pub(crate) struct AdminCleanupRunRecord {
|
|||||||
pub(crate) error: Option<String>,
|
pub(crate) error: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub(crate) enum AdminCleanupTaskKind {
|
||||||
|
Config,
|
||||||
|
Users,
|
||||||
|
RequestBodies,
|
||||||
|
Usage,
|
||||||
|
AuditLogs,
|
||||||
|
Stats,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AdminCleanupTaskKind {
|
||||||
|
fn record_kind(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Config => "config_purge",
|
||||||
|
Self::Users => "users_purge",
|
||||||
|
Self::RequestBodies => "request_bodies",
|
||||||
|
Self::Usage => "usage_purge",
|
||||||
|
Self::AuditLogs => "audit_logs_purge",
|
||||||
|
Self::Stats => "stats_purge",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn target(self) -> Option<AdminSystemPurgeTarget> {
|
||||||
|
match self {
|
||||||
|
Self::RequestBodies => None,
|
||||||
|
Self::Config => Some(AdminSystemPurgeTarget::Config),
|
||||||
|
Self::Users => Some(AdminSystemPurgeTarget::Users),
|
||||||
|
Self::Usage => Some(AdminSystemPurgeTarget::Usage),
|
||||||
|
Self::AuditLogs => Some(AdminSystemPurgeTarget::AuditLogs),
|
||||||
|
Self::Stats => Some(AdminSystemPurgeTarget::Stats),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn start_message(self, batch_size: Option<usize>) -> String {
|
||||||
|
match self {
|
||||||
|
Self::Config => "系统配置后台清空已开始".to_string(),
|
||||||
|
Self::Users => "非管理员用户后台清空已开始".to_string(),
|
||||||
|
Self::RequestBodies => format!(
|
||||||
|
"请求/响应体后台清理已开始,每批 {} 条",
|
||||||
|
batch_size.unwrap_or(1)
|
||||||
|
),
|
||||||
|
Self::Usage => "使用记录后台清空已开始".to_string(),
|
||||||
|
Self::AuditLogs => "审计日志后台清空已开始".to_string(),
|
||||||
|
Self::Stats => "统计聚合后台清空和重建已开始".to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn failure_message(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Config => "系统配置后台清空失败",
|
||||||
|
Self::Users => "非管理员用户后台清空失败",
|
||||||
|
Self::RequestBodies => "请求/响应体后台清理失败",
|
||||||
|
Self::Usage => "使用记录后台清空失败",
|
||||||
|
Self::AuditLogs => "审计日志后台清空失败",
|
||||||
|
Self::Stats => "统计聚合后台清空或重建失败",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn list_admin_cleanup_run_records(
|
pub(crate) async fn list_admin_cleanup_run_records(
|
||||||
data: &GatewayDataState,
|
data: &GatewayDataState,
|
||||||
) -> Result<Vec<AdminCleanupRunRecord>, DataLayerError> {
|
) -> Result<Vec<AdminCleanupRunRecord>, DataLayerError> {
|
||||||
@@ -43,6 +102,13 @@ pub(crate) async fn list_admin_cleanup_run_records(
|
|||||||
|
|
||||||
pub(crate) async fn start_admin_request_body_cleanup_task(
|
pub(crate) async fn start_admin_request_body_cleanup_task(
|
||||||
app: AppState,
|
app: AppState,
|
||||||
|
) -> Result<AdminCleanupRunRecord, GatewayError> {
|
||||||
|
start_admin_system_purge_task(app, AdminCleanupTaskKind::RequestBodies).await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn start_admin_system_purge_task(
|
||||||
|
app: AppState,
|
||||||
|
kind: AdminCleanupTaskKind,
|
||||||
) -> Result<AdminCleanupRunRecord, GatewayError> {
|
) -> Result<AdminCleanupRunRecord, GatewayError> {
|
||||||
let data = app.data.clone();
|
let data = app.data.clone();
|
||||||
let records = list_admin_cleanup_run_records(&data)
|
let records = list_admin_cleanup_run_records(&data)
|
||||||
@@ -50,42 +116,55 @@ pub(crate) async fn start_admin_request_body_cleanup_task(
|
|||||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||||
if let Some(existing) = records
|
if let Some(existing) = records
|
||||||
.iter()
|
.iter()
|
||||||
.find(|record| record.kind == "request_bodies" && record.status == "processing")
|
.find(|record| record.kind == kind.record_kind() && record.status == "processing")
|
||||||
.cloned()
|
.cloned()
|
||||||
{
|
{
|
||||||
return Ok(existing);
|
return Ok(existing);
|
||||||
}
|
}
|
||||||
|
|
||||||
let batch_size = system_config_usize(&data, "cleanup_batch_size", 1_000)
|
let batch_size = if kind == AdminCleanupTaskKind::RequestBodies {
|
||||||
.await
|
Some(
|
||||||
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
system_config_usize(&data, "cleanup_batch_size", 1_000)
|
||||||
.max(1);
|
.await
|
||||||
|
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||||
|
.max(1),
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
let started_at = now_unix_secs();
|
let started_at = now_unix_secs();
|
||||||
let record = AdminCleanupRunRecord {
|
let record = AdminCleanupRunRecord {
|
||||||
id: uuid::Uuid::new_v4().to_string(),
|
id: uuid::Uuid::new_v4().to_string(),
|
||||||
kind: "request_bodies".to_string(),
|
kind: kind.record_kind().to_string(),
|
||||||
trigger: "manual".to_string(),
|
trigger: "manual".to_string(),
|
||||||
status: "processing".to_string(),
|
status: "processing".to_string(),
|
||||||
message: format!("请求/响应体后台清理已开始,每批 {batch_size} 条"),
|
message: kind.start_message(batch_size),
|
||||||
started_at_unix_secs: started_at,
|
started_at_unix_secs: started_at,
|
||||||
completed_at_unix_secs: None,
|
completed_at_unix_secs: None,
|
||||||
duration_ms: None,
|
duration_ms: None,
|
||||||
summary: json!({
|
summary: initial_task_summary(kind, batch_size),
|
||||||
"batch_size": batch_size,
|
|
||||||
"batches": 0,
|
|
||||||
"cleaned": {},
|
|
||||||
}),
|
|
||||||
error: None,
|
error: None,
|
||||||
};
|
};
|
||||||
record_cleanup_run(&data, record.clone())
|
record_cleanup_run(&data, record.clone())
|
||||||
.await
|
.await
|
||||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||||
|
|
||||||
tokio::spawn(run_request_body_cleanup_task(
|
match kind {
|
||||||
data,
|
AdminCleanupTaskKind::RequestBodies => {
|
||||||
record.clone(),
|
tokio::spawn(run_request_body_cleanup_task(
|
||||||
batch_size,
|
data,
|
||||||
));
|
record.clone(),
|
||||||
|
batch_size.unwrap_or(1),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
AdminCleanupTaskKind::Usage
|
||||||
|
| AdminCleanupTaskKind::AuditLogs
|
||||||
|
| AdminCleanupTaskKind::Config
|
||||||
|
| AdminCleanupTaskKind::Users
|
||||||
|
| AdminCleanupTaskKind::Stats => {
|
||||||
|
tokio::spawn(run_admin_system_purge_task(app, record.clone(), kind));
|
||||||
|
}
|
||||||
|
}
|
||||||
Ok(record)
|
Ok(record)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -235,6 +314,149 @@ async fn run_request_body_cleanup_task(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn run_admin_system_purge_task(
|
||||||
|
app: AppState,
|
||||||
|
initial_record: AdminCleanupRunRecord,
|
||||||
|
kind: AdminCleanupTaskKind,
|
||||||
|
) {
|
||||||
|
let started_at = Instant::now();
|
||||||
|
let data = app.data.clone();
|
||||||
|
match run_admin_system_purge_task_once(&app, kind).await {
|
||||||
|
Ok((summary, message)) => {
|
||||||
|
let completed = cleanup_task_record(
|
||||||
|
&initial_record,
|
||||||
|
"completed",
|
||||||
|
message,
|
||||||
|
summary,
|
||||||
|
Some(started_at),
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
if let Err(err) = record_cleanup_run(&data, completed).await {
|
||||||
|
warn!(error = %err, "failed to record admin system purge task completion");
|
||||||
|
}
|
||||||
|
info!(
|
||||||
|
event_name = "admin_system_purge_task_completed",
|
||||||
|
log_type = "ops",
|
||||||
|
worker = "admin_system_purge_task",
|
||||||
|
kind = ?kind,
|
||||||
|
"gateway finished admin system purge task"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
let failed = cleanup_task_record(
|
||||||
|
&initial_record,
|
||||||
|
"failed",
|
||||||
|
kind.failure_message().to_string(),
|
||||||
|
json!({}),
|
||||||
|
Some(started_at),
|
||||||
|
Some(format!("{err:?}")),
|
||||||
|
);
|
||||||
|
if let Err(record_err) = record_cleanup_run(&data, failed).await {
|
||||||
|
warn!(error = %record_err, "failed to record admin system purge task failure");
|
||||||
|
}
|
||||||
|
warn!(error = ?err, kind = ?kind, "admin system purge task failed");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn run_admin_system_purge_task_once(
|
||||||
|
app: &AppState,
|
||||||
|
kind: AdminCleanupTaskKind,
|
||||||
|
) -> Result<(Value, String), GatewayError> {
|
||||||
|
let target = kind
|
||||||
|
.target()
|
||||||
|
.expect("non-request-body cleanup task should map to purge target");
|
||||||
|
let purge = app.purge_admin_system_data(target).await?;
|
||||||
|
let deleted_total = purge.total();
|
||||||
|
let affected = purge.affected.clone();
|
||||||
|
|
||||||
|
if kind == AdminCleanupTaskKind::Stats {
|
||||||
|
let rebuild = app.rebuild_admin_stats_once().await?;
|
||||||
|
let message = if rebuild.capped {
|
||||||
|
format!(
|
||||||
|
"统计聚合后台清空完成,已重建 {} 个小时桶和 {} 个日桶,仍有历史统计待后台任务继续重建",
|
||||||
|
rebuild.hourly_buckets, rebuild.daily_buckets
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
format!(
|
||||||
|
"统计聚合后台清空并重建完成,删除 {} 行,重建 {} 个小时桶和 {} 个日桶",
|
||||||
|
deleted_total, rebuild.hourly_buckets, rebuild.daily_buckets
|
||||||
|
)
|
||||||
|
};
|
||||||
|
return Ok((
|
||||||
|
json!({
|
||||||
|
"deleted": affected,
|
||||||
|
"rebuilt": {
|
||||||
|
"hourly_buckets": rebuild.hourly_buckets,
|
||||||
|
"daily_buckets": rebuild.daily_buckets,
|
||||||
|
"capped": rebuild.capped,
|
||||||
|
},
|
||||||
|
"total": deleted_total,
|
||||||
|
}),
|
||||||
|
message,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
let message = match kind {
|
||||||
|
AdminCleanupTaskKind::Config => format!("系统配置后台清空完成,影响 {deleted_total} 行"),
|
||||||
|
AdminCleanupTaskKind::Users => format!("非管理员用户后台清空完成,影响 {deleted_total} 行"),
|
||||||
|
AdminCleanupTaskKind::Usage => format!("使用记录后台清空完成,影响 {deleted_total} 行"),
|
||||||
|
AdminCleanupTaskKind::AuditLogs => format!("审计日志后台清空完成,影响 {deleted_total} 行"),
|
||||||
|
AdminCleanupTaskKind::RequestBodies | AdminCleanupTaskKind::Stats => unreachable!(),
|
||||||
|
};
|
||||||
|
Ok((
|
||||||
|
json!({
|
||||||
|
"deleted": affected,
|
||||||
|
"total": deleted_total,
|
||||||
|
}),
|
||||||
|
message,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn initial_task_summary(kind: AdminCleanupTaskKind, batch_size: Option<usize>) -> Value {
|
||||||
|
match kind {
|
||||||
|
AdminCleanupTaskKind::RequestBodies => json!({
|
||||||
|
"batch_size": batch_size.unwrap_or(1),
|
||||||
|
"batches": 0,
|
||||||
|
"cleaned": {},
|
||||||
|
}),
|
||||||
|
AdminCleanupTaskKind::Stats => json!({
|
||||||
|
"deleted": {},
|
||||||
|
"rebuilt": {},
|
||||||
|
}),
|
||||||
|
AdminCleanupTaskKind::Config
|
||||||
|
| AdminCleanupTaskKind::Users
|
||||||
|
| AdminCleanupTaskKind::Usage
|
||||||
|
| AdminCleanupTaskKind::AuditLogs => json!({
|
||||||
|
"deleted": {},
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn cleanup_task_record(
|
||||||
|
initial: &AdminCleanupRunRecord,
|
||||||
|
status: &str,
|
||||||
|
message: String,
|
||||||
|
summary: Value,
|
||||||
|
started_at: Option<Instant>,
|
||||||
|
error: Option<String>,
|
||||||
|
) -> AdminCleanupRunRecord {
|
||||||
|
let completed = matches!(status, "completed" | "failed");
|
||||||
|
AdminCleanupRunRecord {
|
||||||
|
id: initial.id.clone(),
|
||||||
|
kind: initial.kind.clone(),
|
||||||
|
trigger: initial.trigger.clone(),
|
||||||
|
status: status.to_string(),
|
||||||
|
message,
|
||||||
|
started_at_unix_secs: initial.started_at_unix_secs,
|
||||||
|
completed_at_unix_secs: completed.then(now_unix_secs),
|
||||||
|
duration_ms: started_at
|
||||||
|
.map(|value| value.elapsed().as_millis().try_into().unwrap_or(u64::MAX)),
|
||||||
|
summary,
|
||||||
|
error,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn request_body_cleanup_record(
|
fn request_body_cleanup_record(
|
||||||
initial: &AdminCleanupRunRecord,
|
initial: &AdminCleanupRunRecord,
|
||||||
status: &str,
|
status: &str,
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ use super::{
|
|||||||
advance_proxy_upgrade_rollout_once, cleanup_audit_logs_once,
|
advance_proxy_upgrade_rollout_once, cleanup_audit_logs_once,
|
||||||
cleanup_expired_gemini_file_mappings_once, cleanup_proxy_node_metrics_once,
|
cleanup_expired_gemini_file_mappings_once, cleanup_proxy_node_metrics_once,
|
||||||
cleanup_request_candidates_once, cleanup_stale_pending_requests_once,
|
cleanup_request_candidates_once, cleanup_stale_pending_requests_once,
|
||||||
cleanup_stale_proxy_nodes_once, collect_proxy_upgrade_rollout_probes,
|
cleanup_stale_proxy_nodes_once, collect_proxy_upgrade_rollout_probes, now_unix_secs,
|
||||||
perform_db_maintenance_once, perform_provider_checkin_once, perform_stats_aggregation_once,
|
perform_db_maintenance_once, perform_provider_checkin_once, perform_stats_aggregation_once,
|
||||||
perform_stats_hourly_aggregation_once, perform_usage_cleanup_once,
|
perform_stats_hourly_aggregation_once, perform_usage_cleanup_once,
|
||||||
perform_wallet_daily_usage_aggregation_once, record_completed_cleanup_run,
|
perform_wallet_daily_usage_aggregation_once, record_completed_cleanup_run,
|
||||||
|
|||||||
@@ -434,6 +434,99 @@ VALUES
|
|||||||
assert_eq!(admin_exists, 1);
|
assert_eq!(admin_exists, 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn admin_system_request_bodies_purge_clears_inline_usage_body_fields() {
|
||||||
|
let config = SqlDatabaseConfig {
|
||||||
|
driver: DatabaseDriver::Sqlite,
|
||||||
|
url: "sqlite::memory:".to_string(),
|
||||||
|
pool: SqlPoolConfig {
|
||||||
|
max_connections: 1,
|
||||||
|
..SqlPoolConfig::default()
|
||||||
|
},
|
||||||
|
};
|
||||||
|
let backend = SqliteBackend::from_config(config).expect("backend should build");
|
||||||
|
run_sqlite_migrations(backend.pool())
|
||||||
|
.await
|
||||||
|
.expect("sqlite migrations should run");
|
||||||
|
|
||||||
|
for (column, ty) in [
|
||||||
|
("request_body", "TEXT"),
|
||||||
|
("response_body", "TEXT"),
|
||||||
|
("provider_request_body", "TEXT"),
|
||||||
|
("client_response_body", "TEXT"),
|
||||||
|
("request_body_compressed", "BLOB"),
|
||||||
|
("response_body_compressed", "BLOB"),
|
||||||
|
("provider_request_body_compressed", "BLOB"),
|
||||||
|
("client_response_body_compressed", "BLOB"),
|
||||||
|
] {
|
||||||
|
sqlx::query(&format!(r#"ALTER TABLE "usage" ADD COLUMN {column} {ty}"#))
|
||||||
|
.execute(backend.pool())
|
||||||
|
.await
|
||||||
|
.expect("legacy body column should be added");
|
||||||
|
}
|
||||||
|
|
||||||
|
sqlx::query(
|
||||||
|
r#"
|
||||||
|
INSERT INTO "usage" (
|
||||||
|
request_id,
|
||||||
|
provider_name,
|
||||||
|
model,
|
||||||
|
request_body,
|
||||||
|
response_body,
|
||||||
|
provider_request_body,
|
||||||
|
client_response_body,
|
||||||
|
request_body_compressed,
|
||||||
|
response_body_compressed,
|
||||||
|
provider_request_body_compressed,
|
||||||
|
client_response_body_compressed,
|
||||||
|
created_at_unix_ms
|
||||||
|
)
|
||||||
|
VALUES (
|
||||||
|
'request-1',
|
||||||
|
'openai',
|
||||||
|
'gpt-4.1',
|
||||||
|
'client request',
|
||||||
|
'provider response',
|
||||||
|
'provider request',
|
||||||
|
'client response',
|
||||||
|
X'01',
|
||||||
|
X'02',
|
||||||
|
X'03',
|
||||||
|
X'04',
|
||||||
|
1
|
||||||
|
)
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.execute(backend.pool())
|
||||||
|
.await
|
||||||
|
.expect("usage row should insert");
|
||||||
|
|
||||||
|
let summary = backend
|
||||||
|
.purge_admin_system_data(AdminSystemPurgeTarget::RequestBodies)
|
||||||
|
.await
|
||||||
|
.expect("request body purge should run");
|
||||||
|
|
||||||
|
assert_eq!(summary.affected.get("usage_body_fields_cleaned"), Some(&1));
|
||||||
|
let remaining: i64 = sqlx::query_scalar(
|
||||||
|
r#"
|
||||||
|
SELECT COUNT(*)
|
||||||
|
FROM "usage"
|
||||||
|
WHERE request_body IS NOT NULL
|
||||||
|
OR response_body IS NOT NULL
|
||||||
|
OR provider_request_body IS NOT NULL
|
||||||
|
OR client_response_body IS NOT NULL
|
||||||
|
OR request_body_compressed IS NOT NULL
|
||||||
|
OR response_body_compressed IS NOT NULL
|
||||||
|
OR provider_request_body_compressed IS NOT NULL
|
||||||
|
OR client_response_body_compressed IS NOT NULL
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.fetch_one(backend.pool())
|
||||||
|
.await
|
||||||
|
.expect("remaining body count should load");
|
||||||
|
assert_eq!(remaining, 0);
|
||||||
|
}
|
||||||
|
|
||||||
async fn sqlite_count(pool: &sqlx::SqlitePool, table: &str) -> i64 {
|
async fn sqlite_count(pool: &sqlx::SqlitePool, table: &str) -> i64 {
|
||||||
let sql = format!("SELECT COUNT(*) FROM \"{table}\"");
|
let sql = format!("SELECT COUNT(*) FROM \"{table}\"");
|
||||||
sqlx::query_scalar::<_, i64>(&sql)
|
sqlx::query_scalar::<_, i64>(&sql)
|
||||||
|
|||||||
@@ -561,6 +561,17 @@ const ADMIN_USAGE_CHILD_TABLES: &[&str] = &[
|
|||||||
"usage_settlement_snapshots",
|
"usage_settlement_snapshots",
|
||||||
];
|
];
|
||||||
|
|
||||||
|
const USAGE_BODY_FIELD_COLUMNS: &[&str] = &[
|
||||||
|
"request_body",
|
||||||
|
"response_body",
|
||||||
|
"provider_request_body",
|
||||||
|
"client_response_body",
|
||||||
|
"request_body_compressed",
|
||||||
|
"response_body_compressed",
|
||||||
|
"provider_request_body_compressed",
|
||||||
|
"client_response_body_compressed",
|
||||||
|
];
|
||||||
|
|
||||||
const ADMIN_USER_SCOPED_TABLES: &[&str] = &[
|
const ADMIN_USER_SCOPED_TABLES: &[&str] = &[
|
||||||
"stats_user_daily_cost_savings_model_provider",
|
"stats_user_daily_cost_savings_model_provider",
|
||||||
"stats_user_daily_cost_savings_model",
|
"stats_user_daily_cost_savings_model",
|
||||||
@@ -720,9 +731,10 @@ WHERE request_count <> 0
|
|||||||
}
|
}
|
||||||
AdminSystemPurgeTarget::RequestBodies => {
|
AdminSystemPurgeTarget::RequestBodies => {
|
||||||
pg_delete_table(tx, "usage_body_blobs", summary).await?;
|
pg_delete_table(tx, "usage_body_blobs", summary).await?;
|
||||||
pg_execute_if_table(
|
pg_execute_if_table_has_columns(
|
||||||
tx,
|
tx,
|
||||||
"usage",
|
"usage",
|
||||||
|
USAGE_BODY_FIELD_COLUMNS,
|
||||||
"usage_body_fields_cleaned",
|
"usage_body_fields_cleaned",
|
||||||
r#"
|
r#"
|
||||||
UPDATE public.usage
|
UPDATE public.usage
|
||||||
@@ -1080,6 +1092,33 @@ WHERE request_count <> 0
|
|||||||
}
|
}
|
||||||
AdminSystemPurgeTarget::RequestBodies => {
|
AdminSystemPurgeTarget::RequestBodies => {
|
||||||
mysql_delete_table(tx, "usage_body_blobs", summary).await?;
|
mysql_delete_table(tx, "usage_body_blobs", summary).await?;
|
||||||
|
mysql_execute_if_table_has_columns(
|
||||||
|
tx,
|
||||||
|
"usage",
|
||||||
|
USAGE_BODY_FIELD_COLUMNS,
|
||||||
|
"usage_body_fields_cleaned",
|
||||||
|
r#"
|
||||||
|
UPDATE `usage`
|
||||||
|
SET request_body = NULL,
|
||||||
|
response_body = NULL,
|
||||||
|
provider_request_body = NULL,
|
||||||
|
client_response_body = NULL,
|
||||||
|
request_body_compressed = NULL,
|
||||||
|
response_body_compressed = NULL,
|
||||||
|
provider_request_body_compressed = NULL,
|
||||||
|
client_response_body_compressed = NULL
|
||||||
|
WHERE request_body IS NOT NULL
|
||||||
|
OR response_body IS NOT NULL
|
||||||
|
OR provider_request_body IS NOT NULL
|
||||||
|
OR client_response_body IS NOT NULL
|
||||||
|
OR request_body_compressed IS NOT NULL
|
||||||
|
OR response_body_compressed IS NOT NULL
|
||||||
|
OR provider_request_body_compressed IS NOT NULL
|
||||||
|
OR client_response_body_compressed IS NOT NULL
|
||||||
|
"#,
|
||||||
|
summary,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
mysql_execute_if_table(
|
mysql_execute_if_table(
|
||||||
tx,
|
tx,
|
||||||
"usage_http_audits",
|
"usage_http_audits",
|
||||||
@@ -1375,6 +1414,33 @@ WHERE request_count <> 0
|
|||||||
}
|
}
|
||||||
AdminSystemPurgeTarget::RequestBodies => {
|
AdminSystemPurgeTarget::RequestBodies => {
|
||||||
sqlite_delete_table(tx, "usage_body_blobs", summary).await?;
|
sqlite_delete_table(tx, "usage_body_blobs", summary).await?;
|
||||||
|
sqlite_execute_if_table_has_columns(
|
||||||
|
tx,
|
||||||
|
"usage",
|
||||||
|
USAGE_BODY_FIELD_COLUMNS,
|
||||||
|
"usage_body_fields_cleaned",
|
||||||
|
r#"
|
||||||
|
UPDATE "usage"
|
||||||
|
SET request_body = NULL,
|
||||||
|
response_body = NULL,
|
||||||
|
provider_request_body = NULL,
|
||||||
|
client_response_body = NULL,
|
||||||
|
request_body_compressed = NULL,
|
||||||
|
response_body_compressed = NULL,
|
||||||
|
provider_request_body_compressed = NULL,
|
||||||
|
client_response_body_compressed = NULL
|
||||||
|
WHERE request_body IS NOT NULL
|
||||||
|
OR response_body IS NOT NULL
|
||||||
|
OR provider_request_body IS NOT NULL
|
||||||
|
OR client_response_body IS NOT NULL
|
||||||
|
OR request_body_compressed IS NOT NULL
|
||||||
|
OR response_body_compressed IS NOT NULL
|
||||||
|
OR provider_request_body_compressed IS NOT NULL
|
||||||
|
OR client_response_body_compressed IS NOT NULL
|
||||||
|
"#,
|
||||||
|
summary,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
sqlite_execute_if_table(
|
sqlite_execute_if_table(
|
||||||
tx,
|
tx,
|
||||||
"usage_http_audits",
|
"usage_http_audits",
|
||||||
@@ -1573,9 +1639,10 @@ WHERE blobs.body_ref = doomed.body_ref
|
|||||||
limit,
|
limit,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
pg_execute_batch_if_table(
|
pg_execute_batch_if_table_has_columns(
|
||||||
tx,
|
tx,
|
||||||
"usage",
|
"usage",
|
||||||
|
USAGE_BODY_FIELD_COLUMNS,
|
||||||
"usage_body_fields_cleaned",
|
"usage_body_fields_cleaned",
|
||||||
r#"
|
r#"
|
||||||
WITH batch AS (
|
WITH batch AS (
|
||||||
@@ -1674,9 +1741,10 @@ WHERE body_ref IN (
|
|||||||
limit,
|
limit,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
mysql_execute_batch_if_table(
|
mysql_execute_batch_if_table_has_columns(
|
||||||
tx,
|
tx,
|
||||||
"usage",
|
"usage",
|
||||||
|
USAGE_BODY_FIELD_COLUMNS,
|
||||||
"usage_body_fields_cleaned",
|
"usage_body_fields_cleaned",
|
||||||
r#"
|
r#"
|
||||||
UPDATE `usage`
|
UPDATE `usage`
|
||||||
@@ -1772,9 +1840,10 @@ WHERE body_ref IN (
|
|||||||
limit,
|
limit,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
sqlite_execute_batch_if_table(
|
sqlite_execute_batch_if_table_has_columns(
|
||||||
tx,
|
tx,
|
||||||
"usage",
|
"usage",
|
||||||
|
USAGE_BODY_FIELD_COLUMNS,
|
||||||
"usage_body_fields_cleaned",
|
"usage_body_fields_cleaned",
|
||||||
r#"
|
r#"
|
||||||
UPDATE "usage"
|
UPDATE "usage"
|
||||||
@@ -1923,6 +1992,26 @@ async fn pg_execute_if_table(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn pg_execute_if_table_has_columns(
|
||||||
|
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
|
||||||
|
table: &str,
|
||||||
|
columns: &[&str],
|
||||||
|
key: &str,
|
||||||
|
sql: &str,
|
||||||
|
summary: &mut AdminSystemPurgeSummary,
|
||||||
|
) -> Result<(), DataLayerError> {
|
||||||
|
if !pg_table_has_columns(tx, checked_sql_identifier(table)?, columns).await? {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let rows = sqlx::query(sql)
|
||||||
|
.execute(&mut **tx)
|
||||||
|
.await
|
||||||
|
.map_postgres_err()?
|
||||||
|
.rows_affected();
|
||||||
|
summary.add(key, rows);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
async fn pg_execute_batch_if_table(
|
async fn pg_execute_batch_if_table(
|
||||||
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
|
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
|
||||||
table: &str,
|
table: &str,
|
||||||
@@ -1944,6 +2033,28 @@ async fn pg_execute_batch_if_table(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn pg_execute_batch_if_table_has_columns(
|
||||||
|
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
|
||||||
|
table: &str,
|
||||||
|
columns: &[&str],
|
||||||
|
key: &str,
|
||||||
|
sql: &str,
|
||||||
|
summary: &mut AdminSystemPurgeSummary,
|
||||||
|
limit: i64,
|
||||||
|
) -> Result<(), DataLayerError> {
|
||||||
|
if !pg_table_has_columns(tx, checked_sql_identifier(table)?, columns).await? {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let rows = sqlx::query(sql)
|
||||||
|
.bind(limit)
|
||||||
|
.execute(&mut **tx)
|
||||||
|
.await
|
||||||
|
.map_postgres_err()?
|
||||||
|
.rows_affected();
|
||||||
|
summary.add(key, rows);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
async fn pg_table_exists(
|
async fn pg_table_exists(
|
||||||
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
|
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
|
||||||
table: &str,
|
table: &str,
|
||||||
@@ -1956,6 +2067,38 @@ async fn pg_table_exists(
|
|||||||
.map_postgres_err()
|
.map_postgres_err()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn pg_table_has_columns(
|
||||||
|
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
|
||||||
|
table: &str,
|
||||||
|
columns: &[&str],
|
||||||
|
) -> Result<bool, DataLayerError> {
|
||||||
|
let table = checked_sql_identifier(table)?;
|
||||||
|
if !pg_table_exists(tx, table).await? {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
for column in columns {
|
||||||
|
let column = checked_sql_identifier(column)?;
|
||||||
|
let exists: bool = sqlx::query_scalar(
|
||||||
|
"SELECT EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM information_schema.columns
|
||||||
|
WHERE table_schema = 'public'
|
||||||
|
AND table_name = $1
|
||||||
|
AND column_name = $2
|
||||||
|
)",
|
||||||
|
)
|
||||||
|
.bind(table)
|
||||||
|
.bind(column)
|
||||||
|
.fetch_one(&mut **tx)
|
||||||
|
.await
|
||||||
|
.map_postgres_err()?;
|
||||||
|
if !exists {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
|
||||||
async fn mysql_delete_table(
|
async fn mysql_delete_table(
|
||||||
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
|
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
|
||||||
table: &str,
|
table: &str,
|
||||||
@@ -2036,6 +2179,26 @@ async fn mysql_execute_if_table(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn mysql_execute_if_table_has_columns(
|
||||||
|
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
|
||||||
|
table: &str,
|
||||||
|
columns: &[&str],
|
||||||
|
key: &str,
|
||||||
|
sql: &str,
|
||||||
|
summary: &mut AdminSystemPurgeSummary,
|
||||||
|
) -> Result<(), DataLayerError> {
|
||||||
|
if !mysql_table_has_columns(tx, checked_sql_identifier(table)?, columns).await? {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let rows = sqlx::query(sql)
|
||||||
|
.execute(&mut **tx)
|
||||||
|
.await
|
||||||
|
.map_sql_err()?
|
||||||
|
.rows_affected();
|
||||||
|
summary.add(key, rows);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
async fn mysql_execute_batch_if_table(
|
async fn mysql_execute_batch_if_table(
|
||||||
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
|
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
|
||||||
table: &str,
|
table: &str,
|
||||||
@@ -2057,6 +2220,28 @@ async fn mysql_execute_batch_if_table(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn mysql_execute_batch_if_table_has_columns(
|
||||||
|
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
|
||||||
|
table: &str,
|
||||||
|
columns: &[&str],
|
||||||
|
key: &str,
|
||||||
|
sql: &str,
|
||||||
|
summary: &mut AdminSystemPurgeSummary,
|
||||||
|
limit: i64,
|
||||||
|
) -> Result<(), DataLayerError> {
|
||||||
|
if !mysql_table_has_columns(tx, checked_sql_identifier(table)?, columns).await? {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let rows = sqlx::query(sql)
|
||||||
|
.bind(limit)
|
||||||
|
.execute(&mut **tx)
|
||||||
|
.await
|
||||||
|
.map_sql_err()?
|
||||||
|
.rows_affected();
|
||||||
|
summary.add(key, rows);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
async fn mysql_table_exists(
|
async fn mysql_table_exists(
|
||||||
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
|
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
|
||||||
table: &str,
|
table: &str,
|
||||||
@@ -2072,6 +2257,36 @@ async fn mysql_table_exists(
|
|||||||
Ok(total > 0)
|
Ok(total > 0)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn mysql_table_has_columns(
|
||||||
|
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
|
||||||
|
table: &str,
|
||||||
|
columns: &[&str],
|
||||||
|
) -> Result<bool, DataLayerError> {
|
||||||
|
let table = checked_sql_identifier(table)?;
|
||||||
|
if !mysql_table_exists(tx, table).await? {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
for column in columns {
|
||||||
|
let column = checked_sql_identifier(column)?;
|
||||||
|
let total: i64 = sqlx::query_scalar(
|
||||||
|
"SELECT COUNT(*)
|
||||||
|
FROM information_schema.columns
|
||||||
|
WHERE table_schema = DATABASE()
|
||||||
|
AND table_name = ?
|
||||||
|
AND column_name = ?",
|
||||||
|
)
|
||||||
|
.bind(table)
|
||||||
|
.bind(column)
|
||||||
|
.fetch_one(&mut **tx)
|
||||||
|
.await
|
||||||
|
.map_sql_err()?;
|
||||||
|
if total == 0 {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
|
||||||
async fn sqlite_delete_table(
|
async fn sqlite_delete_table(
|
||||||
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
|
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
|
||||||
table: &str,
|
table: &str,
|
||||||
@@ -2152,6 +2367,26 @@ async fn sqlite_execute_if_table(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn sqlite_execute_if_table_has_columns(
|
||||||
|
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
|
||||||
|
table: &str,
|
||||||
|
columns: &[&str],
|
||||||
|
key: &str,
|
||||||
|
sql: &str,
|
||||||
|
summary: &mut AdminSystemPurgeSummary,
|
||||||
|
) -> Result<(), DataLayerError> {
|
||||||
|
if !sqlite_table_has_columns(tx, checked_sql_identifier(table)?, columns).await? {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let rows = sqlx::query(sql)
|
||||||
|
.execute(&mut **tx)
|
||||||
|
.await
|
||||||
|
.map_sql_err()?
|
||||||
|
.rows_affected();
|
||||||
|
summary.add(key, rows);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
async fn sqlite_execute_batch_if_table(
|
async fn sqlite_execute_batch_if_table(
|
||||||
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
|
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
|
||||||
table: &str,
|
table: &str,
|
||||||
@@ -2173,6 +2408,28 @@ async fn sqlite_execute_batch_if_table(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn sqlite_execute_batch_if_table_has_columns(
|
||||||
|
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
|
||||||
|
table: &str,
|
||||||
|
columns: &[&str],
|
||||||
|
key: &str,
|
||||||
|
sql: &str,
|
||||||
|
summary: &mut AdminSystemPurgeSummary,
|
||||||
|
limit: i64,
|
||||||
|
) -> Result<(), DataLayerError> {
|
||||||
|
if !sqlite_table_has_columns(tx, checked_sql_identifier(table)?, columns).await? {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let rows = sqlx::query(sql)
|
||||||
|
.bind(limit)
|
||||||
|
.execute(&mut **tx)
|
||||||
|
.await
|
||||||
|
.map_sql_err()?
|
||||||
|
.rows_affected();
|
||||||
|
summary.add(key, rows);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
async fn sqlite_table_exists(
|
async fn sqlite_table_exists(
|
||||||
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
|
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
|
||||||
table: &str,
|
table: &str,
|
||||||
@@ -2187,6 +2444,31 @@ async fn sqlite_table_exists(
|
|||||||
Ok(total > 0)
|
Ok(total > 0)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn sqlite_table_has_columns(
|
||||||
|
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
|
||||||
|
table: &str,
|
||||||
|
columns: &[&str],
|
||||||
|
) -> Result<bool, DataLayerError> {
|
||||||
|
let table = checked_sql_identifier(table)?;
|
||||||
|
if !sqlite_table_exists(tx, table).await? {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
for column in columns {
|
||||||
|
let column = checked_sql_identifier(column)?;
|
||||||
|
let total: i64 =
|
||||||
|
sqlx::query_scalar("SELECT COUNT(*) FROM pragma_table_info(?) WHERE name = ?")
|
||||||
|
.bind(table)
|
||||||
|
.bind(column)
|
||||||
|
.fetch_one(&mut **tx)
|
||||||
|
.await
|
||||||
|
.map_sql_err()?;
|
||||||
|
if total == 0 {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
|
||||||
fn current_unix_secs() -> u64 {
|
fn current_unix_secs() -> u64 {
|
||||||
chrono::Utc::now().timestamp().max(0) as u64
|
chrono::Utc::now().timestamp().max(0) as u64
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -253,7 +253,7 @@ export interface CleanupRunListResponse {
|
|||||||
items: CleanupRunRecord[]
|
items: CleanupRunRecord[]
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RequestBodyCleanupTaskResponse {
|
export interface CleanupTaskResponse {
|
||||||
message: string
|
message: string
|
||||||
task: CleanupRunRecord
|
task: CleanupRunRecord
|
||||||
}
|
}
|
||||||
@@ -1073,16 +1073,25 @@ export const adminApi = {
|
|||||||
},
|
},
|
||||||
|
|
||||||
// 数据清空
|
// 数据清空
|
||||||
purgeConfig: () => purge<{ message: string; deleted: Record<string, number> }>('config'),
|
purgeConfig: () => purge<CleanupTaskResponse>('config'),
|
||||||
purgeUsers: () => purge<{ message: string; deleted: Record<string, number> }>('users'),
|
purgeUsers: () => purge<CleanupTaskResponse>('users'),
|
||||||
purgeUsage: () => purge<{ message: string; deleted: Record<string, number> }>('usage'),
|
async purgeUsage(): Promise<CleanupTaskResponse> {
|
||||||
purgeAuditLogs: () => purge<{ message: string; deleted: Record<string, number> }>('audit-logs'),
|
const response = await apiClient.post<CleanupTaskResponse>('/api/admin/system/purge/usage')
|
||||||
purgeRequestBodies: () => purge<{ message: string; cleaned: Record<string, number> }>('request-bodies'),
|
return response.data
|
||||||
async purgeRequestBodiesAsync(): Promise<RequestBodyCleanupTaskResponse> {
|
},
|
||||||
const response = await apiClient.post<RequestBodyCleanupTaskResponse>('/api/admin/system/purge/request-bodies/task')
|
async purgeAuditLogs(): Promise<CleanupTaskResponse> {
|
||||||
|
const response = await apiClient.post<CleanupTaskResponse>('/api/admin/system/purge/audit-logs')
|
||||||
|
return response.data
|
||||||
|
},
|
||||||
|
purgeRequestBodies: () => purge<CleanupTaskResponse>('request-bodies'),
|
||||||
|
async purgeRequestBodiesAsync(): Promise<CleanupTaskResponse> {
|
||||||
|
const response = await apiClient.post<CleanupTaskResponse>('/api/admin/system/purge/request-bodies/task')
|
||||||
|
return response.data
|
||||||
|
},
|
||||||
|
async purgeStats(): Promise<CleanupTaskResponse> {
|
||||||
|
const response = await apiClient.post<CleanupTaskResponse>('/api/admin/system/purge/stats')
|
||||||
return response.data
|
return response.data
|
||||||
},
|
},
|
||||||
purgeStats: () => purge<{ message: string }>('stats'),
|
|
||||||
async getCleanupRuns(): Promise<CleanupRunListResponse> {
|
async getCleanupRuns(): Promise<CleanupRunListResponse> {
|
||||||
const response = await apiClient.get<CleanupRunListResponse>('/api/admin/system/cleanup/runs')
|
const response = await apiClient.get<CleanupRunListResponse>('/api/admin/system/cleanup/runs')
|
||||||
return response.data
|
return response.data
|
||||||
|
|||||||
@@ -432,6 +432,11 @@ function cleanupKindLabel(kind: string): string {
|
|||||||
audit_cleanup: '审计日志',
|
audit_cleanup: '审计日志',
|
||||||
request_candidate_cleanup: '候选记录',
|
request_candidate_cleanup: '候选记录',
|
||||||
request_bodies: '请求体',
|
request_bodies: '请求体',
|
||||||
|
config_purge: '配置清空',
|
||||||
|
users_purge: '用户清空',
|
||||||
|
usage_purge: '使用记录清空',
|
||||||
|
audit_logs_purge: '审计日志清空',
|
||||||
|
stats_purge: '统计聚合清空',
|
||||||
system_cleanup: '系统清理',
|
system_cleanup: '系统清理',
|
||||||
}
|
}
|
||||||
return labels[kind] || kind
|
return labels[kind] || kind
|
||||||
|
|||||||
@@ -62,37 +62,37 @@ const purgeItems: PurgeItem[] = [
|
|||||||
{
|
{
|
||||||
key: 'config',
|
key: 'config',
|
||||||
title: '清空配置',
|
title: '清空配置',
|
||||||
description: '删除所有提供商、端点、API Key 和模型配置',
|
description: '后台删除所有提供商、端点、API Key 和模型配置',
|
||||||
buttonText: '清空配置',
|
buttonText: '清空配置',
|
||||||
icon: markRaw(Settings),
|
icon: markRaw(Settings),
|
||||||
confirmMessage: '确定要清空所有提供商配置吗?这将删除所有提供商、端点、API Key 和模型配置,操作不可逆。',
|
confirmMessage: '确定要后台清空所有提供商配置吗?这将删除所有提供商、端点、API Key 和模型配置,操作不可逆。',
|
||||||
action: () => adminApi.purgeConfig(),
|
action: () => adminApi.purgeConfig(),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
key: 'users',
|
key: 'users',
|
||||||
title: '清空用户',
|
title: '清空用户',
|
||||||
description: '删除所有非管理员用户及其 API Keys',
|
description: '后台删除所有非管理员用户及其 API Keys',
|
||||||
buttonText: '清空用户',
|
buttonText: '清空用户',
|
||||||
icon: markRaw(Users),
|
icon: markRaw(Users),
|
||||||
confirmMessage: '确定要清空所有非管理员用户吗?管理员账户将被保留,操作不可逆。',
|
confirmMessage: '确定要后台清空所有非管理员用户吗?管理员账户将被保留,操作不可逆。',
|
||||||
action: () => adminApi.purgeUsers(),
|
action: () => adminApi.purgeUsers(),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
key: 'usage',
|
key: 'usage',
|
||||||
title: '清空使用记录',
|
title: '清空使用记录',
|
||||||
description: '删除全部使用记录和请求候选记录',
|
description: '后台清空全部使用记录和请求候选记录',
|
||||||
buttonText: '清空记录',
|
buttonText: '清空记录',
|
||||||
icon: markRaw(BarChart3),
|
icon: markRaw(BarChart3),
|
||||||
confirmMessage: '确定要清空全部使用记录吗?所有请求统计数据将被永久删除,操作不可逆。',
|
confirmMessage: '确定要后台清空全部使用记录吗?所有请求统计数据将被永久删除,操作不可逆。',
|
||||||
action: () => adminApi.purgeUsage(),
|
action: () => adminApi.purgeUsage(),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
key: 'audit-logs',
|
key: 'audit-logs',
|
||||||
title: '清空审计日志',
|
title: '清空审计日志',
|
||||||
description: '删除全部审计日志记录',
|
description: '后台删除全部审计日志记录',
|
||||||
buttonText: '清空日志',
|
buttonText: '清空日志',
|
||||||
icon: markRaw(Shield),
|
icon: markRaw(Shield),
|
||||||
confirmMessage: '确定要清空全部审计日志吗?所有安全事件记录将被永久删除,操作不可逆。',
|
confirmMessage: '确定要后台清空全部审计日志吗?所有安全事件记录将被永久删除,操作不可逆。',
|
||||||
action: () => adminApi.purgeAuditLogs(),
|
action: () => adminApi.purgeAuditLogs(),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -107,10 +107,10 @@ const purgeItems: PurgeItem[] = [
|
|||||||
{
|
{
|
||||||
key: 'stats',
|
key: 'stats',
|
||||||
title: '清空统计聚合',
|
title: '清空统计聚合',
|
||||||
description: '删除统计聚合数据,保留原始使用记录;统计可从原始使用记录重新构建',
|
description: '后台删除统计聚合数据并重建,保留原始使用记录',
|
||||||
buttonText: '清空统计聚合',
|
buttonText: '清空统计聚合',
|
||||||
icon: markRaw(PieChart),
|
icon: markRaw(PieChart),
|
||||||
confirmMessage: '确定要清空全部统计聚合数据吗?原始使用记录会保留,仪表盘和累计统计可从原始记录重新构建。',
|
confirmMessage: '确定要后台清空全部统计聚合数据吗?原始使用记录会保留,仪表盘和累计统计会从原始记录重新构建。',
|
||||||
action: () => adminApi.purgeStats(),
|
action: () => adminApi.purgeStats(),
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
|
|||||||
Reference in New Issue
Block a user