i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-01 17:00:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(admin): run destructive purges as cleanup tasks

Browse Source
This commit is contained in:
fawney19
2026-05-10 00:41:21 +08:00
parent 391c2fbe5d
commit a81053e6ff
9 changed files with 682 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

110
apps/aether-gateway/src/handlers/admin/system/core/system_routes.rs
View File

@@ -18,7 +18,6 @@ use crate::handlers::admin::system::shared::settings::{
};
use crate::handlers::admin::system::shared::smtp::build_admin_smtp_test_payload;
use crate::GatewayError;
use aether_data::repository::system::AdminSystemPurgeTarget;
use axum::{
body::{Body, Bytes},
http,
@@ -205,34 +204,21 @@ pub(super) async fn maybe_build_local_admin_core_system_response(
return Ok(Some(Json(json!({ "items": records })).into_response()));
}
if decision.route_kind.as_deref() == Some("purge_request_bodies_task")
&& request_method == http::Method::POST
&& request_path == "/api/admin/system/purge/request-bodies/task"
if let Some((task_kind, action, object_type, object_id)) =
admin_system_purge_task_for_route_kind(decision.route_kind.as_deref())
{
let task =
crate::maintenance::start_admin_request_body_cleanup_task(state.cloned_app()).await?;
if request_method != http::Method::POST {
return Ok(None);
}
let task = crate::maintenance::start_admin_system_purge_task(state.cloned_app(), task_kind)
.await?;
return Ok(Some(attach_admin_audit_response(
Json(json!({
"message": task.message.clone(),
"task": task,
}))
.into_response(),
"admin_system_request_body_cleanup_task_started",
"purge_request_bodies_async",
"request_bodies",
"all",
)));
}
if let Some((target, action, object_type, object_id)) =
admin_system_purge_target_for_route_kind(decision.route_kind.as_deref())
{
if request_method != http::Method::POST {
return Ok(None);
}
return Ok(Some(attach_admin_audit_response(
Json(build_admin_system_purge_payload(state, target).await?).into_response(),
"admin_system_data_purged",
"admin_system_purge_task_started",
action,
object_type,
object_id,
@@ -465,97 +451,55 @@ pub(super) async fn maybe_build_local_admin_core_system_response(
Ok(None)
}
fn admin_system_purge_target_for_route_kind(
fn admin_system_purge_task_for_route_kind(
route_kind: Option<&str>,
) -> Option<(
AdminSystemPurgeTarget,
crate::maintenance::AdminCleanupTaskKind,
&'static str,
&'static str,
&'static str,
)> {
match route_kind {
Some("purge_config") => Some((
AdminSystemPurgeTarget::Config,
"purge_system_config",
crate::maintenance::AdminCleanupTaskKind::Config,
"purge_system_config_async",
"system_config",
"global",
)),
Some("purge_users") => Some((
AdminSystemPurgeTarget::Users,
"purge_non_admin_users",
crate::maintenance::AdminCleanupTaskKind::Users,
"purge_non_admin_users_async",
"users",
"non_admin",
)),
Some("purge_usage") => Some((
AdminSystemPurgeTarget::Usage,
"purge_usage_records",
crate::maintenance::AdminCleanupTaskKind::Usage,
"purge_usage_records_async",
"usage",
"all",
)),
Some("purge_audit_logs") => Some((
AdminSystemPurgeTarget::AuditLogs,
"purge_audit_logs",
crate::maintenance::AdminCleanupTaskKind::AuditLogs,
"purge_audit_logs_async",
"audit_logs",
"all",
)),
Some("purge_request_bodies") => Some((
AdminSystemPurgeTarget::RequestBodies,
"purge_request_bodies",
Some("purge_request_bodies") | Some("purge_request_bodies_task") => Some((
crate::maintenance::AdminCleanupTaskKind::RequestBodies,
"purge_request_bodies_async",
"request_bodies",
"all",
)),
Some("purge_stats") => Some((AdminSystemPurgeTarget::Stats, "purge_stats", "stats", "all")),
Some("purge_stats") => Some((
crate::maintenance::AdminCleanupTaskKind::Stats,
"purge_stats_async",
"stats",
"all",
)),
_ => None,
}
}
async fn build_admin_system_purge_payload(
state: &AdminAppState<'_>,
target: AdminSystemPurgeTarget,
) -> Result<serde_json::Value, GatewayError> {
let summary = state.purge_admin_system_data(target).await?;
let total = summary.total();
let affected = summary.affected.clone();
if target == AdminSystemPurgeTarget::Stats {
let rebuild = state.rebuild_admin_stats_once().await?;
let message = if rebuild.capped {
format!(
"统计聚合已清空，已重建 {} 个小时桶和 {} 个日桶，仍有历史统计待后台任务继续重建",
rebuild.hourly_buckets, rebuild.daily_buckets
)
} else {
format!(
"统计聚合已清空并重建，删除 {} 行，重建 {} 个小时桶和 {} 个日桶",
total, rebuild.hourly_buckets, rebuild.daily_buckets
)
};
return Ok(json!({
"message": message,
"deleted": affected,
"rebuilt": {
"hourly_buckets": rebuild.hourly_buckets,
"daily_buckets": rebuild.daily_buckets,
"capped": rebuild.capped,
},
}));
}
let (message, count_key) = match target {
AdminSystemPurgeTarget::Config => ("系统配置已清空", "deleted"),
AdminSystemPurgeTarget::Users => ("非管理员用户已清空", "deleted"),
AdminSystemPurgeTarget::Usage => ("使用记录已清空", "deleted"),
AdminSystemPurgeTarget::AuditLogs => ("审计日志已清空", "deleted"),
AdminSystemPurgeTarget::RequestBodies => ("请求/响应体已清空", "cleaned"),
AdminSystemPurgeTarget::Stats => unreachable!("stats handled above"),
};
Ok(json!({
"message": format!("{message}，影响 {} 行", total),
count_key: affected,
}))
}
async fn build_admin_system_cleanup_payload(
state: &AdminAppState<'_>,
) -> Result<serde_json::Value, GatewayError> {

3
apps/aether-gateway/src/maintenance/runtime.rs
View File

@@ -53,7 +53,8 @@ pub(crate) use aether_data_contracts::repository::usage::{
use audit_cleanup::*;
pub(crate) use cleanup_runs::{
list_admin_cleanup_run_records, record_completed_cleanup_run, record_failed_cleanup_run,
start_admin_request_body_cleanup_task, AdminCleanupRunRecord,
start_admin_request_body_cleanup_task, start_admin_system_purge_task, AdminCleanupRunRecord,
AdminCleanupTaskKind,
};
use config::*;
use db_maintenance::*;

258
apps/aether-gateway/src/maintenance/runtime/cleanup_runs.rs
View File

@@ -1,6 +1,6 @@
use std::time::Instant;
use aether_data::repository::system::AdminSystemPurgeSummary;
use aether_data::repository::system::{AdminSystemPurgeSummary, AdminSystemPurgeTarget};
use aether_data_contracts::DataLayerError;
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
@@ -29,6 +29,65 @@ pub(crate) struct AdminCleanupRunRecord {
pub(crate) error: Option<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum AdminCleanupTaskKind {
Config,
Users,
RequestBodies,
Usage,
AuditLogs,
Stats,
}
impl AdminCleanupTaskKind {
fn record_kind(self) -> &'static str {
match self {
Self::Config => "config_purge",
Self::Users => "users_purge",
Self::RequestBodies => "request_bodies",
Self::Usage => "usage_purge",
Self::AuditLogs => "audit_logs_purge",
Self::Stats => "stats_purge",
}
}
fn target(self) -> Option<AdminSystemPurgeTarget> {
match self {
Self::RequestBodies => None,
Self::Config => Some(AdminSystemPurgeTarget::Config),
Self::Users => Some(AdminSystemPurgeTarget::Users),
Self::Usage => Some(AdminSystemPurgeTarget::Usage),
Self::AuditLogs => Some(AdminSystemPurgeTarget::AuditLogs),
Self::Stats => Some(AdminSystemPurgeTarget::Stats),
}
}
fn start_message(self, batch_size: Option<usize>) -> String {
match self {
Self::Config => "系统配置后台清空已开始".to_string(),
Self::Users => "非管理员用户后台清空已开始".to_string(),
Self::RequestBodies => format!(
"请求/响应体后台清理已开始，每批 {} 条",
batch_size.unwrap_or(1)
),
Self::Usage => "使用记录后台清空已开始".to_string(),
Self::AuditLogs => "审计日志后台清空已开始".to_string(),
Self::Stats => "统计聚合后台清空和重建已开始".to_string(),
}
}
fn failure_message(self) -> &'static str {
match self {
Self::Config => "系统配置后台清空失败",
Self::Users => "非管理员用户后台清空失败",
Self::RequestBodies => "请求/响应体后台清理失败",
Self::Usage => "使用记录后台清空失败",
Self::AuditLogs => "审计日志后台清空失败",
Self::Stats => "统计聚合后台清空或重建失败",
}
}
}
pub(crate) async fn list_admin_cleanup_run_records(
data: &GatewayDataState,
) -> Result<Vec<AdminCleanupRunRecord>, DataLayerError> {
@@ -43,6 +102,13 @@ pub(crate) async fn list_admin_cleanup_run_records(
pub(crate) async fn start_admin_request_body_cleanup_task(
app: AppState,
) -> Result<AdminCleanupRunRecord, GatewayError> {
start_admin_system_purge_task(app, AdminCleanupTaskKind::RequestBodies).await
}
pub(crate) async fn start_admin_system_purge_task(
app: AppState,
kind: AdminCleanupTaskKind,
) -> Result<AdminCleanupRunRecord, GatewayError> {
let data = app.data.clone();
let records = list_admin_cleanup_run_records(&data)
@@ -50,42 +116,55 @@ pub(crate) async fn start_admin_request_body_cleanup_task(
.map_err(|err| GatewayError::Internal(err.to_string()))?;
if let Some(existing) = records
.iter()
.find(|record| record.kind == "request_bodies" && record.status == "processing")
.find(|record| record.kind == kind.record_kind() && record.status == "processing")
.cloned()
{
return Ok(existing);
}
let batch_size = system_config_usize(&data, "cleanup_batch_size", 1_000)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?
.max(1);
let batch_size = if kind == AdminCleanupTaskKind::RequestBodies {
Some(
system_config_usize(&data, "cleanup_batch_size", 1_000)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?
.max(1),
)
} else {
None
};
let started_at = now_unix_secs();
let record = AdminCleanupRunRecord {
id: uuid::Uuid::new_v4().to_string(),
kind: "request_bodies".to_string(),
kind: kind.record_kind().to_string(),
trigger: "manual".to_string(),
status: "processing".to_string(),
message: format!("请求/响应体后台清理已开始，每批 {batch_size} 条"),
message: kind.start_message(batch_size),
started_at_unix_secs: started_at,
completed_at_unix_secs: None,
duration_ms: None,
summary: json!({
"batch_size": batch_size,
"batches": 0,
"cleaned": {},
}),
summary: initial_task_summary(kind, batch_size),
error: None,
};
record_cleanup_run(&data, record.clone())
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?;
tokio::spawn(run_request_body_cleanup_task(
data,
record.clone(),
batch_size,
));
match kind {
AdminCleanupTaskKind::RequestBodies => {
tokio::spawn(run_request_body_cleanup_task(
data,
record.clone(),
batch_size.unwrap_or(1),
));
}
AdminCleanupTaskKind::Usage
| AdminCleanupTaskKind::AuditLogs
| AdminCleanupTaskKind::Config
| AdminCleanupTaskKind::Users
| AdminCleanupTaskKind::Stats => {
tokio::spawn(run_admin_system_purge_task(app, record.clone(), kind));
}
}
Ok(record)
}
@@ -235,6 +314,149 @@ async fn run_request_body_cleanup_task(
);
}
async fn run_admin_system_purge_task(
app: AppState,
initial_record: AdminCleanupRunRecord,
kind: AdminCleanupTaskKind,
) {
let started_at = Instant::now();
let data = app.data.clone();
match run_admin_system_purge_task_once(&app, kind).await {
Ok((summary, message)) => {
let completed = cleanup_task_record(
&initial_record,
"completed",
message,
summary,
Some(started_at),
None,
);
if let Err(err) = record_cleanup_run(&data, completed).await {
warn!(error = %err, "failed to record admin system purge task completion");
}
info!(
event_name = "admin_system_purge_task_completed",
log_type = "ops",
worker = "admin_system_purge_task",
kind = ?kind,
"gateway finished admin system purge task"
);
}
Err(err) => {
let failed = cleanup_task_record(
&initial_record,
"failed",
kind.failure_message().to_string(),
json!({}),
Some(started_at),
Some(format!("{err:?}")),
);
if let Err(record_err) = record_cleanup_run(&data, failed).await {
warn!(error = %record_err, "failed to record admin system purge task failure");
}
warn!(error = ?err, kind = ?kind, "admin system purge task failed");
}
}
}
async fn run_admin_system_purge_task_once(
app: &AppState,
kind: AdminCleanupTaskKind,
) -> Result<(Value, String), GatewayError> {
let target = kind
.target()
.expect("non-request-body cleanup task should map to purge target");
let purge = app.purge_admin_system_data(target).await?;
let deleted_total = purge.total();
let affected = purge.affected.clone();
if kind == AdminCleanupTaskKind::Stats {
let rebuild = app.rebuild_admin_stats_once().await?;
let message = if rebuild.capped {
format!(
"统计聚合后台清空完成，已重建 {} 个小时桶和 {} 个日桶，仍有历史统计待后台任务继续重建",
rebuild.hourly_buckets, rebuild.daily_buckets
)
} else {
format!(
"统计聚合后台清空并重建完成，删除 {} 行，重建 {} 个小时桶和 {} 个日桶",
deleted_total, rebuild.hourly_buckets, rebuild.daily_buckets
)
};
return Ok((
json!({
"deleted": affected,
"rebuilt": {
"hourly_buckets": rebuild.hourly_buckets,
"daily_buckets": rebuild.daily_buckets,
"capped": rebuild.capped,
},
"total": deleted_total,
}),
message,
));
}
let message = match kind {
AdminCleanupTaskKind::Config => format!("系统配置后台清空完成，影响 {deleted_total} 行"),
AdminCleanupTaskKind::Users => format!("非管理员用户后台清空完成，影响 {deleted_total} 行"),
AdminCleanupTaskKind::Usage => format!("使用记录后台清空完成，影响 {deleted_total} 行"),
AdminCleanupTaskKind::AuditLogs => format!("审计日志后台清空完成，影响 {deleted_total} 行"),
AdminCleanupTaskKind::RequestBodies | AdminCleanupTaskKind::Stats => unreachable!(),
};
Ok((
json!({
"deleted": affected,
"total": deleted_total,
}),
message,
))
}
fn initial_task_summary(kind: AdminCleanupTaskKind, batch_size: Option<usize>) -> Value {
match kind {
AdminCleanupTaskKind::RequestBodies => json!({
"batch_size": batch_size.unwrap_or(1),
"batches": 0,
"cleaned": {},
}),
AdminCleanupTaskKind::Stats => json!({
"deleted": {},
"rebuilt": {},
}),
AdminCleanupTaskKind::Config
| AdminCleanupTaskKind::Users
| AdminCleanupTaskKind::Usage
| AdminCleanupTaskKind::AuditLogs => json!({
"deleted": {},
}),
}
}
fn cleanup_task_record(
initial: &AdminCleanupRunRecord,
status: &str,
message: String,
summary: Value,
started_at: Option<Instant>,
error: Option<String>,
) -> AdminCleanupRunRecord {
let completed = matches!(status, "completed" | "failed");
AdminCleanupRunRecord {
id: initial.id.clone(),
kind: initial.kind.clone(),
trigger: initial.trigger.clone(),
status: status.to_string(),
message,
started_at_unix_secs: initial.started_at_unix_secs,
completed_at_unix_secs: completed.then(now_unix_secs),
duration_ms: started_at
.map(|value| value.elapsed().as_millis().try_into().unwrap_or(u64::MAX)),
summary,
error,
}
}
fn request_body_cleanup_record(
initial: &AdminCleanupRunRecord,
status: &str,

2
apps/aether-gateway/src/maintenance/runtime/runners.rs
View File

@@ -10,7 +10,7 @@ use super::{
advance_proxy_upgrade_rollout_once, cleanup_audit_logs_once,
cleanup_expired_gemini_file_mappings_once, cleanup_proxy_node_metrics_once,
cleanup_request_candidates_once, cleanup_stale_pending_requests_once,
cleanup_stale_proxy_nodes_once, collect_proxy_upgrade_rollout_probes,
cleanup_stale_proxy_nodes_once, collect_proxy_upgrade_rollout_probes, now_unix_secs,
perform_db_maintenance_once, perform_provider_checkin_once, perform_stats_aggregation_once,
perform_stats_hourly_aggregation_once, perform_usage_cleanup_once,
perform_wallet_daily_usage_aggregation_once, record_completed_cleanup_run,

93
crates/aether-data/src/backend/sqlite.rs
View File

@@ -434,6 +434,99 @@ VALUES
assert_eq!(admin_exists, 1);
}
#[tokio::test]
async fn admin_system_request_bodies_purge_clears_inline_usage_body_fields() {
let config = SqlDatabaseConfig {
driver: DatabaseDriver::Sqlite,
url: "sqlite::memory:".to_string(),
pool: SqlPoolConfig {
max_connections: 1,
..SqlPoolConfig::default()
},
};
let backend = SqliteBackend::from_config(config).expect("backend should build");
run_sqlite_migrations(backend.pool())
.await
.expect("sqlite migrations should run");
for (column, ty) in [
("request_body", "TEXT"),
("response_body", "TEXT"),
("provider_request_body", "TEXT"),
("client_response_body", "TEXT"),
("request_body_compressed", "BLOB"),
("response_body_compressed", "BLOB"),
("provider_request_body_compressed", "BLOB"),
("client_response_body_compressed", "BLOB"),
] {
sqlx::query(&format!(r#"ALTER TABLE "usage" ADD COLUMN {column} {ty}"#))
.execute(backend.pool())
.await
.expect("legacy body column should be added");
}
sqlx::query(
r#"
INSERT INTO "usage" (
request_id,
provider_name,
model,
request_body,
response_body,
provider_request_body,
client_response_body,
request_body_compressed,
response_body_compressed,
provider_request_body_compressed,
client_response_body_compressed,
created_at_unix_ms
)
VALUES (
'request-1',
'openai',
'gpt-4.1',
'client request',
'provider response',
'provider request',
'client response',
X'01',
X'02',
X'03',
X'04',
1
)
"#,
)
.execute(backend.pool())
.await
.expect("usage row should insert");
let summary = backend
.purge_admin_system_data(AdminSystemPurgeTarget::RequestBodies)
.await
.expect("request body purge should run");
assert_eq!(summary.affected.get("usage_body_fields_cleaned"), Some(&1));
let remaining: i64 = sqlx::query_scalar(
r#"
SELECT COUNT(*)
FROM "usage"
WHERE request_body IS NOT NULL
OR response_body IS NOT NULL
OR provider_request_body IS NOT NULL
OR client_response_body IS NOT NULL
OR request_body_compressed IS NOT NULL
OR response_body_compressed IS NOT NULL
OR provider_request_body_compressed IS NOT NULL
OR client_response_body_compressed IS NOT NULL
"#,
)
.fetch_one(backend.pool())
.await
.expect("remaining body count should load");
assert_eq!(remaining, 0);
}
async fn sqlite_count(pool: &sqlx::SqlitePool, table: &str) -> i64 {
let sql = format!("SELECT COUNT(*) FROM \"{table}\"");
sqlx::query_scalar::<_, i64>(&sql)

290
crates/aether-data/src/backend/system.rs
View File

@@ -561,6 +561,17 @@ const ADMIN_USAGE_CHILD_TABLES: &[&str] = &[
"usage_settlement_snapshots",
];
const USAGE_BODY_FIELD_COLUMNS: &[&str] = &[
"request_body",
"response_body",
"provider_request_body",
"client_response_body",
"request_body_compressed",
"response_body_compressed",
"provider_request_body_compressed",
"client_response_body_compressed",
];
const ADMIN_USER_SCOPED_TABLES: &[&str] = &[
"stats_user_daily_cost_savings_model_provider",
"stats_user_daily_cost_savings_model",
@@ -720,9 +731,10 @@ WHERE request_count <> 0
}
AdminSystemPurgeTarget::RequestBodies => {
pg_delete_table(tx, "usage_body_blobs", summary).await?;
pg_execute_if_table(
pg_execute_if_table_has_columns(
tx,
"usage",
USAGE_BODY_FIELD_COLUMNS,
"usage_body_fields_cleaned",
r#"
UPDATE public.usage
@@ -1080,6 +1092,33 @@ WHERE request_count <> 0
}
AdminSystemPurgeTarget::RequestBodies => {
mysql_delete_table(tx, "usage_body_blobs", summary).await?;
mysql_execute_if_table_has_columns(
tx,
"usage",
USAGE_BODY_FIELD_COLUMNS,
"usage_body_fields_cleaned",
r#"
UPDATE `usage`
SET request_body = NULL,
response_body = NULL,
provider_request_body = NULL,
client_response_body = NULL,
request_body_compressed = NULL,
response_body_compressed = NULL,
provider_request_body_compressed = NULL,
client_response_body_compressed = NULL
WHERE request_body IS NOT NULL
OR response_body IS NOT NULL
OR provider_request_body IS NOT NULL
OR client_response_body IS NOT NULL
OR request_body_compressed IS NOT NULL
OR response_body_compressed IS NOT NULL
OR provider_request_body_compressed IS NOT NULL
OR client_response_body_compressed IS NOT NULL
"#,
summary,
)
.await?;
mysql_execute_if_table(
tx,
"usage_http_audits",
@@ -1375,6 +1414,33 @@ WHERE request_count <> 0
}
AdminSystemPurgeTarget::RequestBodies => {
sqlite_delete_table(tx, "usage_body_blobs", summary).await?;
sqlite_execute_if_table_has_columns(
tx,
"usage",
USAGE_BODY_FIELD_COLUMNS,
"usage_body_fields_cleaned",
r#"
UPDATE "usage"
SET request_body = NULL,
response_body = NULL,
provider_request_body = NULL,
client_response_body = NULL,
request_body_compressed = NULL,
response_body_compressed = NULL,
provider_request_body_compressed = NULL,
client_response_body_compressed = NULL
WHERE request_body IS NOT NULL
OR response_body IS NOT NULL
OR provider_request_body IS NOT NULL
OR client_response_body IS NOT NULL
OR request_body_compressed IS NOT NULL
OR response_body_compressed IS NOT NULL
OR provider_request_body_compressed IS NOT NULL
OR client_response_body_compressed IS NOT NULL
"#,
summary,
)
.await?;
sqlite_execute_if_table(
tx,
"usage_http_audits",
@@ -1573,9 +1639,10 @@ WHERE blobs.body_ref = doomed.body_ref
limit,
)
.await?;
pg_execute_batch_if_table(
pg_execute_batch_if_table_has_columns(
tx,
"usage",
USAGE_BODY_FIELD_COLUMNS,
"usage_body_fields_cleaned",
r#"
WITH batch AS (
@@ -1674,9 +1741,10 @@ WHERE body_ref IN (
limit,
)
.await?;
mysql_execute_batch_if_table(
mysql_execute_batch_if_table_has_columns(
tx,
"usage",
USAGE_BODY_FIELD_COLUMNS,
"usage_body_fields_cleaned",
r#"
UPDATE `usage`
@@ -1772,9 +1840,10 @@ WHERE body_ref IN (
limit,
)
.await?;
sqlite_execute_batch_if_table(
sqlite_execute_batch_if_table_has_columns(
tx,
"usage",
USAGE_BODY_FIELD_COLUMNS,
"usage_body_fields_cleaned",
r#"
UPDATE "usage"
@@ -1923,6 +1992,26 @@ async fn pg_execute_if_table(
Ok(())
}
async fn pg_execute_if_table_has_columns(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
table: &str,
columns: &[&str],
key: &str,
sql: &str,
summary: &mut AdminSystemPurgeSummary,
) -> Result<(), DataLayerError> {
if !pg_table_has_columns(tx, checked_sql_identifier(table)?, columns).await? {
return Ok(());
}
let rows = sqlx::query(sql)
.execute(&mut **tx)
.await
.map_postgres_err()?
.rows_affected();
summary.add(key, rows);
Ok(())
}
async fn pg_execute_batch_if_table(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
table: &str,
@@ -1944,6 +2033,28 @@ async fn pg_execute_batch_if_table(
Ok(())
}
async fn pg_execute_batch_if_table_has_columns(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
table: &str,
columns: &[&str],
key: &str,
sql: &str,
summary: &mut AdminSystemPurgeSummary,
limit: i64,
) -> Result<(), DataLayerError> {
if !pg_table_has_columns(tx, checked_sql_identifier(table)?, columns).await? {
return Ok(());
}
let rows = sqlx::query(sql)
.bind(limit)
.execute(&mut **tx)
.await
.map_postgres_err()?
.rows_affected();
summary.add(key, rows);
Ok(())
}
async fn pg_table_exists(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
table: &str,
@@ -1956,6 +2067,38 @@ async fn pg_table_exists(
.map_postgres_err()
}
async fn pg_table_has_columns(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
table: &str,
columns: &[&str],
) -> Result<bool, DataLayerError> {
let table = checked_sql_identifier(table)?;
if !pg_table_exists(tx, table).await? {
return Ok(false);
}
for column in columns {
let column = checked_sql_identifier(column)?;
let exists: bool = sqlx::query_scalar(
"SELECT EXISTS (
SELECT 1
FROM information_schema.columns
WHERE table_schema = 'public'
AND table_name = $1
AND column_name = $2
)",
)
.bind(table)
.bind(column)
.fetch_one(&mut **tx)
.await
.map_postgres_err()?;
if !exists {
return Ok(false);
}
}
Ok(true)
}
async fn mysql_delete_table(
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
table: &str,
@@ -2036,6 +2179,26 @@ async fn mysql_execute_if_table(
Ok(())
}
async fn mysql_execute_if_table_has_columns(
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
table: &str,
columns: &[&str],
key: &str,
sql: &str,
summary: &mut AdminSystemPurgeSummary,
) -> Result<(), DataLayerError> {
if !mysql_table_has_columns(tx, checked_sql_identifier(table)?, columns).await? {
return Ok(());
}
let rows = sqlx::query(sql)
.execute(&mut **tx)
.await
.map_sql_err()?
.rows_affected();
summary.add(key, rows);
Ok(())
}
async fn mysql_execute_batch_if_table(
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
table: &str,
@@ -2057,6 +2220,28 @@ async fn mysql_execute_batch_if_table(
Ok(())
}
async fn mysql_execute_batch_if_table_has_columns(
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
table: &str,
columns: &[&str],
key: &str,
sql: &str,
summary: &mut AdminSystemPurgeSummary,
limit: i64,
) -> Result<(), DataLayerError> {
if !mysql_table_has_columns(tx, checked_sql_identifier(table)?, columns).await? {
return Ok(());
}
let rows = sqlx::query(sql)
.bind(limit)
.execute(&mut **tx)
.await
.map_sql_err()?
.rows_affected();
summary.add(key, rows);
Ok(())
}
async fn mysql_table_exists(
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
table: &str,
@@ -2072,6 +2257,36 @@ async fn mysql_table_exists(
Ok(total > 0)
}
async fn mysql_table_has_columns(
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
table: &str,
columns: &[&str],
) -> Result<bool, DataLayerError> {
let table = checked_sql_identifier(table)?;
if !mysql_table_exists(tx, table).await? {
return Ok(false);
}
for column in columns {
let column = checked_sql_identifier(column)?;
let total: i64 = sqlx::query_scalar(
"SELECT COUNT(*)
FROM information_schema.columns
WHERE table_schema = DATABASE()
AND table_name = ?
AND column_name = ?",
)
.bind(table)
.bind(column)
.fetch_one(&mut **tx)
.await
.map_sql_err()?;
if total == 0 {
return Ok(false);
}
}
Ok(true)
}
async fn sqlite_delete_table(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
table: &str,
@@ -2152,6 +2367,26 @@ async fn sqlite_execute_if_table(
Ok(())
}
async fn sqlite_execute_if_table_has_columns(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
table: &str,
columns: &[&str],
key: &str,
sql: &str,
summary: &mut AdminSystemPurgeSummary,
) -> Result<(), DataLayerError> {
if !sqlite_table_has_columns(tx, checked_sql_identifier(table)?, columns).await? {
return Ok(());
}
let rows = sqlx::query(sql)
.execute(&mut **tx)
.await
.map_sql_err()?
.rows_affected();
summary.add(key, rows);
Ok(())
}
async fn sqlite_execute_batch_if_table(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
table: &str,
@@ -2173,6 +2408,28 @@ async fn sqlite_execute_batch_if_table(
Ok(())
}
async fn sqlite_execute_batch_if_table_has_columns(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
table: &str,
columns: &[&str],
key: &str,
sql: &str,
summary: &mut AdminSystemPurgeSummary,
limit: i64,
) -> Result<(), DataLayerError> {
if !sqlite_table_has_columns(tx, checked_sql_identifier(table)?, columns).await? {
return Ok(());
}
let rows = sqlx::query(sql)
.bind(limit)
.execute(&mut **tx)
.await
.map_sql_err()?
.rows_affected();
summary.add(key, rows);
Ok(())
}
async fn sqlite_table_exists(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
table: &str,
@@ -2187,6 +2444,31 @@ async fn sqlite_table_exists(
Ok(total > 0)
}
async fn sqlite_table_has_columns(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
table: &str,
columns: &[&str],
) -> Result<bool, DataLayerError> {
let table = checked_sql_identifier(table)?;
if !sqlite_table_exists(tx, table).await? {
return Ok(false);
}
for column in columns {
let column = checked_sql_identifier(column)?;
let total: i64 =
sqlx::query_scalar("SELECT COUNT(*) FROM pragma_table_info(?) WHERE name = ?")
.bind(table)
.bind(column)
.fetch_one(&mut **tx)
.await
.map_sql_err()?;
if total == 0 {
return Ok(false);
}
}
Ok(true)
}
fn current_unix_secs() -> u64 {
chrono::Utc::now().timestamp().max(0) as u64
}

27
frontend/src/api/admin.ts
View File

@@ -253,7 +253,7 @@ export interface CleanupRunListResponse {
items: CleanupRunRecord[]
}
export interface RequestBodyCleanupTaskResponse {
export interface CleanupTaskResponse {
message: string
task: CleanupRunRecord
}
@@ -1073,16 +1073,25 @@ export const adminApi = {
},
// 数据清空
purgeConfig: () => purge<{ message: string; deleted: Record<string, number> }>('config'),
purgeUsers: () => purge<{ message: string; deleted: Record<string, number> }>('users'),
purgeUsage: () => purge<{ message: string; deleted: Record<string, number> }>('usage'),
purgeAuditLogs: () => purge<{ message: string; deleted: Record<string, number> }>('audit-logs'),
purgeRequestBodies: () => purge<{ message: string; cleaned: Record<string, number> }>('request-bodies'),
async purgeRequestBodiesAsync(): Promise<RequestBodyCleanupTaskResponse> {
const response = await apiClient.post<RequestBodyCleanupTaskResponse>('/api/admin/system/purge/request-bodies/task')
purgeConfig: () => purge<CleanupTaskResponse>('config'),
purgeUsers: () => purge<CleanupTaskResponse>('users'),
async purgeUsage(): Promise<CleanupTaskResponse> {
const response = await apiClient.post<CleanupTaskResponse>('/api/admin/system/purge/usage')
return response.data
},
async purgeAuditLogs(): Promise<CleanupTaskResponse> {
const response = await apiClient.post<CleanupTaskResponse>('/api/admin/system/purge/audit-logs')
return response.data
},
purgeRequestBodies: () => purge<CleanupTaskResponse>('request-bodies'),
async purgeRequestBodiesAsync(): Promise<CleanupTaskResponse> {
const response = await apiClient.post<CleanupTaskResponse>('/api/admin/system/purge/request-bodies/task')
return response.data
},
async purgeStats(): Promise<CleanupTaskResponse> {
const response = await apiClient.post<CleanupTaskResponse>('/api/admin/system/purge/stats')
return response.data
},
purgeStats: () => purge<{ message: string }>('stats'),
async getCleanupRuns(): Promise<CleanupRunListResponse> {
const response = await apiClient.get<CleanupRunListResponse>('/api/admin/system/cleanup/runs')
return response.data

5
frontend/src/views/admin/system-settings/CleanupPolicySection.vue
View File

@@ -432,6 +432,11 @@ function cleanupKindLabel(kind: string): string {
audit_cleanup: '审计日志',
request_candidate_cleanup: '候选记录',
request_bodies: '请求体',
config_purge: '配置清空',
users_purge: '用户清空',
usage_purge: '使用记录清空',
audit_logs_purge: '审计日志清空',
stats_purge: '统计聚合清空',
system_cleanup: '系统清理',
}
return labels[kind] || kind

20
frontend/src/views/admin/system-settings/DataManagementSection.vue
View File

@@ -62,37 +62,37 @@ const purgeItems: PurgeItem[] = [
{
key: 'config',
title: '清空配置',
description: '删除所有提供商、端点、API Key 和模型配置',
description: '后台删除所有提供商、端点、API Key 和模型配置',
buttonText: '清空配置',
icon: markRaw(Settings),
confirmMessage: '确定要清空所有提供商配置吗？这将删除所有提供商、端点、API Key 和模型配置，操作不可逆。',
confirmMessage: '确定要后台清空所有提供商配置吗？这将删除所有提供商、端点、API Key 和模型配置，操作不可逆。',
action: () => adminApi.purgeConfig(),
},
{
key: 'users',
title: '清空用户',
description: '删除所有非管理员用户及其 API Keys',
description: '后台删除所有非管理员用户及其 API Keys',
buttonText: '清空用户',
icon: markRaw(Users),
confirmMessage: '确定要清空所有非管理员用户吗？管理员账户将被保留，操作不可逆。',
confirmMessage: '确定要后台清空所有非管理员用户吗？管理员账户将被保留，操作不可逆。',
action: () => adminApi.purgeUsers(),
},
{
key: 'usage',
title: '清空使用记录',
description: '删除全部使用记录和请求候选记录',
description: '后台清空全部使用记录和请求候选记录',
buttonText: '清空记录',
icon: markRaw(BarChart3),
confirmMessage: '确定要清空全部使用记录吗？所有请求统计数据将被永久删除，操作不可逆。',
confirmMessage: '确定要后台清空全部使用记录吗？所有请求统计数据将被永久删除，操作不可逆。',
action: () => adminApi.purgeUsage(),
},
{
key: 'audit-logs',
title: '清空审计日志',
description: '删除全部审计日志记录',
description: '后台删除全部审计日志记录',
buttonText: '清空日志',
icon: markRaw(Shield),
confirmMessage: '确定要清空全部审计日志吗？所有安全事件记录将被永久删除，操作不可逆。',
confirmMessage: '确定要后台清空全部审计日志吗？所有安全事件记录将被永久删除，操作不可逆。',
action: () => adminApi.purgeAuditLogs(),
},
{
@@ -107,10 +107,10 @@ const purgeItems: PurgeItem[] = [
{
key: 'stats',
title: '清空统计聚合',
description: '删除统计聚合数据，保留原始使用记录；统计可从原始使用记录重新构建',
description: '后台删除统计聚合数据并重建，保留原始使用记录',
buttonText: '清空统计聚合',
icon: markRaw(PieChart),
confirmMessage: '确定要清空全部统计聚合数据吗？原始使用记录会保留，仪表盘和累计统计可从原始记录重新构建。',
confirmMessage: '确定要后台清空全部统计聚合数据吗？原始使用记录会保留，仪表盘和累计统计会从原始记录重新构建。',
action: () => adminApi.purgeStats(),
},
]