i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 01:10:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add async request body cleanup records

Browse Source
This commit is contained in:
Entropy.Xu
2026-05-08 13:50:18 +08:00
parent fa22384f24
commit 3f29335fd6
15 changed files with 1151 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
apps/aether-gateway/src/control/route/admin/system_families.rs
View File

@@ -95,6 +95,14 @@ pub(super) fn classify_admin_system_family_route(
"admin:system",
false,
))
} else if method == http::Method::GET && normalized_path == "/api/admin/system/cleanup/runs" {
Some(classified(
"admin_proxy",
"system_manage",
"cleanup_runs",
"admin:system",
false,
))
} else if method == http::Method::POST && normalized_path == "/api/admin/system/purge/config" {
Some(classified(
"admin_proxy",
@@ -139,6 +147,16 @@ pub(super) fn classify_admin_system_family_route(
"admin:system",
false,
))
} else if method == http::Method::POST
&& normalized_path == "/api/admin/system/purge/request-bodies/task"
{
Some(classified(
"admin_proxy",
"system_manage",
"purge_request_bodies_task",
"admin:system",
false,
))
} else if method == http::Method::POST && normalized_path == "/api/admin/system/purge/stats" {
Some(classified(
"admin_proxy",

23
apps/aether-gateway/src/control/tests/admin_core.rs
View File

@@ -172,6 +172,10 @@ fn classifies_admin_system_maintenance_write_routes_as_admin_proxy_route() {
"/api/admin/system/purge/request-bodies",
"purge_request_bodies",
),
(
"/api/admin/system/purge/request-bodies/task",
"purge_request_bodies_task",
),
("/api/admin/system/purge/stats", "purge_stats"),
];
@@ -246,6 +250,25 @@ fn classifies_admin_system_stats_as_admin_proxy_route() {
assert!(!decision.is_execution_runtime_candidate());
}
#[test]
fn classifies_admin_system_cleanup_runs_as_admin_proxy_route() {
let headers = headers(&[]);
let uri: Uri = "/api/admin/system/cleanup/runs"
.parse()
.expect("uri should parse");
let decision =
classify_control_route(&http::Method::GET, &uri, &headers).expect("route should classify");
assert_eq!(decision.route_class.as_deref(), Some("admin_proxy"));
assert_eq!(decision.route_family.as_deref(), Some("system_manage"));
assert_eq!(decision.route_kind.as_deref(), Some("cleanup_runs"));
assert_eq!(
decision.auth_endpoint_signature.as_deref(),
Some("admin:system")
);
assert!(!decision.is_execution_runtime_candidate());
}
#[test]
fn classifies_admin_system_settings_set_as_admin_proxy_route() {
let headers = headers(&[]);

10
apps/aether-gateway/src/data/state/core.rs
View File

@@ -447,4 +447,14 @@ impl GatewayDataState {
None => Ok(aether_data::repository::system::AdminSystemPurgeSummary::default()),
}
}
pub(crate) async fn purge_admin_request_bodies_batch(
&self,
batch_size: usize,
) -> Result<aether_data::repository::system::AdminSystemPurgeSummary, DataLayerError> {
match self.backends.as_ref() {
Some(backends) => backends.purge_admin_request_bodies_batch(batch_size).await,
None => Ok(aether_data::repository::system::AdminSystemPurgeSummary::default()),
}
}
}

43
apps/aether-gateway/src/handlers/admin/system/core/system_routes.rs
View File

@@ -26,6 +26,7 @@ use axum::{
Json,
};
use serde_json::json;
use std::time::Instant;
pub(super) async fn maybe_build_local_admin_core_system_response(
state: &AdminAppState<'_>,
@@ -194,6 +195,35 @@ pub(super) async fn maybe_build_local_admin_core_system_response(
)));
}
if decision.route_kind.as_deref() == Some("cleanup_runs")
&& request_method == http::Method::GET
&& request_path == "/api/admin/system/cleanup/runs"
{
let records = crate::maintenance::list_admin_cleanup_run_records(&state.app().data)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?;
return Ok(Some(Json(json!({ "items": records })).into_response()));
}
if decision.route_kind.as_deref() == Some("purge_request_bodies_task")
&& request_method == http::Method::POST
&& request_path == "/api/admin/system/purge/request-bodies/task"
{
let task =
crate::maintenance::start_admin_request_body_cleanup_task(state.cloned_app()).await?;
return Ok(Some(attach_admin_audit_response(
Json(json!({
"message": task.message.clone(),
"task": task,
}))
.into_response(),
"admin_system_request_body_cleanup_task_started",
"purge_request_bodies_async",
"request_bodies",
"all",
)));
}
if let Some((target, action, object_type, object_id)) =
admin_system_purge_target_for_route_kind(decision.route_kind.as_deref())
{
@@ -529,6 +559,8 @@ async fn build_admin_system_purge_payload(
async fn build_admin_system_cleanup_payload(
state: &AdminAppState<'_>,
) -> Result<serde_json::Value, GatewayError> {
let started_at_unix_secs = chrono::Utc::now().timestamp().max(0) as u64;
let started_at = Instant::now();
let summary = state.run_admin_system_cleanup_once().await?;
let cleaned = json!({
"audit_logs": summary.audit_logs_deleted,
@@ -554,6 +586,17 @@ async fn build_admin_system_cleanup_payload(
.saturating_add(summary.usage.keys_cleaned)
.saturating_add(summary.usage.records_deleted);
crate::maintenance::record_completed_cleanup_run(
&state.app().data,
"system_cleanup",
"manual",
started_at_unix_secs,
started_at,
cleaned.clone(),
format!("系统清理已执行，影响 {total} 项"),
)
.await;
Ok(json!({
"message": format!("系统清理已执行，影响 {} 项", total),
"cleaned": cleaned,

12
apps/aether-gateway/src/maintenance/mod.rs
View File

@@ -4,8 +4,9 @@ mod tests;
pub(crate) use runtime::{
cancel_proxy_upgrade_rollout, clear_proxy_upgrade_rollout_conflicts,
inspect_proxy_upgrade_rollout, perform_oauth_token_refresh_once, perform_pool_quota_probe_once,
perform_provider_checkin_once, rebuild_admin_stats_once, record_proxy_upgrade_traffic_success,
inspect_proxy_upgrade_rollout, list_admin_cleanup_run_records,
perform_oauth_token_refresh_once, perform_pool_quota_probe_once, perform_provider_checkin_once,
rebuild_admin_stats_once, record_completed_cleanup_run, record_proxy_upgrade_traffic_success,
restore_proxy_upgrade_rollout_skipped_nodes, retry_proxy_upgrade_rollout_node,
run_admin_system_cleanup_once, skip_proxy_upgrade_rollout_node, spawn_audit_cleanup_worker,
spawn_db_maintenance_worker, spawn_gemini_file_mapping_cleanup_worker,
@@ -14,9 +15,10 @@ pub(crate) use runtime::{
spawn_proxy_node_stale_cleanup_worker, spawn_proxy_upgrade_rollout_worker,
spawn_request_candidate_cleanup_worker, spawn_stats_aggregation_worker,
spawn_stats_hourly_aggregation_worker, spawn_usage_cleanup_worker,
spawn_wallet_daily_usage_aggregation_worker, start_proxy_upgrade_rollout,
AdminStatsRebuildSummary, AdminSystemCleanupSummary, OAuthTokenRefreshRunSummary,
PoolQuotaProbeRunSummary, ProviderCheckinRunSummary, ProxyUpgradeRolloutCancelSummary,
spawn_wallet_daily_usage_aggregation_worker, start_admin_request_body_cleanup_task,
start_proxy_upgrade_rollout, AdminCleanupRunRecord, AdminStatsRebuildSummary,
AdminSystemCleanupSummary, OAuthTokenRefreshRunSummary, PoolQuotaProbeRunSummary,
ProviderCheckinRunSummary, ProxyUpgradeRolloutCancelSummary,
ProxyUpgradeRolloutConflictClearSummary, ProxyUpgradeRolloutNodeActionSummary,
ProxyUpgradeRolloutProbeConfig, ProxyUpgradeRolloutSkippedRestoreSummary,
ProxyUpgradeRolloutStatus, ProxyUpgradeRolloutTrackedNodeState,

6
apps/aether-gateway/src/maintenance/runtime.rs
View File

@@ -8,6 +8,8 @@ use crate::{AppState, GatewayError};
#[path = "runtime/audit_cleanup.rs"]
mod audit_cleanup;
#[path = "runtime/cleanup_runs.rs"]
mod cleanup_runs;
#[path = "runtime/config.rs"]
mod config;
#[path = "runtime/db_maintenance.rs"]
@@ -47,6 +49,10 @@ pub(crate) use aether_data_contracts::repository::usage::{
UsageCleanupSummary, UsageCleanupWindow,
};
use audit_cleanup::*;
pub(crate) use cleanup_runs::{
list_admin_cleanup_run_records, record_completed_cleanup_run, record_failed_cleanup_run,
start_admin_request_body_cleanup_task, AdminCleanupRunRecord,
};
use config::*;
use db_maintenance::*;
pub(crate) use oauth_token_refresh::{

296
apps/aether-gateway/src/maintenance/runtime/cleanup_runs.rs Normal file
View File

@@ -0,0 +1,296 @@
use std::time::Instant;
use aether_data::repository::system::AdminSystemPurgeSummary;
use aether_data_contracts::DataLayerError;
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use tracing::{info, warn};
use crate::data::GatewayDataState;
use crate::{AppState, GatewayError};
use super::{now_unix_secs, system_config_usize};
const CLEANUP_RUN_HISTORY_KEY: &str = "admin_cleanup_run_history";
const CLEANUP_RUN_HISTORY_LIMIT: usize = 50;
const REQUEST_BODY_PROGRESS_UPDATE_BATCHES: usize = 10;
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub(crate) struct AdminCleanupRunRecord {
pub(crate) id: String,
pub(crate) kind: String,
pub(crate) trigger: String,
pub(crate) status: String,
pub(crate) message: String,
pub(crate) started_at_unix_secs: u64,
pub(crate) completed_at_unix_secs: Option<u64>,
pub(crate) duration_ms: Option<u64>,
pub(crate) summary: Value,
pub(crate) error: Option<String>,
}
pub(crate) async fn list_admin_cleanup_run_records(
data: &GatewayDataState,
) -> Result<Vec<AdminCleanupRunRecord>, DataLayerError> {
let Some(value) = data
.find_system_config_value(CLEANUP_RUN_HISTORY_KEY)
.await?
else {
return Ok(Vec::new());
};
Ok(parse_cleanup_run_records(value))
}
pub(crate) async fn start_admin_request_body_cleanup_task(
app: AppState,
) -> Result<AdminCleanupRunRecord, GatewayError> {
let data = app.data.clone();
let records = list_admin_cleanup_run_records(&data)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?;
if let Some(existing) = records
.iter()
.find(|record| record.kind == "request_bodies" && record.status == "processing")
.cloned()
{
return Ok(existing);
}
let batch_size = system_config_usize(&data, "cleanup_batch_size", 1_000)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?
.max(1);
let started_at = now_unix_secs();
let record = AdminCleanupRunRecord {
id: uuid::Uuid::new_v4().to_string(),
kind: "request_bodies".to_string(),
trigger: "manual".to_string(),
status: "processing".to_string(),
message: format!("请求/响应体后台清理已开始，每批 {batch_size} 条"),
started_at_unix_secs: started_at,
completed_at_unix_secs: None,
duration_ms: None,
summary: json!({
"batch_size": batch_size,
"batches": 0,
"cleaned": {},
}),
error: None,
};
record_cleanup_run(&data, record.clone())
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?;
tokio::spawn(run_request_body_cleanup_task(
data,
record.clone(),
batch_size,
));
Ok(record)
}
pub(crate) async fn record_completed_cleanup_run(
data: &GatewayDataState,
kind: &str,
trigger: &str,
started_at_unix_secs: u64,
started_at: Instant,
summary: Value,
message: impl Into<String>,
) {
let record = AdminCleanupRunRecord {
id: uuid::Uuid::new_v4().to_string(),
kind: kind.to_string(),
trigger: trigger.to_string(),
status: "completed".to_string(),
message: message.into(),
started_at_unix_secs,
completed_at_unix_secs: Some(now_unix_secs()),
duration_ms: Some(
started_at
.elapsed()
.as_millis()
.try_into()
.unwrap_or(u64::MAX),
),
summary,
error: None,
};
if let Err(err) = record_cleanup_run(data, record).await {
warn!(error = %err, kind, "failed to record cleanup run");
}
}
pub(crate) async fn record_failed_cleanup_run(
data: &GatewayDataState,
kind: &str,
trigger: &str,
started_at_unix_secs: u64,
started_at: Instant,
error: &DataLayerError,
) {
let record = AdminCleanupRunRecord {
id: uuid::Uuid::new_v4().to_string(),
kind: kind.to_string(),
trigger: trigger.to_string(),
status: "failed".to_string(),
message: "清理执行失败".to_string(),
started_at_unix_secs,
completed_at_unix_secs: Some(now_unix_secs()),
duration_ms: Some(
started_at
.elapsed()
.as_millis()
.try_into()
.unwrap_or(u64::MAX),
),
summary: json!({}),
error: Some(error.to_string()),
};
if let Err(err) = record_cleanup_run(data, record).await {
warn!(error = %err, kind, "failed to record failed cleanup run");
}
}
async fn run_request_body_cleanup_task(
data: std::sync::Arc<GatewayDataState>,
initial_record: AdminCleanupRunRecord,
batch_size: usize,
) {
let started_at = Instant::now();
let mut total = AdminSystemPurgeSummary::default();
let mut batches = 0usize;
loop {
match data.purge_admin_request_bodies_batch(batch_size).await {
Ok(batch) if batch.total() == 0 => break,
Ok(batch) => {
batches = batches.saturating_add(1);
total.merge(&batch);
if batches.is_multiple_of(REQUEST_BODY_PROGRESS_UPDATE_BATCHES) {
let progress = request_body_cleanup_record(
&initial_record,
"processing",
format!(
"请求/响应体后台清理中，已处理 {} 批，影响 {} 行",
batches,
total.total()
),
batches,
batch_size,
&total,
Some(started_at),
None,
);
if let Err(err) = record_cleanup_run(&data, progress).await {
warn!(error = %err, "failed to update request body cleanup progress");
}
}
tokio::task::yield_now().await;
}
Err(err) => {
let failed = request_body_cleanup_record(
&initial_record,
"failed",
"请求/响应体后台清理失败".to_string(),
batches,
batch_size,
&total,
Some(started_at),
Some(err.to_string()),
);
if let Err(record_err) = record_cleanup_run(&data, failed).await {
warn!(error = %record_err, "failed to record request body cleanup failure");
}
warn!(error = %err, "request body cleanup task failed");
return;
}
}
}
let completed = request_body_cleanup_record(
&initial_record,
"completed",
format!(
"请求/响应体后台清理完成，影响 {} 行，共 {} 批",
total.total(),
batches
),
batches,
batch_size,
&total,
Some(started_at),
None,
);
if let Err(err) = record_cleanup_run(&data, completed).await {
warn!(error = %err, "failed to record request body cleanup completion");
}
info!(
event_name = "request_body_cleanup_task_completed",
log_type = "ops",
worker = "request_body_cleanup_task",
batches,
affected = total.total(),
"gateway finished request body cleanup task"
);
}
fn request_body_cleanup_record(
initial: &AdminCleanupRunRecord,
status: &str,
message: String,
batches: usize,
batch_size: usize,
total: &AdminSystemPurgeSummary,
started_at: Option<Instant>,
error: Option<String>,
) -> AdminCleanupRunRecord {
let completed = matches!(status, "completed" | "failed");
AdminCleanupRunRecord {
id: initial.id.clone(),
kind: initial.kind.clone(),
trigger: initial.trigger.clone(),
status: status.to_string(),
message,
started_at_unix_secs: initial.started_at_unix_secs,
completed_at_unix_secs: completed.then(now_unix_secs),
duration_ms: started_at
.map(|value| value.elapsed().as_millis().try_into().unwrap_or(u64::MAX)),
summary: json!({
"batch_size": batch_size,
"batches": batches,
"cleaned": total.affected,
"total": total.total(),
}),
error,
}
}
async fn record_cleanup_run(
data: &GatewayDataState,
record: AdminCleanupRunRecord,
) -> Result<(), DataLayerError> {
let mut records = list_admin_cleanup_run_records(data).await?;
records.retain(|existing| existing.id != record.id);
records.insert(0, record);
records.truncate(CLEANUP_RUN_HISTORY_LIMIT);
let value = serde_json::to_value(records).map_err(|err| {
DataLayerError::UnexpectedValue(format!("invalid cleanup run history: {err}"))
})?;
data.upsert_system_config_entry(
CLEANUP_RUN_HISTORY_KEY,
&value,
Some("最近的系统清理执行记录"),
)
.await?;
Ok(())
}
fn parse_cleanup_run_records(value: Value) -> Vec<AdminCleanupRunRecord> {
value
.as_array()
.into_iter()
.flat_map(|items| items.iter())
.filter_map(|item| serde_json::from_value::<AdminCleanupRunRecord>(item.clone()).ok())
.collect()
}

35
apps/aether-gateway/src/maintenance/runtime/request_candidate_cleanup.rs
View File

@@ -1,9 +1,14 @@
use aether_data_contracts::DataLayerError;
use serde_json::json;
use std::time::Instant;
use tracing::info;
use crate::data::GatewayDataState;
use super::{now_unix_secs, system_config_bool, system_config_u64, system_config_usize};
use super::{
now_unix_secs, record_completed_cleanup_run, record_failed_cleanup_run, system_config_bool,
system_config_u64, system_config_usize,
};
pub(crate) async fn cleanup_request_candidates_once(
data: &GatewayDataState,
@@ -47,7 +52,33 @@ pub(crate) async fn cleanup_request_candidates_once(
pub(super) async fn run_request_candidate_cleanup_once(
data: &GatewayDataState,
) -> Result<(), DataLayerError> {
let deleted = cleanup_request_candidates_once(data).await?;
let started_at_unix_secs = now_unix_secs();
let started_at = Instant::now();
let deleted = match cleanup_request_candidates_once(data).await {
Ok(deleted) => deleted,
Err(err) => {
record_failed_cleanup_run(
data,
"request_candidate_cleanup",
"auto",
started_at_unix_secs,
started_at,
&err,
)
.await;
return Err(err);
}
};
record_completed_cleanup_run(
data,
"request_candidate_cleanup",
"auto",
started_at_unix_secs,
started_at,
json!({ "request_candidates_deleted": deleted }),
format!("候选记录自动清理完成，删除 {deleted} 行"),
)
.await;
if deleted > 0 {
info!(deleted, "gateway deleted expired request candidates");
}

80
apps/aether-gateway/src/maintenance/runtime/runners.rs
View File

@@ -1,4 +1,6 @@
use aether_data_contracts::DataLayerError;
use serde_json::json;
use std::time::Instant;
use tracing::{info, warn};
use crate::data::GatewayDataState;
@@ -8,15 +10,41 @@ use super::{
advance_proxy_upgrade_rollout_once, cleanup_audit_logs_once,
cleanup_expired_gemini_file_mappings_once, cleanup_request_candidates_once,
cleanup_stale_pending_requests_once, cleanup_stale_proxy_nodes_once,
collect_proxy_upgrade_rollout_probes, perform_db_maintenance_once,
collect_proxy_upgrade_rollout_probes, now_unix_secs, perform_db_maintenance_once,
perform_provider_checkin_once, perform_stats_aggregation_once,
perform_stats_hourly_aggregation_once, perform_usage_cleanup_once,
perform_wallet_daily_usage_aggregation_once, record_proxy_upgrade_traffic_success,
summarize_database_pool,
perform_wallet_daily_usage_aggregation_once, record_completed_cleanup_run,
record_failed_cleanup_run, record_proxy_upgrade_traffic_success, summarize_database_pool,
};
pub(super) async fn run_audit_cleanup_once(data: &GatewayDataState) -> Result<(), DataLayerError> {
let deleted = cleanup_audit_logs_once(data).await?;
let started_at_unix_secs = now_unix_secs();
let started_at = Instant::now();
let deleted = match cleanup_audit_logs_once(data).await {
Ok(deleted) => deleted,
Err(err) => {
record_failed_cleanup_run(
data,
"audit_cleanup",
"auto",
started_at_unix_secs,
started_at,
&err,
)
.await;
return Err(err);
}
};
record_completed_cleanup_run(
data,
"audit_cleanup",
"auto",
started_at_unix_secs,
started_at,
json!({ "audit_logs_deleted": deleted }),
format!("审计日志自动清理完成，删除 {deleted} 行"),
)
.await;
if deleted > 0 {
info!(
event_name = "audit_cleanup_completed",
@@ -187,7 +215,49 @@ pub(super) async fn run_stats_aggregation_once(
}
pub(super) async fn run_usage_cleanup_once(data: &GatewayDataState) -> Result<(), DataLayerError> {
let summary = perform_usage_cleanup_once(data).await?;
let started_at_unix_secs = now_unix_secs();
let started_at = Instant::now();
let summary = match perform_usage_cleanup_once(data).await {
Ok(summary) => summary,
Err(err) => {
record_failed_cleanup_run(
data,
"usage_cleanup",
"auto",
started_at_unix_secs,
started_at,
&err,
)
.await;
return Err(err);
}
};
record_completed_cleanup_run(
data,
"usage_cleanup",
"auto",
started_at_unix_secs,
started_at,
json!({
"body_externalized": summary.body_externalized,
"legacy_body_refs_migrated": summary.legacy_body_refs_migrated,
"body_cleaned": summary.body_cleaned,
"header_cleaned": summary.header_cleaned,
"keys_cleaned": summary.keys_cleaned,
"records_deleted": summary.records_deleted,
}),
format!(
"请求记录自动清理完成，影响 {} 项",
summary
.body_externalized
.saturating_add(summary.legacy_body_refs_migrated)
.saturating_add(summary.body_cleaned)
.saturating_add(summary.header_cleaned)
.saturating_add(summary.keys_cleaned)
.saturating_add(summary.records_deleted)
),
)
.await;
if summary.body_externalized > 0
|| summary.legacy_body_refs_migrated > 0
|| summary.body_cleaned > 0