chore(proxy): guard metrics retention cleanup

This commit is contained in:
fawney19
2026-05-08 22:57:00 +08:00
parent a703acd1fe
commit e21fb58479
16 changed files with 562 additions and 75 deletions

View File

@@ -1073,11 +1073,16 @@ impl GatewayDataState {
&self, &self,
retain_1m_from_unix_secs: u64, retain_1m_from_unix_secs: u64,
retain_1h_from_unix_secs: u64, retain_1h_from_unix_secs: u64,
delete_limit: usize,
) -> Result<super::ProxyNodeMetricsCleanupSummary, DataLayerError> { ) -> Result<super::ProxyNodeMetricsCleanupSummary, DataLayerError> {
match &self.proxy_node_writer { match &self.proxy_node_writer {
Some(repository) => { Some(repository) => {
repository repository
.cleanup_proxy_node_metrics(retain_1m_from_unix_secs, retain_1h_from_unix_secs) .cleanup_proxy_node_metrics(
retain_1m_from_unix_secs,
retain_1h_from_unix_secs,
delete_limit,
)
.await .await
} }
None => Ok(super::ProxyNodeMetricsCleanupSummary::default()), None => Ok(super::ProxyNodeMetricsCleanupSummary::default()),

View File

@@ -533,6 +533,8 @@ async fn build_admin_system_cleanup_payload(
let cleaned = json!({ let cleaned = json!({
"audit_logs": summary.audit_logs_deleted, "audit_logs": summary.audit_logs_deleted,
"request_candidates": summary.request_candidates_deleted, "request_candidates": summary.request_candidates_deleted,
"proxy_node_metrics_1m": summary.proxy_node_metrics.deleted_1m_rows,
"proxy_node_metrics_1h": summary.proxy_node_metrics.deleted_1h_rows,
"pending_failed": summary.pending_failed, "pending_failed": summary.pending_failed,
"pending_recovered": summary.pending_recovered, "pending_recovered": summary.pending_recovered,
"usage_body_externalized": summary.usage.body_externalized, "usage_body_externalized": summary.usage.body_externalized,
@@ -545,6 +547,8 @@ async fn build_admin_system_cleanup_payload(
let total = summary let total = summary
.audit_logs_deleted .audit_logs_deleted
.saturating_add(summary.request_candidates_deleted) .saturating_add(summary.request_candidates_deleted)
.saturating_add(summary.proxy_node_metrics.deleted_1m_rows)
.saturating_add(summary.proxy_node_metrics.deleted_1h_rows)
.saturating_add(summary.pending_failed) .saturating_add(summary.pending_failed)
.saturating_add(summary.pending_recovered) .saturating_add(summary.pending_recovered)
.saturating_add(summary.usage.body_externalized) .saturating_add(summary.usage.body_externalized)

View File

@@ -132,6 +132,8 @@ struct UsageCleanupSettings {
pub(crate) struct AdminSystemCleanupSummary { pub(crate) struct AdminSystemCleanupSummary {
pub(crate) audit_logs_deleted: usize, pub(crate) audit_logs_deleted: usize,
pub(crate) request_candidates_deleted: usize, pub(crate) request_candidates_deleted: usize,
pub(crate) proxy_node_metrics:
aether_data::repository::proxy_nodes::ProxyNodeMetricsCleanupSummary,
pub(crate) pending_failed: usize, pub(crate) pending_failed: usize,
pub(crate) pending_recovered: usize, pub(crate) pending_recovered: usize,
pub(crate) usage: UsageCleanupSummary, pub(crate) usage: UsageCleanupSummary,
@@ -149,12 +151,14 @@ pub(crate) async fn run_admin_system_cleanup_once(
) -> Result<AdminSystemCleanupSummary, aether_data::DataLayerError> { ) -> Result<AdminSystemCleanupSummary, aether_data::DataLayerError> {
let audit_logs_deleted = cleanup_audit_logs_once(data).await?; let audit_logs_deleted = cleanup_audit_logs_once(data).await?;
let request_candidates_deleted = cleanup_request_candidates_once(data).await?; let request_candidates_deleted = cleanup_request_candidates_once(data).await?;
let proxy_node_metrics = cleanup_proxy_node_metrics_once(data).await?;
let pending = cleanup_stale_pending_requests_once(data).await?; let pending = cleanup_stale_pending_requests_once(data).await?;
let usage = perform_usage_cleanup_once(data).await?; let usage = perform_usage_cleanup_once(data).await?;
Ok(AdminSystemCleanupSummary { Ok(AdminSystemCleanupSummary {
audit_logs_deleted, audit_logs_deleted,
request_candidates_deleted, request_candidates_deleted,
proxy_node_metrics,
pending_failed: pending.failed, pending_failed: pending.failed,
pending_recovered: pending.recovered, pending_recovered: pending.recovered,
usage, usage,

View File

@@ -3,18 +3,107 @@ use aether_data_contracts::DataLayerError;
use crate::data::GatewayDataState; use crate::data::GatewayDataState;
use super::now_unix_secs; use super::{now_unix_secs, system_config_bool, system_config_u64, system_config_usize};
const PROXY_NODE_METRICS_1M_RETENTION_SECS: u64 = 30 * 24 * 60 * 60; const SECS_PER_DAY: u64 = 24 * 60 * 60;
const PROXY_NODE_METRICS_1H_RETENTION_SECS: u64 = 180 * 24 * 60 * 60; const PROXY_NODE_METRICS_1M_RETENTION_DAYS_DEFAULT: u64 = 30;
const PROXY_NODE_METRICS_1H_RETENTION_DAYS_DEFAULT: u64 = 180;
const PROXY_NODE_METRICS_RETENTION_DAYS_MIN: u64 = 1;
const PROXY_NODE_METRICS_1M_RETENTION_DAYS_MAX: u64 = 365;
const PROXY_NODE_METRICS_1H_RETENTION_DAYS_MAX: u64 = 1_095;
const PROXY_NODE_METRICS_CLEANUP_BATCH_SIZE_DEFAULT: usize = 5_000;
const PROXY_NODE_METRICS_CLEANUP_BATCH_SIZE_MAX: usize = 50_000;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) struct ProxyNodeMetricsCleanupSettings {
pub retain_1m_days: u64,
pub retain_1h_days: u64,
pub batch_size: usize,
}
pub(super) async fn proxy_node_metrics_cleanup_settings(
data: &GatewayDataState,
) -> Result<ProxyNodeMetricsCleanupSettings, DataLayerError> {
let retain_1m_days = system_config_u64(
data,
"proxy_node_metrics_1m_retention_days",
PROXY_NODE_METRICS_1M_RETENTION_DAYS_DEFAULT,
)
.await?
.clamp(
PROXY_NODE_METRICS_RETENTION_DAYS_MIN,
PROXY_NODE_METRICS_1M_RETENTION_DAYS_MAX,
);
let retain_1h_days = system_config_u64(
data,
"proxy_node_metrics_1h_retention_days",
PROXY_NODE_METRICS_1H_RETENTION_DAYS_DEFAULT,
)
.await?
.clamp(retain_1m_days, PROXY_NODE_METRICS_1H_RETENTION_DAYS_MAX);
let cleanup_batch_size =
system_config_usize(data, "proxy_node_metrics_cleanup_batch_size", 0).await?;
let fallback_batch_size = system_config_usize(
data,
"cleanup_batch_size",
PROXY_NODE_METRICS_CLEANUP_BATCH_SIZE_DEFAULT,
)
.await?;
let batch_size = (if cleanup_batch_size > 0 {
cleanup_batch_size
} else {
fallback_batch_size
})
.clamp(1, PROXY_NODE_METRICS_CLEANUP_BATCH_SIZE_MAX);
Ok(ProxyNodeMetricsCleanupSettings {
retain_1m_days,
retain_1h_days,
batch_size,
})
}
pub(super) async fn cleanup_proxy_node_metrics_once( pub(super) async fn cleanup_proxy_node_metrics_once(
data: &GatewayDataState, data: &GatewayDataState,
) -> Result<ProxyNodeMetricsCleanupSummary, DataLayerError> { ) -> Result<ProxyNodeMetricsCleanupSummary, DataLayerError> {
let now = now_unix_secs(); cleanup_proxy_node_metrics_at(data, now_unix_secs()).await
data.cleanup_proxy_node_metrics( }
now.saturating_sub(PROXY_NODE_METRICS_1M_RETENTION_SECS),
now.saturating_sub(PROXY_NODE_METRICS_1H_RETENTION_SECS), pub(super) async fn cleanup_proxy_node_metrics_at(
) data: &GatewayDataState,
.await now_unix_secs: u64,
) -> Result<ProxyNodeMetricsCleanupSummary, DataLayerError> {
if !system_config_bool(data, "enable_auto_cleanup", true).await? {
return Ok(ProxyNodeMetricsCleanupSummary::default());
}
let settings = proxy_node_metrics_cleanup_settings(data).await?;
let retain_1m_from_unix_secs =
now_unix_secs.saturating_sub(settings.retain_1m_days.saturating_mul(SECS_PER_DAY));
let retain_1h_from_unix_secs =
now_unix_secs.saturating_sub(settings.retain_1h_days.saturating_mul(SECS_PER_DAY));
let mut summary = ProxyNodeMetricsCleanupSummary::default();
loop {
let deleted = data
.cleanup_proxy_node_metrics(
retain_1m_from_unix_secs,
retain_1h_from_unix_secs,
settings.batch_size,
)
.await?;
summary.deleted_1m_rows = summary
.deleted_1m_rows
.saturating_add(deleted.deleted_1m_rows);
summary.deleted_1h_rows = summary
.deleted_1h_rows
.saturating_add(deleted.deleted_1h_rows);
if deleted.deleted_1m_rows < settings.batch_size
&& deleted.deleted_1h_rows < settings.batch_size
{
break;
}
}
Ok(summary)
} }

View File

@@ -3,8 +3,8 @@ use std::sync::{Arc, Mutex};
use std::time::Duration; use std::time::Duration;
use aether_data::repository::proxy_nodes::{ use aether_data::repository::proxy_nodes::{
InMemoryProxyNodeRepository, ProxyNodeHeartbeatMutation, ProxyNodeReadRepository, bucket_start_unix_secs, InMemoryProxyNodeRepository, ProxyNodeHeartbeatMutation,
ProxyNodeWriteRepository, StoredProxyNode, ProxyNodeMetricsStep, ProxyNodeReadRepository, ProxyNodeWriteRepository, StoredProxyNode,
}; };
use aether_runtime::bounded_queue; use aether_runtime::bounded_queue;
use axum::extract::ws::Message; use axum::extract::ws::Message;
@@ -14,23 +14,25 @@ use serde_json::json;
use tokio::sync::watch; use tokio::sync::watch;
use super::{ use super::{
advance_proxy_upgrade_rollout_once, cleanup_audit_logs_with, cleanup_stale_proxy_nodes_once, advance_proxy_upgrade_rollout_once, cleanup_audit_logs_with, cleanup_proxy_node_metrics_at,
inspect_proxy_upgrade_rollout, next_daily_run_after, next_db_maintenance_run_after, cleanup_proxy_node_metrics_once, cleanup_stale_proxy_nodes_once, inspect_proxy_upgrade_rollout,
next_stats_aggregation_run_after, next_stats_hourly_aggregation_run_after, next_daily_run_after, next_db_maintenance_run_after, next_stats_aggregation_run_after,
pending_cleanup_batch_size, pending_cleanup_timeout_minutes, plan_pending_cleanup_batch, next_stats_hourly_aggregation_run_after, pending_cleanup_batch_size,
provider_checkin_schedule, record_proxy_upgrade_traffic_success, run_db_maintenance_with, pending_cleanup_timeout_minutes, plan_pending_cleanup_batch, provider_checkin_schedule,
run_proxy_upgrade_rollout_once, spawn_audit_cleanup_worker, spawn_db_maintenance_worker, proxy_node_metrics_cleanup_settings, record_proxy_upgrade_traffic_success,
spawn_oauth_token_refresh_worker, spawn_pending_cleanup_worker, spawn_pool_monitor_worker, run_db_maintenance_with, run_proxy_upgrade_rollout_once, spawn_audit_cleanup_worker,
spawn_pool_quota_probe_worker, spawn_provider_checkin_worker, spawn_db_maintenance_worker, spawn_oauth_token_refresh_worker, spawn_pending_cleanup_worker,
spawn_pool_monitor_worker, spawn_pool_quota_probe_worker, spawn_provider_checkin_worker,
spawn_proxy_node_stale_cleanup_worker, spawn_proxy_upgrade_rollout_worker, spawn_proxy_node_stale_cleanup_worker, spawn_proxy_upgrade_rollout_worker,
spawn_stats_aggregation_worker, spawn_stats_hourly_aggregation_worker, spawn_stats_aggregation_worker, spawn_stats_hourly_aggregation_worker,
spawn_usage_cleanup_worker, spawn_wallet_daily_usage_aggregation_worker, spawn_usage_cleanup_worker, spawn_wallet_daily_usage_aggregation_worker,
start_proxy_upgrade_rollout, stats_aggregation_target_day, start_proxy_upgrade_rollout, stats_aggregation_target_day,
stats_hourly_aggregation_target_hour, summarize_database_pool, usage_cleanup_settings, stats_hourly_aggregation_target_hour, summarize_database_pool, usage_cleanup_settings,
usage_cleanup_window, wallet_daily_usage_aggregation_target, AppState, DbMaintenanceRunSummary, usage_cleanup_window, wallet_daily_usage_aggregation_target, AppState, DbMaintenanceRunSummary,
FailedPendingUsageRow, GatewayDataState, ProxyUpgradeRolloutProbeConfig, StalePendingUsageRow, FailedPendingUsageRow, GatewayDataState, ProxyNodeMetricsCleanupSettings,
UsageCleanupSettings, USAGE_CLEANUP_HOUR, USAGE_CLEANUP_MINUTE, ProxyUpgradeRolloutProbeConfig, StalePendingUsageRow, UsageCleanupSettings, USAGE_CLEANUP_HOUR,
WALLET_DAILY_USAGE_AGGREGATION_HOUR, WALLET_DAILY_USAGE_AGGREGATION_MINUTE, USAGE_CLEANUP_MINUTE, WALLET_DAILY_USAGE_AGGREGATION_HOUR,
WALLET_DAILY_USAGE_AGGREGATION_MINUTE,
}; };
#[tokio::test] #[tokio::test]
@@ -734,6 +736,176 @@ async fn usage_cleanup_settings_resolve_batch_and_delete_toggle() {
); );
} }
#[tokio::test]
async fn proxy_node_metrics_cleanup_settings_use_dedicated_retention_and_batch_limits() {
let data = GatewayDataState::disabled().with_system_config_values_for_tests([
("cleanup_batch_size".to_string(), json!(250)),
("proxy_node_metrics_1m_retention_days".to_string(), json!(0)),
("proxy_node_metrics_1h_retention_days".to_string(), json!(7)),
(
"proxy_node_metrics_cleanup_batch_size".to_string(),
json!(100_000),
),
]);
let settings = proxy_node_metrics_cleanup_settings(&data)
.await
.expect("proxy metrics cleanup settings should resolve");
assert_eq!(
settings,
ProxyNodeMetricsCleanupSettings {
retain_1m_days: 1,
retain_1h_days: 7,
batch_size: 50_000,
}
);
}
#[tokio::test]
async fn proxy_node_metrics_cleanup_settings_fallback_to_global_batch_size() {
let data = GatewayDataState::disabled().with_system_config_values_for_tests([
("cleanup_batch_size".to_string(), json!(250)),
(
"proxy_node_metrics_cleanup_batch_size".to_string(),
json!(0),
),
]);
let settings = proxy_node_metrics_cleanup_settings(&data)
.await
.expect("proxy metrics cleanup settings should resolve");
assert_eq!(
settings,
ProxyNodeMetricsCleanupSettings {
retain_1m_days: 30,
retain_1h_days: 180,
batch_size: 250,
}
);
}
#[tokio::test]
async fn proxy_node_metrics_cleanup_deletes_expired_buckets_in_batches() {
let repository = Arc::new(InMemoryProxyNodeRepository::seed(vec![
sample_connected_proxy_node("node-metrics-1", 30, 1),
sample_connected_proxy_node("node-metrics-2", 30, 1),
sample_connected_proxy_node("node-metrics-3", 30, 1),
]));
let data = GatewayDataState::with_proxy_node_repository_for_tests(Arc::clone(&repository))
.with_system_config_values_for_tests([
("enable_auto_cleanup".to_string(), json!(true)),
("proxy_node_metrics_1m_retention_days".to_string(), json!(1)),
("proxy_node_metrics_1h_retention_days".to_string(), json!(1)),
(
"proxy_node_metrics_cleanup_batch_size".to_string(),
json!(1),
),
]);
for (idx, node_id) in ["node-metrics-1", "node-metrics-2", "node-metrics-3"]
.into_iter()
.enumerate()
{
repository
.apply_heartbeat(&ProxyNodeHeartbeatMutation {
node_id: node_id.to_string(),
heartbeat_interval: Some(30),
active_connections: Some(i32::try_from(idx + 1).unwrap()),
total_requests_delta: None,
avg_latency_ms: None,
failed_requests_delta: None,
dns_failures_delta: None,
stream_errors_delta: None,
proxy_metadata: Some(json!({
"tunnel_metrics": {
"connect_errors": idx + 1,
"disconnects": 0,
"error_events_total": 0,
"ws_in_bytes": idx + 1,
"ws_out_bytes": idx + 1,
"ws_in_frames": idx + 1,
"ws_out_frames": idx + 1,
"heartbeat_rtt_last_ms": 10
}
})),
proxy_version: Some("1.0.0".to_string()),
})
.await
.expect("heartbeat should write metrics");
}
let now = chrono::Utc::now().timestamp().max(0) as u64;
let old_bucket = bucket_start_unix_secs(now, ProxyNodeMetricsStep::OneMinute);
let cleanup = data
.cleanup_proxy_node_metrics(
old_bucket.saturating_add(60),
old_bucket.saturating_add(3_600),
1,
)
.await
.expect("direct cleanup should delete a limited batch");
assert_eq!(cleanup.deleted_1m_rows, 1);
assert_eq!(cleanup.deleted_1h_rows, 1);
let cleanup = cleanup_proxy_node_metrics_at(&data, now.saturating_add(2 * 86_400))
.await
.expect("runtime cleanup should loop over batches");
assert_eq!(cleanup.deleted_1m_rows, 2);
assert_eq!(cleanup.deleted_1h_rows, 2);
}
#[tokio::test]
async fn proxy_node_metrics_cleanup_respects_auto_cleanup_toggle() {
let repository = Arc::new(InMemoryProxyNodeRepository::seed(vec![
sample_connected_proxy_node("node-metrics-disabled", 30, 1),
]));
let data = GatewayDataState::with_proxy_node_repository_for_tests(Arc::clone(&repository))
.with_system_config_values_for_tests([
("enable_auto_cleanup".to_string(), json!(false)),
("proxy_node_metrics_1m_retention_days".to_string(), json!(1)),
("proxy_node_metrics_1h_retention_days".to_string(), json!(1)),
(
"proxy_node_metrics_cleanup_batch_size".to_string(),
json!(1),
),
]);
repository
.apply_heartbeat(&ProxyNodeHeartbeatMutation {
node_id: "node-metrics-disabled".to_string(),
heartbeat_interval: Some(30),
active_connections: Some(1),
total_requests_delta: None,
avg_latency_ms: None,
failed_requests_delta: None,
dns_failures_delta: None,
stream_errors_delta: None,
proxy_metadata: Some(json!({
"tunnel_metrics": {
"connect_errors": 1,
"disconnects": 0,
"error_events_total": 0,
"ws_in_bytes": 1,
"ws_out_bytes": 1,
"ws_in_frames": 1,
"ws_out_frames": 1,
"heartbeat_rtt_last_ms": 10
}
})),
proxy_version: Some("1.0.0".to_string()),
})
.await
.expect("heartbeat should write metrics");
let cleanup = cleanup_proxy_node_metrics_once(&data)
.await
.expect("runtime cleanup should short-circuit");
assert_eq!(cleanup.deleted_1m_rows, 0);
assert_eq!(cleanup.deleted_1h_rows, 0);
}
#[test] #[test]
fn usage_cleanup_window_uses_non_overlapping_ranges() { fn usage_cleanup_window_uses_non_overlapping_ranges() {
let now_utc = "2026-03-18T03:00:00Z" let now_utc = "2026-03-18T03:00:00Z"

View File

@@ -675,10 +675,15 @@ impl AppState {
&self, &self,
retain_1m_from_unix_secs: u64, retain_1m_from_unix_secs: u64,
retain_1h_from_unix_secs: u64, retain_1h_from_unix_secs: u64,
delete_limit: usize,
) -> Result<aether_data::repository::proxy_nodes::ProxyNodeMetricsCleanupSummary, GatewayError> ) -> Result<aether_data::repository::proxy_nodes::ProxyNodeMetricsCleanupSummary, GatewayError>
{ {
self.data self.data
.cleanup_proxy_node_metrics(retain_1m_from_unix_secs, retain_1h_from_unix_secs) .cleanup_proxy_node_metrics(
retain_1m_from_unix_secs,
retain_1h_from_unix_secs,
delete_limit,
)
.await .await
.map_err(|err| GatewayError::Internal(err.to_string())) .map_err(|err| GatewayError::Internal(err.to_string()))
} }

View File

@@ -1258,6 +1258,9 @@ pub fn admin_system_config_default_value(key: &str) -> Option<serde_json::Value>
"cleanup_batch_size" => Some(json!(1000)), "cleanup_batch_size" => Some(json!(1000)),
"request_candidates_retention_days" => Some(json!(30)), "request_candidates_retention_days" => Some(json!(30)),
"request_candidates_cleanup_batch_size" => Some(json!(5000)), "request_candidates_cleanup_batch_size" => Some(json!(5000)),
"proxy_node_metrics_1m_retention_days" => Some(json!(30)),
"proxy_node_metrics_1h_retention_days" => Some(json!(180)),
"proxy_node_metrics_cleanup_batch_size" => Some(json!(5000)),
"enable_provider_checkin" => Some(json!(true)), "enable_provider_checkin" => Some(json!(true)),
"provider_checkin_time" => Some(json!("01:05")), "provider_checkin_time" => Some(json!("01:05")),
"provider_priority_mode" => Some(json!("provider")), "provider_priority_mode" => Some(json!("provider")),

View File

@@ -840,17 +840,33 @@ impl ProxyNodeWriteRepository for InMemoryProxyNodeRepository {
&self, &self,
retain_1m_from_unix_secs: u64, retain_1m_from_unix_secs: u64,
retain_1h_from_unix_secs: u64, retain_1h_from_unix_secs: u64,
delete_limit: usize,
) -> Result<ProxyNodeMetricsCleanupSummary, DataLayerError> { ) -> Result<ProxyNodeMetricsCleanupSummary, DataLayerError> {
let delete_limit = delete_limit.max(1);
let mut metrics_1m = self.metrics_1m.write().expect("proxy node repository lock"); let mut metrics_1m = self.metrics_1m.write().expect("proxy node repository lock");
let before_1m = metrics_1m.len(); let expired_1m_keys = metrics_1m
metrics_1m.retain(|(_, bucket_start), _| *bucket_start >= retain_1m_from_unix_secs); .keys()
let deleted_1m_rows = before_1m.saturating_sub(metrics_1m.len()); .filter(|(_, bucket_start)| *bucket_start < retain_1m_from_unix_secs)
.take(delete_limit)
.cloned()
.collect::<Vec<_>>();
let deleted_1m_rows = expired_1m_keys
.iter()
.filter(|key| metrics_1m.remove(key).is_some())
.count();
drop(metrics_1m); drop(metrics_1m);
let mut metrics_1h = self.metrics_1h.write().expect("proxy node repository lock"); let mut metrics_1h = self.metrics_1h.write().expect("proxy node repository lock");
let before_1h = metrics_1h.len(); let expired_1h_keys = metrics_1h
metrics_1h.retain(|(_, bucket_start), _| *bucket_start >= retain_1h_from_unix_secs); .keys()
let deleted_1h_rows = before_1h.saturating_sub(metrics_1h.len()); .filter(|(_, bucket_start)| *bucket_start < retain_1h_from_unix_secs)
.take(delete_limit)
.cloned()
.collect::<Vec<_>>();
let deleted_1h_rows = expired_1h_keys
.iter()
.filter(|key| metrics_1h.remove(key).is_some())
.count();
Ok(ProxyNodeMetricsCleanupSummary { Ok(ProxyNodeMetricsCleanupSummary {
deleted_1m_rows, deleted_1m_rows,

View File

@@ -1031,22 +1031,38 @@ WHERE is_manual = 0
&self, &self,
retain_1m_from_unix_secs: u64, retain_1m_from_unix_secs: u64,
retain_1h_from_unix_secs: u64, retain_1h_from_unix_secs: u64,
delete_limit: usize,
) -> Result<ProxyNodeMetricsCleanupSummary, DataLayerError> { ) -> Result<ProxyNodeMetricsCleanupSummary, DataLayerError> {
let deleted_1m = let delete_limit_i64 = i64::try_from(delete_limit.max(1)).unwrap_or(i64::MAX);
sqlx::query("DELETE FROM proxy_node_metrics_1m WHERE bucket_start_unix_secs < ?") let deleted_1m = sqlx::query(
.bind(i64::try_from(retain_1m_from_unix_secs).unwrap_or(i64::MAX)) r#"
.execute(&self.pool) DELETE FROM proxy_node_metrics_1m
.await WHERE bucket_start_unix_secs < ?
.map_sql_err()? ORDER BY bucket_start_unix_secs ASC
.rows_affected() as usize; LIMIT ?
"#,
)
.bind(i64::try_from(retain_1m_from_unix_secs).unwrap_or(i64::MAX))
.bind(delete_limit_i64)
.execute(&self.pool)
.await
.map_sql_err()?
.rows_affected() as usize;
let deleted_1h = let deleted_1h = sqlx::query(
sqlx::query("DELETE FROM proxy_node_metrics_1h WHERE bucket_start_unix_secs < ?") r#"
.bind(i64::try_from(retain_1h_from_unix_secs).unwrap_or(i64::MAX)) DELETE FROM proxy_node_metrics_1h
.execute(&self.pool) WHERE bucket_start_unix_secs < ?
.await ORDER BY bucket_start_unix_secs ASC
.map_sql_err()? LIMIT ?
.rows_affected() as usize; "#,
)
.bind(i64::try_from(retain_1h_from_unix_secs).unwrap_or(i64::MAX))
.bind(delete_limit_i64)
.execute(&self.pool)
.await
.map_sql_err()?
.rows_affected() as usize;
Ok(ProxyNodeMetricsCleanupSummary { Ok(ProxyNodeMetricsCleanupSummary {
deleted_1m_rows: deleted_1m, deleted_1m_rows: deleted_1m,

View File

@@ -1448,22 +1448,52 @@ WHERE id = $1
&self, &self,
retain_1m_from_unix_secs: u64, retain_1m_from_unix_secs: u64,
retain_1h_from_unix_secs: u64, retain_1h_from_unix_secs: u64,
delete_limit: usize,
) -> Result<ProxyNodeMetricsCleanupSummary, DataLayerError> { ) -> Result<ProxyNodeMetricsCleanupSummary, DataLayerError> {
let deleted_1m = let delete_limit_i64 = i64::try_from(delete_limit.max(1)).unwrap_or(i64::MAX);
sqlx::query("DELETE FROM proxy_node_metrics_1m WHERE bucket_start_unix_secs < $1") let deleted_1m = sqlx::query(
.bind(i64::try_from(retain_1m_from_unix_secs).unwrap_or(i64::MAX)) r#"
.execute(&self.pool) WITH expired AS (
.await SELECT node_id, bucket_start_unix_secs
.map_postgres_err()? FROM proxy_node_metrics_1m
.rows_affected() as usize; WHERE bucket_start_unix_secs < $1
ORDER BY bucket_start_unix_secs ASC
LIMIT $2
)
DELETE FROM proxy_node_metrics_1m metrics
USING expired
WHERE metrics.node_id = expired.node_id
AND metrics.bucket_start_unix_secs = expired.bucket_start_unix_secs
"#,
)
.bind(i64::try_from(retain_1m_from_unix_secs).unwrap_or(i64::MAX))
.bind(delete_limit_i64)
.execute(&self.pool)
.await
.map_postgres_err()?
.rows_affected() as usize;
let deleted_1h = let deleted_1h = sqlx::query(
sqlx::query("DELETE FROM proxy_node_metrics_1h WHERE bucket_start_unix_secs < $1") r#"
.bind(i64::try_from(retain_1h_from_unix_secs).unwrap_or(i64::MAX)) WITH expired AS (
.execute(&self.pool) SELECT node_id, bucket_start_unix_secs
.await FROM proxy_node_metrics_1h
.map_postgres_err()? WHERE bucket_start_unix_secs < $1
.rows_affected() as usize; ORDER BY bucket_start_unix_secs ASC
LIMIT $2
)
DELETE FROM proxy_node_metrics_1h metrics
USING expired
WHERE metrics.node_id = expired.node_id
AND metrics.bucket_start_unix_secs = expired.bucket_start_unix_secs
"#,
)
.bind(i64::try_from(retain_1h_from_unix_secs).unwrap_or(i64::MAX))
.bind(delete_limit_i64)
.execute(&self.pool)
.await
.map_postgres_err()?
.rows_affected() as usize;
Ok(ProxyNodeMetricsCleanupSummary { Ok(ProxyNodeMetricsCleanupSummary {
deleted_1m_rows: deleted_1m, deleted_1m_rows: deleted_1m,

View File

@@ -1034,22 +1034,46 @@ WHERE is_manual = 0
&self, &self,
retain_1m_from_unix_secs: u64, retain_1m_from_unix_secs: u64,
retain_1h_from_unix_secs: u64, retain_1h_from_unix_secs: u64,
delete_limit: usize,
) -> Result<ProxyNodeMetricsCleanupSummary, DataLayerError> { ) -> Result<ProxyNodeMetricsCleanupSummary, DataLayerError> {
let deleted_1m = let delete_limit_i64 = i64::try_from(delete_limit.max(1)).unwrap_or(i64::MAX);
sqlx::query("DELETE FROM proxy_node_metrics_1m WHERE bucket_start_unix_secs < ?") let deleted_1m = sqlx::query(
.bind(i64::try_from(retain_1m_from_unix_secs).unwrap_or(i64::MAX)) r#"
.execute(&self.pool) DELETE FROM proxy_node_metrics_1m
.await WHERE (node_id, bucket_start_unix_secs) IN (
.map_sql_err()? SELECT node_id, bucket_start_unix_secs
.rows_affected() as usize; FROM proxy_node_metrics_1m
WHERE bucket_start_unix_secs < ?
ORDER BY bucket_start_unix_secs ASC
LIMIT ?
)
"#,
)
.bind(i64::try_from(retain_1m_from_unix_secs).unwrap_or(i64::MAX))
.bind(delete_limit_i64)
.execute(&self.pool)
.await
.map_sql_err()?
.rows_affected() as usize;
let deleted_1h = let deleted_1h = sqlx::query(
sqlx::query("DELETE FROM proxy_node_metrics_1h WHERE bucket_start_unix_secs < ?") r#"
.bind(i64::try_from(retain_1h_from_unix_secs).unwrap_or(i64::MAX)) DELETE FROM proxy_node_metrics_1h
.execute(&self.pool) WHERE (node_id, bucket_start_unix_secs) IN (
.await SELECT node_id, bucket_start_unix_secs
.map_sql_err()? FROM proxy_node_metrics_1h
.rows_affected() as usize; WHERE bucket_start_unix_secs < ?
ORDER BY bucket_start_unix_secs ASC
LIMIT ?
)
"#,
)
.bind(i64::try_from(retain_1h_from_unix_secs).unwrap_or(i64::MAX))
.bind(delete_limit_i64)
.execute(&self.pool)
.await
.map_sql_err()?
.rows_affected() as usize;
Ok(ProxyNodeMetricsCleanupSummary { Ok(ProxyNodeMetricsCleanupSummary {
deleted_1m_rows: deleted_1m, deleted_1m_rows: deleted_1m,
@@ -1627,7 +1651,7 @@ VALUES ('node-1', 'registered', 'ok', 3)
); );
let cleanup = repository let cleanup = repository
.cleanup_proxy_node_metrics(now.saturating_add(1), now.saturating_add(1)) .cleanup_proxy_node_metrics(now.saturating_add(1), now.saturating_add(1), 10)
.await .await
.expect("cleanup should run"); .expect("cleanup should run");
assert_eq!(cleanup.deleted_1m_rows, 1); assert_eq!(cleanup.deleted_1m_rows, 1);

View File

@@ -717,6 +717,7 @@ pub trait ProxyNodeWriteRepository: Send + Sync {
&self, &self,
retain_1m_from_unix_secs: u64, retain_1m_from_unix_secs: u64,
retain_1h_from_unix_secs: u64, retain_1h_from_unix_secs: u64,
delete_limit: usize,
) -> Result<ProxyNodeMetricsCleanupSummary, crate::DataLayerError>; ) -> Result<ProxyNodeMetricsCleanupSummary, crate::DataLayerError>;
} }

View File

@@ -908,7 +908,10 @@ export const MOCK_SYSTEM_CONFIGS = [
{ key: 'cache_enabled', value: true, description: '是否启用缓存' }, { key: 'cache_enabled', value: true, description: '是否启用缓存' },
{ key: 'default_cache_ttl', value: 3600, description: '默认缓存 TTL' }, { key: 'default_cache_ttl', value: 3600, description: '默认缓存 TTL' },
{ key: 'fallback_enabled', value: true, description: '是否启用故障转移' }, { key: 'fallback_enabled', value: true, description: '是否启用故障转移' },
{ key: 'max_fallback_attempts', value: 3, description: '最大故障转移次数' } { key: 'max_fallback_attempts', value: 3, description: '最大故障转移次数' },
{ key: 'proxy_node_metrics_1m_retention_days', value: 30, description: '代理节点 1m 指标保留天数' },
{ key: 'proxy_node_metrics_1h_retention_days', value: 180, description: '代理节点 1h 指标保留天数' },
{ key: 'proxy_node_metrics_cleanup_batch_size', value: 5000, description: '代理节点指标每批次清理条数' }
] ]
// ========== API 格式 ========== // ========== API 格式 ==========

View File

@@ -102,6 +102,9 @@
:audit-log-retention-days="systemConfig.audit_log_retention_days" :audit-log-retention-days="systemConfig.audit_log_retention_days"
:request-candidates-retention-days="systemConfig.request_candidates_retention_days" :request-candidates-retention-days="systemConfig.request_candidates_retention_days"
:request-candidates-cleanup-batch-size="systemConfig.request_candidates_cleanup_batch_size" :request-candidates-cleanup-batch-size="systemConfig.request_candidates_cleanup_batch_size"
:proxy-node-metrics-1m-retention-days="systemConfig.proxy_node_metrics_1m_retention_days"
:proxy-node-metrics-1h-retention-days="systemConfig.proxy_node_metrics_1h_retention_days"
:proxy-node-metrics-cleanup-batch-size="systemConfig.proxy_node_metrics_cleanup_batch_size"
:loading="cleanupConfigLoading" :loading="cleanupConfigLoading"
:has-changes="hasCleanupConfigChanges" :has-changes="hasCleanupConfigChanges"
@save="saveCleanupConfig" @save="saveCleanupConfig"
@@ -114,6 +117,9 @@
@update:audit-log-retention-days="systemConfig.audit_log_retention_days = $event" @update:audit-log-retention-days="systemConfig.audit_log_retention_days = $event"
@update:request-candidates-retention-days="systemConfig.request_candidates_retention_days = $event" @update:request-candidates-retention-days="systemConfig.request_candidates_retention_days = $event"
@update:request-candidates-cleanup-batch-size="systemConfig.request_candidates_cleanup_batch_size = $event" @update:request-candidates-cleanup-batch-size="systemConfig.request_candidates_cleanup_batch_size = $event"
@update:proxy-node-metrics-1m-retention-days="systemConfig.proxy_node_metrics_1m_retention_days = $event"
@update:proxy-node-metrics-1h-retention-days="systemConfig.proxy_node_metrics_1h_retention_days = $event"
@update:proxy-node-metrics-cleanup-batch-size="systemConfig.proxy_node_metrics_cleanup_batch_size = $event"
/> />
<!-- 定时任务 --> <!-- 定时任务 -->

View File

@@ -192,6 +192,72 @@
独立控制候选记录大表清理节奏不再跟随 Key 删除联动 独立控制候选记录大表清理节奏不再跟随 Key 删除联动
</p> </p>
</div> </div>
<div>
<Label
for="proxy-node-metrics-1m-retention-days"
class="block text-sm font-medium"
>
代理 1m 指标保留天数
</Label>
<Input
id="proxy-node-metrics-1m-retention-days"
:model-value="proxyNodeMetrics1mRetentionDays"
type="number"
min="1"
max="365"
placeholder="30"
class="mt-1"
@update:model-value="$emit('update:proxyNodeMetrics1mRetentionDays', Number($event))"
/>
<p class="mt-1 text-xs text-muted-foreground">
用于代理节点稳定性分钟级图表后端最少保留 1
</p>
</div>
<div>
<Label
for="proxy-node-metrics-1h-retention-days"
class="block text-sm font-medium"
>
代理 1h 指标保留天数
</Label>
<Input
id="proxy-node-metrics-1h-retention-days"
:model-value="proxyNodeMetrics1hRetentionDays"
type="number"
min="1"
max="1095"
placeholder="180"
class="mt-1"
@update:model-value="$emit('update:proxyNodeMetrics1hRetentionDays', Number($event))"
/>
<p class="mt-1 text-xs text-muted-foreground">
小时级聚合用于长期趋势不能短于 1m 指标保留天数
</p>
</div>
<div>
<Label
for="proxy-node-metrics-cleanup-batch-size"
class="block text-sm font-medium"
>
代理指标清理批次
</Label>
<Input
id="proxy-node-metrics-cleanup-batch-size"
:model-value="proxyNodeMetricsCleanupBatchSize"
type="number"
min="1"
max="50000"
placeholder="5000"
class="mt-1"
@update:model-value="$emit('update:proxyNodeMetricsCleanupBatchSize', Number($event))"
/>
<p class="mt-1 text-xs text-muted-foreground">
独立限制 proxy_node_metrics 表的单批删除数量
</p>
</div>
</div> </div>
<!-- 清理策略说明 --> <!-- 清理策略说明 -->
@@ -206,6 +272,7 @@
<p>4. <strong>归档删除</strong>: 超过保留期限后完全删除记录</p> <p>4. <strong>归档删除</strong>: 超过保留期限后完全删除记录</p>
<p>5. <strong>候选记录</strong>: 独立按保留天数清理 request_candidates 审计记录不再跟随 Key 删除联动</p> <p>5. <strong>候选记录</strong>: 独立按保留天数清理 request_candidates 审计记录不再跟随 Key 删除联动</p>
<p>6. <strong>审计日志</strong>: 独立清理记录用户登录操作等安全事件</p> <p>6. <strong>审计日志</strong>: 独立清理记录用户登录操作等安全事件</p>
<p>7. <strong>代理指标</strong>: 仅保留 1m/1h 聚合桶清理任务按批次删除过期桶</p>
</div> </div>
</div> </div>
</CardSection> </CardSection>
@@ -228,6 +295,9 @@ defineProps<{
auditLogRetentionDays: number auditLogRetentionDays: number
requestCandidatesRetentionDays: number requestCandidatesRetentionDays: number
requestCandidatesCleanupBatchSize: number requestCandidatesCleanupBatchSize: number
proxyNodeMetrics1mRetentionDays: number
proxyNodeMetrics1hRetentionDays: number
proxyNodeMetricsCleanupBatchSize: number
loading: boolean loading: boolean
hasChanges: boolean hasChanges: boolean
}>() }>()
@@ -243,5 +313,8 @@ defineEmits<{
'update:auditLogRetentionDays': [value: number] 'update:auditLogRetentionDays': [value: number]
'update:requestCandidatesRetentionDays': [value: number] 'update:requestCandidatesRetentionDays': [value: number]
'update:requestCandidatesCleanupBatchSize': [value: number] 'update:requestCandidatesCleanupBatchSize': [value: number]
'update:proxyNodeMetrics1mRetentionDays': [value: number]
'update:proxyNodeMetrics1hRetentionDays': [value: number]
'update:proxyNodeMetricsCleanupBatchSize': [value: number]
}>() }>()
</script> </script>

View File

@@ -34,6 +34,9 @@ export interface SystemConfig {
audit_log_retention_days: number audit_log_retention_days: number
request_candidates_retention_days: number request_candidates_retention_days: number
request_candidates_cleanup_batch_size: number request_candidates_cleanup_batch_size: number
proxy_node_metrics_1m_retention_days: number
proxy_node_metrics_1h_retention_days: number
proxy_node_metrics_cleanup_batch_size: number
// 定时任务 // 定时任务
enable_provider_checkin: boolean enable_provider_checkin: boolean
provider_checkin_time: string provider_checkin_time: string
@@ -70,6 +73,9 @@ const CONFIG_KEYS = [
'audit_log_retention_days', 'audit_log_retention_days',
'request_candidates_retention_days', 'request_candidates_retention_days',
'request_candidates_cleanup_batch_size', 'request_candidates_cleanup_batch_size',
'proxy_node_metrics_1m_retention_days',
'proxy_node_metrics_1h_retention_days',
'proxy_node_metrics_cleanup_batch_size',
// 定时任务 // 定时任务
'enable_provider_checkin', 'enable_provider_checkin',
'provider_checkin_time', 'provider_checkin_time',
@@ -107,6 +113,9 @@ function createDefaultConfig(): SystemConfig {
audit_log_retention_days: 30, audit_log_retention_days: 30,
request_candidates_retention_days: 30, request_candidates_retention_days: 30,
request_candidates_cleanup_batch_size: 5000, request_candidates_cleanup_batch_size: 5000,
proxy_node_metrics_1m_retention_days: 30,
proxy_node_metrics_1h_retention_days: 180,
proxy_node_metrics_cleanup_batch_size: 5000,
// 定时任务 // 定时任务
enable_provider_checkin: true, enable_provider_checkin: true,
provider_checkin_time: '01:05', provider_checkin_time: '01:05',
@@ -181,7 +190,13 @@ export function useSystemConfig() {
systemConfig.value.request_candidates_retention_days !== systemConfig.value.request_candidates_retention_days !==
originalConfig.value.request_candidates_retention_days || originalConfig.value.request_candidates_retention_days ||
systemConfig.value.request_candidates_cleanup_batch_size !== systemConfig.value.request_candidates_cleanup_batch_size !==
originalConfig.value.request_candidates_cleanup_batch_size originalConfig.value.request_candidates_cleanup_batch_size ||
systemConfig.value.proxy_node_metrics_1m_retention_days !==
originalConfig.value.proxy_node_metrics_1m_retention_days ||
systemConfig.value.proxy_node_metrics_1h_retention_days !==
originalConfig.value.proxy_node_metrics_1h_retention_days ||
systemConfig.value.proxy_node_metrics_cleanup_batch_size !==
originalConfig.value.proxy_node_metrics_cleanup_batch_size
) )
}) })
@@ -441,6 +456,21 @@ export function useSystemConfig() {
value: systemConfig.value.request_candidates_cleanup_batch_size, value: systemConfig.value.request_candidates_cleanup_batch_size,
description: '请求候选记录每批次清理条数', description: '请求候选记录每批次清理条数',
}, },
{
key: 'proxy_node_metrics_1m_retention_days',
value: systemConfig.value.proxy_node_metrics_1m_retention_days,
description: '代理节点 1m 指标保留天数',
},
{
key: 'proxy_node_metrics_1h_retention_days',
value: systemConfig.value.proxy_node_metrics_1h_retention_days,
description: '代理节点 1h 指标保留天数',
},
{
key: 'proxy_node_metrics_cleanup_batch_size',
value: systemConfig.value.proxy_node_metrics_cleanup_batch_size,
description: '代理节点指标每批次清理条数',
},
] ]
await Promise.all( await Promise.all(
@@ -462,6 +492,12 @@ export function useSystemConfig() {
systemConfig.value.request_candidates_retention_days systemConfig.value.request_candidates_retention_days
originalConfig.value.request_candidates_cleanup_batch_size = originalConfig.value.request_candidates_cleanup_batch_size =
systemConfig.value.request_candidates_cleanup_batch_size systemConfig.value.request_candidates_cleanup_batch_size
originalConfig.value.proxy_node_metrics_1m_retention_days =
systemConfig.value.proxy_node_metrics_1m_retention_days
originalConfig.value.proxy_node_metrics_1h_retention_days =
systemConfig.value.proxy_node_metrics_1h_retention_days
originalConfig.value.proxy_node_metrics_cleanup_batch_size =
systemConfig.value.proxy_node_metrics_cleanup_batch_size
} }
success('请求记录清理配置已保存') success('请求记录清理配置已保存')
} catch (err) { } catch (err) {