i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-03 01:40:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/aether-rust-pioneer' into codex/async-cleanup-records

Browse Source 
# Conflicts:
#	apps/aether-gateway/src/maintenance/mod.rs
#	apps/aether-gateway/src/maintenance/runtime/runners.rs
This commit is contained in:
fawney19
2026-05-10 00:28:39 +08:00
parent 3f29335fd6 209322b499
commit 391c2fbe5d
315 changed files with 19750 additions and 3183 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
apps/aether-gateway/src/maintenance/mod.rs
View File

@@ -12,13 +12,14 @@ pub(crate) use runtime::{
spawn_db_maintenance_worker, spawn_gemini_file_mapping_cleanup_worker,
spawn_oauth_token_refresh_worker, spawn_pending_cleanup_worker, spawn_pool_monitor_worker,
spawn_pool_quota_probe_worker, spawn_provider_checkin_worker,
spawn_proxy_node_stale_cleanup_worker, spawn_proxy_upgrade_rollout_worker,
spawn_request_candidate_cleanup_worker, spawn_stats_aggregation_worker,
spawn_stats_hourly_aggregation_worker, spawn_usage_cleanup_worker,
spawn_wallet_daily_usage_aggregation_worker, start_admin_request_body_cleanup_task,
start_proxy_upgrade_rollout, AdminCleanupRunRecord, AdminStatsRebuildSummary,
AdminSystemCleanupSummary, OAuthTokenRefreshRunSummary, PoolQuotaProbeRunSummary,
ProviderCheckinRunSummary, ProxyUpgradeRolloutCancelSummary,
spawn_proxy_node_metrics_cleanup_worker, spawn_proxy_node_stale_cleanup_worker,
spawn_proxy_upgrade_rollout_worker, spawn_request_candidate_cleanup_worker,
spawn_stats_aggregation_worker, spawn_stats_hourly_aggregation_worker,
spawn_usage_cleanup_worker, spawn_wallet_daily_usage_aggregation_worker,
start_admin_request_body_cleanup_task, start_admin_system_purge_task,
start_proxy_upgrade_rollout, AdminCleanupRunRecord, AdminCleanupTaskKind,
AdminStatsRebuildSummary, AdminSystemCleanupSummary, OAuthTokenRefreshRunSummary,
PoolQuotaProbeRunSummary, ProviderCheckinRunSummary, ProxyUpgradeRolloutCancelSummary,
ProxyUpgradeRolloutConflictClearSummary, ProxyUpgradeRolloutNodeActionSummary,
ProxyUpgradeRolloutProbeConfig, ProxyUpgradeRolloutSkippedRestoreSummary,
ProxyUpgradeRolloutStatus, ProxyUpgradeRolloutTrackedNodeState,

9
apps/aether-gateway/src/maintenance/runtime.rs
View File

@@ -22,6 +22,8 @@ mod pending_cleanup;
mod pool_quota_probe;
#[path = "runtime/provider_checkin.rs"]
mod provider_checkin;
#[path = "runtime/proxy_node_metrics_cleanup.rs"]
mod proxy_node_metrics_cleanup;
#[path = "runtime/proxy_node_staleness.rs"]
mod proxy_node_staleness;
#[path = "runtime/proxy_upgrade_rollout.rs"]
@@ -65,6 +67,7 @@ pub(crate) use pool_quota_probe::{
PoolQuotaProbeWorkerConfig,
};
pub(crate) use provider_checkin::{perform_provider_checkin_once, ProviderCheckinRunSummary};
use proxy_node_metrics_cleanup::*;
use proxy_node_staleness::*;
use proxy_upgrade_rollout::*;
pub(crate) use proxy_upgrade_rollout::{
@@ -96,6 +99,8 @@ const AUDIT_LOG_CLEANUP_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
const GEMINI_FILE_MAPPING_CLEANUP_INTERVAL: Duration = Duration::from_secs(60 * 60);
const PENDING_CLEANUP_INTERVAL: Duration = Duration::from_secs(5 * 60);
const PROXY_NODE_STALE_SWEEP_INTERVAL: Duration = Duration::from_secs(5);
const PROXY_NODE_METRICS_CLEANUP_HOUR: u32 = 2;
const PROXY_NODE_METRICS_CLEANUP_MINUTE: u32 = 10;
const PROXY_UPGRADE_ROLLOUT_INTERVAL: Duration = Duration::from_secs(15);
const PROXY_NODE_STALE_MIN_GRACE_SECS: u64 = 15;
const PROXY_NODE_STALE_MISSED_HEARTBEATS: u64 = 3;
@@ -133,6 +138,8 @@ struct UsageCleanupSettings {
pub(crate) struct AdminSystemCleanupSummary {
pub(crate) audit_logs_deleted: usize,
pub(crate) request_candidates_deleted: usize,
pub(crate) proxy_node_metrics:
aether_data::repository::proxy_nodes::ProxyNodeMetricsCleanupSummary,
pub(crate) pending_failed: usize,
pub(crate) pending_recovered: usize,
pub(crate) usage: UsageCleanupSummary,
@@ -150,12 +157,14 @@ pub(crate) async fn run_admin_system_cleanup_once(
) -> Result<AdminSystemCleanupSummary, aether_data::DataLayerError> {
let audit_logs_deleted = cleanup_audit_logs_once(data).await?;
let request_candidates_deleted = cleanup_request_candidates_once(data).await?;
let proxy_node_metrics = cleanup_proxy_node_metrics_once(data).await?;
let pending = cleanup_stale_pending_requests_once(data).await?;
let usage = perform_usage_cleanup_once(data).await?;
Ok(AdminSystemCleanupSummary {
audit_logs_deleted,
request_candidates_deleted,
proxy_node_metrics,
pending_failed: pending.failed,
pending_recovered: pending.recovered,
usage,

109
apps/aether-gateway/src/maintenance/runtime/proxy_node_metrics_cleanup.rs Normal file
View File

@@ -0,0 +1,109 @@
use aether_data::repository::proxy_nodes::ProxyNodeMetricsCleanupSummary;
use aether_data_contracts::DataLayerError;
use crate::data::GatewayDataState;
use super::{now_unix_secs, system_config_bool, system_config_u64, system_config_usize};
const SECS_PER_DAY: u64 = 24 * 60 * 60;
const PROXY_NODE_METRICS_1M_RETENTION_DAYS_DEFAULT: u64 = 30;
const PROXY_NODE_METRICS_1H_RETENTION_DAYS_DEFAULT: u64 = 180;
const PROXY_NODE_METRICS_RETENTION_DAYS_MIN: u64 = 1;
const PROXY_NODE_METRICS_1M_RETENTION_DAYS_MAX: u64 = 365;
const PROXY_NODE_METRICS_1H_RETENTION_DAYS_MAX: u64 = 1_095;
const PROXY_NODE_METRICS_CLEANUP_BATCH_SIZE_DEFAULT: usize = 5_000;
const PROXY_NODE_METRICS_CLEANUP_BATCH_SIZE_MAX: usize = 50_000;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) struct ProxyNodeMetricsCleanupSettings {
pub retain_1m_days: u64,
pub retain_1h_days: u64,
pub batch_size: usize,
}
pub(super) async fn proxy_node_metrics_cleanup_settings(
data: &GatewayDataState,
) -> Result<ProxyNodeMetricsCleanupSettings, DataLayerError> {
let retain_1m_days = system_config_u64(
data,
"proxy_node_metrics_1m_retention_days",
PROXY_NODE_METRICS_1M_RETENTION_DAYS_DEFAULT,
)
.await?
.clamp(
PROXY_NODE_METRICS_RETENTION_DAYS_MIN,
PROXY_NODE_METRICS_1M_RETENTION_DAYS_MAX,
);
let retain_1h_days = system_config_u64(
data,
"proxy_node_metrics_1h_retention_days",
PROXY_NODE_METRICS_1H_RETENTION_DAYS_DEFAULT,
)
.await?
.clamp(retain_1m_days, PROXY_NODE_METRICS_1H_RETENTION_DAYS_MAX);
let cleanup_batch_size =
system_config_usize(data, "proxy_node_metrics_cleanup_batch_size", 0).await?;
let fallback_batch_size = system_config_usize(
data,
"cleanup_batch_size",
PROXY_NODE_METRICS_CLEANUP_BATCH_SIZE_DEFAULT,
)
.await?;
let batch_size = (if cleanup_batch_size > 0 {
cleanup_batch_size
} else {
fallback_batch_size
})
.clamp(1, PROXY_NODE_METRICS_CLEANUP_BATCH_SIZE_MAX);
Ok(ProxyNodeMetricsCleanupSettings {
retain_1m_days,
retain_1h_days,
batch_size,
})
}
pub(super) async fn cleanup_proxy_node_metrics_once(
data: &GatewayDataState,
) -> Result<ProxyNodeMetricsCleanupSummary, DataLayerError> {
cleanup_proxy_node_metrics_at(data, now_unix_secs()).await
}
pub(super) async fn cleanup_proxy_node_metrics_at(
data: &GatewayDataState,
now_unix_secs: u64,
) -> Result<ProxyNodeMetricsCleanupSummary, DataLayerError> {
if !system_config_bool(data, "enable_auto_cleanup", true).await? {
return Ok(ProxyNodeMetricsCleanupSummary::default());
}
let settings = proxy_node_metrics_cleanup_settings(data).await?;
let retain_1m_from_unix_secs =
now_unix_secs.saturating_sub(settings.retain_1m_days.saturating_mul(SECS_PER_DAY));
let retain_1h_from_unix_secs =
now_unix_secs.saturating_sub(settings.retain_1h_days.saturating_mul(SECS_PER_DAY));
let mut summary = ProxyNodeMetricsCleanupSummary::default();
loop {
let deleted = data
.cleanup_proxy_node_metrics(
retain_1m_from_unix_secs,
retain_1h_from_unix_secs,
settings.batch_size,
)
.await?;
summary.deleted_1m_rows = summary
.deleted_1m_rows
.saturating_add(deleted.deleted_1m_rows);
summary.deleted_1h_rows = summary
.deleted_1h_rows
.saturating_add(deleted.deleted_1h_rows);
if deleted.deleted_1m_rows < settings.batch_size
&& deleted.deleted_1h_rows < settings.batch_size
{
break;
}
}
Ok(summary)
}

25
apps/aether-gateway/src/maintenance/runtime/runners.rs
View File

@@ -8,10 +8,10 @@ use crate::{AppState, GatewayError};
use super::{
advance_proxy_upgrade_rollout_once, cleanup_audit_logs_once,
cleanup_expired_gemini_file_mappings_once, cleanup_request_candidates_once,
cleanup_stale_pending_requests_once, cleanup_stale_proxy_nodes_once,
collect_proxy_upgrade_rollout_probes, now_unix_secs, perform_db_maintenance_once,
perform_provider_checkin_once, perform_stats_aggregation_once,
cleanup_expired_gemini_file_mappings_once, cleanup_proxy_node_metrics_once,
cleanup_request_candidates_once, cleanup_stale_pending_requests_once,
cleanup_stale_proxy_nodes_once, collect_proxy_upgrade_rollout_probes,
perform_db_maintenance_once, perform_provider_checkin_once, perform_stats_aggregation_once,
perform_stats_hourly_aggregation_once, perform_usage_cleanup_once,
perform_wallet_daily_usage_aggregation_once, record_completed_cleanup_run,
record_failed_cleanup_run, record_proxy_upgrade_traffic_success, summarize_database_pool,
@@ -89,6 +89,23 @@ pub(super) async fn run_proxy_node_stale_cleanup_once(
Ok(())
}
pub(super) async fn run_proxy_node_metrics_cleanup_once(
data: &GatewayDataState,
) -> Result<(), DataLayerError> {
let summary = cleanup_proxy_node_metrics_once(data).await?;
if summary.deleted_1m_rows > 0 || summary.deleted_1h_rows > 0 {
info!(
event_name = "proxy_node_metrics_cleanup_completed",
log_type = "ops",
worker = "proxy_node_metrics_cleanup",
deleted_1m_rows = summary.deleted_1m_rows,
deleted_1h_rows = summary.deleted_1h_rows,
"gateway deleted expired proxy node metrics buckets"
);
}
Ok(())
}
pub(super) async fn run_proxy_upgrade_rollout_once(state: &AppState) -> Result<(), DataLayerError> {
let mut summary = advance_proxy_upgrade_rollout_once(&state.data).await?;
let probes = collect_proxy_upgrade_rollout_probes(&state.data).await?;

198
apps/aether-gateway/src/maintenance/runtime/tests.rs
View File

@@ -3,8 +3,8 @@ use std::sync::{Arc, Mutex};
use std::time::Duration;
use aether_data::repository::proxy_nodes::{
InMemoryProxyNodeRepository, ProxyNodeHeartbeatMutation, ProxyNodeReadRepository,
ProxyNodeWriteRepository, StoredProxyNode,
bucket_start_unix_secs, InMemoryProxyNodeRepository, ProxyNodeHeartbeatMutation,
ProxyNodeMetricsStep, ProxyNodeReadRepository, ProxyNodeWriteRepository, StoredProxyNode,
};
use aether_runtime::bounded_queue;
use axum::extract::ws::Message;
@@ -14,23 +14,25 @@ use serde_json::json;
use tokio::sync::watch;
use super::{
advance_proxy_upgrade_rollout_once, cleanup_audit_logs_with, cleanup_stale_proxy_nodes_once,
inspect_proxy_upgrade_rollout, next_daily_run_after, next_db_maintenance_run_after,
next_stats_aggregation_run_after, next_stats_hourly_aggregation_run_after,
pending_cleanup_batch_size, pending_cleanup_timeout_minutes, plan_pending_cleanup_batch,
provider_checkin_schedule, record_proxy_upgrade_traffic_success, run_db_maintenance_with,
run_proxy_upgrade_rollout_once, spawn_audit_cleanup_worker, spawn_db_maintenance_worker,
spawn_oauth_token_refresh_worker, spawn_pending_cleanup_worker, spawn_pool_monitor_worker,
spawn_pool_quota_probe_worker, spawn_provider_checkin_worker,
advance_proxy_upgrade_rollout_once, cleanup_audit_logs_with, cleanup_proxy_node_metrics_at,
cleanup_proxy_node_metrics_once, cleanup_stale_proxy_nodes_once, inspect_proxy_upgrade_rollout,
next_daily_run_after, next_db_maintenance_run_after, next_stats_aggregation_run_after,
next_stats_hourly_aggregation_run_after, pending_cleanup_batch_size,
pending_cleanup_timeout_minutes, plan_pending_cleanup_batch, provider_checkin_schedule,
proxy_node_metrics_cleanup_settings, record_proxy_upgrade_traffic_success,
run_db_maintenance_with, run_proxy_upgrade_rollout_once, spawn_audit_cleanup_worker,
spawn_db_maintenance_worker, spawn_oauth_token_refresh_worker, spawn_pending_cleanup_worker,
spawn_pool_monitor_worker, spawn_pool_quota_probe_worker, spawn_provider_checkin_worker,
spawn_proxy_node_stale_cleanup_worker, spawn_proxy_upgrade_rollout_worker,
spawn_stats_aggregation_worker, spawn_stats_hourly_aggregation_worker,
spawn_usage_cleanup_worker, spawn_wallet_daily_usage_aggregation_worker,
start_proxy_upgrade_rollout, stats_aggregation_target_day,
stats_hourly_aggregation_target_hour, summarize_database_pool, usage_cleanup_settings,
usage_cleanup_window, wallet_daily_usage_aggregation_target, AppState, DbMaintenanceRunSummary,
FailedPendingUsageRow, GatewayDataState, ProxyUpgradeRolloutProbeConfig, StalePendingUsageRow,
UsageCleanupSettings, USAGE_CLEANUP_HOUR, USAGE_CLEANUP_MINUTE,
WALLET_DAILY_USAGE_AGGREGATION_HOUR, WALLET_DAILY_USAGE_AGGREGATION_MINUTE,
FailedPendingUsageRow, GatewayDataState, ProxyNodeMetricsCleanupSettings,
ProxyUpgradeRolloutProbeConfig, StalePendingUsageRow, UsageCleanupSettings, USAGE_CLEANUP_HOUR,
USAGE_CLEANUP_MINUTE, WALLET_DAILY_USAGE_AGGREGATION_HOUR,
WALLET_DAILY_USAGE_AGGREGATION_MINUTE,
};
#[tokio::test]
@@ -734,6 +736,176 @@ async fn usage_cleanup_settings_resolve_batch_and_delete_toggle() {
);
}
#[tokio::test]
async fn proxy_node_metrics_cleanup_settings_use_dedicated_retention_and_batch_limits() {
let data = GatewayDataState::disabled().with_system_config_values_for_tests([
("cleanup_batch_size".to_string(), json!(250)),
("proxy_node_metrics_1m_retention_days".to_string(), json!(0)),
("proxy_node_metrics_1h_retention_days".to_string(), json!(7)),
(
"proxy_node_metrics_cleanup_batch_size".to_string(),
json!(100_000),
),
]);
let settings = proxy_node_metrics_cleanup_settings(&data)
.await
.expect("proxy metrics cleanup settings should resolve");
assert_eq!(
settings,
ProxyNodeMetricsCleanupSettings {
retain_1m_days: 1,
retain_1h_days: 7,
batch_size: 50_000,
}
);
}
#[tokio::test]
async fn proxy_node_metrics_cleanup_settings_fallback_to_global_batch_size() {
let data = GatewayDataState::disabled().with_system_config_values_for_tests([
("cleanup_batch_size".to_string(), json!(250)),
(
"proxy_node_metrics_cleanup_batch_size".to_string(),
json!(0),
),
]);
let settings = proxy_node_metrics_cleanup_settings(&data)
.await
.expect("proxy metrics cleanup settings should resolve");
assert_eq!(
settings,
ProxyNodeMetricsCleanupSettings {
retain_1m_days: 30,
retain_1h_days: 180,
batch_size: 250,
}
);
}
#[tokio::test]
async fn proxy_node_metrics_cleanup_deletes_expired_buckets_in_batches() {
let repository = Arc::new(InMemoryProxyNodeRepository::seed(vec![
sample_connected_proxy_node("node-metrics-1", 30, 1),
sample_connected_proxy_node("node-metrics-2", 30, 1),
sample_connected_proxy_node("node-metrics-3", 30, 1),
]));
let data = GatewayDataState::with_proxy_node_repository_for_tests(Arc::clone(&repository))
.with_system_config_values_for_tests([
("enable_auto_cleanup".to_string(), json!(true)),
("proxy_node_metrics_1m_retention_days".to_string(), json!(1)),
("proxy_node_metrics_1h_retention_days".to_string(), json!(1)),
(
"proxy_node_metrics_cleanup_batch_size".to_string(),
json!(1),
),
]);
for (idx, node_id) in ["node-metrics-1", "node-metrics-2", "node-metrics-3"]
.into_iter()
.enumerate()
{
repository
.apply_heartbeat(&ProxyNodeHeartbeatMutation {
node_id: node_id.to_string(),
heartbeat_interval: Some(30),
active_connections: Some(i32::try_from(idx + 1).unwrap()),
total_requests_delta: None,
avg_latency_ms: None,
failed_requests_delta: None,
dns_failures_delta: None,
stream_errors_delta: None,
proxy_metadata: Some(json!({
"tunnel_metrics": {
"connect_errors": idx + 1,
"disconnects": 0,
"error_events_total": 0,
"ws_in_bytes": idx + 1,
"ws_out_bytes": idx + 1,
"ws_in_frames": idx + 1,
"ws_out_frames": idx + 1,
"heartbeat_rtt_last_ms": 10
}
})),
proxy_version: Some("1.0.0".to_string()),
})
.await
.expect("heartbeat should write metrics");
}
let now = chrono::Utc::now().timestamp().max(0) as u64;
let old_bucket = bucket_start_unix_secs(now, ProxyNodeMetricsStep::OneMinute);
let cleanup = data
.cleanup_proxy_node_metrics(
old_bucket.saturating_add(60),
old_bucket.saturating_add(3_600),
1,
)
.await
.expect("direct cleanup should delete a limited batch");
assert_eq!(cleanup.deleted_1m_rows, 1);
assert_eq!(cleanup.deleted_1h_rows, 1);
let cleanup = cleanup_proxy_node_metrics_at(&data, now.saturating_add(2 * 86_400))
.await
.expect("runtime cleanup should loop over batches");
assert_eq!(cleanup.deleted_1m_rows, 2);
assert_eq!(cleanup.deleted_1h_rows, 2);
}
#[tokio::test]
async fn proxy_node_metrics_cleanup_respects_auto_cleanup_toggle() {
let repository = Arc::new(InMemoryProxyNodeRepository::seed(vec![
sample_connected_proxy_node("node-metrics-disabled", 30, 1),
]));
let data = GatewayDataState::with_proxy_node_repository_for_tests(Arc::clone(&repository))
.with_system_config_values_for_tests([
("enable_auto_cleanup".to_string(), json!(false)),
("proxy_node_metrics_1m_retention_days".to_string(), json!(1)),
("proxy_node_metrics_1h_retention_days".to_string(), json!(1)),
(
"proxy_node_metrics_cleanup_batch_size".to_string(),
json!(1),
),
]);
repository
.apply_heartbeat(&ProxyNodeHeartbeatMutation {
node_id: "node-metrics-disabled".to_string(),
heartbeat_interval: Some(30),
active_connections: Some(1),
total_requests_delta: None,
avg_latency_ms: None,
failed_requests_delta: None,
dns_failures_delta: None,
stream_errors_delta: None,
proxy_metadata: Some(json!({
"tunnel_metrics": {
"connect_errors": 1,
"disconnects": 0,
"error_events_total": 0,
"ws_in_bytes": 1,
"ws_out_bytes": 1,
"ws_in_frames": 1,
"ws_out_frames": 1,
"heartbeat_rtt_last_ms": 10
}
})),
proxy_version: Some("1.0.0".to_string()),
})
.await
.expect("heartbeat should write metrics");
let cleanup = cleanup_proxy_node_metrics_once(&data)
.await
.expect("runtime cleanup should short-circuit");
assert_eq!(cleanup.deleted_1m_rows, 0);
assert_eq!(cleanup.deleted_1h_rows, 0);
}
#[test]
fn usage_cleanup_window_uses_non_overlapping_ranges() {
let now_utc = "2026-03-18T03:00:00Z"

30
apps/aether-gateway/src/maintenance/runtime/workers.rs
View File

@@ -12,12 +12,14 @@ use super::{
maintenance_timezone, parse_hhmm_time, perform_oauth_token_refresh_once,
provider_checkin_schedule, run_audit_cleanup_once, run_db_maintenance_once,
run_gemini_file_mapping_cleanup_once, run_pending_cleanup_once, run_pool_monitor_once,
run_provider_checkin_once, run_proxy_node_stale_cleanup_once, run_proxy_upgrade_rollout_once,
run_provider_checkin_once, run_proxy_node_metrics_cleanup_once,
run_proxy_node_stale_cleanup_once, run_proxy_upgrade_rollout_once,
run_request_candidate_cleanup_once, run_stats_aggregation_once,
run_stats_hourly_aggregation_once, run_usage_cleanup_once,
run_wallet_daily_usage_aggregation_once, AUDIT_LOG_CLEANUP_INTERVAL,
GEMINI_FILE_MAPPING_CLEANUP_INTERVAL, OAUTH_TOKEN_REFRESH_INTERVAL, PENDING_CLEANUP_INTERVAL,
POOL_MONITOR_INTERVAL, PROVIDER_CHECKIN_DEFAULT_TIME, PROXY_NODE_STALE_SWEEP_INTERVAL,
POOL_MONITOR_INTERVAL, PROVIDER_CHECKIN_DEFAULT_TIME, PROXY_NODE_METRICS_CLEANUP_HOUR,
PROXY_NODE_METRICS_CLEANUP_MINUTE, PROXY_NODE_STALE_SWEEP_INTERVAL,
PROXY_UPGRADE_ROLLOUT_INTERVAL, REQUEST_CANDIDATE_CLEANUP_INTERVAL, USAGE_CLEANUP_HOUR,
USAGE_CLEANUP_MINUTE, WALLET_DAILY_USAGE_AGGREGATION_HOUR,
WALLET_DAILY_USAGE_AGGREGATION_MINUTE,
@@ -292,6 +294,30 @@ pub(crate) fn spawn_proxy_node_stale_cleanup_worker(
}))
}
pub(crate) fn spawn_proxy_node_metrics_cleanup_worker(
data: Arc<GatewayDataState>,
) -> Option<tokio::task::JoinHandle<()>> {
if !data.has_proxy_node_writer() {
return None;
}
let timezone = maintenance_timezone();
Some(tokio::spawn(async move {
loop {
tokio::time::sleep(duration_until_next_daily_run(
Utc::now(),
timezone,
PROXY_NODE_METRICS_CLEANUP_HOUR,
PROXY_NODE_METRICS_CLEANUP_MINUTE,
))
.await;
if let Err(err) = run_proxy_node_metrics_cleanup_once(&data).await {
log_maintenance_worker_failure("proxy_node_metrics_cleanup", "tick", &err);
}
}
}))
}
pub(crate) fn spawn_proxy_upgrade_rollout_worker(
state: AppState,
) -> Option<tokio::task::JoinHandle<()>> {

4
apps/aether-gateway/src/maintenance/tests.rs
View File

@@ -119,9 +119,7 @@ async fn gateway_background_request_candidate_cleanup_deletes_expired_entries_in
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
for handle in background_tasks {
handle.abort();
}
background_tasks.shutdown().await;
}
#[tokio::test]