mirror of
https://github.com/fawney19/Aether.git
synced 2026-10-07 01:47:47 +08:00
Consolidate subscription usage policy enforcement, privacy-safe persistence, and gateway security hardening into one reviewable change. Includes bounded HTTP and execution envelopes, header and protocol guards, DNS and relay validation, authentication and secret projection hardening, secure backup/install paths, and regression coverage.
1501 lines
52 KiB
Rust
1501 lines
52 KiB
Rust
use aether_contracts::tunnel_security::TUNNEL_SECURITY_NON_TLS_REQUIRED;
|
|
use async_trait::async_trait;
|
|
use serde_json::Value;
|
|
|
|
const PROXY_NODE_BOUND_TUNNEL_SECRET_PREFIX: &str =
|
|
"aether-proxy-node-secret-v2:aether-runtime-secret-v1:";
|
|
|
|
#[derive(Clone, PartialEq, serde::Serialize, serde::Deserialize)]
|
|
pub struct StoredProxyNode {
|
|
pub id: String,
|
|
#[serde(default = "new_proxy_node_tunnel_generation")]
|
|
pub tunnel_generation: String,
|
|
pub name: String,
|
|
pub ip: String,
|
|
pub port: i32,
|
|
pub region: Option<String>,
|
|
pub is_manual: bool,
|
|
pub proxy_url: Option<String>,
|
|
pub proxy_username: Option<String>,
|
|
pub proxy_password: Option<String>,
|
|
pub status: String,
|
|
pub registered_by: Option<String>,
|
|
pub last_heartbeat_at_unix_secs: Option<u64>,
|
|
pub heartbeat_interval: i32,
|
|
pub active_connections: i32,
|
|
pub total_requests: i64,
|
|
pub avg_latency_ms: Option<f64>,
|
|
pub failed_requests: i64,
|
|
pub dns_failures: i64,
|
|
pub stream_errors: i64,
|
|
pub proxy_metadata: Option<serde_json::Value>,
|
|
pub hardware_info: Option<serde_json::Value>,
|
|
pub estimated_max_concurrency: Option<i32>,
|
|
pub tunnel_mode: bool,
|
|
pub tunnel_connected: bool,
|
|
pub tunnel_connected_at_unix_secs: Option<u64>,
|
|
pub remote_config: Option<serde_json::Value>,
|
|
pub config_version: i32,
|
|
pub created_at_unix_ms: Option<u64>,
|
|
pub updated_at_unix_secs: Option<u64>,
|
|
}
|
|
|
|
impl std::fmt::Debug for StoredProxyNode {
|
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
formatter
|
|
.debug_struct("StoredProxyNode")
|
|
.field("id", &self.id)
|
|
.field("tunnel_generation", &self.tunnel_generation)
|
|
.field("name", &self.name)
|
|
.field("ip", &self.ip)
|
|
.field("port", &self.port)
|
|
.field("region", &self.region)
|
|
.field("is_manual", &self.is_manual)
|
|
.field("proxy_url", &self.proxy_url.as_ref().map(|_| "[REDACTED]"))
|
|
.field("proxy_username", &self.proxy_username)
|
|
.field(
|
|
"proxy_password",
|
|
&self.proxy_password.as_ref().map(|_| "[REDACTED]"),
|
|
)
|
|
.field("status", &self.status)
|
|
.field("tunnel_mode", &self.tunnel_mode)
|
|
.field("tunnel_connected", &self.tunnel_connected)
|
|
.field(
|
|
"proxy_metadata",
|
|
&self.proxy_metadata.as_ref().map(|_| "[REDACTED]"),
|
|
)
|
|
.field(
|
|
"hardware_info",
|
|
&self.hardware_info.as_ref().map(|_| "[REDACTED]"),
|
|
)
|
|
.field(
|
|
"remote_config",
|
|
&self.remote_config.as_ref().map(|_| "[REDACTED]"),
|
|
)
|
|
.finish_non_exhaustive()
|
|
}
|
|
}
|
|
|
|
impl StoredProxyNode {
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub fn new(
|
|
id: String,
|
|
name: String,
|
|
ip: String,
|
|
port: i32,
|
|
is_manual: bool,
|
|
status: String,
|
|
heartbeat_interval: i32,
|
|
active_connections: i32,
|
|
total_requests: i64,
|
|
failed_requests: i64,
|
|
dns_failures: i64,
|
|
stream_errors: i64,
|
|
tunnel_mode: bool,
|
|
tunnel_connected: bool,
|
|
config_version: i32,
|
|
) -> Result<Self, crate::DataLayerError> {
|
|
if id.trim().is_empty() {
|
|
return Err(crate::DataLayerError::UnexpectedValue(
|
|
"proxy_nodes.id is empty".to_string(),
|
|
));
|
|
}
|
|
if name.trim().is_empty() {
|
|
return Err(crate::DataLayerError::UnexpectedValue(
|
|
"proxy_nodes.name is empty".to_string(),
|
|
));
|
|
}
|
|
if ip.trim().is_empty() {
|
|
return Err(crate::DataLayerError::UnexpectedValue(
|
|
"proxy_nodes.ip is empty".to_string(),
|
|
));
|
|
}
|
|
if status.trim().is_empty() {
|
|
return Err(crate::DataLayerError::UnexpectedValue(
|
|
"proxy_nodes.status is empty".to_string(),
|
|
));
|
|
}
|
|
|
|
Ok(Self {
|
|
id,
|
|
tunnel_generation: new_proxy_node_tunnel_generation(),
|
|
name,
|
|
ip,
|
|
port,
|
|
region: None,
|
|
is_manual,
|
|
proxy_url: None,
|
|
proxy_username: None,
|
|
proxy_password: None,
|
|
status,
|
|
registered_by: None,
|
|
last_heartbeat_at_unix_secs: None,
|
|
heartbeat_interval,
|
|
active_connections,
|
|
total_requests,
|
|
avg_latency_ms: None,
|
|
failed_requests,
|
|
dns_failures,
|
|
stream_errors,
|
|
proxy_metadata: None,
|
|
hardware_info: None,
|
|
estimated_max_concurrency: None,
|
|
tunnel_mode,
|
|
tunnel_connected,
|
|
tunnel_connected_at_unix_secs: None,
|
|
remote_config: None,
|
|
config_version,
|
|
created_at_unix_ms: None,
|
|
updated_at_unix_secs: None,
|
|
})
|
|
}
|
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub fn with_runtime_fields(
|
|
mut self,
|
|
region: Option<String>,
|
|
registered_by: Option<String>,
|
|
last_heartbeat_at_unix_secs: Option<u64>,
|
|
avg_latency_ms: Option<f64>,
|
|
proxy_metadata: Option<serde_json::Value>,
|
|
hardware_info: Option<serde_json::Value>,
|
|
estimated_max_concurrency: Option<i32>,
|
|
tunnel_connected_at_unix_secs: Option<u64>,
|
|
remote_config: Option<serde_json::Value>,
|
|
created_at_unix_ms: Option<u64>,
|
|
updated_at_unix_secs: Option<u64>,
|
|
) -> Self {
|
|
self.region = region;
|
|
self.registered_by = registered_by;
|
|
self.last_heartbeat_at_unix_secs = last_heartbeat_at_unix_secs;
|
|
self.avg_latency_ms = avg_latency_ms;
|
|
self.proxy_metadata = proxy_metadata;
|
|
self.hardware_info = hardware_info;
|
|
self.estimated_max_concurrency = estimated_max_concurrency;
|
|
self.tunnel_connected_at_unix_secs = tunnel_connected_at_unix_secs;
|
|
self.remote_config = remote_config;
|
|
self.created_at_unix_ms = created_at_unix_ms;
|
|
self.updated_at_unix_secs = updated_at_unix_secs;
|
|
self
|
|
}
|
|
|
|
pub fn with_manual_proxy_fields(
|
|
mut self,
|
|
proxy_url: Option<String>,
|
|
proxy_username: Option<String>,
|
|
proxy_password: Option<String>,
|
|
) -> Self {
|
|
self.proxy_url = proxy_url;
|
|
self.proxy_username = proxy_username;
|
|
self.proxy_password = proxy_password;
|
|
self
|
|
}
|
|
|
|
pub fn with_tunnel_generation(mut self, tunnel_generation: String) -> Self {
|
|
self.tunnel_generation = tunnel_generation;
|
|
self
|
|
}
|
|
}
|
|
|
|
pub fn new_proxy_node_tunnel_generation() -> String {
|
|
uuid::Uuid::new_v4().to_string()
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
|
|
pub struct ProxyNodeHeartbeatMutation {
|
|
pub node_id: String,
|
|
#[serde(default)]
|
|
pub expected_tunnel_generation: Option<String>,
|
|
pub heartbeat_interval: Option<i32>,
|
|
pub active_connections: Option<i32>,
|
|
pub total_requests_delta: Option<i64>,
|
|
pub avg_latency_ms: Option<f64>,
|
|
pub failed_requests_delta: Option<i64>,
|
|
pub dns_failures_delta: Option<i64>,
|
|
pub stream_errors_delta: Option<i64>,
|
|
pub proxy_metadata: Option<serde_json::Value>,
|
|
pub proxy_version: Option<String>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
|
|
pub struct ProxyNodeTrafficMutation {
|
|
pub node_id: String,
|
|
/// Incarnation fence captured when the request plan selected this node.
|
|
/// Missing fences are rejected by the gateway path so a stale plan cannot
|
|
/// update a node recreated under the same id.
|
|
#[serde(default)]
|
|
pub expected_tunnel_generation: Option<String>,
|
|
pub total_requests_delta: i64,
|
|
pub failed_requests_delta: i64,
|
|
pub dns_failures_delta: i64,
|
|
pub stream_errors_delta: i64,
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
|
|
pub struct ProxyNodeRegistrationMutation {
|
|
/// The stable identity selected by the caller before any secret is
|
|
/// protected. Re-registration of an existing endpoint must use its
|
|
/// existing id; repositories reject attempts to replace it.
|
|
#[serde(default)]
|
|
pub node_id: Option<String>,
|
|
pub name: String,
|
|
pub ip: String,
|
|
pub port: i32,
|
|
pub region: Option<String>,
|
|
pub heartbeat_interval: i32,
|
|
pub active_connections: Option<i32>,
|
|
pub total_requests: Option<i64>,
|
|
pub avg_latency_ms: Option<f64>,
|
|
pub hardware_info: Option<serde_json::Value>,
|
|
pub estimated_max_concurrency: Option<i32>,
|
|
pub proxy_metadata: Option<serde_json::Value>,
|
|
pub proxy_version: Option<String>,
|
|
pub registered_by: Option<String>,
|
|
pub tunnel_mode: bool,
|
|
}
|
|
|
|
#[derive(Clone, PartialEq, serde::Serialize, serde::Deserialize)]
|
|
pub struct ProxyNodeManualCreateMutation {
|
|
/// Optional caller-selected id used to bind credentials before the row is
|
|
/// inserted. Repositories generate one only for legacy callers that do
|
|
/// not provide it.
|
|
#[serde(default)]
|
|
pub node_id: Option<String>,
|
|
pub name: String,
|
|
pub ip: String,
|
|
pub port: i32,
|
|
pub region: Option<String>,
|
|
pub proxy_url: String,
|
|
pub proxy_username: Option<String>,
|
|
pub proxy_password: Option<String>,
|
|
pub registered_by: Option<String>,
|
|
}
|
|
|
|
impl std::fmt::Debug for ProxyNodeManualCreateMutation {
|
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
formatter
|
|
.debug_struct("ProxyNodeManualCreateMutation")
|
|
.field("node_id", &self.node_id)
|
|
.field("name", &self.name)
|
|
.field("ip", &self.ip)
|
|
.field("port", &self.port)
|
|
.field("region", &self.region)
|
|
.field("proxy_url", &"[REDACTED]")
|
|
.field("proxy_username", &self.proxy_username)
|
|
.field(
|
|
"proxy_password",
|
|
&self.proxy_password.as_ref().map(|_| "[REDACTED]"),
|
|
)
|
|
.field("registered_by", &self.registered_by)
|
|
.finish()
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, PartialEq, serde::Serialize, serde::Deserialize)]
|
|
pub struct ProxyNodeManualUpdateMutation {
|
|
pub node_id: String,
|
|
pub name: Option<String>,
|
|
pub ip: Option<String>,
|
|
pub port: Option<i32>,
|
|
pub region: Option<String>,
|
|
pub proxy_url: Option<String>,
|
|
pub proxy_username: Option<String>,
|
|
pub proxy_password: Option<String>,
|
|
}
|
|
|
|
impl std::fmt::Debug for ProxyNodeManualUpdateMutation {
|
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
formatter
|
|
.debug_struct("ProxyNodeManualUpdateMutation")
|
|
.field("node_id", &self.node_id)
|
|
.field("name", &self.name)
|
|
.field("ip", &self.ip)
|
|
.field("port", &self.port)
|
|
.field("region", &self.region)
|
|
.field("proxy_url", &self.proxy_url.as_ref().map(|_| "[REDACTED]"))
|
|
.field("proxy_username", &self.proxy_username)
|
|
.field(
|
|
"proxy_password",
|
|
&self.proxy_password.as_ref().map(|_| "[REDACTED]"),
|
|
)
|
|
.finish()
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
|
|
pub struct ProxyNodeTunnelStatusMutation {
|
|
pub node_id: String,
|
|
#[serde(default)]
|
|
pub expected_tunnel_generation: Option<String>,
|
|
pub connected: bool,
|
|
pub conn_count: i32,
|
|
pub detail: Option<String>,
|
|
pub observed_at_unix_secs: Option<u64>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
|
|
pub struct ProxyNodeRemoteConfigMutation {
|
|
pub node_id: String,
|
|
#[serde(default)]
|
|
pub expected_tunnel_generation: Option<String>,
|
|
pub node_name: Option<String>,
|
|
pub allowed_ports: Option<Vec<u16>>,
|
|
pub log_level: Option<String>,
|
|
pub heartbeat_interval: Option<i32>,
|
|
pub scheduling_state: Option<Option<String>>,
|
|
pub upgrade_to: Option<Option<String>>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
|
|
pub struct StoredProxyNodeEvent {
|
|
pub id: i64,
|
|
pub node_id: String,
|
|
pub event_type: String,
|
|
pub detail: Option<String>,
|
|
pub event_metadata: Option<serde_json::Value>,
|
|
pub created_at_unix_ms: Option<u64>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
|
pub struct ProxyNodeEventQuery {
|
|
pub limit: usize,
|
|
pub from_unix_secs: Option<u64>,
|
|
pub to_unix_secs: Option<u64>,
|
|
pub event_type: Option<String>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
|
pub enum ProxyNodeMetricsStep {
|
|
OneMinute,
|
|
OneHour,
|
|
}
|
|
|
|
impl ProxyNodeMetricsStep {
|
|
pub fn bucket_size_secs(self) -> u64 {
|
|
match self {
|
|
Self::OneMinute => 60,
|
|
Self::OneHour => 3_600,
|
|
}
|
|
}
|
|
|
|
pub fn as_api_value(self) -> &'static str {
|
|
match self {
|
|
Self::OneMinute => "1m",
|
|
Self::OneHour => "1h",
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
|
|
pub struct StoredProxyNodeMetricsBucket {
|
|
pub node_id: String,
|
|
pub bucket_start_unix_secs: u64,
|
|
pub samples: i64,
|
|
pub uptime_samples: i64,
|
|
pub active_connections_sum: i64,
|
|
pub active_connections_max: i64,
|
|
pub heartbeat_rtt_ms_sum: i64,
|
|
pub heartbeat_rtt_ms_max: i64,
|
|
pub connect_errors_delta: i64,
|
|
pub disconnects_delta: i64,
|
|
pub error_events_delta: i64,
|
|
pub ws_in_bytes_delta: i64,
|
|
pub ws_out_bytes_delta: i64,
|
|
pub ws_in_frames_delta: i64,
|
|
pub ws_out_frames_delta: i64,
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
|
|
pub struct StoredProxyFleetMetricsBucket {
|
|
pub bucket_start_unix_secs: u64,
|
|
pub samples: i64,
|
|
pub uptime_samples: i64,
|
|
pub active_connections_sum: i64,
|
|
pub active_connections_max: i64,
|
|
pub heartbeat_rtt_ms_sum: i64,
|
|
pub heartbeat_rtt_ms_max: i64,
|
|
pub connect_errors_delta: i64,
|
|
pub disconnects_delta: i64,
|
|
pub error_events_delta: i64,
|
|
pub ws_in_bytes_delta: i64,
|
|
pub ws_out_bytes_delta: i64,
|
|
pub ws_in_frames_delta: i64,
|
|
pub ws_out_frames_delta: i64,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)]
|
|
pub struct ProxyNodeMetricsCleanupSummary {
|
|
pub deleted_1m_rows: usize,
|
|
pub deleted_1h_rows: usize,
|
|
}
|
|
|
|
pub const PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR: &str = "tunnel_err";
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
|
pub struct TunnelErrorEventRecord {
|
|
pub timestamp_unix_secs: u64,
|
|
pub timestamp_unix_ms: Option<u64>,
|
|
pub category: String,
|
|
pub message: String,
|
|
pub severity: Option<String>,
|
|
pub component: Option<String>,
|
|
pub summary: Option<String>,
|
|
pub operator_action: Option<String>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
|
pub struct TunnelMetricsCounters {
|
|
pub connect_errors: u64,
|
|
pub disconnects: u64,
|
|
pub error_events_total: u64,
|
|
pub ws_in_bytes: u64,
|
|
pub ws_out_bytes: u64,
|
|
pub ws_in_frames: u64,
|
|
pub ws_out_frames: u64,
|
|
pub heartbeat_rtt_last_ms: u64,
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct TunnelMetricsSample {
|
|
pub samples: i64,
|
|
pub uptime_samples: i64,
|
|
pub active_connections_sum: i64,
|
|
pub active_connections_max: i64,
|
|
pub heartbeat_rtt_ms_sum: i64,
|
|
pub heartbeat_rtt_ms_max: i64,
|
|
pub connect_errors_delta: i64,
|
|
pub disconnects_delta: i64,
|
|
pub error_events_delta: i64,
|
|
pub ws_in_bytes_delta: i64,
|
|
pub ws_out_bytes_delta: i64,
|
|
pub ws_in_frames_delta: i64,
|
|
pub ws_out_frames_delta: i64,
|
|
pub recent_error_events: Vec<TunnelErrorEventRecord>,
|
|
}
|
|
|
|
pub fn bucket_start_unix_secs(timestamp_unix_secs: u64, step: ProxyNodeMetricsStep) -> u64 {
|
|
let size = step.bucket_size_secs();
|
|
timestamp_unix_secs / size * size
|
|
}
|
|
|
|
pub fn build_tunnel_metrics_sample(
|
|
previous_proxy_metadata: Option<&Value>,
|
|
current_proxy_metadata: Option<&Value>,
|
|
active_connections: i32,
|
|
tunnel_connected: bool,
|
|
) -> Option<TunnelMetricsSample> {
|
|
let current = extract_tunnel_metrics_counters(current_proxy_metadata)?;
|
|
let previous = extract_tunnel_metrics_counters(previous_proxy_metadata);
|
|
let current_recent_errors = extract_recent_tunnel_errors(current_proxy_metadata);
|
|
|
|
let connect_errors_delta =
|
|
counter_delta_u64(previous.map(|v| v.connect_errors), current.connect_errors);
|
|
let disconnects_delta = counter_delta_u64(previous.map(|v| v.disconnects), current.disconnects);
|
|
let error_events_delta = counter_delta_u64(
|
|
previous.map(|v| v.error_events_total),
|
|
current.error_events_total,
|
|
);
|
|
let ws_in_bytes_delta = counter_delta_u64(previous.map(|v| v.ws_in_bytes), current.ws_in_bytes);
|
|
let ws_out_bytes_delta =
|
|
counter_delta_u64(previous.map(|v| v.ws_out_bytes), current.ws_out_bytes);
|
|
let ws_in_frames_delta =
|
|
counter_delta_u64(previous.map(|v| v.ws_in_frames), current.ws_in_frames);
|
|
let ws_out_frames_delta =
|
|
counter_delta_u64(previous.map(|v| v.ws_out_frames), current.ws_out_frames);
|
|
|
|
let take_recent = usize::try_from(error_events_delta).unwrap_or(usize::MAX);
|
|
let recent_error_events = if take_recent == 0 {
|
|
Vec::new()
|
|
} else {
|
|
let capture = take_recent.min(current_recent_errors.len());
|
|
let from = current_recent_errors.len().saturating_sub(capture);
|
|
current_recent_errors[from..].to_vec()
|
|
};
|
|
|
|
let active_connections = i64::from(active_connections.max(0));
|
|
let heartbeat_rtt_last_ms = i64::try_from(current.heartbeat_rtt_last_ms).unwrap_or(i64::MAX);
|
|
|
|
Some(TunnelMetricsSample {
|
|
samples: 1,
|
|
uptime_samples: if tunnel_connected { 1 } else { 0 },
|
|
active_connections_sum: active_connections,
|
|
active_connections_max: active_connections,
|
|
heartbeat_rtt_ms_sum: heartbeat_rtt_last_ms,
|
|
heartbeat_rtt_ms_max: heartbeat_rtt_last_ms,
|
|
connect_errors_delta: i64::try_from(connect_errors_delta).unwrap_or(i64::MAX),
|
|
disconnects_delta: i64::try_from(disconnects_delta).unwrap_or(i64::MAX),
|
|
error_events_delta: i64::try_from(error_events_delta).unwrap_or(i64::MAX),
|
|
ws_in_bytes_delta: i64::try_from(ws_in_bytes_delta).unwrap_or(i64::MAX),
|
|
ws_out_bytes_delta: i64::try_from(ws_out_bytes_delta).unwrap_or(i64::MAX),
|
|
ws_in_frames_delta: i64::try_from(ws_in_frames_delta).unwrap_or(i64::MAX),
|
|
ws_out_frames_delta: i64::try_from(ws_out_frames_delta).unwrap_or(i64::MAX),
|
|
recent_error_events,
|
|
})
|
|
}
|
|
|
|
pub fn build_tunnel_error_event_detail(event: &TunnelErrorEventRecord) -> String {
|
|
format!(
|
|
"[{}] {}",
|
|
event.category,
|
|
event.summary.as_deref().unwrap_or(event.message.as_str())
|
|
)
|
|
}
|
|
|
|
pub fn normalize_proxy_metadata(
|
|
proxy_metadata: Option<&serde_json::Value>,
|
|
proxy_version: Option<&str>,
|
|
) -> Option<serde_json::Value> {
|
|
let mut normalized = match proxy_metadata {
|
|
Some(serde_json::Value::Object(map)) => map.clone(),
|
|
Some(_) | None => serde_json::Map::new(),
|
|
};
|
|
|
|
let raw_version = normalized
|
|
.remove("version")
|
|
.and_then(|value| value.as_str().map(str::to_string));
|
|
let version = proxy_version
|
|
.map(str::trim)
|
|
.filter(|value| !value.is_empty())
|
|
.map(|value| value.chars().take(20).collect::<String>())
|
|
.or_else(|| {
|
|
raw_version
|
|
.as_deref()
|
|
.map(str::trim)
|
|
.filter(|value| !value.is_empty())
|
|
.map(|value| value.chars().take(20).collect::<String>())
|
|
});
|
|
if let Some(version) = version {
|
|
normalized.insert("version".to_string(), serde_json::Value::String(version));
|
|
}
|
|
|
|
if normalized.is_empty() {
|
|
None
|
|
} else {
|
|
Some(serde_json::Value::Object(normalized))
|
|
}
|
|
}
|
|
|
|
pub fn normalize_heartbeat_proxy_metadata(
|
|
previous_proxy_metadata: Option<&Value>,
|
|
proxy_metadata: Option<&Value>,
|
|
proxy_version: Option<&str>,
|
|
) -> Option<Value> {
|
|
let Some(Value::Object(mut normalized)) =
|
|
normalize_proxy_metadata(proxy_metadata, proxy_version)
|
|
else {
|
|
return None;
|
|
};
|
|
|
|
// Tunnel security is control-plane state. A heartbeat may refresh runtime
|
|
// metadata, but it must never introduce or replace this trusted field.
|
|
normalized.remove("tunnel_security");
|
|
let merged = preserve_proxy_metadata_tunnel_security(
|
|
previous_proxy_metadata,
|
|
Some(Value::Object(normalized)),
|
|
);
|
|
merged.filter(|value| {
|
|
value
|
|
.as_object()
|
|
.is_some_and(|metadata| !metadata.is_empty())
|
|
})
|
|
}
|
|
|
|
pub fn preserve_proxy_metadata_tunnel_security(
|
|
previous_proxy_metadata: Option<&Value>,
|
|
next_proxy_metadata: Option<Value>,
|
|
) -> Option<Value> {
|
|
let Some(tunnel_security) = previous_proxy_metadata
|
|
.and_then(|value| value.get("tunnel_security"))
|
|
.filter(|value| value.is_object())
|
|
.cloned()
|
|
else {
|
|
return next_proxy_metadata;
|
|
};
|
|
|
|
match next_proxy_metadata {
|
|
Some(Value::Object(mut metadata)) => {
|
|
metadata.insert("tunnel_security".to_string(), tunnel_security);
|
|
Some(Value::Object(metadata))
|
|
}
|
|
Some(value) => Some(value),
|
|
None => {
|
|
let mut metadata = serde_json::Map::new();
|
|
metadata.insert("tunnel_security".to_string(), tunnel_security);
|
|
Some(Value::Object(metadata))
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Merge metadata received during a trusted registration/re-registration.
|
|
///
|
|
/// Registration is the control-plane path that may rotate a tunnel PSK. A
|
|
/// registration payload that omits `tunnel_security` is therefore a partial
|
|
/// metadata refresh and must not clear the previously trusted security state.
|
|
/// Only a non-empty, gateway-bound v2 ciphertext proves that the registration
|
|
/// passed through the gateway credential-binding path, and that ciphertext is
|
|
/// accepted only with the required non-TLS security mode. Mode-only,
|
|
/// plaintext, malformed, null, scalar, empty, and disabled security values are
|
|
/// treated as omission and cannot clear a previously trusted object.
|
|
pub fn merge_proxy_metadata_for_registration(
|
|
previous_proxy_metadata: Option<&Value>,
|
|
next_proxy_metadata: Option<Value>,
|
|
) -> Option<Value> {
|
|
let Some(next_proxy_metadata) = next_proxy_metadata else {
|
|
return previous_proxy_metadata.cloned();
|
|
};
|
|
let incoming_security_is_explicit =
|
|
proxy_metadata_has_explicit_tunnel_security(Some(&next_proxy_metadata));
|
|
|
|
let Value::Object(mut metadata) = next_proxy_metadata else {
|
|
// `normalize_proxy_metadata` normally prevents this branch. Keep a
|
|
// malformed replacement from erasing trusted control-plane state.
|
|
return previous_proxy_metadata.cloned();
|
|
};
|
|
|
|
if incoming_security_is_explicit {
|
|
return Some(Value::Object(metadata));
|
|
}
|
|
|
|
// Null, scalar, and empty security objects are not valid replacements.
|
|
// Remove them before restoring the previous trusted object so malformed
|
|
// input cannot mask or downgrade the registered security policy.
|
|
metadata.remove("tunnel_security");
|
|
if let Some(previous_security) = previous_proxy_metadata
|
|
.and_then(|value| value.get("tunnel_security"))
|
|
.filter(|value| value.is_object())
|
|
.cloned()
|
|
{
|
|
metadata.insert("tunnel_security".to_string(), previous_security);
|
|
}
|
|
|
|
(!metadata.is_empty()).then_some(Value::Object(metadata))
|
|
}
|
|
|
|
/// Return whether metadata contains a complete gateway-bound tunnel security
|
|
/// replacement that a trusted registration may persist.
|
|
pub fn proxy_metadata_has_explicit_tunnel_security(proxy_metadata: Option<&Value>) -> bool {
|
|
proxy_metadata
|
|
.and_then(Value::as_object)
|
|
.and_then(|metadata| metadata.get("tunnel_security"))
|
|
.and_then(Value::as_object)
|
|
.is_some_and(|security| {
|
|
security.get("mode").and_then(Value::as_str) == Some(TUNNEL_SECURITY_NON_TLS_REQUIRED)
|
|
&& security
|
|
.get("encryption_key_encrypted")
|
|
.and_then(Value::as_str)
|
|
.and_then(|encrypted| {
|
|
encrypted.strip_prefix(PROXY_NODE_BOUND_TUNNEL_SECRET_PREFIX)
|
|
})
|
|
.is_some_and(|ciphertext| !ciphertext.is_empty())
|
|
})
|
|
}
|
|
|
|
fn extract_tunnel_metrics_counters(
|
|
proxy_metadata: Option<&Value>,
|
|
) -> Option<TunnelMetricsCounters> {
|
|
let tunnel_metrics = proxy_metadata
|
|
.and_then(Value::as_object)
|
|
.and_then(|metadata| metadata.get("tunnel_metrics"))
|
|
.and_then(Value::as_object)?;
|
|
|
|
Some(TunnelMetricsCounters {
|
|
connect_errors: json_u64(tunnel_metrics.get("connect_errors")).unwrap_or(0),
|
|
disconnects: json_u64(tunnel_metrics.get("disconnects")).unwrap_or(0),
|
|
error_events_total: json_u64(tunnel_metrics.get("error_events_total")).unwrap_or(0),
|
|
ws_in_bytes: json_u64(tunnel_metrics.get("ws_in_bytes")).unwrap_or(0),
|
|
ws_out_bytes: json_u64(tunnel_metrics.get("ws_out_bytes")).unwrap_or(0),
|
|
ws_in_frames: json_u64(tunnel_metrics.get("ws_in_frames")).unwrap_or(0),
|
|
ws_out_frames: json_u64(tunnel_metrics.get("ws_out_frames")).unwrap_or(0),
|
|
heartbeat_rtt_last_ms: json_u64(tunnel_metrics.get("heartbeat_rtt_last_ms")).unwrap_or(0),
|
|
})
|
|
}
|
|
|
|
fn extract_recent_tunnel_errors(proxy_metadata: Option<&Value>) -> Vec<TunnelErrorEventRecord> {
|
|
proxy_metadata
|
|
.and_then(Value::as_object)
|
|
.and_then(|metadata| metadata.get("recent_tunnel_errors"))
|
|
.and_then(Value::as_array)
|
|
.map(|items| {
|
|
items
|
|
.iter()
|
|
.filter_map(|item| {
|
|
let item = item.as_object()?;
|
|
Some(TunnelErrorEventRecord {
|
|
timestamp_unix_secs: json_u64(item.get("timestamp_unix_secs"))
|
|
.unwrap_or_default(),
|
|
timestamp_unix_ms: json_u64(item.get("timestamp_unix_ms")),
|
|
category: item
|
|
.get("category")
|
|
.and_then(Value::as_str)
|
|
.unwrap_or("unknown")
|
|
.to_string(),
|
|
message: item
|
|
.get("message")
|
|
.and_then(Value::as_str)
|
|
.unwrap_or("n/a")
|
|
.to_string(),
|
|
severity: json_string(item.get("severity")),
|
|
component: json_string(item.get("component")),
|
|
summary: json_string(item.get("summary")),
|
|
operator_action: json_string(item.get("operator_action")),
|
|
})
|
|
})
|
|
.collect::<Vec<_>>()
|
|
})
|
|
.unwrap_or_default()
|
|
}
|
|
|
|
fn json_u64(value: Option<&Value>) -> Option<u64> {
|
|
value.and_then(|value| {
|
|
value
|
|
.as_u64()
|
|
.or_else(|| value.as_i64().and_then(|n| (n >= 0).then_some(n as u64)))
|
|
})
|
|
}
|
|
|
|
fn json_string(value: Option<&Value>) -> Option<String> {
|
|
value
|
|
.and_then(Value::as_str)
|
|
.map(str::trim)
|
|
.filter(|value| !value.is_empty())
|
|
.map(ToString::to_string)
|
|
}
|
|
|
|
fn counter_delta_u64(previous: Option<u64>, current: u64) -> u64 {
|
|
match previous {
|
|
Some(previous) if current >= previous => current - previous,
|
|
Some(_) => current,
|
|
None => 0,
|
|
}
|
|
}
|
|
|
|
fn normalize_proxy_version_label(value: &str) -> Option<String> {
|
|
let trimmed = value.trim();
|
|
if trimmed.is_empty() {
|
|
return None;
|
|
}
|
|
Some(
|
|
trimmed
|
|
.strip_prefix("tunnel-v")
|
|
.or_else(|| trimmed.strip_prefix("proxy-v"))
|
|
.unwrap_or(trimmed)
|
|
.to_ascii_lowercase(),
|
|
)
|
|
}
|
|
|
|
pub const PROXY_NODE_SCHEDULING_STATE_DRAINING: &str = "draining";
|
|
pub const PROXY_NODE_SCHEDULING_STATE_CORDONED: &str = "cordoned";
|
|
|
|
pub fn normalize_proxy_node_scheduling_state(value: &str) -> Option<&'static str> {
|
|
let trimmed = value.trim();
|
|
if trimmed.eq_ignore_ascii_case(PROXY_NODE_SCHEDULING_STATE_DRAINING) {
|
|
return Some(PROXY_NODE_SCHEDULING_STATE_DRAINING);
|
|
}
|
|
if trimmed.eq_ignore_ascii_case(PROXY_NODE_SCHEDULING_STATE_CORDONED) {
|
|
return Some(PROXY_NODE_SCHEDULING_STATE_CORDONED);
|
|
}
|
|
None
|
|
}
|
|
|
|
pub fn remote_config_scheduling_state(
|
|
remote_config: Option<&serde_json::Value>,
|
|
) -> Option<&'static str> {
|
|
remote_config
|
|
.and_then(serde_json::Value::as_object)
|
|
.and_then(|value| value.get("scheduling_state"))
|
|
.and_then(serde_json::Value::as_str)
|
|
.and_then(normalize_proxy_node_scheduling_state)
|
|
}
|
|
|
|
pub fn proxy_node_accepts_new_tunnels(node: &StoredProxyNode) -> bool {
|
|
remote_config_scheduling_state(node.remote_config.as_ref()).is_none()
|
|
}
|
|
|
|
pub fn proxy_reported_version(proxy_metadata: Option<&serde_json::Value>) -> Option<String> {
|
|
proxy_metadata
|
|
.and_then(serde_json::Value::as_object)
|
|
.and_then(|value| value.get("version"))
|
|
.and_then(serde_json::Value::as_str)
|
|
.and_then(normalize_proxy_version_label)
|
|
}
|
|
|
|
pub fn remote_config_upgrade_target(remote_config: Option<&serde_json::Value>) -> Option<String> {
|
|
remote_config
|
|
.and_then(serde_json::Value::as_object)
|
|
.and_then(|value| value.get("upgrade_to"))
|
|
.and_then(serde_json::Value::as_str)
|
|
.and_then(normalize_proxy_version_label)
|
|
}
|
|
|
|
pub fn reconcile_remote_config_after_heartbeat(
|
|
remote_config: Option<&serde_json::Value>,
|
|
proxy_version: Option<&str>,
|
|
) -> Option<serde_json::Value> {
|
|
let Some(mut config) = remote_config
|
|
.and_then(serde_json::Value::as_object)
|
|
.cloned()
|
|
else {
|
|
return remote_config.cloned();
|
|
};
|
|
let Some(target_version) = config
|
|
.get("upgrade_to")
|
|
.and_then(serde_json::Value::as_str)
|
|
.and_then(normalize_proxy_version_label)
|
|
else {
|
|
return Some(serde_json::Value::Object(config));
|
|
};
|
|
let Some(reported_version) = proxy_version.and_then(normalize_proxy_version_label) else {
|
|
return Some(serde_json::Value::Object(config));
|
|
};
|
|
|
|
if reported_version == target_version {
|
|
config.remove("upgrade_to");
|
|
}
|
|
|
|
(!config.is_empty()).then_some(serde_json::Value::Object(config))
|
|
}
|
|
|
|
#[async_trait]
|
|
pub trait ProxyNodeReadRepository: Send + Sync {
|
|
async fn list_proxy_nodes(&self) -> Result<Vec<StoredProxyNode>, crate::DataLayerError>;
|
|
|
|
async fn find_proxy_node(
|
|
&self,
|
|
node_id: &str,
|
|
) -> Result<Option<StoredProxyNode>, crate::DataLayerError>;
|
|
|
|
async fn list_proxy_node_events(
|
|
&self,
|
|
node_id: &str,
|
|
limit: usize,
|
|
) -> Result<Vec<StoredProxyNodeEvent>, crate::DataLayerError>;
|
|
|
|
async fn list_proxy_node_events_filtered(
|
|
&self,
|
|
node_id: &str,
|
|
query: &ProxyNodeEventQuery,
|
|
) -> Result<Vec<StoredProxyNodeEvent>, crate::DataLayerError> {
|
|
let mut items = self.list_proxy_node_events(node_id, query.limit).await?;
|
|
if let Some(from_unix_secs) = query.from_unix_secs {
|
|
items.retain(|item| item.created_at_unix_ms.unwrap_or(0) >= from_unix_secs);
|
|
}
|
|
if let Some(to_unix_secs) = query.to_unix_secs {
|
|
items.retain(|item| item.created_at_unix_ms.unwrap_or(u64::MAX) <= to_unix_secs);
|
|
}
|
|
if let Some(event_type) = query.event_type.as_deref() {
|
|
items.retain(|item| item.event_type.eq_ignore_ascii_case(event_type));
|
|
}
|
|
items.truncate(query.limit);
|
|
Ok(items)
|
|
}
|
|
|
|
async fn list_proxy_node_metrics(
|
|
&self,
|
|
node_id: &str,
|
|
step: ProxyNodeMetricsStep,
|
|
from_unix_secs: u64,
|
|
to_unix_secs: u64,
|
|
limit: usize,
|
|
) -> Result<Vec<StoredProxyNodeMetricsBucket>, crate::DataLayerError>;
|
|
|
|
async fn list_proxy_fleet_metrics(
|
|
&self,
|
|
step: ProxyNodeMetricsStep,
|
|
from_unix_secs: u64,
|
|
to_unix_secs: u64,
|
|
limit: usize,
|
|
) -> Result<Vec<StoredProxyFleetMetricsBucket>, crate::DataLayerError>;
|
|
}
|
|
|
|
#[async_trait]
|
|
pub trait ProxyNodeWriteRepository: Send + Sync {
|
|
async fn reset_stale_tunnel_statuses(&self) -> Result<usize, crate::DataLayerError>;
|
|
|
|
async fn compare_and_set_proxy_password(
|
|
&self,
|
|
node_id: &str,
|
|
expected: &str,
|
|
replacement: &str,
|
|
) -> Result<bool, crate::DataLayerError>;
|
|
|
|
async fn compare_and_set_proxy_metadata(
|
|
&self,
|
|
node_id: &str,
|
|
expected: &serde_json::Value,
|
|
replacement: &serde_json::Value,
|
|
) -> Result<bool, crate::DataLayerError>;
|
|
|
|
async fn create_manual_node(
|
|
&self,
|
|
mutation: &ProxyNodeManualCreateMutation,
|
|
) -> Result<StoredProxyNode, crate::DataLayerError>;
|
|
|
|
async fn update_manual_node(
|
|
&self,
|
|
mutation: &ProxyNodeManualUpdateMutation,
|
|
) -> Result<Option<StoredProxyNode>, crate::DataLayerError>;
|
|
|
|
async fn register_node(
|
|
&self,
|
|
mutation: &ProxyNodeRegistrationMutation,
|
|
) -> Result<StoredProxyNode, crate::DataLayerError>;
|
|
|
|
async fn apply_heartbeat(
|
|
&self,
|
|
mutation: &ProxyNodeHeartbeatMutation,
|
|
) -> Result<Option<StoredProxyNode>, crate::DataLayerError>;
|
|
|
|
async fn record_traffic(
|
|
&self,
|
|
mutation: &ProxyNodeTrafficMutation,
|
|
) -> Result<bool, crate::DataLayerError>;
|
|
|
|
async fn update_tunnel_status(
|
|
&self,
|
|
mutation: &ProxyNodeTunnelStatusMutation,
|
|
) -> Result<Option<StoredProxyNode>, crate::DataLayerError>;
|
|
|
|
async fn unregister_node(
|
|
&self,
|
|
node_id: &str,
|
|
) -> Result<Option<StoredProxyNode>, crate::DataLayerError>;
|
|
|
|
async fn delete_node(
|
|
&self,
|
|
node_id: &str,
|
|
) -> Result<Option<StoredProxyNode>, crate::DataLayerError>;
|
|
|
|
async fn update_remote_config(
|
|
&self,
|
|
mutation: &ProxyNodeRemoteConfigMutation,
|
|
) -> Result<Option<StoredProxyNode>, crate::DataLayerError>;
|
|
|
|
async fn increment_manual_node_requests(
|
|
&self,
|
|
node_id: &str,
|
|
total_delta: i64,
|
|
failed_delta: i64,
|
|
latency_ms: Option<i64>,
|
|
) -> Result<(), crate::DataLayerError>;
|
|
|
|
async fn cleanup_proxy_node_metrics(
|
|
&self,
|
|
retain_1m_from_unix_secs: u64,
|
|
retain_1h_from_unix_secs: u64,
|
|
delete_limit: usize,
|
|
) -> Result<ProxyNodeMetricsCleanupSummary, crate::DataLayerError>;
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use serde_json::json;
|
|
|
|
use super::{
|
|
bucket_start_unix_secs, build_tunnel_error_event_detail, build_tunnel_metrics_sample,
|
|
merge_proxy_metadata_for_registration, normalize_heartbeat_proxy_metadata,
|
|
normalize_proxy_node_scheduling_state, preserve_proxy_metadata_tunnel_security,
|
|
proxy_node_accepts_new_tunnels, proxy_reported_version,
|
|
reconcile_remote_config_after_heartbeat, remote_config_scheduling_state,
|
|
remote_config_upgrade_target, ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation,
|
|
ProxyNodeMetricsStep, StoredProxyNode,
|
|
};
|
|
|
|
#[test]
|
|
fn proxy_node_debug_output_redacts_credentials_and_untrusted_metadata() {
|
|
let password = "debug-secret-proxy-password";
|
|
let proxy_url = "https://user:[email protected]";
|
|
let metadata_secret = "debug-secret-proxy-metadata";
|
|
let mut stored = StoredProxyNode::new(
|
|
"node-1".to_string(),
|
|
"node".to_string(),
|
|
"127.0.0.1".to_string(),
|
|
8080,
|
|
true,
|
|
"online".to_string(),
|
|
30,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
false,
|
|
false,
|
|
1,
|
|
)
|
|
.expect("proxy node should build")
|
|
.with_manual_proxy_fields(
|
|
Some(proxy_url.to_string()),
|
|
Some("user".to_string()),
|
|
Some(password.to_string()),
|
|
);
|
|
stored.proxy_metadata = Some(json!({"secret": metadata_secret}));
|
|
let create = ProxyNodeManualCreateMutation {
|
|
node_id: Some("node-1".to_string()),
|
|
name: "node".to_string(),
|
|
ip: "127.0.0.1".to_string(),
|
|
port: 8080,
|
|
region: None,
|
|
proxy_url: proxy_url.to_string(),
|
|
proxy_username: Some("user".to_string()),
|
|
proxy_password: Some(password.to_string()),
|
|
registered_by: None,
|
|
};
|
|
let update = ProxyNodeManualUpdateMutation {
|
|
node_id: "node-1".to_string(),
|
|
name: None,
|
|
ip: None,
|
|
port: None,
|
|
region: None,
|
|
proxy_url: Some(proxy_url.to_string()),
|
|
proxy_username: Some("user".to_string()),
|
|
proxy_password: Some(password.to_string()),
|
|
};
|
|
|
|
for rendered in [
|
|
format!("{stored:?}"),
|
|
format!("{create:?}"),
|
|
format!("{update:?}"),
|
|
] {
|
|
for secret in [password, proxy_url, metadata_secret] {
|
|
assert!(!rendered.contains(secret), "Debug output leaked {secret}");
|
|
}
|
|
assert!(rendered.contains("[REDACTED]"));
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn normalizes_reported_versions_and_clears_completed_upgrade_targets() {
|
|
let remote_config = json!({
|
|
"node_name": "edge-1",
|
|
"upgrade_to": "tunnel-v2.0.0",
|
|
});
|
|
let proxy_metadata = json!({
|
|
"version": "2.0.0",
|
|
"arch": "arm64",
|
|
});
|
|
|
|
assert_eq!(
|
|
proxy_reported_version(Some(&proxy_metadata)).as_deref(),
|
|
Some("2.0.0")
|
|
);
|
|
assert_eq!(
|
|
remote_config_upgrade_target(Some(&remote_config)).as_deref(),
|
|
Some("2.0.0")
|
|
);
|
|
|
|
let reconciled =
|
|
reconcile_remote_config_after_heartbeat(Some(&remote_config), Some("tunnel-v2.0.0"))
|
|
.expect("reconciled config should remain an object");
|
|
assert_eq!(reconciled.get("upgrade_to"), None);
|
|
assert_eq!(reconciled.get("node_name"), Some(&json!("edge-1")));
|
|
}
|
|
|
|
#[test]
|
|
fn normalizes_proxy_node_scheduling_state_and_detects_unschedulable_nodes() {
|
|
assert_eq!(
|
|
normalize_proxy_node_scheduling_state("draining"),
|
|
Some("draining")
|
|
);
|
|
assert_eq!(
|
|
normalize_proxy_node_scheduling_state(" CORDONED "),
|
|
Some("cordoned")
|
|
);
|
|
assert_eq!(normalize_proxy_node_scheduling_state("active"), None);
|
|
|
|
let remote_config = json!({
|
|
"node_name": "edge-1",
|
|
"scheduling_state": "draining",
|
|
});
|
|
assert_eq!(
|
|
remote_config_scheduling_state(Some(&remote_config)),
|
|
Some("draining")
|
|
);
|
|
|
|
let node = StoredProxyNode::new(
|
|
"node-1".to_string(),
|
|
"edge-1".to_string(),
|
|
"127.0.0.1".to_string(),
|
|
0,
|
|
false,
|
|
"online".to_string(),
|
|
30,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
true,
|
|
true,
|
|
0,
|
|
)
|
|
.expect("node should build")
|
|
.with_runtime_fields(
|
|
None,
|
|
None,
|
|
Some(1_800_000_000),
|
|
None,
|
|
None,
|
|
None,
|
|
None,
|
|
Some(1_800_000_001),
|
|
Some(remote_config),
|
|
Some(1_800_000_000),
|
|
Some(1_800_000_001),
|
|
);
|
|
|
|
assert!(!proxy_node_accepts_new_tunnels(&node));
|
|
}
|
|
|
|
#[test]
|
|
fn builds_tunnel_metrics_sample_with_reset_safe_counter_deltas() {
|
|
let previous = json!({
|
|
"tunnel_metrics": {
|
|
"connect_errors": 10,
|
|
"disconnects": 5,
|
|
"error_events_total": 7,
|
|
"ws_in_bytes": 1_000,
|
|
"ws_out_bytes": 2_000,
|
|
"ws_in_frames": 10,
|
|
"ws_out_frames": 20,
|
|
"heartbeat_rtt_last_ms": 30
|
|
}
|
|
});
|
|
let current = json!({
|
|
"tunnel_metrics": {
|
|
"connect_errors": 12,
|
|
"disconnects": 2,
|
|
"error_events_total": 9,
|
|
"ws_in_bytes": 1_500,
|
|
"ws_out_bytes": 100,
|
|
"ws_in_frames": 11,
|
|
"ws_out_frames": 3,
|
|
"heartbeat_rtt_last_ms": 44
|
|
},
|
|
"recent_tunnel_errors": [
|
|
{"timestamp_unix_secs": 100, "category": "older", "message": "old"},
|
|
{
|
|
"timestamp_unix_secs": 101,
|
|
"timestamp_unix_ms": 101_999,
|
|
"category": "newer",
|
|
"message": "new",
|
|
"severity": "error",
|
|
"component": "tunnel_write",
|
|
"summary": "WebSocket write failed because the peer closed or reset the connection",
|
|
"operator_action": "Check gateway restarts and network resets."
|
|
}
|
|
]
|
|
});
|
|
|
|
let sample = build_tunnel_metrics_sample(Some(&previous), Some(¤t), 4, true)
|
|
.expect("sample should build");
|
|
assert_eq!(sample.samples, 1);
|
|
assert_eq!(sample.uptime_samples, 1);
|
|
assert_eq!(sample.active_connections_sum, 4);
|
|
assert_eq!(sample.heartbeat_rtt_ms_sum, 44);
|
|
assert_eq!(sample.connect_errors_delta, 2);
|
|
assert_eq!(sample.disconnects_delta, 2);
|
|
assert_eq!(sample.error_events_delta, 2);
|
|
assert_eq!(sample.ws_in_bytes_delta, 500);
|
|
assert_eq!(sample.ws_out_bytes_delta, 100);
|
|
assert_eq!(sample.ws_out_frames_delta, 3);
|
|
assert_eq!(sample.recent_error_events.len(), 2);
|
|
assert_eq!(sample.recent_error_events[0].category, "older");
|
|
assert_eq!(sample.recent_error_events[0].summary, None);
|
|
assert_eq!(
|
|
sample.recent_error_events[1].severity.as_deref(),
|
|
Some("error")
|
|
);
|
|
assert_eq!(
|
|
sample.recent_error_events[1].component.as_deref(),
|
|
Some("tunnel_write")
|
|
);
|
|
assert_eq!(
|
|
sample.recent_error_events[1].timestamp_unix_ms,
|
|
Some(101_999)
|
|
);
|
|
assert_eq!(
|
|
build_tunnel_error_event_detail(&sample.recent_error_events[1]),
|
|
"[newer] WebSocket write failed because the peer closed or reset the connection"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn builds_tunnel_metrics_sample_uses_first_counter_report_as_baseline() {
|
|
let current = json!({
|
|
"tunnel_metrics": {
|
|
"connect_errors": 12,
|
|
"disconnects": 5,
|
|
"error_events_total": 7,
|
|
"ws_in_bytes": 1_500,
|
|
"ws_out_bytes": 2_500,
|
|
"ws_in_frames": 15,
|
|
"ws_out_frames": 25,
|
|
"heartbeat_rtt_last_ms": 44
|
|
},
|
|
"recent_tunnel_errors": [
|
|
{"timestamp_unix_secs": 101, "category": "newer", "message": "new"}
|
|
]
|
|
});
|
|
|
|
let sample = build_tunnel_metrics_sample(None, Some(¤t), 4, true)
|
|
.expect("sample should build");
|
|
|
|
assert_eq!(sample.samples, 1);
|
|
assert_eq!(sample.heartbeat_rtt_ms_sum, 44);
|
|
assert_eq!(sample.connect_errors_delta, 0);
|
|
assert_eq!(sample.disconnects_delta, 0);
|
|
assert_eq!(sample.error_events_delta, 0);
|
|
assert_eq!(sample.ws_in_bytes_delta, 0);
|
|
assert_eq!(sample.ws_out_bytes_delta, 0);
|
|
assert_eq!(sample.ws_in_frames_delta, 0);
|
|
assert_eq!(sample.ws_out_frames_delta, 0);
|
|
assert!(sample.recent_error_events.is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn preserves_secure_tunnel_metadata_across_heartbeat_metadata_refresh() {
|
|
let previous = json!({
|
|
"version": "1.0.0",
|
|
"tunnel_security": {
|
|
"mode": "non_tls_required",
|
|
"encryption_key": "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc="
|
|
}
|
|
});
|
|
let next = json!({
|
|
"version": "1.0.1",
|
|
"tunnel_metrics": {"connect_successes": 1},
|
|
"tunnel_security": {
|
|
"mode": "disabled",
|
|
"encryption_key": "attacker-controlled"
|
|
}
|
|
});
|
|
|
|
let merged = preserve_proxy_metadata_tunnel_security(Some(&previous), Some(next))
|
|
.expect("metadata should remain present");
|
|
assert_eq!(
|
|
merged
|
|
.pointer("/tunnel_security/mode")
|
|
.and_then(|v| v.as_str()),
|
|
Some("non_tls_required")
|
|
);
|
|
assert_eq!(
|
|
merged
|
|
.pointer("/tunnel_security/encryption_key")
|
|
.and_then(|v| v.as_str()),
|
|
Some("BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc=")
|
|
);
|
|
assert_eq!(
|
|
merged.pointer("/tunnel_metrics/connect_successes"),
|
|
Some(&json!(1))
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn registration_metadata_preserves_omitted_tunnel_security() {
|
|
let previous = json!({
|
|
"version": "1.0.0",
|
|
"tunnel_security": {
|
|
"mode": "non_tls_required",
|
|
"encryption_key_encrypted": "aether-proxy-node-secret-v2:aether-runtime-secret-v1:sealed-old"
|
|
}
|
|
});
|
|
let next = json!({
|
|
"version": "1.1.0",
|
|
"tunnel_metrics": {"connect_successes": 2}
|
|
});
|
|
|
|
let merged = merge_proxy_metadata_for_registration(Some(&previous), Some(next))
|
|
.expect("registration metadata should remain present");
|
|
assert_eq!(merged.get("version"), Some(&json!("1.1.0")));
|
|
assert_eq!(
|
|
merged.pointer("/tunnel_security/encryption_key_encrypted"),
|
|
Some(&json!(
|
|
"aether-proxy-node-secret-v2:aether-runtime-secret-v1:sealed-old"
|
|
))
|
|
);
|
|
assert_eq!(
|
|
merged.pointer("/tunnel_metrics/connect_successes"),
|
|
Some(&json!(2))
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn registration_metadata_accepts_explicit_tunnel_security_rotation() {
|
|
let previous = json!({
|
|
"tunnel_security": {
|
|
"mode": "non_tls_required",
|
|
"encryption_key_encrypted": "aether-proxy-node-secret-v2:aether-runtime-secret-v1:sealed-old"
|
|
}
|
|
});
|
|
let next = json!({
|
|
"tunnel_security": {
|
|
"mode": "non_tls_required",
|
|
"encryption_key_encrypted": "aether-proxy-node-secret-v2:aether-runtime-secret-v1:sealed-new"
|
|
}
|
|
});
|
|
|
|
let merged = merge_proxy_metadata_for_registration(Some(&previous), Some(next))
|
|
.expect("rotated registration metadata should remain present");
|
|
assert_eq!(
|
|
merged.pointer("/tunnel_security/encryption_key_encrypted"),
|
|
Some(&json!(
|
|
"aether-proxy-node-secret-v2:aether-runtime-secret-v1:sealed-new"
|
|
))
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn registration_metadata_rejects_invalid_security_replacement() {
|
|
let previous = json!({
|
|
"tunnel_security": {
|
|
"mode": "non_tls_required",
|
|
"encryption_key_encrypted": "aether-proxy-node-secret-v2:aether-runtime-secret-v1:sealed-old"
|
|
}
|
|
});
|
|
let next = json!({
|
|
"version": "1.2.0",
|
|
"tunnel_security": null
|
|
});
|
|
|
|
let merged = merge_proxy_metadata_for_registration(Some(&previous), Some(next))
|
|
.expect("previous security should be retained");
|
|
assert_eq!(merged.get("version"), Some(&json!("1.2.0")));
|
|
assert_eq!(
|
|
merged.pointer("/tunnel_security/encryption_key_encrypted"),
|
|
Some(&json!(
|
|
"aether-proxy-node-secret-v2:aether-runtime-secret-v1:sealed-old"
|
|
))
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn registration_metadata_rejects_mode_only_security_downgrade() {
|
|
let previous = json!({
|
|
"tunnel_security": {
|
|
"mode": "non_tls_required",
|
|
"encryption_key_encrypted": "aether-proxy-node-secret-v2:aether-runtime-secret-v1:sealed-old"
|
|
}
|
|
});
|
|
let next = json!({
|
|
"version": "1.3.0",
|
|
"tunnel_security": {"mode": "disabled"}
|
|
});
|
|
|
|
let merged = merge_proxy_metadata_for_registration(Some(&previous), Some(next))
|
|
.expect("mode-only security must not replace the registered credential");
|
|
assert_eq!(merged.get("version"), Some(&json!("1.3.0")));
|
|
assert_eq!(
|
|
merged.pointer("/tunnel_security/encryption_key_encrypted"),
|
|
Some(&json!(
|
|
"aether-proxy-node-secret-v2:aether-runtime-secret-v1:sealed-old"
|
|
))
|
|
);
|
|
assert_eq!(
|
|
merged.pointer("/tunnel_security/mode"),
|
|
Some(&json!("non_tls_required"))
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn registration_metadata_rejects_bound_ciphertext_with_disabled_mode() {
|
|
let previous = json!({
|
|
"tunnel_security": {
|
|
"mode": "non_tls_required",
|
|
"encryption_key_encrypted": "aether-proxy-node-secret-v2:aether-runtime-secret-v1:sealed-old"
|
|
}
|
|
});
|
|
let next = json!({
|
|
"version": "1.3.1",
|
|
"tunnel_security": {
|
|
"mode": "disabled",
|
|
"encryption_key_encrypted": "aether-proxy-node-secret-v2:aether-runtime-secret-v1:sealed-attacker"
|
|
}
|
|
});
|
|
|
|
let merged = merge_proxy_metadata_for_registration(Some(&previous), Some(next))
|
|
.expect("disabled security must not replace the registered credential");
|
|
assert_eq!(merged.get("version"), Some(&json!("1.3.1")));
|
|
assert_eq!(
|
|
merged.pointer("/tunnel_security/encryption_key_encrypted"),
|
|
Some(&json!(
|
|
"aether-proxy-node-secret-v2:aether-runtime-secret-v1:sealed-old"
|
|
))
|
|
);
|
|
assert_eq!(
|
|
merged.pointer("/tunnel_security/mode"),
|
|
Some(&json!("non_tls_required"))
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn new_registration_drops_invalid_tunnel_security_fields() {
|
|
for invalid_security in [
|
|
json!(null),
|
|
json!("disabled"),
|
|
json!({}),
|
|
json!({"mode": "disabled"}),
|
|
json!({"mode": "disabled", "encryption_key_encrypted": "aether-proxy-node-secret-v2:aether-runtime-secret-v1:sealed-attacker"}),
|
|
json!({"mode": "non_tls_required", "encryption_key_encrypted": ""}),
|
|
json!({"mode": "non_tls_required", "encryption_key_encrypted": "not-gateway-bound"}),
|
|
] {
|
|
let next = json!({
|
|
"version": "1.4.0",
|
|
"tunnel_security": invalid_security
|
|
});
|
|
let merged = merge_proxy_metadata_for_registration(None, Some(next))
|
|
.expect("valid non-security metadata should remain");
|
|
assert_eq!(merged.get("version"), Some(&json!("1.4.0")));
|
|
assert!(merged.get("tunnel_security").is_none());
|
|
}
|
|
|
|
assert_eq!(
|
|
merge_proxy_metadata_for_registration(
|
|
None,
|
|
Some(json!({"tunnel_security": {"mode": "disabled"}})),
|
|
),
|
|
None
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn heartbeat_metadata_cannot_introduce_tunnel_security() {
|
|
let injected = json!({
|
|
"tunnel_security": {
|
|
"mode": "disabled",
|
|
"encryption_key": "attacker-controlled"
|
|
}
|
|
});
|
|
assert_eq!(
|
|
normalize_heartbeat_proxy_metadata(None, Some(&injected), None),
|
|
None
|
|
);
|
|
|
|
let injected_with_runtime_metadata = json!({
|
|
"version": "1.2.3",
|
|
"arch": "arm64",
|
|
"tunnel_security": {
|
|
"mode": "disabled",
|
|
"encryption_key": "attacker-controlled"
|
|
}
|
|
});
|
|
let normalized =
|
|
normalize_heartbeat_proxy_metadata(None, Some(&injected_with_runtime_metadata), None)
|
|
.expect("runtime metadata should remain present");
|
|
assert_eq!(normalized.get("version"), Some(&json!("1.2.3")));
|
|
assert_eq!(normalized.get("arch"), Some(&json!("arm64")));
|
|
assert!(normalized.get("tunnel_security").is_none());
|
|
}
|
|
|
|
#[test]
|
|
fn maps_timestamps_to_metric_buckets() {
|
|
assert_eq!(
|
|
bucket_start_unix_secs(1_710_000_119, ProxyNodeMetricsStep::OneMinute),
|
|
1_710_000_060
|
|
);
|
|
assert_eq!(
|
|
bucket_start_unix_secs(1_710_003_999, ProxyNodeMetricsStep::OneHour),
|
|
1_710_003_600
|
|
);
|
|
}
|
|
}
|