mirror of
https://github.com/fawney19/Aether.git
synced 2026-10-07 18:07:47 +08:00
1501 lines
52 KiB
Rust
1501 lines
52 KiB
Rust
use aether_contracts::tunnel_security::TUNNEL_SECURITY_NON_TLS_REQUIRED;
|
|||
use async_trait::async_trait;
|
|||
use serde_json::Value;
|
|||
|
|||
const PROXY_NODE_BOUND_TUNNEL_SECRET_PREFIX: &str =
|
|||
|
|
"aether-proxy-node-secret-v2:aether-runtime-secret-v1:";
|
||
|
|
|
||
|
|
#[derive(Clone, PartialEq, serde::Serialize, serde::Deserialize)]
|
||
pub struct StoredProxyNode {
|
|||
|
|
pub id: String,
|
||
#[serde(default = "new_proxy_node_tunnel_generation")]
|
|||
|
|
pub tunnel_generation: String,
|
||
pub name: String,
|
|||
|
|
pub ip: String,
|
||
|
|
pub port: i32,
|
||
|
|
pub region: Option<String>,
|
||
|
|
pub is_manual: bool,
|
||
|
|
pub proxy_url: Option<String>,
|
||
|
|
pub proxy_username: Option<String>,
|
||
|
|
pub proxy_password: Option<String>,
|
||
|
|
pub status: String,
|
||
|
|
pub registered_by: Option<String>,
|
||
|
|
pub last_heartbeat_at_unix_secs: Option<u64>,
|
||
|
|
pub heartbeat_interval: i32,
|
||
|
|
pub active_connections: i32,
|
||
|
|
pub total_requests: i64,
|
||
|
|
pub avg_latency_ms: Option<f64>,
|
||
|
|
pub failed_requests: i64,
|
||
|
|
pub dns_failures: i64,
|
||
|
|
pub stream_errors: i64,
|
||
|
|
pub proxy_metadata: Option<serde_json::Value>,
|
||
|
|
pub hardware_info: Option<serde_json::Value>,
|
||
|
|
pub estimated_max_concurrency: Option<i32>,
|
||
|
|
pub tunnel_mode: bool,
|
||
|
|
pub tunnel_connected: bool,
|
||
|
|
pub tunnel_connected_at_unix_secs: Option<u64>,
|
||
|
|
pub remote_config: Option<serde_json::Value>,
|
||
|
|
pub config_version: i32,
|
||
pub created_at_unix_ms: Option<u64>,
|
|||
pub updated_at_unix_secs: Option<u64>,
|
|||
|
|
}
|
||
|
|
|
||
impl std::fmt::Debug for StoredProxyNode {
|
|||
|
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||
|
|
formatter
|
||
|
|
.debug_struct("StoredProxyNode")
|
||
|
|
.field("id", &self.id)
|
||
|
|
.field("tunnel_generation", &self.tunnel_generation)
|
||
|
|
.field("name", &self.name)
|
||
|
|
.field("ip", &self.ip)
|
||
|
|
.field("port", &self.port)
|
||
|
|
.field("region", &self.region)
|
||
|
|
.field("is_manual", &self.is_manual)
|
||
|
|
.field("proxy_url", &self.proxy_url.as_ref().map(|_| "[REDACTED]"))
|
||
|
|
.field("proxy_username", &self.proxy_username)
|
||
|
|
.field(
|
||
|
|
"proxy_password",
|
||
|
|
&self.proxy_password.as_ref().map(|_| "[REDACTED]"),
|
||
|
|
)
|
||
|
|
.field("status", &self.status)
|
||
|
|
.field("tunnel_mode", &self.tunnel_mode)
|
||
|
|
.field("tunnel_connected", &self.tunnel_connected)
|
||
|
|
.field(
|
||
|
|
"proxy_metadata",
|
||
|
|
&self.proxy_metadata.as_ref().map(|_| "[REDACTED]"),
|
||
|
|
)
|
||
|
|
.field(
|
||
|
|
"hardware_info",
|
||
|
|
&self.hardware_info.as_ref().map(|_| "[REDACTED]"),
|
||
|
|
)
|
||
|
|
.field(
|
||
|
|
"remote_config",
|
||
|
|
&self.remote_config.as_ref().map(|_| "[REDACTED]"),
|
||
|
|
)
|
||
|
|
.finish_non_exhaustive()
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
impl StoredProxyNode {
|
|||
|
|
#[allow(clippy::too_many_arguments)]
|
||
|
|
pub fn new(
|
||
|
|
id: String,
|
||
|
|
name: String,
|
||
|
|
ip: String,
|
||
|
|
port: i32,
|
||
|
|
is_manual: bool,
|
||
|
|
status: String,
|
||
|
|
heartbeat_interval: i32,
|
||
|
|
active_connections: i32,
|
||
|
|
total_requests: i64,
|
||
|
|
failed_requests: i64,
|
||
|
|
dns_failures: i64,
|
||
|
|
stream_errors: i64,
|
||
|
|
tunnel_mode: bool,
|
||
|
|
tunnel_connected: bool,
|
||
|
|
config_version: i32,
|
||
|
|
) -> Result<Self, crate::DataLayerError> {
|
||
|
|
if id.trim().is_empty() {
|
||
|
|
return Err(crate::DataLayerError::UnexpectedValue(
|
||
|
|
"proxy_nodes.id is empty".to_string(),
|
||
|
|
));
|
||
|
|
}
|
||
|
|
if name.trim().is_empty() {
|
||
|
|
return Err(crate::DataLayerError::UnexpectedValue(
|
||
|
|
"proxy_nodes.name is empty".to_string(),
|
||
|
|
));
|
||
|
|
}
|
||
|
|
if ip.trim().is_empty() {
|
||
|
|
return Err(crate::DataLayerError::UnexpectedValue(
|
||
|
|
"proxy_nodes.ip is empty".to_string(),
|
||
|
|
));
|
||
|
|
}
|
||
|
|
if status.trim().is_empty() {
|
||
|
|
return Err(crate::DataLayerError::UnexpectedValue(
|
||
|
|
"proxy_nodes.status is empty".to_string(),
|
||
|
|
));
|
||
|
|
}
|
||
|
|
|
||
|
|
Ok(Self {
|
||
|
|
id,
|
||
tunnel_generation: new_proxy_node_tunnel_generation(),
|
|||
name,
|
|||
|
|
ip,
|
||
|
|
port,
|
||
|
|
region: None,
|
||
|
|
is_manual,
|
||
|
|
proxy_url: None,
|
||
|
|
proxy_username: None,
|
||
|
|
proxy_password: None,
|
||
|
|
status,
|
||
|
|
registered_by: None,
|
||
|
|
last_heartbeat_at_unix_secs: None,
|
||
|
|
heartbeat_interval,
|
||
|
|
active_connections,
|
||
|
|
total_requests,
|
||
|
|
avg_latency_ms: None,
|
||
|
|
failed_requests,
|
||
|
|
dns_failures,
|
||
|
|
stream_errors,
|
||
|
|
proxy_metadata: None,
|
||
|
|
hardware_info: None,
|
||
|
|
estimated_max_concurrency: None,
|
||
|
|
tunnel_mode,
|
||
|
|
tunnel_connected,
|
||
|
|
tunnel_connected_at_unix_secs: None,
|
||
|
|
remote_config: None,
|
||
|
|
config_version,
|
||
created_at_unix_ms: None,
|
|||
updated_at_unix_secs: None,
|
|||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
#[allow(clippy::too_many_arguments)]
|
||
|
|
pub fn with_runtime_fields(
|
||
|
|
mut self,
|
||
|
|
region: Option<String>,
|
||
|
|
registered_by: Option<String>,
|
||
|
|
last_heartbeat_at_unix_secs: Option<u64>,
|
||
|
|
avg_latency_ms: Option<f64>,
|
||
|
|
proxy_metadata: Option<serde_json::Value>,
|
||
|
|
hardware_info: Option<serde_json::Value>,
|
||
|
|
estimated_max_concurrency: Option<i32>,
|
||
|
|
tunnel_connected_at_unix_secs: Option<u64>,
|
||
|
|
remote_config: Option<serde_json::Value>,
|
||
created_at_unix_ms: Option<u64>,
|
|||
updated_at_unix_secs: Option<u64>,
|
|||
|
|
) -> Self {
|
||
|
|
self.region = region;
|
||
|
|
self.registered_by = registered_by;
|
||
|
|
self.last_heartbeat_at_unix_secs = last_heartbeat_at_unix_secs;
|
||
|
|
self.avg_latency_ms = avg_latency_ms;
|
||
|
|
self.proxy_metadata = proxy_metadata;
|
||
|
|
self.hardware_info = hardware_info;
|
||
|
|
self.estimated_max_concurrency = estimated_max_concurrency;
|
||
|
|
self.tunnel_connected_at_unix_secs = tunnel_connected_at_unix_secs;
|
||
|
|
self.remote_config = remote_config;
|
||
self.created_at_unix_ms = created_at_unix_ms;
|
|||
self.updated_at_unix_secs = updated_at_unix_secs;
|
|||
|
|
self
|
||
|
|
}
|
||
|
|
|
||
|
|
pub fn with_manual_proxy_fields(
|
||
|
|
mut self,
|
||
|
|
proxy_url: Option<String>,
|
||
|
|
proxy_username: Option<String>,
|
||
|
|
proxy_password: Option<String>,
|
||
|
|
) -> Self {
|
||
|
|
self.proxy_url = proxy_url;
|
||
|
|
self.proxy_username = proxy_username;
|
||
|
|
self.proxy_password = proxy_password;
|
||
|
|
self
|
||
|
|
}
|
||
|
|||
|
|
pub fn with_tunnel_generation(mut self, tunnel_generation: String) -> Self {
|
||
|
|
self.tunnel_generation = tunnel_generation;
|
||
|
|
self
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
pub fn new_proxy_node_tunnel_generation() -> String {
|
||
|
|
uuid::Uuid::new_v4().to_string()
|
||
}
|
|||
|
|
|
||
|
|
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
|
||
|
|
pub struct ProxyNodeHeartbeatMutation {
|
||
|
|
pub node_id: String,
|
||
#[serde(default)]
|
|||
|
|
pub expected_tunnel_generation: Option<String>,
|
||
pub heartbeat_interval: Option<i32>,
|
|||
|
|
pub active_connections: Option<i32>,
|
||
|
|
pub total_requests_delta: Option<i64>,
|
||
|
|
pub avg_latency_ms: Option<f64>,
|
||
|
|
pub failed_requests_delta: Option<i64>,
|
||
|
|
pub dns_failures_delta: Option<i64>,
|
||
|
|
pub stream_errors_delta: Option<i64>,
|
||
|
|
pub proxy_metadata: Option<serde_json::Value>,
|
||
|
|
pub proxy_version: Option<String>,
|
||
|
|
}
|
||
|
|
|
||
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
|
|||
|
|
pub struct ProxyNodeTrafficMutation {
|
||
|
|
pub node_id: String,
|
||
/// Incarnation fence captured when the request plan selected this node.
|
|||
|
|
/// Missing fences are rejected by the gateway path so a stale plan cannot
|
||
|
|
/// update a node recreated under the same id.
|
||
|
|
#[serde(default)]
|
||
|
|
pub expected_tunnel_generation: Option<String>,
|
||
pub total_requests_delta: i64,
|
|||
|
|
pub failed_requests_delta: i64,
|
||
|
|
pub dns_failures_delta: i64,
|
||
|
|
pub stream_errors_delta: i64,
|
||
|
|
}
|
||
|
|
|
||
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
|
|||
|
|
pub struct ProxyNodeRegistrationMutation {
|
||
/// The stable identity selected by the caller before any secret is
|
|||
|
|
/// protected. Re-registration of an existing endpoint must use its
|
||
|
|
/// existing id; repositories reject attempts to replace it.
|
||
|
|
#[serde(default)]
|
||
|
|
pub node_id: Option<String>,
|
||
pub name: String,
|
|||
|
|
pub ip: String,
|
||
|
|
pub port: i32,
|
||
|
|
pub region: Option<String>,
|
||
|
|
pub heartbeat_interval: i32,
|
||
|
|
pub active_connections: Option<i32>,
|
||
|
|
pub total_requests: Option<i64>,
|
||
|
|
pub avg_latency_ms: Option<f64>,
|
||
|
|
pub hardware_info: Option<serde_json::Value>,
|
||
|
|
pub estimated_max_concurrency: Option<i32>,
|
||
|
|
pub proxy_metadata: Option<serde_json::Value>,
|
||
|
|
pub proxy_version: Option<String>,
|
||
|
|
pub registered_by: Option<String>,
|
||
|
|
pub tunnel_mode: bool,
|
||
|
|
}
|
||
|
|
|
||
#[derive(Clone, PartialEq, serde::Serialize, serde::Deserialize)]
|
|||
pub struct ProxyNodeManualCreateMutation {
|
|||
/// Optional caller-selected id used to bind credentials before the row is
|
|||
|
|
/// inserted. Repositories generate one only for legacy callers that do
|
||
|
|
/// not provide it.
|
||
|
|
#[serde(default)]
|
||
|
|
pub node_id: Option<String>,
|
||
pub name: String,
|
|||
|
|
pub ip: String,
|
||
|
|
pub port: i32,
|
||
|
|
pub region: Option<String>,
|
||
|
|
pub proxy_url: String,
|
||
|
|
pub proxy_username: Option<String>,
|
||
|
|
pub proxy_password: Option<String>,
|
||
|
|
pub registered_by: Option<String>,
|
||
|
|
}
|
||
|
|
|
||
impl std::fmt::Debug for ProxyNodeManualCreateMutation {
|
|||
|
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||
|
|
formatter
|
||
|
|
.debug_struct("ProxyNodeManualCreateMutation")
|
||
|
|
.field("node_id", &self.node_id)
|
||
|
|
.field("name", &self.name)
|
||
|
|
.field("ip", &self.ip)
|
||
|
|
.field("port", &self.port)
|
||
|
|
.field("region", &self.region)
|
||
|
|
.field("proxy_url", &"[REDACTED]")
|
||
|
|
.field("proxy_username", &self.proxy_username)
|
||
|
|
.field(
|
||
|
|
"proxy_password",
|
||
|
|
&self.proxy_password.as_ref().map(|_| "[REDACTED]"),
|
||
|
|
)
|
||
|
|
.field("registered_by", &self.registered_by)
|
||
|
|
.finish()
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
#[derive(Clone, PartialEq, serde::Serialize, serde::Deserialize)]
|
||
pub struct ProxyNodeManualUpdateMutation {
|
|||
|
|
pub node_id: String,
|
||
|
|
pub name: Option<String>,
|
||
|
|
pub ip: Option<String>,
|
||
|
|
pub port: Option<i32>,
|
||
|
|
pub region: Option<String>,
|
||
|
|
pub proxy_url: Option<String>,
|
||
|
|
pub proxy_username: Option<String>,
|
||
|
|
pub proxy_password: Option<String>,
|
||
|
|
}
|
||
|
|
|
||
impl std::fmt::Debug for ProxyNodeManualUpdateMutation {
|
|||
|
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||
|
|
formatter
|
||
|
|
.debug_struct("ProxyNodeManualUpdateMutation")
|
||
|
|
.field("node_id", &self.node_id)
|
||
|
|
.field("name", &self.name)
|
||
|
|
.field("ip", &self.ip)
|
||
|
|
.field("port", &self.port)
|
||
|
|
.field("region", &self.region)
|
||
|
|
.field("proxy_url", &self.proxy_url.as_ref().map(|_| "[REDACTED]"))
|
||
|
|
.field("proxy_username", &self.proxy_username)
|
||
|
|
.field(
|
||
|
|
"proxy_password",
|
||
|
|
&self.proxy_password.as_ref().map(|_| "[REDACTED]"),
|
||
|
|
)
|
||
|
|
.finish()
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
|
|||
|
|
pub struct ProxyNodeTunnelStatusMutation {
|
||
|
|
pub node_id: String,
|
||
#[serde(default)]
|
|||
|
|
pub expected_tunnel_generation: Option<String>,
|
||
pub connected: bool,
|
|||
|
|
pub conn_count: i32,
|
||
|
|
pub detail: Option<String>,
|
||
|
|
pub observed_at_unix_secs: Option<u64>,
|
||
|
|
}
|
||
|
|
|
||
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
|
|||
|
|
pub struct ProxyNodeRemoteConfigMutation {
|
||
|
|
pub node_id: String,
|
||
#[serde(default)]
|
|||
|
|
pub expected_tunnel_generation: Option<String>,
|
||
pub node_name: Option<String>,
|
|||
|
|
pub allowed_ports: Option<Vec<u16>>,
|
||
|
|
pub log_level: Option<String>,
|
||
|
|
pub heartbeat_interval: Option<i32>,
|
||
|
|
pub scheduling_state: Option<Option<String>>,
|
||
|
|
pub upgrade_to: Option<Option<String>>,
|
||
|
|
}
|
||
|
|
|
||
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
|
|||
|
|
pub struct StoredProxyNodeEvent {
|
||
|
|
pub id: i64,
|
||
|
|
pub node_id: String,
|
||
|
|
pub event_type: String,
|
||
|
|
pub detail: Option<String>,
|
||
pub event_metadata: Option<serde_json::Value>,
|
|||
pub created_at_unix_ms: Option<u64>,
|
|||
}
|
|||
|
|
|
||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
|||
|
|
pub struct ProxyNodeEventQuery {
|
||
|
|
pub limit: usize,
|
||
|
|
pub from_unix_secs: Option<u64>,
|
||
|
|
pub to_unix_secs: Option<u64>,
|
||
|
|
pub event_type: Option<String>,
|
||
|
|
}
|
||
|
|
|
||
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||
|
|
pub enum ProxyNodeMetricsStep {
|
||
|
|
OneMinute,
|
||
|
|
OneHour,
|
||
|
|
}
|
||
|
|
|
||
|
|
impl ProxyNodeMetricsStep {
|
||
|
|
pub fn bucket_size_secs(self) -> u64 {
|
||
|
|
match self {
|
||
|
|
Self::OneMinute => 60,
|
||
|
|
Self::OneHour => 3_600,
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
pub fn as_api_value(self) -> &'static str {
|
||
|
|
match self {
|
||
|
|
Self::OneMinute => "1m",
|
||
|
|
Self::OneHour => "1h",
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
|
||
|
|
pub struct StoredProxyNodeMetricsBucket {
|
||
|
|
pub node_id: String,
|
||
|
|
pub bucket_start_unix_secs: u64,
|
||
|
|
pub samples: i64,
|
||
|
|
pub uptime_samples: i64,
|
||
|
|
pub active_connections_sum: i64,
|
||
|
|
pub active_connections_max: i64,
|
||
|
|
pub heartbeat_rtt_ms_sum: i64,
|
||
|
|
pub heartbeat_rtt_ms_max: i64,
|
||
|
|
pub connect_errors_delta: i64,
|
||
|
|
pub disconnects_delta: i64,
|
||
|
|
pub error_events_delta: i64,
|
||
|
|
pub ws_in_bytes_delta: i64,
|
||
|
|
pub ws_out_bytes_delta: i64,
|
||
|
|
pub ws_in_frames_delta: i64,
|
||
|
|
pub ws_out_frames_delta: i64,
|
||
|
|
}
|
||
|
|
|
||
|
|
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
|
||
|
|
pub struct StoredProxyFleetMetricsBucket {
|
||
|
|
pub bucket_start_unix_secs: u64,
|
||
|
|
pub samples: i64,
|
||
|
|
pub uptime_samples: i64,
|
||
|
|
pub active_connections_sum: i64,
|
||
|
|
pub active_connections_max: i64,
|
||
|
|
pub heartbeat_rtt_ms_sum: i64,
|
||
|
|
pub heartbeat_rtt_ms_max: i64,
|
||
|
|
pub connect_errors_delta: i64,
|
||
|
|
pub disconnects_delta: i64,
|
||
|
|
pub error_events_delta: i64,
|
||
|
|
pub ws_in_bytes_delta: i64,
|
||
|
|
pub ws_out_bytes_delta: i64,
|
||
|
|
pub ws_in_frames_delta: i64,
|
||
|
|
pub ws_out_frames_delta: i64,
|
||
|
|
}
|
||
|
|
|
||
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)]
|
||
|
|
pub struct ProxyNodeMetricsCleanupSummary {
|
||
|
|
pub deleted_1m_rows: usize,
|
||
|
|
pub deleted_1h_rows: usize,
|
||
|
|
}
|
||
|
|
|
||
|
|
pub const PROXY_NODE_EVENT_TYPE_TUNNEL_ERROR: &str = "tunnel_err";
|
||
|
|
|
||
|
|
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||
|
|
pub struct TunnelErrorEventRecord {
|
||
|
|
pub timestamp_unix_secs: u64,
|
||
pub timestamp_unix_ms: Option<u64>,
|
|||
pub category: String,
|
|||
|
|
pub message: String,
|
||
pub severity: Option<String>,
|
|||
|
|
pub component: Option<String>,
|
||
|
|
pub summary: Option<String>,
|
||
|
|
pub operator_action: Option<String>,
|
||
}
|
|||
|
|
|
||
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||
|
|
pub struct TunnelMetricsCounters {
|
||
|
|
pub connect_errors: u64,
|
||
|
|
pub disconnects: u64,
|
||
|
|
pub error_events_total: u64,
|
||
|
|
pub ws_in_bytes: u64,
|
||
|
|
pub ws_out_bytes: u64,
|
||
|
|
pub ws_in_frames: u64,
|
||
|
|
pub ws_out_frames: u64,
|
||
|
|
pub heartbeat_rtt_last_ms: u64,
|
||
|
|
}
|
||
|
|
|
||
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||
|
|
pub struct TunnelMetricsSample {
|
||
|
|
pub samples: i64,
|
||
|
|
pub uptime_samples: i64,
|
||
|
|
pub active_connections_sum: i64,
|
||
|
|
pub active_connections_max: i64,
|
||
|
|
pub heartbeat_rtt_ms_sum: i64,
|
||
|
|
pub heartbeat_rtt_ms_max: i64,
|
||
|
|
pub connect_errors_delta: i64,
|
||
|
|
pub disconnects_delta: i64,
|
||
|
|
pub error_events_delta: i64,
|
||
|
|
pub ws_in_bytes_delta: i64,
|
||
|
|
pub ws_out_bytes_delta: i64,
|
||
|
|
pub ws_in_frames_delta: i64,
|
||
|
|
pub ws_out_frames_delta: i64,
|
||
|
|
pub recent_error_events: Vec<TunnelErrorEventRecord>,
|
||
|
|
}
|
||
|
|
|
||
|
|
pub fn bucket_start_unix_secs(timestamp_unix_secs: u64, step: ProxyNodeMetricsStep) -> u64 {
|
||
|
|
let size = step.bucket_size_secs();
|
||
|
|
timestamp_unix_secs / size * size
|
||
|
|
}
|
||
|
|
|
||
|
|
pub fn build_tunnel_metrics_sample(
|
||
|
|
previous_proxy_metadata: Option<&Value>,
|
||
|
|
current_proxy_metadata: Option<&Value>,
|
||
|
|
active_connections: i32,
|
||
|
|
tunnel_connected: bool,
|
||
|
|
) -> Option<TunnelMetricsSample> {
|
||
|
|
let current = extract_tunnel_metrics_counters(current_proxy_metadata)?;
|
||
|
|
let previous = extract_tunnel_metrics_counters(previous_proxy_metadata);
|
||
|
|
let current_recent_errors = extract_recent_tunnel_errors(current_proxy_metadata);
|
||
|
|
|
||
|
|
let connect_errors_delta =
|
||
|
|
counter_delta_u64(previous.map(|v| v.connect_errors), current.connect_errors);
|
||
|
|
let disconnects_delta = counter_delta_u64(previous.map(|v| v.disconnects), current.disconnects);
|
||
|
|
let error_events_delta = counter_delta_u64(
|
||
|
|
previous.map(|v| v.error_events_total),
|
||
|
|
current.error_events_total,
|
||
|
|
);
|
||
|
|
let ws_in_bytes_delta = counter_delta_u64(previous.map(|v| v.ws_in_bytes), current.ws_in_bytes);
|
||
|
|
let ws_out_bytes_delta =
|
||
|
|
counter_delta_u64(previous.map(|v| v.ws_out_bytes), current.ws_out_bytes);
|
||
|
|
let ws_in_frames_delta =
|
||
|
|
counter_delta_u64(previous.map(|v| v.ws_in_frames), current.ws_in_frames);
|
||
|
|
let ws_out_frames_delta =
|
||
|
|
counter_delta_u64(previous.map(|v| v.ws_out_frames), current.ws_out_frames);
|
||
|
|
|
||
|
|
let take_recent = usize::try_from(error_events_delta).unwrap_or(usize::MAX);
|
||
|
|
let recent_error_events = if take_recent == 0 {
|
||
|
|
Vec::new()
|
||
|
|
} else {
|
||
|
|
let capture = take_recent.min(current_recent_errors.len());
|
||
|
|
let from = current_recent_errors.len().saturating_sub(capture);
|
||
|
|
current_recent_errors[from..].to_vec()
|
||
|
|
};
|
||
|
|
|
||
|
|
let active_connections = i64::from(active_connections.max(0));
|
||
|
|
let heartbeat_rtt_last_ms = i64::try_from(current.heartbeat_rtt_last_ms).unwrap_or(i64::MAX);
|
||
|
|
|
||
|
|
Some(TunnelMetricsSample {
|
||
|
|
samples: 1,
|
||
|
|
uptime_samples: if tunnel_connected { 1 } else { 0 },
|
||
|
|
active_connections_sum: active_connections,
|
||
|
|
active_connections_max: active_connections,
|
||
|
|
heartbeat_rtt_ms_sum: heartbeat_rtt_last_ms,
|
||
|
|
heartbeat_rtt_ms_max: heartbeat_rtt_last_ms,
|
||
|
|
connect_errors_delta: i64::try_from(connect_errors_delta).unwrap_or(i64::MAX),
|
||
|
|
disconnects_delta: i64::try_from(disconnects_delta).unwrap_or(i64::MAX),
|
||
|
|
error_events_delta: i64::try_from(error_events_delta).unwrap_or(i64::MAX),
|
||
|
|
ws_in_bytes_delta: i64::try_from(ws_in_bytes_delta).unwrap_or(i64::MAX),
|
||
|
|
ws_out_bytes_delta: i64::try_from(ws_out_bytes_delta).unwrap_or(i64::MAX),
|
||
|
|
ws_in_frames_delta: i64::try_from(ws_in_frames_delta).unwrap_or(i64::MAX),
|
||
|
|
ws_out_frames_delta: i64::try_from(ws_out_frames_delta).unwrap_or(i64::MAX),
|
||
|
|
recent_error_events,
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
pub fn build_tunnel_error_event_detail(event: &TunnelErrorEventRecord) -> String {
|
||
format!(
|
|||
|
|
"[{}] {}",
|
||
|
|
event.category,
|
||
|
|
event.summary.as_deref().unwrap_or(event.message.as_str())
|
||
|
|
)
|
||
}
|
|||
|
|
|
||
pub fn normalize_proxy_metadata(
|
|||
|
|
proxy_metadata: Option<&serde_json::Value>,
|
||
|
|
proxy_version: Option<&str>,
|
||
|
|
) -> Option<serde_json::Value> {
|
||
|
|
let mut normalized = match proxy_metadata {
|
||
|
|
Some(serde_json::Value::Object(map)) => map.clone(),
|
||
|
|
Some(_) | None => serde_json::Map::new(),
|
||
|
|
};
|
||
|
|
|
||
|
|
let raw_version = normalized
|
||
|
|
.remove("version")
|
||
|
|
.and_then(|value| value.as_str().map(str::to_string));
|
||
|
|
let version = proxy_version
|
||
|
|
.map(str::trim)
|
||
|
|
.filter(|value| !value.is_empty())
|
||
|
|
.map(|value| value.chars().take(20).collect::<String>())
|
||
|
|
.or_else(|| {
|
||
|
|
raw_version
|
||
|
|
.as_deref()
|
||
|
|
.map(str::trim)
|
||
|
|
.filter(|value| !value.is_empty())
|
||
|
|
.map(|value| value.chars().take(20).collect::<String>())
|
||
|
|
});
|
||
|
|
if let Some(version) = version {
|
||
|
|
normalized.insert("version".to_string(), serde_json::Value::String(version));
|
||
|
|
}
|
||
|
|
|
||
|
|
if normalized.is_empty() {
|
||
|
|
None
|
||
|
|
} else {
|
||
|
|
Some(serde_json::Value::Object(normalized))
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
pub fn normalize_heartbeat_proxy_metadata(
|
|||
|
|
previous_proxy_metadata: Option<&Value>,
|
||
|
|
proxy_metadata: Option<&Value>,
|
||
|
|
proxy_version: Option<&str>,
|
||
|
|
) -> Option<Value> {
|
||
|
|
let Some(Value::Object(mut normalized)) =
|
||
|
|
normalize_proxy_metadata(proxy_metadata, proxy_version)
|
||
|
|
else {
|
||
|
|
return None;
|
||
|
|
};
|
||
|
|
|
||
|
|
// Tunnel security is control-plane state. A heartbeat may refresh runtime
|
||
|
|
// metadata, but it must never introduce or replace this trusted field.
|
||
|
|
normalized.remove("tunnel_security");
|
||
|
|
let merged = preserve_proxy_metadata_tunnel_security(
|
||
|
|
previous_proxy_metadata,
|
||
|
|
Some(Value::Object(normalized)),
|
||
|
|
);
|
||
|
|
merged.filter(|value| {
|
||
|
|
value
|
||
|
|
.as_object()
|
||
|
|
.is_some_and(|metadata| !metadata.is_empty())
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
pub fn preserve_proxy_metadata_tunnel_security(
|
|||
|
|
previous_proxy_metadata: Option<&Value>,
|
||
|
|
next_proxy_metadata: Option<Value>,
|
||
|
|
) -> Option<Value> {
|
||
|
|
let Some(tunnel_security) = previous_proxy_metadata
|
||
|
|
.and_then(|value| value.get("tunnel_security"))
|
||
|
|
.filter(|value| value.is_object())
|
||
|
|
.cloned()
|
||
|
|
else {
|
||
|
|
return next_proxy_metadata;
|
||
|
|
};
|
||
|
|
|
||
|
|
match next_proxy_metadata {
|
||
|
|
Some(Value::Object(mut metadata)) => {
|
||
metadata.insert("tunnel_security".to_string(), tunnel_security);
|
|||
Some(Value::Object(metadata))
|
|||
|
|
}
|
||
|
|
Some(value) => Some(value),
|
||
|
|
None => {
|
||
|
|
let mut metadata = serde_json::Map::new();
|
||
|
|
metadata.insert("tunnel_security".to_string(), tunnel_security);
|
||
|
|
Some(Value::Object(metadata))
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
/// Merge metadata received during a trusted registration/re-registration.
|
|||
|
|
///
|
||
|
|
/// Registration is the control-plane path that may rotate a tunnel PSK. A
|
||
|
|
/// registration payload that omits `tunnel_security` is therefore a partial
|
||
|
|
/// metadata refresh and must not clear the previously trusted security state.
|
||
|
|
/// Only a non-empty, gateway-bound v2 ciphertext proves that the registration
|
||
|
|
/// passed through the gateway credential-binding path, and that ciphertext is
|
||
|
|
/// accepted only with the required non-TLS security mode. Mode-only,
|
||
|
|
/// plaintext, malformed, null, scalar, empty, and disabled security values are
|
||
|
|
/// treated as omission and cannot clear a previously trusted object.
|
||
|
|
pub fn merge_proxy_metadata_for_registration(
|
||
|
|
previous_proxy_metadata: Option<&Value>,
|
||
|
|
next_proxy_metadata: Option<Value>,
|
||
|
|
) -> Option<Value> {
|
||
|
|
let Some(next_proxy_metadata) = next_proxy_metadata else {
|
||
|
|
return previous_proxy_metadata.cloned();
|
||
|
|
};
|
||
|
|
let incoming_security_is_explicit =
|
||
|
|
proxy_metadata_has_explicit_tunnel_security(Some(&next_proxy_metadata));
|
||
|
|
|
||
|
|
let Value::Object(mut metadata) = next_proxy_metadata else {
|
||
|
|
// `normalize_proxy_metadata` normally prevents this branch. Keep a
|
||
|
|
// malformed replacement from erasing trusted control-plane state.
|
||
|
|
return previous_proxy_metadata.cloned();
|
||
|
|
};
|
||
|
|
|
||
|
|
if incoming_security_is_explicit {
|
||
|
|
return Some(Value::Object(metadata));
|
||
|
|
}
|
||
|
|
|
||
|
|
// Null, scalar, and empty security objects are not valid replacements.
|
||
|
|
// Remove them before restoring the previous trusted object so malformed
|
||
|
|
// input cannot mask or downgrade the registered security policy.
|
||
|
|
metadata.remove("tunnel_security");
|
||
|
|
if let Some(previous_security) = previous_proxy_metadata
|
||
|
|
.and_then(|value| value.get("tunnel_security"))
|
||
|
|
.filter(|value| value.is_object())
|
||
|
|
.cloned()
|
||
|
|
{
|
||
|
|
metadata.insert("tunnel_security".to_string(), previous_security);
|
||
|
|
}
|
||
|
|
|
||
|
|
(!metadata.is_empty()).then_some(Value::Object(metadata))
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Return whether metadata contains a complete gateway-bound tunnel security
|
||
|
|
/// replacement that a trusted registration may persist.
|
||
|
|
pub fn proxy_metadata_has_explicit_tunnel_security(proxy_metadata: Option<&Value>) -> bool {
|
||
|
|
proxy_metadata
|
||
|
|
.and_then(Value::as_object)
|
||
|
|
.and_then(|metadata| metadata.get("tunnel_security"))
|
||
|
|
.and_then(Value::as_object)
|
||
|
|
.is_some_and(|security| {
|
||
|
|
security.get("mode").and_then(Value::as_str) == Some(TUNNEL_SECURITY_NON_TLS_REQUIRED)
|
||
|
|
&& security
|
||
|
|
.get("encryption_key_encrypted")
|
||
|
|
.and_then(Value::as_str)
|
||
|
|
.and_then(|encrypted| {
|
||
|
|
encrypted.strip_prefix(PROXY_NODE_BOUND_TUNNEL_SECRET_PREFIX)
|
||
|
|
})
|
||
|
|
.is_some_and(|ciphertext| !ciphertext.is_empty())
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
fn extract_tunnel_metrics_counters(
|
|||
|
|
proxy_metadata: Option<&Value>,
|
||
|
|
) -> Option<TunnelMetricsCounters> {
|
||
|
|
let tunnel_metrics = proxy_metadata
|
||
|
|
.and_then(Value::as_object)
|
||
|
|
.and_then(|metadata| metadata.get("tunnel_metrics"))
|
||
|
|
.and_then(Value::as_object)?;
|
||
|
|
|
||
|
|
Some(TunnelMetricsCounters {
|
||
|
|
connect_errors: json_u64(tunnel_metrics.get("connect_errors")).unwrap_or(0),
|
||
|
|
disconnects: json_u64(tunnel_metrics.get("disconnects")).unwrap_or(0),
|
||
|
|
error_events_total: json_u64(tunnel_metrics.get("error_events_total")).unwrap_or(0),
|
||
|
|
ws_in_bytes: json_u64(tunnel_metrics.get("ws_in_bytes")).unwrap_or(0),
|
||
|
|
ws_out_bytes: json_u64(tunnel_metrics.get("ws_out_bytes")).unwrap_or(0),
|
||
|
|
ws_in_frames: json_u64(tunnel_metrics.get("ws_in_frames")).unwrap_or(0),
|
||
|
|
ws_out_frames: json_u64(tunnel_metrics.get("ws_out_frames")).unwrap_or(0),
|
||
|
|
heartbeat_rtt_last_ms: json_u64(tunnel_metrics.get("heartbeat_rtt_last_ms")).unwrap_or(0),
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
fn extract_recent_tunnel_errors(proxy_metadata: Option<&Value>) -> Vec<TunnelErrorEventRecord> {
|
||
|
|
proxy_metadata
|
||
|
|
.and_then(Value::as_object)
|
||
|
|
.and_then(|metadata| metadata.get("recent_tunnel_errors"))
|
||
|
|
.and_then(Value::as_array)
|
||
|
|
.map(|items| {
|
||
|
|
items
|
||
|
|
.iter()
|
||
|
|
.filter_map(|item| {
|
||
|
|
let item = item.as_object()?;
|
||
|
|
Some(TunnelErrorEventRecord {
|
||
|
|
timestamp_unix_secs: json_u64(item.get("timestamp_unix_secs"))
|
||
|
|
.unwrap_or_default(),
|
||
timestamp_unix_ms: json_u64(item.get("timestamp_unix_ms")),
|
|||
category: item
|
|||
|
|
.get("category")
|
||
|
|
.and_then(Value::as_str)
|
||
|
|
.unwrap_or("unknown")
|
||
|
|
.to_string(),
|
||
|
|
message: item
|
||
|
|
.get("message")
|
||
|
|
.and_then(Value::as_str)
|
||
|
|
.unwrap_or("n/a")
|
||
|
|
.to_string(),
|
||
severity: json_string(item.get("severity")),
|
|||
|
|
component: json_string(item.get("component")),
|
||
|
|
summary: json_string(item.get("summary")),
|
||
|
|
operator_action: json_string(item.get("operator_action")),
|
||
})
|
|||
|
|
})
|
||
|
|
.collect::<Vec<_>>()
|
||
|
|
})
|
||
|
|
.unwrap_or_default()
|
||
|
|
}
|
||
|
|
|
||
|
|
fn json_u64(value: Option<&Value>) -> Option<u64> {
|
||
|
|
value.and_then(|value| {
|
||
|
|
value
|
||
|
|
.as_u64()
|
||
|
|
.or_else(|| value.as_i64().and_then(|n| (n >= 0).then_some(n as u64)))
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
fn json_string(value: Option<&Value>) -> Option<String> {
|
|||
|
|
value
|
||
|
|
.and_then(Value::as_str)
|
||
|
|
.map(str::trim)
|
||
|
|
.filter(|value| !value.is_empty())
|
||
|
|
.map(ToString::to_string)
|
||
|
|
}
|
||
|
|
|
||
fn counter_delta_u64(previous: Option<u64>, current: u64) -> u64 {
|
|||
|
|
match previous {
|
||
|
|
Some(previous) if current >= previous => current - previous,
|
||
Some(_) => current,
|
|||
|
|
None => 0,
|
||
}
|
|||
|
|
}
|
||
|
|
|
||
fn normalize_proxy_version_label(value: &str) -> Option<String> {
|
|||
|
|
let trimmed = value.trim();
|
||
|
|
if trimmed.is_empty() {
|
||
|
|
return None;
|
||
|
|
}
|
||
|
|
Some(
|
||
|
|
trimmed
|
||
.strip_prefix("tunnel-v")
|
|||
|
|
.or_else(|| trimmed.strip_prefix("proxy-v"))
|
||
.unwrap_or(trimmed)
|
|||
|
|
.to_ascii_lowercase(),
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
pub const PROXY_NODE_SCHEDULING_STATE_DRAINING: &str = "draining";
|
||
|
|
pub const PROXY_NODE_SCHEDULING_STATE_CORDONED: &str = "cordoned";
|
||
|
|
|
||
|
|
pub fn normalize_proxy_node_scheduling_state(value: &str) -> Option<&'static str> {
|
||
|
|
let trimmed = value.trim();
|
||
|
|
if trimmed.eq_ignore_ascii_case(PROXY_NODE_SCHEDULING_STATE_DRAINING) {
|
||
|
|
return Some(PROXY_NODE_SCHEDULING_STATE_DRAINING);
|
||
|
|
}
|
||
|
|
if trimmed.eq_ignore_ascii_case(PROXY_NODE_SCHEDULING_STATE_CORDONED) {
|
||
|
|
return Some(PROXY_NODE_SCHEDULING_STATE_CORDONED);
|
||
|
|
}
|
||
|
|
None
|
||
|
|
}
|
||
|
|
|
||
|
|
pub fn remote_config_scheduling_state(
|
||
|
|
remote_config: Option<&serde_json::Value>,
|
||
|
|
) -> Option<&'static str> {
|
||
|
|
remote_config
|
||
|
|
.and_then(serde_json::Value::as_object)
|
||
|
|
.and_then(|value| value.get("scheduling_state"))
|
||
|
|
.and_then(serde_json::Value::as_str)
|
||
|
|
.and_then(normalize_proxy_node_scheduling_state)
|
||
|
|
}
|
||
|
|
|
||
|
|
pub fn proxy_node_accepts_new_tunnels(node: &StoredProxyNode) -> bool {
|
||
|
|
remote_config_scheduling_state(node.remote_config.as_ref()).is_none()
|
||
|
|
}
|
||
|
|
|
||
|
|
pub fn proxy_reported_version(proxy_metadata: Option<&serde_json::Value>) -> Option<String> {
|
||
|
|
proxy_metadata
|
||
|
|
.and_then(serde_json::Value::as_object)
|
||
|
|
.and_then(|value| value.get("version"))
|
||
|
|
.and_then(serde_json::Value::as_str)
|
||
|
|
.and_then(normalize_proxy_version_label)
|
||
|
|
}
|
||
|
|
|
||
|
|
pub fn remote_config_upgrade_target(remote_config: Option<&serde_json::Value>) -> Option<String> {
|
||
|
|
remote_config
|
||
|
|
.and_then(serde_json::Value::as_object)
|
||
|
|
.and_then(|value| value.get("upgrade_to"))
|
||
|
|
.and_then(serde_json::Value::as_str)
|
||
|
|
.and_then(normalize_proxy_version_label)
|
||
|
|
}
|
||
|
|
|
||
|
|
pub fn reconcile_remote_config_after_heartbeat(
|
||
|
|
remote_config: Option<&serde_json::Value>,
|
||
|
|
proxy_version: Option<&str>,
|
||
|
|
) -> Option<serde_json::Value> {
|
||
|
|
let Some(mut config) = remote_config
|
||
|
|
.and_then(serde_json::Value::as_object)
|
||
|
|
.cloned()
|
||
|
|
else {
|
||
|
|
return remote_config.cloned();
|
||
|
|
};
|
||
|
|
let Some(target_version) = config
|
||
|
|
.get("upgrade_to")
|
||
|
|
.and_then(serde_json::Value::as_str)
|
||
|
|
.and_then(normalize_proxy_version_label)
|
||
|
|
else {
|
||
|
|
return Some(serde_json::Value::Object(config));
|
||
|
|
};
|
||
|
|
let Some(reported_version) = proxy_version.and_then(normalize_proxy_version_label) else {
|
||
|
|
return Some(serde_json::Value::Object(config));
|
||
|
|
};
|
||
|
|
|
||
|
|
if reported_version == target_version {
|
||
|
|
config.remove("upgrade_to");
|
||
|
|
}
|
||
|
|
|
||
|
|
(!config.is_empty()).then_some(serde_json::Value::Object(config))
|
||
|
|
}
|
||
|
|
|
||
#[async_trait]
|
|||
|
|
pub trait ProxyNodeReadRepository: Send + Sync {
|
||
|
|
async fn list_proxy_nodes(&self) -> Result<Vec<StoredProxyNode>, crate::DataLayerError>;
|
||
|
|
|
||
|
|
async fn find_proxy_node(
|
||
|
|
&self,
|
||
|
|
node_id: &str,
|
||
|
|
) -> Result<Option<StoredProxyNode>, crate::DataLayerError>;
|
||
|
|
|
||
|
|
async fn list_proxy_node_events(
|
||
|
|
&self,
|
||
|
|
node_id: &str,
|
||
|
|
limit: usize,
|
||
|
|
) -> Result<Vec<StoredProxyNodeEvent>, crate::DataLayerError>;
|
||
|
|||
|
|
async fn list_proxy_node_events_filtered(
|
||
|
|
&self,
|
||
|
|
node_id: &str,
|
||
|
|
query: &ProxyNodeEventQuery,
|
||
|
|
) -> Result<Vec<StoredProxyNodeEvent>, crate::DataLayerError> {
|
||
|
|
let mut items = self.list_proxy_node_events(node_id, query.limit).await?;
|
||
|
|
if let Some(from_unix_secs) = query.from_unix_secs {
|
||
|
|
items.retain(|item| item.created_at_unix_ms.unwrap_or(0) >= from_unix_secs);
|
||
|
|
}
|
||
|
|
if let Some(to_unix_secs) = query.to_unix_secs {
|
||
|
|
items.retain(|item| item.created_at_unix_ms.unwrap_or(u64::MAX) <= to_unix_secs);
|
||
|
|
}
|
||
|
|
if let Some(event_type) = query.event_type.as_deref() {
|
||
|
|
items.retain(|item| item.event_type.eq_ignore_ascii_case(event_type));
|
||
|
|
}
|
||
|
|
items.truncate(query.limit);
|
||
|
|
Ok(items)
|
||
|
|
}
|
||
|
|
|
||
|
|
async fn list_proxy_node_metrics(
|
||
|
|
&self,
|
||
|
|
node_id: &str,
|
||
|
|
step: ProxyNodeMetricsStep,
|
||
|
|
from_unix_secs: u64,
|
||
|
|
to_unix_secs: u64,
|
||
|
|
limit: usize,
|
||
|
|
) -> Result<Vec<StoredProxyNodeMetricsBucket>, crate::DataLayerError>;
|
||
|
|
|
||
|
|
async fn list_proxy_fleet_metrics(
|
||
|
|
&self,
|
||
|
|
step: ProxyNodeMetricsStep,
|
||
|
|
from_unix_secs: u64,
|
||
|
|
to_unix_secs: u64,
|
||
|
|
limit: usize,
|
||
|
|
) -> Result<Vec<StoredProxyFleetMetricsBucket>, crate::DataLayerError>;
|
||
}
|
|||
|
|
|
||
|
|
#[async_trait]
|
||
|
|
pub trait ProxyNodeWriteRepository: Send + Sync {
|
||
async fn reset_stale_tunnel_statuses(&self) -> Result<usize, crate::DataLayerError>;
|
|||
|
|
|
||
async fn compare_and_set_proxy_password(
|
|||
|
|
&self,
|
||
|
|
node_id: &str,
|
||
|
|
expected: &str,
|
||
|
|
replacement: &str,
|
||
|
|
) -> Result<bool, crate::DataLayerError>;
|
||
|
|
|
||
|
|
async fn compare_and_set_proxy_metadata(
|
||
|
|
&self,
|
||
|
|
node_id: &str,
|
||
|
|
expected: &serde_json::Value,
|
||
|
|
replacement: &serde_json::Value,
|
||
|
|
) -> Result<bool, crate::DataLayerError>;
|
||
|
|
|
||
async fn create_manual_node(
|
|||
|
|
&self,
|
||
|
|
mutation: &ProxyNodeManualCreateMutation,
|
||
|
|
) -> Result<StoredProxyNode, crate::DataLayerError>;
|
||
|
|
|
||
|
|
async fn update_manual_node(
|
||
|
|
&self,
|
||
|
|
mutation: &ProxyNodeManualUpdateMutation,
|
||
|
|
) -> Result<Option<StoredProxyNode>, crate::DataLayerError>;
|
||
|
|
|
||
async fn register_node(
|
|||
|
|
&self,
|
||
|
|
mutation: &ProxyNodeRegistrationMutation,
|
||
|
|
) -> Result<StoredProxyNode, crate::DataLayerError>;
|
||
|
|
|
||
async fn apply_heartbeat(
|
|||
|
|
&self,
|
||
|
|
mutation: &ProxyNodeHeartbeatMutation,
|
||
|
|
) -> Result<Option<StoredProxyNode>, crate::DataLayerError>;
|
||
|
|
|
||
async fn record_traffic(
|
|||
|
|
&self,
|
||
|
|
mutation: &ProxyNodeTrafficMutation,
|
||
|
|
) -> Result<bool, crate::DataLayerError>;
|
||
|
|
|
||
async fn update_tunnel_status(
|
|||
|
|
&self,
|
||
|
|
mutation: &ProxyNodeTunnelStatusMutation,
|
||
|
|
) -> Result<Option<StoredProxyNode>, crate::DataLayerError>;
|
||
|
|||
|
|
async fn unregister_node(
|
||
|
|
&self,
|
||
|
|
node_id: &str,
|
||
|
|
) -> Result<Option<StoredProxyNode>, crate::DataLayerError>;
|
||
|
|
|
||
async fn delete_node(
|
|||
|
|
&self,
|
||
|
|
node_id: &str,
|
||
|
|
) -> Result<Option<StoredProxyNode>, crate::DataLayerError>;
|
||
|
|
|
||
async fn update_remote_config(
|
|||
|
|
&self,
|
||
|
|
mutation: &ProxyNodeRemoteConfigMutation,
|
||
|
|
) -> Result<Option<StoredProxyNode>, crate::DataLayerError>;
|
||
|
|||
|
|
async fn increment_manual_node_requests(
|
||
|
|
&self,
|
||
|
|
node_id: &str,
|
||
|
|
total_delta: i64,
|
||
|
|
failed_delta: i64,
|
||
|
|
latency_ms: Option<i64>,
|
||
|
|
) -> Result<(), crate::DataLayerError>;
|
||
|
|||
|
|
async fn cleanup_proxy_node_metrics(
|
||
|
|
&self,
|
||
|
|
retain_1m_from_unix_secs: u64,
|
||
|
|
retain_1h_from_unix_secs: u64,
|
||
delete_limit: usize,
|
|||
) -> Result<ProxyNodeMetricsCleanupSummary, crate::DataLayerError>;
|
|||
}
|
|||
|
|
|
||
|
|
#[cfg(test)]
|
||
|
|
mod tests {
|
||
|
|
use serde_json::json;
|
||
|
|
|
||
|
|
use super::{
|
||
bucket_start_unix_secs, build_tunnel_error_event_detail, build_tunnel_metrics_sample,
|
|||
merge_proxy_metadata_for_registration, normalize_heartbeat_proxy_metadata,
|
|||
normalize_proxy_node_scheduling_state, preserve_proxy_metadata_tunnel_security,
|
|||
|
|
proxy_node_accepts_new_tunnels, proxy_reported_version,
|
||
|
|
reconcile_remote_config_after_heartbeat, remote_config_scheduling_state,
|
||
remote_config_upgrade_target, ProxyNodeManualCreateMutation, ProxyNodeManualUpdateMutation,
|
|||
|
|
ProxyNodeMetricsStep, StoredProxyNode,
|
||
};
|
|||
|
|
|
||
#[test]
|
|||
|
|
fn proxy_node_debug_output_redacts_credentials_and_untrusted_metadata() {
|
||
|
|
let password = "debug-secret-proxy-password";
|
||
|
|
let proxy_url = "https://user:[email protected]";
|
||
|
|
let metadata_secret = "debug-secret-proxy-metadata";
|
||
|
|
let mut stored = StoredProxyNode::new(
|
||
|
|
"node-1".to_string(),
|
||
|
|
"node".to_string(),
|
||
|
|
"127.0.0.1".to_string(),
|
||
|
|
8080,
|
||
|
|
true,
|
||
|
|
"online".to_string(),
|
||
|
|
30,
|
||
|
|
0,
|
||
|
|
0,
|
||
|
|
0,
|
||
|
|
0,
|
||
|
|
0,
|
||
|
|
false,
|
||
|
|
false,
|
||
|
|
1,
|
||
|
|
)
|
||
|
|
.expect("proxy node should build")
|
||
|
|
.with_manual_proxy_fields(
|
||
|
|
Some(proxy_url.to_string()),
|
||
|
|
Some("user".to_string()),
|
||
|
|
Some(password.to_string()),
|
||
|
|
);
|
||
|
|
stored.proxy_metadata = Some(json!({"secret": metadata_secret}));
|
||
|
|
let create = ProxyNodeManualCreateMutation {
|
||
|
|
node_id: Some("node-1".to_string()),
|
||
|
|
name: "node".to_string(),
|
||
|
|
ip: "127.0.0.1".to_string(),
|
||
|
|
port: 8080,
|
||
|
|
region: None,
|
||
|
|
proxy_url: proxy_url.to_string(),
|
||
|
|
proxy_username: Some("user".to_string()),
|
||
|
|
proxy_password: Some(password.to_string()),
|
||
|
|
registered_by: None,
|
||
|
|
};
|
||
|
|
let update = ProxyNodeManualUpdateMutation {
|
||
|
|
node_id: "node-1".to_string(),
|
||
|
|
name: None,
|
||
|
|
ip: None,
|
||
|
|
port: None,
|
||
|
|
region: None,
|
||
|
|
proxy_url: Some(proxy_url.to_string()),
|
||
|
|
proxy_username: Some("user".to_string()),
|
||
|
|
proxy_password: Some(password.to_string()),
|
||
|
|
};
|
||
|
|
|
||
|
|
for rendered in [
|
||
|
|
format!("{stored:?}"),
|
||
|
|
format!("{create:?}"),
|
||
|
|
format!("{update:?}"),
|
||
|
|
] {
|
||
|
|
for secret in [password, proxy_url, metadata_secret] {
|
||
|
|
assert!(!rendered.contains(secret), "Debug output leaked {secret}");
|
||
|
|
}
|
||
|
|
assert!(rendered.contains("[REDACTED]"));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
#[test]
|
|||
|
|
fn normalizes_reported_versions_and_clears_completed_upgrade_targets() {
|
||
|
|
let remote_config = json!({
|
||
|
|
"node_name": "edge-1",
|
||
"upgrade_to": "tunnel-v2.0.0",
|
|||
});
|
|||
|
|
let proxy_metadata = json!({
|
||
|
|
"version": "2.0.0",
|
||
|
|
"arch": "arm64",
|
||
|
|
});
|
||
|
|
|
||
|
|
assert_eq!(
|
||
|
|
proxy_reported_version(Some(&proxy_metadata)).as_deref(),
|
||
|
|
Some("2.0.0")
|
||
|
|
);
|
||
|
|
assert_eq!(
|
||
|
|
remote_config_upgrade_target(Some(&remote_config)).as_deref(),
|
||
|
|
Some("2.0.0")
|
||
|
|
);
|
||
|
|
|
||
|
|
let reconciled =
|
||
reconcile_remote_config_after_heartbeat(Some(&remote_config), Some("tunnel-v2.0.0"))
|
|||
.expect("reconciled config should remain an object");
|
|||
|
|
assert_eq!(reconciled.get("upgrade_to"), None);
|
||
|
|
assert_eq!(reconciled.get("node_name"), Some(&json!("edge-1")));
|
||
|
|
}
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn normalizes_proxy_node_scheduling_state_and_detects_unschedulable_nodes() {
|
||
|
|
assert_eq!(
|
||
|
|
normalize_proxy_node_scheduling_state("draining"),
|
||
|
|
Some("draining")
|
||
|
|
);
|
||
|
|
assert_eq!(
|
||
|
|
normalize_proxy_node_scheduling_state(" CORDONED "),
|
||
|
|
Some("cordoned")
|
||
|
|
);
|
||
|
|
assert_eq!(normalize_proxy_node_scheduling_state("active"), None);
|
||
|
|
|
||
|
|
let remote_config = json!({
|
||
|
|
"node_name": "edge-1",
|
||
|
|
"scheduling_state": "draining",
|
||
|
|
});
|
||
|
|
assert_eq!(
|
||
|
|
remote_config_scheduling_state(Some(&remote_config)),
|
||
|
|
Some("draining")
|
||
|
|
);
|
||
|
|
|
||
|
|
let node = StoredProxyNode::new(
|
||
|
|
"node-1".to_string(),
|
||
|
|
"edge-1".to_string(),
|
||
|
|
"127.0.0.1".to_string(),
|
||
|
|
0,
|
||
|
|
false,
|
||
|
|
"online".to_string(),
|
||
|
|
30,
|
||
|
|
0,
|
||
|
|
0,
|
||
|
|
0,
|
||
|
|
0,
|
||
|
|
0,
|
||
|
|
true,
|
||
|
|
true,
|
||
|
|
0,
|
||
|
|
)
|
||
|
|
.expect("node should build")
|
||
|
|
.with_runtime_fields(
|
||
|
|
None,
|
||
|
|
None,
|
||
|
|
Some(1_800_000_000),
|
||
|
|
None,
|
||
|
|
None,
|
||
|
|
None,
|
||
|
|
None,
|
||
|
|
Some(1_800_000_001),
|
||
|
|
Some(remote_config),
|
||
|
|
Some(1_800_000_000),
|
||
|
|
Some(1_800_000_001),
|
||
|
|
);
|
||
|
|
|
||
|
|
assert!(!proxy_node_accepts_new_tunnels(&node));
|
||
|
|
}
|
||
|
|||
|
|
#[test]
|
||
|
|
fn builds_tunnel_metrics_sample_with_reset_safe_counter_deltas() {
|
||
|
|
let previous = json!({
|
||
|
|
"tunnel_metrics": {
|
||
|
|
"connect_errors": 10,
|
||
|
|
"disconnects": 5,
|
||
|
|
"error_events_total": 7,
|
||
|
|
"ws_in_bytes": 1_000,
|
||
|
|
"ws_out_bytes": 2_000,
|
||
|
|
"ws_in_frames": 10,
|
||
|
|
"ws_out_frames": 20,
|
||
|
|
"heartbeat_rtt_last_ms": 30
|
||
|
|
}
|
||
|
|
});
|
||
|
|
let current = json!({
|
||
|
|
"tunnel_metrics": {
|
||
|
|
"connect_errors": 12,
|
||
|
|
"disconnects": 2,
|
||
|
|
"error_events_total": 9,
|
||
|
|
"ws_in_bytes": 1_500,
|
||
|
|
"ws_out_bytes": 100,
|
||
|
|
"ws_in_frames": 11,
|
||
|
|
"ws_out_frames": 3,
|
||
|
|
"heartbeat_rtt_last_ms": 44
|
||
|
|
},
|
||
|
|
"recent_tunnel_errors": [
|
||
|
|
{"timestamp_unix_secs": 100, "category": "older", "message": "old"},
|
||
{
|
|||
|
|
"timestamp_unix_secs": 101,
|
||
"timestamp_unix_ms": 101_999,
|
|||
"category": "newer",
|
|||
|
|
"message": "new",
|
||
|
|
"severity": "error",
|
||
|
|
"component": "tunnel_write",
|
||
|
|
"summary": "WebSocket write failed because the peer closed or reset the connection",
|
||
|
|
"operator_action": "Check gateway restarts and network resets."
|
||
|
|
}
|
||
]
|
|||
|
|
});
|
||
|
|
|
||
|
|
let sample = build_tunnel_metrics_sample(Some(&previous), Some(¤t), 4, true)
|
||
|
|
.expect("sample should build");
|
||
|
|
assert_eq!(sample.samples, 1);
|
||
|
|
assert_eq!(sample.uptime_samples, 1);
|
||
|
|
assert_eq!(sample.active_connections_sum, 4);
|
||
|
|
assert_eq!(sample.heartbeat_rtt_ms_sum, 44);
|
||
|
|
assert_eq!(sample.connect_errors_delta, 2);
|
||
|
|
assert_eq!(sample.disconnects_delta, 2);
|
||
|
|
assert_eq!(sample.error_events_delta, 2);
|
||
|
|
assert_eq!(sample.ws_in_bytes_delta, 500);
|
||
|
|
assert_eq!(sample.ws_out_bytes_delta, 100);
|
||
|
|
assert_eq!(sample.ws_out_frames_delta, 3);
|
||
|
|
assert_eq!(sample.recent_error_events.len(), 2);
|
||
|
|
assert_eq!(sample.recent_error_events[0].category, "older");
|
||
assert_eq!(sample.recent_error_events[0].summary, None);
|
|||
|
|
assert_eq!(
|
||
|
|
sample.recent_error_events[1].severity.as_deref(),
|
||
|
|
Some("error")
|
||
|
|
);
|
||
|
|
assert_eq!(
|
||
|
|
sample.recent_error_events[1].component.as_deref(),
|
||
|
|
Some("tunnel_write")
|
||
|
|
);
|
||
assert_eq!(
|
|||
|
|
sample.recent_error_events[1].timestamp_unix_ms,
|
||
|
|
Some(101_999)
|
||
|
|
);
|
||
assert_eq!(
|
|||
|
|
build_tunnel_error_event_detail(&sample.recent_error_events[1]),
|
||
|
|
"[newer] WebSocket write failed because the peer closed or reset the connection"
|
||
|
|
);
|
||
}
|
|||
|
|
|
||
#[test]
|
|||
|
|
fn builds_tunnel_metrics_sample_uses_first_counter_report_as_baseline() {
|
||
|
|
let current = json!({
|
||
|
|
"tunnel_metrics": {
|
||
|
|
"connect_errors": 12,
|
||
|
|
"disconnects": 5,
|
||
|
|
"error_events_total": 7,
|
||
|
|
"ws_in_bytes": 1_500,
|
||
|
|
"ws_out_bytes": 2_500,
|
||
|
|
"ws_in_frames": 15,
|
||
|
|
"ws_out_frames": 25,
|
||
|
|
"heartbeat_rtt_last_ms": 44
|
||
|
|
},
|
||
|
|
"recent_tunnel_errors": [
|
||
|
|
{"timestamp_unix_secs": 101, "category": "newer", "message": "new"}
|
||
|
|
]
|
||
|
|
});
|
||
|
|
|
||
|
|
let sample = build_tunnel_metrics_sample(None, Some(¤t), 4, true)
|
||
|
|
.expect("sample should build");
|
||
|
|
|
||
|
|
assert_eq!(sample.samples, 1);
|
||
|
|
assert_eq!(sample.heartbeat_rtt_ms_sum, 44);
|
||
|
|
assert_eq!(sample.connect_errors_delta, 0);
|
||
|
|
assert_eq!(sample.disconnects_delta, 0);
|
||
|
|
assert_eq!(sample.error_events_delta, 0);
|
||
|
|
assert_eq!(sample.ws_in_bytes_delta, 0);
|
||
|
|
assert_eq!(sample.ws_out_bytes_delta, 0);
|
||
|
|
assert_eq!(sample.ws_in_frames_delta, 0);
|
||
|
|
assert_eq!(sample.ws_out_frames_delta, 0);
|
||
|
|
assert!(sample.recent_error_events.is_empty());
|
||
|
|
}
|
||
|
|
|
||
#[test]
|
|||
|
|
fn preserves_secure_tunnel_metadata_across_heartbeat_metadata_refresh() {
|
||
|
|
let previous = json!({
|
||
|
|
"version": "1.0.0",
|
||
|
|
"tunnel_security": {
|
||
|
|
"mode": "non_tls_required",
|
||
|
|
"encryption_key": "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc="
|
||
|
|
}
|
||
|
|
});
|
||
|
|
let next = json!({
|
||
|
|
"version": "1.0.1",
|
||
"tunnel_metrics": {"connect_successes": 1},
|
|||
|
|
"tunnel_security": {
|
||
|
|
"mode": "disabled",
|
||
|
|
"encryption_key": "attacker-controlled"
|
||
|
|
}
|
||
});
|
|||
|
|
|
||
|
|
let merged = preserve_proxy_metadata_tunnel_security(Some(&previous), Some(next))
|
||
|
|
.expect("metadata should remain present");
|
||
|
|
assert_eq!(
|
||
|
|
merged
|
||
|
|
.pointer("/tunnel_security/mode")
|
||
|
|
.and_then(|v| v.as_str()),
|
||
|
|
Some("non_tls_required")
|
||
|
|
);
|
||
|
|
assert_eq!(
|
||
|
|
merged
|
||
|
|
.pointer("/tunnel_security/encryption_key")
|
||
|
|
.and_then(|v| v.as_str()),
|
||
|
|
Some("BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc=")
|
||
|
|
);
|
||
|
|
assert_eq!(
|
||
|
|
merged.pointer("/tunnel_metrics/connect_successes"),
|
||
|
|
Some(&json!(1))
|
||
|
|
);
|
||
|
|
}
|
||
|
|
|
||
#[test]
|
|||
|
|
fn registration_metadata_preserves_omitted_tunnel_security() {
|
||
|
|
let previous = json!({
|
||
|
|
"version": "1.0.0",
|
||
|
|
"tunnel_security": {
|
||
|
|
"mode": "non_tls_required",
|
||
|
|
"encryption_key_encrypted": "aether-proxy-node-secret-v2:aether-runtime-secret-v1:sealed-old"
|
||
|
|
}
|
||
|
|
});
|
||
|
|
let next = json!({
|
||
|
|
"version": "1.1.0",
|
||
|
|
"tunnel_metrics": {"connect_successes": 2}
|
||
|
|
});
|
||
|
|
|
||
|
|
let merged = merge_proxy_metadata_for_registration(Some(&previous), Some(next))
|
||
|
|
.expect("registration metadata should remain present");
|
||
|
|
assert_eq!(merged.get("version"), Some(&json!("1.1.0")));
|
||
|
|
assert_eq!(
|
||
|
|
merged.pointer("/tunnel_security/encryption_key_encrypted"),
|
||
|
|
Some(&json!(
|
||
|
|
"aether-proxy-node-secret-v2:aether-runtime-secret-v1:sealed-old"
|
||
|
|
))
|
||
|
|
);
|
||
|
|
assert_eq!(
|
||
|
|
merged.pointer("/tunnel_metrics/connect_successes"),
|
||
|
|
Some(&json!(2))
|
||
|
|
);
|
||
|
|
}
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn registration_metadata_accepts_explicit_tunnel_security_rotation() {
|
||
|
|
let previous = json!({
|
||
|
|
"tunnel_security": {
|
||
|
|
"mode": "non_tls_required",
|
||
|
|
"encryption_key_encrypted": "aether-proxy-node-secret-v2:aether-runtime-secret-v1:sealed-old"
|
||
|
|
}
|
||
|
|
});
|
||
|
|
let next = json!({
|
||
|
|
"tunnel_security": {
|
||
|
|
"mode": "non_tls_required",
|
||
|
|
"encryption_key_encrypted": "aether-proxy-node-secret-v2:aether-runtime-secret-v1:sealed-new"
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
let merged = merge_proxy_metadata_for_registration(Some(&previous), Some(next))
|
||
|
|
.expect("rotated registration metadata should remain present");
|
||
|
|
assert_eq!(
|
||
|
|
merged.pointer("/tunnel_security/encryption_key_encrypted"),
|
||
|
|
Some(&json!(
|
||
|
|
"aether-proxy-node-secret-v2:aether-runtime-secret-v1:sealed-new"
|
||
|
|
))
|
||
|
|
);
|
||
|
|
}
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn registration_metadata_rejects_invalid_security_replacement() {
|
||
|
|
let previous = json!({
|
||
|
|
"tunnel_security": {
|
||
|
|
"mode": "non_tls_required",
|
||
|
|
"encryption_key_encrypted": "aether-proxy-node-secret-v2:aether-runtime-secret-v1:sealed-old"
|
||
|
|
}
|
||
|
|
});
|
||
|
|
let next = json!({
|
||
|
|
"version": "1.2.0",
|
||
|
|
"tunnel_security": null
|
||
|
|
});
|
||
|
|
|
||
|
|
let merged = merge_proxy_metadata_for_registration(Some(&previous), Some(next))
|
||
|
|
.expect("previous security should be retained");
|
||
|
|
assert_eq!(merged.get("version"), Some(&json!("1.2.0")));
|
||
|
|
assert_eq!(
|
||
|
|
merged.pointer("/tunnel_security/encryption_key_encrypted"),
|
||
|
|
Some(&json!(
|
||
|
|
"aether-proxy-node-secret-v2:aether-runtime-secret-v1:sealed-old"
|
||
|
|
))
|
||
|
|
);
|
||
|
|
}
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn registration_metadata_rejects_mode_only_security_downgrade() {
|
||
|
|
let previous = json!({
|
||
|
|
"tunnel_security": {
|
||
|
|
"mode": "non_tls_required",
|
||
|
|
"encryption_key_encrypted": "aether-proxy-node-secret-v2:aether-runtime-secret-v1:sealed-old"
|
||
|
|
}
|
||
|
|
});
|
||
|
|
let next = json!({
|
||
|
|
"version": "1.3.0",
|
||
|
|
"tunnel_security": {"mode": "disabled"}
|
||
|
|
});
|
||
|
|
|
||
|
|
let merged = merge_proxy_metadata_for_registration(Some(&previous), Some(next))
|
||
|
|
.expect("mode-only security must not replace the registered credential");
|
||
|
|
assert_eq!(merged.get("version"), Some(&json!("1.3.0")));
|
||
|
|
assert_eq!(
|
||
|
|
merged.pointer("/tunnel_security/encryption_key_encrypted"),
|
||
|
|
Some(&json!(
|
||
|
|
"aether-proxy-node-secret-v2:aether-runtime-secret-v1:sealed-old"
|
||
|
|
))
|
||
|
|
);
|
||
|
|
assert_eq!(
|
||
|
|
merged.pointer("/tunnel_security/mode"),
|
||
|
|
Some(&json!("non_tls_required"))
|
||
|
|
);
|
||
|
|
}
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn registration_metadata_rejects_bound_ciphertext_with_disabled_mode() {
|
||
|
|
let previous = json!({
|
||
|
|
"tunnel_security": {
|
||
|
|
"mode": "non_tls_required",
|
||
|
|
"encryption_key_encrypted": "aether-proxy-node-secret-v2:aether-runtime-secret-v1:sealed-old"
|
||
|
|
}
|
||
|
|
});
|
||
|
|
let next = json!({
|
||
|
|
"version": "1.3.1",
|
||
|
|
"tunnel_security": {
|
||
|
|
"mode": "disabled",
|
||
|
|
"encryption_key_encrypted": "aether-proxy-node-secret-v2:aether-runtime-secret-v1:sealed-attacker"
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
let merged = merge_proxy_metadata_for_registration(Some(&previous), Some(next))
|
||
|
|
.expect("disabled security must not replace the registered credential");
|
||
|
|
assert_eq!(merged.get("version"), Some(&json!("1.3.1")));
|
||
|
|
assert_eq!(
|
||
|
|
merged.pointer("/tunnel_security/encryption_key_encrypted"),
|
||
|
|
Some(&json!(
|
||
|
|
"aether-proxy-node-secret-v2:aether-runtime-secret-v1:sealed-old"
|
||
|
|
))
|
||
|
|
);
|
||
|
|
assert_eq!(
|
||
|
|
merged.pointer("/tunnel_security/mode"),
|
||
|
|
Some(&json!("non_tls_required"))
|
||
|
|
);
|
||
|
|
}
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn new_registration_drops_invalid_tunnel_security_fields() {
|
||
|
|
for invalid_security in [
|
||
|
|
json!(null),
|
||
|
|
json!("disabled"),
|
||
|
|
json!({}),
|
||
|
|
json!({"mode": "disabled"}),
|
||
|
|
json!({"mode": "disabled", "encryption_key_encrypted": "aether-proxy-node-secret-v2:aether-runtime-secret-v1:sealed-attacker"}),
|
||
|
|
json!({"mode": "non_tls_required", "encryption_key_encrypted": ""}),
|
||
|
|
json!({"mode": "non_tls_required", "encryption_key_encrypted": "not-gateway-bound"}),
|
||
|
|
] {
|
||
|
|
let next = json!({
|
||
|
|
"version": "1.4.0",
|
||
|
|
"tunnel_security": invalid_security
|
||
|
|
});
|
||
|
|
let merged = merge_proxy_metadata_for_registration(None, Some(next))
|
||
|
|
.expect("valid non-security metadata should remain");
|
||
|
|
assert_eq!(merged.get("version"), Some(&json!("1.4.0")));
|
||
|
|
assert!(merged.get("tunnel_security").is_none());
|
||
|
|
}
|
||
|
|
|
||
|
|
assert_eq!(
|
||
|
|
merge_proxy_metadata_for_registration(
|
||
|
|
None,
|
||
|
|
Some(json!({"tunnel_security": {"mode": "disabled"}})),
|
||
|
|
),
|
||
|
|
None
|
||
|
|
);
|
||
|
|
}
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn heartbeat_metadata_cannot_introduce_tunnel_security() {
|
||
|
|
let injected = json!({
|
||
|
|
"tunnel_security": {
|
||
|
|
"mode": "disabled",
|
||
|
|
"encryption_key": "attacker-controlled"
|
||
|
|
}
|
||
|
|
});
|
||
|
|
assert_eq!(
|
||
|
|
normalize_heartbeat_proxy_metadata(None, Some(&injected), None),
|
||
|
|
None
|
||
|
|
);
|
||
|
|
|
||
|
|
let injected_with_runtime_metadata = json!({
|
||
|
|
"version": "1.2.3",
|
||
|
|
"arch": "arm64",
|
||
|
|
"tunnel_security": {
|
||
|
|
"mode": "disabled",
|
||
|
|
"encryption_key": "attacker-controlled"
|
||
|
|
}
|
||
|
|
});
|
||
|
|
let normalized =
|
||
|
|
normalize_heartbeat_proxy_metadata(None, Some(&injected_with_runtime_metadata), None)
|
||
|
|
.expect("runtime metadata should remain present");
|
||
|
|
assert_eq!(normalized.get("version"), Some(&json!("1.2.3")));
|
||
|
|
assert_eq!(normalized.get("arch"), Some(&json!("arm64")));
|
||
|
|
assert!(normalized.get("tunnel_security").is_none());
|
||
|
|
}
|
||
|
|
|
||
#[test]
|
|||
|
|
fn maps_timestamps_to_metric_buckets() {
|
||
|
|
assert_eq!(
|
||
|
|
bucket_start_unix_secs(1_710_000_119, ProxyNodeMetricsStep::OneMinute),
|
||
|
|
1_710_000_060
|
||
|
|
);
|
||
|
|
assert_eq!(
|
||
|
|
bucket_start_unix_secs(1_710_003_999, ProxyNodeMetricsStep::OneHour),
|
||
|
|
1_710_003_600
|
||
|
|
);
|
||
|
|
}
|
||
}
|