feat(security): harden gateway boundaries and usage policies

Consolidate subscription usage policy enforcement, privacy-safe persistence, and gateway security hardening into one reviewable change. Includes bounded HTTP and execution envelopes, header and protocol guards, DNS and relay validation, authentication and secret projection hardening, secure backup/install paths, and regression coverage.
2026-10-08 18:37:46 +08:00 · 2026-09-04 03:45:52 +08:00
parent ddcbeb3ae9
commit 579f2c7cc1
1019 changed files with 190437 additions and 26080 deletions
@@ -13,7 +13,7 @@ use aether_data::repository::proxy_nodes::{
 use aether_data_contracts::repository::usage::{
    UsageCounterHealthSnapshot, UsageCounterPendingHealthSnapshot,
 };
-use aether_http::{build_http_client, HttpClientConfig};
+use aether_http::{apply_http_client_config, HttpClientConfig};
 use aether_runtime::{
    service_up_sample, AdmissionPermit, ConcurrencyGate, ConcurrencySnapshot, MetricKind,
    MetricLabel, MetricSample,
@@ -104,6 +104,24 @@ const USAGE_COUNTER_EXACT_HEALTH_METRICS_TTL: Duration = Duration::from_secs(5 *
 const USAGE_COUNTER_EXACT_HEALTH_METRICS_MAX_STALENESS: Duration = Duration::from_secs(10 * 60);
 const USAGE_COUNTER_EXACT_HEALTH_METRICS_RETRY_BACKOFF: Duration = Duration::from_secs(5);

+const ADMIN_USAGE_AGGREGATE_INVALID_INPUT_DETAIL: &str =
+    "usage aggregate import payload is invalid";
+
+fn admin_usage_aggregate_import_error(detail: String) -> GatewayError {
+    // Import validation errors can include source row IDs, table names, and
+    // adapter-specific details. Keep those details in process memory only;
+    // the public/admin response receives a stable client-safe message.
+    warn!(
+        event_name = "admin_usage_aggregate_import_rejected",
+        error_length = detail.len(),
+        "usage aggregate import input was rejected"
+    );
+    GatewayError::Client {
+        status: http::StatusCode::BAD_REQUEST,
+        message: ADMIN_USAGE_AGGREGATE_INVALID_INPUT_DETAIL.to_string(),
+    }
+}
+
 fn system_config_key_affects_scheduler(key: &str) -> bool {
    let key = key.trim();
    SCHEDULER_AFFECTING_SYSTEM_CONFIG_KEYS.contains(&key)
@@ -287,18 +305,38 @@ impl AppState {
            GatewayDataState::disabled()
                .with_usage_worker_queue(Self::usage_worker_queue_for(&runtime_state)),
        );
-        let client = build_http_client(&HttpClientConfig {
-            connect_timeout_ms: Some(10_000),
-            request_timeout_ms: Some(300_000),
-            http2_adaptive_window: true,
-            ..HttpClientConfig::default()
-        })?;
-        let owner_forward_client = build_http_client(&HttpClientConfig {
-            connect_timeout_ms: Some(10_000),
-            http2_adaptive_window: true,
-            ..HttpClientConfig::default()
-        })?;
+        let client = apply_http_client_config(
+            reqwest::Client::builder()
+                .no_proxy()
+                .redirect(reqwest::redirect::Policy::none()),
+            &HttpClientConfig {
+                connect_timeout_ms: Some(10_000),
+                request_timeout_ms: Some(300_000),
+                http2_adaptive_window: true,
+                ..HttpClientConfig::default()
+            },
+        )
+        .build()?;
+        let owner_forward_client = apply_http_client_config(
+            reqwest::Client::builder()
+                .no_proxy()
+                .redirect(reqwest::redirect::Policy::none()),
+            &HttpClientConfig {
+                connect_timeout_ms: Some(10_000),
+                http2_adaptive_window: true,
+                ..HttpClientConfig::default()
+            },
+        )
+        .build()?;
        let frontdoor_runtime_guards = Arc::new(FrontdoorRuntimeGuardConfig::from_env());
+        let internal_gateway_auth =
+            Arc::new(crate::internal_gateway_auth::InternalGatewayAuthConfig::for_process());
+        if internal_gateway_auth.status() == "misconfigured" {
+            warn!(
+                environment_variable = crate::internal_gateway_auth::INTERNAL_GATEWAY_AUTH_SECRET_ENV,
+                "internal gateway control plane is fail-closed because its authentication secret is invalid"
+            );
+        }
        Ok(Self {
            #[cfg(test)]
            execution_runtime_override_base_url: execution_runtime_override_base_url
@@ -310,6 +348,7 @@ impl AppState {
            background_data: Arc::clone(&data),
            background_data_isolated: false,
            runtime_state: runtime_state.clone(),
+            internal_gateway_auth,
            usage_runtime: Arc::new(usage::UsageRuntime::disabled()),
            video_tasks: Arc::new(VideoTaskService::new(
                VideoTaskTruthSourceMode::PythonSyncReport,
@@ -349,6 +388,7 @@ impl AppState {
            auth_api_key_force_capabilities_cache: Arc::new(JsonValueCache::default()),
            auth_api_key_feature_settings_cache: Arc::new(JsonValueCache::default()),
            auth_daily_quota_availability_cache: Arc::new(ValueCache::default()),
+            auth_plan_usage_policy_cache: Arc::new(ValueCache::default()),
            auth_wallet_snapshot_cache: Arc::new(ValueCache::default()),
            auth_request_cost_upper_bound_cache: Arc::new(ValueCache::default()),
            provider_quota_snapshot_cache: Arc::new(ValueCache::default()),
@@ -451,6 +491,10 @@ impl AppState {
        true
    }

+    pub(crate) fn internal_gateway_auth_status(&self) -> &'static str {
+        self.internal_gateway_auth.status()
+    }
+
    #[cfg(test)]
    pub(crate) fn execution_runtime_override_base_url(&self) -> Option<&str> {
        self.execution_runtime_override_base_url.as_deref()
@@ -755,6 +799,20 @@ impl AppState {
            .map_err(|err| GatewayError::Internal(err.to_string()))
    }

+    pub(crate) async fn compare_and_set_system_config_string_value(
+        &self,
+        key: &str,
+        expected: &str,
+        replacement: &str,
+    ) -> Result<bool, GatewayError> {
+        let result = self
+            .data
+            .compare_and_set_system_config_string_value(key, expected, replacement)
+            .await;
+        self.system_config_cache.invalidate(key);
+        result.map_err(|err| GatewayError::Internal(err.to_string()))
+    }
+
    async fn read_system_config_json_value_with_cache_windows(
        &self,
        key: &str,
@@ -945,6 +1003,7 @@ impl AppState {
        self.auth_api_key_force_capabilities_cache.clear();
        self.auth_api_key_feature_settings_cache.clear();
        self.auth_daily_quota_availability_cache.clear();
+        self.auth_plan_usage_policy_cache.clear();
        self.auth_wallet_snapshot_cache.clear();
        self.auth_request_cost_upper_bound_cache.clear();
        self.provider_quota_snapshot_cache.clear();
@@ -1030,10 +1089,9 @@ impl AppState {
            .import_admin_system_usage_aggregates(snapshot, user_id_map, api_key_id_map, mode)
            .await
            .map_err(|err| match err {
-                aether_data::DataLayerError::InvalidInput(detail) => GatewayError::Client {
-                    status: http::StatusCode::BAD_REQUEST,
-                    message: detail,
-                },
+                aether_data::DataLayerError::InvalidInput(detail) => {
+                    admin_usage_aggregate_import_error(detail)
+                }
                other => GatewayError::Internal(other.to_string()),
            })
    }
@@ -1124,18 +1182,27 @@ impl AppState {
        &self,
        mutation: &aether_data::repository::proxy_nodes::ProxyNodeRegistrationMutation,
    ) -> Result<Option<StoredProxyNode>, GatewayError> {
-        self.data
-            .register_proxy_node(mutation)
-            .await
-            .map_err(|err| GatewayError::Internal(err.to_string()))
+        self.register_proxy_node_with_bound_secrets(mutation).await
    }

    pub(crate) async fn create_manual_proxy_node(
        &self,
        mutation: &ProxyNodeManualCreateMutation,
    ) -> Result<Option<StoredProxyNode>, GatewayError> {
+        let mut protected = mutation.clone();
+        let node_id = mutation
+            .node_id
+            .as_deref()
+            .map(str::trim)
+            .filter(|value| !value.is_empty())
+            .map(ToOwned::to_owned)
+            .unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
+        protected.node_id = Some(node_id.clone());
+        if let Some(password) = mutation.proxy_password.as_deref() {
+            protected.proxy_password = Some(self.protect_proxy_node_password(&node_id, password)?);
+        }
        self.data
-            .create_manual_proxy_node(mutation)
+            .create_manual_proxy_node(&protected)
            .await
            .map_err(|err| GatewayError::Internal(err.to_string()))
    }
@@ -1144,8 +1211,17 @@ impl AppState {
        &self,
        mutation: &ProxyNodeManualUpdateMutation,
    ) -> Result<Option<StoredProxyNode>, GatewayError> {
+        let Some(existing) = self.find_proxy_node(&mutation.node_id).await? else {
+            return Ok(None);
+        };
+        let mut protected = mutation.clone();
+        protected.node_id = existing.id.clone();
+        if let Some(password) = mutation.proxy_password.as_deref() {
+            protected.proxy_password =
+                Some(self.protect_proxy_node_password(&existing.id, password)?);
+        }
        self.data
-            .update_manual_proxy_node(mutation)
+            .update_manual_proxy_node(&protected)
            .await
            .map_err(|err| GatewayError::Internal(err.to_string()))
    }
@@ -1178,6 +1254,9 @@ impl AppState {
        &self,
        mutation: &ProxyNodeHeartbeatMutation,
    ) -> Result<Option<StoredProxyNode>, GatewayError> {
+        crate::state::decrypt_or_migrate_proxy_tunnel_psk(&self.data, &mutation.node_id)
+            .await
+            .map_err(|err| GatewayError::Internal(err.to_string()))?;
        self.data
            .apply_proxy_node_heartbeat(mutation)
            .await
@@ -2015,9 +2094,16 @@ impl AppState {
        mut self,
        path: impl Into<std::path::PathBuf>,
    ) -> std::io::Result<Self> {
+        let encryption_key = self.data.encryption_key().ok_or_else(|| {
+            std::io::Error::new(
+                std::io::ErrorKind::InvalidInput,
+                "AETHER_GATEWAY_VIDEO_TASK_STORE_PATH requires a configured encryption key",
+            )
+        })?;
        self.video_tasks = Arc::new(VideoTaskService::with_file_store(
            self.video_tasks.truth_source_mode(),
            path,
+            encryption_key,
        )?);
        Ok(self)
    }
@@ -3843,7 +3929,8 @@ mod tests {
    use serde_json::json;

    use super::{
-        database_bounded_auth_load_limit, merge_usage_counter_health_snapshots,
+        admin_usage_aggregate_import_error, database_bounded_auth_load_limit,
+        merge_usage_counter_health_snapshots,
        usage_counter_pending_health_metric_samples_with_timeout,
        usage_queue_health_metric_samples_with_timeout, usage_runtime_metric_samples, AppState,
        MetricKind, MetricSample, METRIC_SNAPSHOT_TTL,
@@ -3852,6 +3939,23 @@ mod tests {
    use crate::cache::SchedulerAffinityTarget;
    use crate::data::{GatewayDataConfig, GatewayDataState};

+    #[test]
+    fn usage_aggregate_import_invalid_input_is_projected_to_a_safe_client_message() {
+        let error = admin_usage_aggregate_import_error(
+            "postgres table stats_user_daily row secret-user contains column password".to_string(),
+        );
+
+        match error {
+            super::GatewayError::Client { status, message } => {
+                assert_eq!(status, http::StatusCode::BAD_REQUEST);
+                assert_eq!(message, "usage aggregate import payload is invalid");
+                assert!(!message.contains("secret-user"));
+                assert!(!message.contains("password"));
+            }
+            other => panic!("expected client-safe import error, got {other:?}"),
+        }
+    }
+
    #[test]
    fn auth_load_gate_reserves_half_of_foreground_database_pool() {
        assert_eq!(