feat(security): harden gateway boundaries and usage policies

Consolidate subscription usage policy enforcement, privacy-safe persistence, and gateway security hardening into one reviewable change.

Includes bounded HTTP and execution envelopes, header and protocol guards, DNS and relay validation, authentication and secret projection hardening, secure backup/install paths, and regression coverage.

apps/aether-gateway/src/handlers/admin/observability/monitoring/cache_affinity_reads.rs

@@ -141,9 +141,8 @@ pub(super) async fn build_admin_monitoring_cache_affinities_response(
         let key = affinity.key_id.as_ref().and_then(|id| key_by_id.get(id));
 
         let user_api_key_name = user_api_key.and_then(|item| item.name.clone());
-        let user_api_key_prefix = user_api_key.and_then(|item| {
-            admin_monitoring_masked_user_api_key_prefix(state, item.key_encrypted.as_deref())
-        });
+        let user_api_key_prefix =
+            user_api_key.and_then(|item| admin_monitoring_masked_user_api_key_prefix(state, item));
         let provider_name = provider.map(|item| item.name.clone());
         let endpoint_url = endpoint
             .map(|item| item.base_url.clone())

apps/aether-gateway/src/handlers/admin/observability/monitoring/cache_payloads.rs

@@ -1,28 +1,16 @@
 use crate::handlers::admin::request::AdminAppState;
+use crate::handlers::shared::{masked_secret_display, open_auth_api_key_secret};
 use crate::provider_key_auth::{
     provider_key_auth_config_is_agent_identity, provider_key_auth_config_uses_header_authorization,
 };
-use aether_crypto::decrypt_python_fernet_ciphertext;
-#[cfg(test)]
-use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
 use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey;
 
 pub(super) fn admin_monitoring_masked_user_api_key_prefix(
     state: &AdminAppState<'_>,
-    ciphertext: Option<&str>,
+    record: &aether_data::repository::auth::StoredAuthApiKeyExportRecord,
 ) -> Option<String> {
-    let Some(ciphertext) = ciphertext.map(str::trim).filter(|value| !value.is_empty()) else {
-        return None;
-    };
-    let full_key = admin_monitoring_try_decrypt_secret(state, ciphertext)?;
-    let prefix_len = full_key.len().min(10);
-    let prefix = &full_key[..prefix_len];
-    let suffix = if full_key.len() >= 4 {
-        &full_key[full_key.len().saturating_sub(4)..]
-    } else {
-        ""
-    };
-    Some(format!("{prefix}...{suffix}"))
+    let projection = open_auth_api_key_secret(state.app(), record).ok()?;
+    Some(masked_secret_display(&projection.plaintext, 10, 4, "..."))
 }
 
 pub(super) fn admin_monitoring_masked_provider_key_prefix(
@@ -43,59 +31,16 @@ pub(super) fn admin_monitoring_masked_provider_key_prefix(
             }
         }
         _ => {
-            let full_key = key
-                .encrypted_api_key
-                .as_deref()
-                .and_then(|ciphertext| admin_monitoring_try_decrypt_secret(state, ciphertext))?;
-            if full_key.len() <= 12 {
-                Some(format!("{full_key}***"))
-            } else {
-                Some(format!(
-                    "{}***{}",
-                    &full_key[..8],
-                    &full_key[full_key.len().saturating_sub(4)..]
-                ))
-            }
+            let full_key = state
+                .app()
+                .decrypt_provider_catalog_key_api_key(key)
+                .ok()
+                .flatten()?;
+            Some(masked_secret_display(&full_key, 8, 4, "***"))
         }
     }
 }
 
-fn admin_monitoring_try_decrypt_secret(
-    state: &AdminAppState<'_>,
-    ciphertext: &str,
-) -> Option<String> {
-    let ciphertext = ciphertext.trim();
-    if ciphertext.is_empty() {
-        return None;
-    }
-    let encryption_key = state.encryption_key().map(str::trim).unwrap_or("");
-    if !encryption_key.is_empty() {
-        if let Ok(value) = decrypt_python_fernet_ciphertext(encryption_key, ciphertext) {
-            return Some(value);
-        }
-    }
-    for env_key in ["AETHER_GATEWAY_DATA_ENCRYPTION_KEY", "ENCRYPTION_KEY"] {
-        let Ok(candidate) = std::env::var(env_key) else {
-            continue;
-        };
-        let candidate = candidate.trim();
-        if candidate.is_empty() || candidate == encryption_key {
-            continue;
-        }
-        if let Ok(value) = decrypt_python_fernet_ciphertext(candidate, ciphertext) {
-            return Some(value);
-        }
-    }
-    #[cfg(test)]
-    if encryption_key != DEVELOPMENT_ENCRYPTION_KEY {
-        if let Ok(value) = decrypt_python_fernet_ciphertext(DEVELOPMENT_ENCRYPTION_KEY, ciphertext)
-        {
-            return Some(value);
-        }
-    }
-    None
-}
-
 pub(super) fn admin_monitoring_cache_affinity_sort_value(value: Option<&serde_json::Value>) -> f64 {
     let Some(value) = value else {
         return 0.0;
@@ -122,11 +67,14 @@ pub(super) fn admin_monitoring_cache_affinity_sort_value(value: Option<&serde_js
 
 #[cfg(test)]
 mod tests {
-    use super::admin_monitoring_masked_provider_key_prefix;
+    use super::{
+        admin_monitoring_masked_provider_key_prefix, admin_monitoring_masked_user_api_key_prefix,
+    };
     use crate::handlers::admin::request::AdminAppState;
     use crate::AppState;
     use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
     use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey;
+    use sha2::{Digest, Sha256};
 
     #[test]
     fn monitoring_labels_agent_identity_instead_of_oauth_token() {
@@ -167,4 +115,41 @@ mod tests {
             Some("[Agent Identity]")
         );
     }
+
+    #[test]
+    fn monitoring_never_exposes_complete_short_credentials() {
+        let app = AppState::new().expect("gateway should build");
+        let state = AdminAppState::new(&app);
+        let plaintext = "short-key";
+        let ciphertext = encrypt_python_fernet_plaintext(DEVELOPMENT_ENCRYPTION_KEY, plaintext)
+            .expect("secret should encrypt");
+        let mut hasher = Sha256::new();
+        hasher.update(plaintext.as_bytes());
+        let record = aether_data::repository::auth::StoredAuthApiKeyExportRecord::new(
+            "owner-1".to_string(),
+            "key-1".to_string(),
+            format!("{:x}", hasher.finalize()),
+            Some(ciphertext),
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            true,
+            None,
+            false,
+            0,
+            0,
+            0.0,
+            false,
+        )
+        .expect("API-key record should build");
+
+        let masked = admin_monitoring_masked_user_api_key_prefix(&state, &record)
+            .expect("secret should decrypt");
+        assert_ne!(masked, plaintext);
+        assert!(!masked.contains(plaintext));
+    }
 }

apps/aether-gateway/src/handlers/admin/observability/monitoring/resilience/snapshot.rs

@@ -218,7 +218,12 @@ pub(super) async fn build_admin_monitoring_resilience_snapshot(
                     "model": item.model,
                     "api_format": item.api_format,
                     "status_code": item.status_code,
-                    "error_message": item.error_message,
+                    "error_message": item
+                        .error_category
+                        .as_deref()
+                        .map(str::trim)
+                        .filter(|value| !value.is_empty())
+                        .unwrap_or("request_failed"),
                 }
             })
         })

apps/aether-gateway/src/handlers/admin/observability/monitoring/tests/mod.rs

@@ -3,6 +3,7 @@ use super::test_support::*;
 use crate::control::GatewayPublicRequestContext;
 use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
 use crate::AppState;
+use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
 use aether_data_contracts::repository::{
     candidates::{RequestCandidateStatus, StoredRequestCandidate},
     provider_catalog::{
@@ -119,9 +120,12 @@ async fn admin_monitoring_cache_affinities_and_affinity_return_local_payload_fro
     let state = AppState::new()
         .expect("state should build")
         .with_data_state_for_tests(
-            crate::data::GatewayDataState::with_provider_catalog_reader_for_tests(provider_catalog)
-                .with_user_reader(user_repository)
-                .with_auth_api_key_reader(auth_repository),
+            crate::data::GatewayDataState::with_provider_catalog_repository_for_tests(
+                provider_catalog,
+            )
+            .with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY)
+            .with_user_reader(user_repository)
+            .with_auth_api_key_reader(auth_repository),
         )
         .with_admin_monitoring_cache_affinity_entry_for_tests(
             "cache_affinity:user-key-1:openai:model-alpha",
@@ -228,9 +232,12 @@ async fn admin_monitoring_cache_affinities_and_delete_use_runtime_scheduler_affi
     let state = AppState::new()
         .expect("state should build")
         .with_data_state_for_tests(
-            crate::data::GatewayDataState::with_provider_catalog_reader_for_tests(provider_catalog)
-                .with_user_reader(user_repository)
-                .with_auth_api_key_reader(auth_repository),
+            crate::data::GatewayDataState::with_provider_catalog_repository_for_tests(
+                provider_catalog,
+            )
+            .with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY)
+            .with_user_reader(user_repository)
+            .with_auth_api_key_reader(auth_repository),
         );
     let affinity_cache_key =
         aether_scheduler_core::build_scheduler_affinity_cache_key_for_api_key_id(
@@ -358,9 +365,12 @@ async fn admin_monitoring_cache_affinities_parse_session_scoped_scheduler_affini
     let state = AppState::new()
         .expect("state should build")
         .with_data_state_for_tests(
-            crate::data::GatewayDataState::with_provider_catalog_reader_for_tests(provider_catalog)
-                .with_user_reader(user_repository)
-                .with_auth_api_key_reader(auth_repository),
+            crate::data::GatewayDataState::with_provider_catalog_repository_for_tests(
+                provider_catalog,
+            )
+            .with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY)
+            .with_user_reader(user_repository)
+            .with_auth_api_key_reader(auth_repository),
         );
     let client_session = aether_scheduler_core::ClientSessionAffinity::new(
         Some("Codex".to_string()),

apps/aether-gateway/src/handlers/admin/observability/monitoring/tests/trace.rs

@@ -69,7 +69,7 @@ async fn admin_monitoring_trace_request_returns_local_payload() {
     assert_eq!(payload["candidates"][0]["provider_name"], json!("OpenAI"));
     assert_eq!(
         payload["candidates"][0]["provider_website"],
-        json!("https://openai.com")
+        json!("https://openai.com/")
     );
     assert_eq!(
         payload["candidates"][0]["endpoint_name"],
@@ -292,10 +292,10 @@ async fn admin_monitoring_trace_request_falls_back_to_usage_routing_snapshot() {
         payload["candidates"][0]["extra_data"]["execution_path"],
         json!("local_execution_runtime_miss")
     );
-    assert_eq!(
-        payload["candidates"][0]["extra_data"]["failure_diagnostic"]["path"],
-        json!("$.reasoning.summary")
-    );
+    assert!(payload["candidates"][0]["error_message"].is_null());
+    assert!(payload["candidates"][0]["extra_data"]
+        .get("failure_diagnostic")
+        .is_none());
 }
 
 #[tokio::test]
@@ -340,11 +340,9 @@ async fn admin_monitoring_trace_request_returns_oauth_account_label_from_auth_co
         vec![sample_endpoint()],
         vec![oauth_key],
     ));
-    let data_state = GatewayDataState::with_decision_trace_readers_for_tests(
-        request_candidates,
-        provider_catalog,
-    )
-    .with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY);
+    let data_state = GatewayDataState::with_request_candidate_reader_for_tests(request_candidates)
+        .attach_provider_catalog_repository_for_tests(provider_catalog)
+        .with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY);
     let state = AppState::new()
         .expect("state should build")
         .with_data_state_for_tests(data_state);
@@ -615,7 +613,7 @@ async fn admin_monitoring_trace_request_exposes_request_path_from_usage_audit()
     usage.candidate_id = Some("cand-used".to_string());
     usage.request_metadata = Some(json!({
         "request_path": "/v1beta/models/gemini-2.5-pro:generateContent",
-        "request_query_string": "alt=sse"
+        "request_query_string": "alt=sse&key=gemini-secret&access_token=oauth-secret"
     }));
     let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(vec![usage]));
     let data_state =
@@ -649,14 +647,19 @@ async fn admin_monitoring_trace_request_exposes_request_path_from_usage_audit()
         payload["request_path_and_query"],
         json!("/v1beta/models/gemini-2.5-pro:generateContent?alt=sse")
     );
-    assert_eq!(
-        payload["candidates"][0]["extra_data"]["request_path_and_query"],
-        json!("/v1beta/models/gemini-2.5-pro:generateContent?alt=sse")
-    );
+    assert!(payload["candidates"][0]["extra_data"]
+        .get("request_path")
+        .is_none());
+    assert!(payload["candidates"][0]["extra_data"]
+        .get("request_query_string")
+        .is_none());
+    assert!(payload["candidates"][0]["extra_data"]
+        .get("request_path_and_query")
+        .is_none());
 }
 
 #[tokio::test]
-async fn admin_monitoring_trace_request_exposes_failed_candidate_upstream_response_boundary() {
+async fn admin_monitoring_trace_request_redacts_failed_candidate_response_payloads() {
     let mut candidate = sample_candidate(
         "cand-used",
         "request-1",
@@ -739,19 +742,18 @@ async fn admin_monitoring_trace_request_exposes_failed_candidate_upstream_respon
     let extra = &payload["candidates"][0]["extra_data"];
     assert_eq!(extra["upstream_response"]["status_code"], json!(302));
     assert_eq!(
-        extra["upstream_response"]["headers"]["location"],
-        json!("/")
-    );
-    assert_eq!(
-        extra["upstream_response"]["body"]["error"]["message"],
-        json!("redirect blocked")
+        extra["upstream_response"]["source"],
+        json!("upstream_response")
     );
+    assert!(extra["upstream_response"].get("headers").is_none());
+    assert!(extra["upstream_response"].get("body").is_none());
+    assert!(extra["upstream_response"].get("body_ref").is_none());
     assert!(extra.get("client_response").is_none());
     assert!(extra.get("provider_response").is_none());
 }
 
 #[tokio::test]
-async fn admin_monitoring_trace_request_prefers_ref_backed_usage_response_body() {
+async fn admin_monitoring_trace_request_does_not_hydrate_ref_backed_usage_response_body() {
     let mut candidate = sample_candidate(
         "cand-used",
         "request-ref-body",
@@ -832,30 +834,16 @@ async fn admin_monitoring_trace_request_prefers_ref_backed_usage_response_body()
         .expect("body should read");
     let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
     let upstream_response = &payload["candidates"][0]["extra_data"]["upstream_response"];
-    assert_eq!(
-        upstream_response["headers"],
-        json!({
-            "content-type": "application/json",
-            "x-request-id": "req_usage-cyber-risk-demo"
-        })
-    );
-    assert_eq!(
-        upstream_response["body"]["error"],
-        json!({
-            "type": "invalid_request",
-            "message": "This content was flagged for possible cybersecurity risk.",
-            "code": 400
-        })
-    );
-    assert!(upstream_response["body"].get("input").is_none());
-    assert_eq!(
-        upstream_response["body_ref"],
-        json!("usage://request/request-ref-body/response_body")
-    );
+    assert_eq!(upstream_response["status_code"], json!(400));
+    assert_eq!(upstream_response["source"], json!("upstream_response"));
+    assert_eq!(upstream_response["body_state"], json!("reference"));
+    assert!(upstream_response.get("headers").is_none());
+    assert!(upstream_response.get("body").is_none());
+    assert!(upstream_response.get("body_ref").is_none());
 }
 
 #[tokio::test]
-async fn admin_monitoring_trace_request_decodes_connect_json_response_body_refs() {
+async fn admin_monitoring_trace_request_does_not_expose_inline_connect_json_response_body() {
     let mut candidate = sample_candidate(
         "cand-used",
         "request-connect",
@@ -922,19 +910,11 @@ async fn admin_monitoring_trace_request_decodes_connect_json_response_body_refs(
     let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
     let upstream_response = &payload["candidates"][0]["extra_data"]["upstream_response"];
     assert_eq!(upstream_response["status_code"], json!(429));
-    assert_eq!(
-        upstream_response["body"]["error"]["code"],
-        json!("resource_exhausted")
-    );
-    assert_eq!(
-        upstream_response["body"]["error"]["message"],
-        json!("quota exhausted")
-    );
-    assert_eq!(
-        upstream_response["body_ref"],
-        json!("usage://request/request-connect/response_body")
-    );
+    assert_eq!(upstream_response["source"], json!("upstream_response"));
     assert_eq!(upstream_response["body_state"], json!("inline"));
+    assert!(upstream_response.get("headers").is_none());
+    assert!(upstream_response.get("body").is_none());
+    assert!(upstream_response.get("body_ref").is_none());
 }
 
 #[tokio::test]

apps/aether-gateway/src/handlers/admin/observability/monitoring/trace.rs

@@ -11,8 +11,9 @@ use aether_admin::observability::monitoring::{
 };
 use aether_data_contracts::repository::{
     candidates::{
-        DecisionTrace, DecisionTraceCandidate, RequestCandidateFinalStatus, RequestCandidateStatus,
-        StoredRequestCandidate,
+        sanitize_request_candidate_error_type, sanitize_request_candidate_extra_data,
+        sanitize_request_candidate_skip_reason, DecisionTrace, DecisionTraceCandidate,
+        RequestCandidateFinalStatus, RequestCandidateStatus, StoredRequestCandidate,
     },
     provider_catalog::StoredProviderCatalogKey,
     usage::StoredRequestUsageAudit,
@@ -30,25 +31,6 @@ struct ResolvedAdminMonitoringTrace {
     usage: Option<StoredRequestUsageAudit>,
 }
 
-async fn hydrate_admin_monitoring_trace_response_body(
-    state: &AdminAppState<'_>,
-    mut usage: StoredRequestUsageAudit,
-) -> Result<StoredRequestUsageAudit, GatewayError> {
-    let is_error_node = !usage.status.eq_ignore_ascii_case("completed")
-        || usage
-            .status_code
-            .is_some_and(|status| !(200..300).contains(&status));
-    let response_body_ref = if is_error_node && usage.response_body.is_none() {
-        usage.response_body_ref.clone()
-    } else {
-        None
-    };
-    if let Some(body_ref) = response_body_ref.as_deref() {
-        usage.response_body = state.resolve_request_usage_body_ref(body_ref).await?;
-    }
-    Ok(usage)
-}
-
 pub(super) async fn build_admin_monitoring_trace_request_response(
     state: &AdminAppState<'_>,
     request_context: &AdminRequestContext<'_>,
@@ -111,10 +93,6 @@ async fn resolve_admin_monitoring_trace(
             .read_request_usage_audit_shallow(request_id)
             .await
             .map_err(|err| GatewayError::Internal(err.to_string()))?;
-        let usage = match usage {
-            Some(usage) => Some(hydrate_admin_monitoring_trace_response_body(state, usage).await?),
-            None => None,
-        };
         return Ok(Some(ResolvedAdminMonitoringTrace { trace, usage }));
     }
 
@@ -125,10 +103,9 @@ async fn resolve_admin_monitoring_trace(
         .await
         .map_err(|err| GatewayError::Internal(err.to_string()))?
     {
-        usage_candidates.push(hydrate_admin_monitoring_trace_response_body(state, usage).await?);
+        usage_candidates.push(usage);
     }
     if let Some(usage) = state.find_request_usage_by_id(request_id).await? {
-        let usage = hydrate_admin_monitoring_trace_response_body(state, usage).await?;
         if !usage_candidates.iter().any(|item| item.id == usage.id) {
             usage_candidates.push(usage);
         }
@@ -203,17 +180,23 @@ fn build_admin_monitoring_usage_routing_snapshot_trace(
         endpoint_id: usage.provider_endpoint_id.clone(),
         key_id: usage.provider_api_key_id.clone(),
         status,
-        skip_reason: usage.routing_candidate_skip_reason().map(ToOwned::to_owned),
+        skip_reason: sanitize_request_candidate_skip_reason(
+            usage.routing_candidate_skip_reason().map(ToOwned::to_owned),
+        ),
         is_cached: false,
         status_code: usage.status_code,
-        error_type: usage
-            .routing_local_execution_runtime_miss_reason()
-            .or(usage.error_category.as_deref())
-            .map(ToOwned::to_owned),
-        error_message: usage.error_message.clone(),
+        error_type: sanitize_request_candidate_error_type(
+            usage
+                .routing_local_execution_runtime_miss_reason()
+                .or(usage.error_category.as_deref())
+                .map(ToOwned::to_owned),
+        ),
+        error_message: None,
         latency_ms: usage.response_time_ms,
         concurrent_requests: None,
-        extra_data: build_admin_monitoring_usage_routing_snapshot_extra_data(usage),
+        extra_data: sanitize_request_candidate_extra_data(
+            build_admin_monitoring_usage_routing_snapshot_extra_data(usage),
+        ),
         required_capabilities: None,
         created_at_unix_ms: usage.created_at_unix_ms,
         started_at_unix_ms: Some(usage.created_at_unix_ms),
@@ -485,8 +468,12 @@ fn parse_admin_monitoring_key_auth_config(
     state: &AdminAppState<'_>,
     key: &StoredProviderCatalogKey,
 ) -> Option<Map<String, Value>> {
-    let ciphertext = key.encrypted_auth_config.as_deref()?;
-    let plaintext = state.decrypt_catalog_secret_with_fallbacks(ciphertext)?;
+    let _ciphertext = key.encrypted_auth_config.as_deref()?;
+    let plaintext = state
+        .app()
+        .decrypt_provider_catalog_key_auth_config(key)
+        .ok()
+        .flatten()?;
     serde_json::from_str::<Value>(&plaintext)
         .ok()?
         .as_object()