feat(security): harden gateway boundaries and usage policies

Consolidate subscription usage policy enforcement, privacy-safe persistence, and gateway security hardening into one reviewable change.

Includes bounded HTTP and execution envelopes, header and protocol guards, DNS and relay validation, authentication and secret projection hardening, secure backup/install paths, and regression coverage.

apps/aether-gateway/src/execution_runtime/sync/execution.rs

@@ -60,8 +60,8 @@ use crate::execution_runtime::transport::{
     build_request_body, collect_response_headers, decode_response_body_bytes_with_limit,
     execution_plan_response_body_limit_bytes, execution_response_body_mode,
     format_hyper_error_chain, format_upstream_request_error, format_wreq_upstream_request_error,
-    response_body_is_json, send_request, DirectHttpResponse, DirectSyncExecutionRuntime,
-    ExecutionRuntimeTransportError,
+    response_body_is_json, safe_transport_error_message, send_request, DirectHttpResponse,
+    DirectSyncExecutionRuntime, ExecutionRuntimeTransportError,
 };
 use crate::execution_runtime::windsurf::maybe_execute_windsurf_sync;
 use crate::execution_runtime::{
@@ -101,7 +101,7 @@ mod policy;
 #[path = "execution/response.rs"]
 mod response;
 
-use policy::decode_execution_result_body;
+use policy::{decode_execution_result_body, decode_execution_result_body_with_limit};
 pub(crate) use response::{
     maybe_build_local_sync_finalize_response, maybe_build_local_video_error_response,
     maybe_build_local_video_success_outcome, resolve_local_sync_error_background_report_kind,
@@ -115,6 +115,11 @@ const SYNC_EXECUTION_IDLE_LOG_INTERVAL: Duration = Duration::from_secs(60);
 const OPENAI_IMAGE_SYNC_JSON_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(15);
 const OPENAI_IMAGE_SYNC_JSON_HEARTBEAT_BYTES: &[u8] = b"\n";
 const OPENAI_IMAGE_SYNC_PROGRESS_WRITE_INTERVAL: Duration = Duration::from_secs(5);
+// Progress parsing must retain only the incomplete SSE record. This is not a
+// response-body limit: the full upstream body is still handled by the normal
+// execution body policy, while malformed streams cannot grow telemetry state
+// forever by withholding a record separator.
+const OPENAI_IMAGE_SYNC_PROGRESS_MAX_BUFFER_BYTES: usize = 16 * 1024 * 1024;
 const INVALID_GEMINI_PROVIDER_SUCCESS_MESSAGE: &str = "Provider returned HTTP 200 but the Gemini response did not contain visible model output; refusing to finalize it as a successful response.";
 
 fn elapsed_ms_since(started_at: Instant) -> u64 {
@@ -174,6 +179,39 @@ struct SyncAttemptTerminalGuard {
     armed: bool,
 }
 
+/// Keep forced-terminal records useful for operations without copying an
+/// arbitrary `GatewayError` into the candidate/usage stores.  Gateway errors
+/// can wrap provider URLs, credentials, query strings, or database details;
+/// those values belong in the internal logging path only.
+fn persisted_sync_abort_message(error: &GatewayError) -> &'static str {
+    match error {
+        GatewayError::UpstreamUnavailable { .. } => {
+            "local sync attempt aborted before terminal finalization: upstream unavailable"
+        }
+        GatewayError::ControlUnavailable { .. } => {
+            "local sync attempt aborted before terminal finalization: control unavailable"
+        }
+        GatewayError::LocalExecutionPlanningTimeout { .. } => {
+            "local sync attempt aborted before terminal finalization: planning timeout"
+        }
+        GatewayError::AdmissionTimeout { .. } => {
+            "local sync attempt aborted before terminal finalization: admission timeout"
+        }
+        GatewayError::Client { .. } => {
+            "local sync attempt aborted before terminal finalization: client error"
+        }
+        GatewayError::PlanUsageLimited(_) => {
+            "local sync attempt aborted before terminal finalization: usage limit"
+        }
+        GatewayError::LastActiveAdminUpdateDenied | GatewayError::LastActiveAdminDeleteDenied => {
+            "local sync attempt aborted before terminal finalization: policy denied"
+        }
+        GatewayError::Internal(_) => {
+            "local sync attempt aborted before terminal finalization: internal error"
+        }
+    }
+}
+
 impl SyncAttemptTerminalGuard {
     fn new(
         state: &AppState,
@@ -213,7 +251,7 @@ impl SyncAttemptTerminalGuard {
             RequestCandidateStatus::Failed,
             StatusCode::INTERNAL_SERVER_ERROR.as_u16(),
             "local_sync_attempt_aborted",
-            format!("Local sync attempt failed before terminal finalization: {error:?}"),
+            persisted_sync_abort_message(error),
         )
         .await;
     }
@@ -346,7 +384,7 @@ impl SyncExecutionFailure {
             error_type: fallback_kind
                 .map(SyncExecutionFailureFallbackKind::error_type)
                 .unwrap_or("execution_runtime_unavailable"),
-            message: err.to_string(),
+            message: safe_transport_error_message(&err),
             status_code: fallback_kind.map(|_| StatusCode::BAD_GATEWAY.as_u16()),
             latency_ms: None,
             fallback_kind,
@@ -1132,10 +1170,42 @@ impl<'a> OpenAiImageSyncProgressRecorder<'a> {
         if chunk.is_empty() {
             return;
         }
-        self.buffer.extend_from_slice(chunk);
+        let mut blocks = Vec::new();
+        let mut remaining = chunk;
+        let mut parser_overflowed = false;
+        loop {
+            while let Some(block_end) = find_sse_block_end(&self.buffer) {
+                blocks.push(self.buffer.drain(..block_end).collect::<Vec<_>>());
+            }
+            if remaining.is_empty() {
+                break;
+            }
+
+            let capacity =
+                OPENAI_IMAGE_SYNC_PROGRESS_MAX_BUFFER_BYTES.saturating_sub(self.buffer.len());
+            if capacity == 0 {
+                // The progress recorder is observational. Drop an incomplete
+                // oversized record and keep the client-facing response alive.
+                self.buffer.clear();
+                parser_overflowed = true;
+                break;
+            }
+            let take = find_sse_block_end(remaining)
+                .map_or(remaining.len(), |block_end| block_end)
+                .min(capacity);
+            if take == 0 {
+                self.buffer.clear();
+                parser_overflowed = true;
+                break;
+            }
+            self.buffer.extend_from_slice(&remaining[..take]);
+            remaining = &remaining[take..];
+        }
+        if parser_overflowed {
+            debug!("openai image sync progress parser dropped an oversized incomplete SSE record");
+        }
         let mut force_persist = false;
-        while let Some(block_end) = find_sse_block_end(&self.buffer) {
-            let block = self.buffer.drain(..block_end).collect::<Vec<_>>();
+        for block in blocks {
             let Some(frame) = parse_openai_image_sync_sse_frame(&block) else {
                 continue;
             };
@@ -1813,12 +1883,16 @@ async fn openai_image_sync_json_heartbeat_final_bytes(
         {
             Ok(bytes) if !bytes.is_empty() => bytes.to_vec(),
             Ok(_) => openai_image_sync_json_heartbeat_error_body("empty sync image response"),
-            Err(err) => openai_image_sync_json_heartbeat_error_body(&err.to_string()),
+            Err(_err) => {
+                openai_image_sync_json_heartbeat_error_body("sync image response read failed")
+            }
         },
         Ok(None) => openai_image_sync_json_heartbeat_error_body(
             "sync image execution ended without a local response",
         ),
-        Err(err) => openai_image_sync_json_heartbeat_error_body(&format!("{err:?}")),
+        // Do not serialize the internal error: Debug output can contain
+        // upstream URLs, credentials, or other request-specific details.
+        Err(_err) => openai_image_sync_json_heartbeat_error_body("sync image execution failed"),
     }
 }
 
@@ -1840,7 +1914,7 @@ async fn apply_sync_success_effects(
 ) {
     if let Some(report_context) = report_context {
         crate::ai_serving::persist_converted_response_history(
-            state.runtime_state(),
+            state,
             report_context,
             payload
                 .client_body_json
@@ -2155,7 +2229,7 @@ async fn execute_execution_runtime_sync_impl(
                         }
                     },
                     Err(err) => {
-                        let transport_error_message = err.to_string();
+                        let transport_error_message = safe_transport_error_message(&err);
                         warn!(
                             event_name = "chatgpt_web_image_execution_unavailable",
                             log_type = "ops",
@@ -2167,7 +2241,7 @@ async fn execute_execution_runtime_sync_impl(
                             key_id,
                             model_name,
                             candidate_index = candidate_index.as_str(),
-                            error = %err,
+                            error = %transport_error_message,
                             "gateway ChatGPT-Web image execution unavailable"
                         );
                         let terminal_unix_secs = current_request_candidate_unix_ms();
@@ -2207,7 +2281,7 @@ async fn execute_execution_runtime_sync_impl(
                 }
             }
             Err(err) => {
-                let transport_error_message = err.to_string();
+                let transport_error_message = safe_transport_error_message(&err);
                 warn!(
                     event_name = "grok_execution_unavailable",
                     log_type = "ops",
@@ -2219,7 +2293,7 @@ async fn execute_execution_runtime_sync_impl(
                     key_id,
                     model_name,
                     candidate_index = candidate_index.as_str(),
-                    error = %err,
+                    error = %transport_error_message,
                     "gateway Grok execution unavailable"
                 );
                 let terminal_unix_secs = current_request_candidate_unix_ms();
@@ -2262,7 +2336,7 @@ async fn execute_execution_runtime_sync_impl(
             match (override_fn.0)(&plan) {
                 Ok(result) => result,
                 Err(err) => {
-                    let transport_error_message = format!("{err:?}");
+                    let transport_error_message = persisted_sync_abort_message(&err).to_string();
                     warn!(
                         event_name = "sync_execution_runtime_test_override_failed",
                         log_type = "ops",
@@ -2408,7 +2482,7 @@ async fn execute_execution_runtime_sync_impl(
                         }
                     },
                     Err(err) => {
-                        let transport_error_message = err.to_string();
+                        let transport_error_message = safe_transport_error_message(&err);
                         warn!(
                             event_name = "chatgpt_web_image_execution_unavailable",
                             log_type = "ops",
@@ -2420,7 +2494,7 @@ async fn execute_execution_runtime_sync_impl(
                             key_id,
                             model_name,
                             candidate_index = candidate_index.as_str(),
-                            error = %err,
+                            error = %transport_error_message,
                             "gateway ChatGPT-Web image execution unavailable"
                         );
                         let terminal_unix_secs = current_request_candidate_unix_ms();
@@ -2459,7 +2533,7 @@ async fn execute_execution_runtime_sync_impl(
                     }
                 },
                 Err(err) => {
-                    let transport_error_message = err.to_string();
+                    let transport_error_message = safe_transport_error_message(&err);
                     warn!(
                         event_name = "grok_execution_unavailable",
                         log_type = "ops",
@@ -2471,7 +2545,7 @@ async fn execute_execution_runtime_sync_impl(
                         key_id,
                         model_name,
                         candidate_index = candidate_index.as_str(),
-                        error = %err,
+                        error = %transport_error_message,
                         "gateway Grok execution unavailable"
                     );
                     let terminal_unix_secs = current_request_candidate_unix_ms();
@@ -2572,8 +2646,26 @@ async fn execute_execution_runtime_sync_impl(
             .as_ref()
             .and_then(|telemetry| telemetry.elapsed_ms);
         let mut headers = std::mem::take(&mut result.headers);
-        let (body_bytes, mut body_json, body_base64) =
-            decode_execution_result_body(result.body.take(), &mut headers)?;
+        let result_body = result.body.take();
+        let chatgpt_web_image_result = plan
+            .provider_api_format
+            .eq_ignore_ascii_case("openai:image")
+            && (plan.headers.iter().any(|(name, value)| {
+                name.eq_ignore_ascii_case("x-aether-chatgpt-web-image") && value == "1"
+            }) || report_context
+                .as_ref()
+                .and_then(|value| value.get("chatgpt_web_image"))
+                .and_then(Value::as_bool)
+                .unwrap_or(false));
+        let (body_bytes, mut body_json, body_base64) = if chatgpt_web_image_result {
+            decode_execution_result_body_with_limit(
+                result_body,
+                &mut headers,
+                crate::execution_runtime::chatgpt_web_image::chatgpt_web_image_sse_envelope_limit_bytes(),
+            )?
+        } else {
+            decode_execution_result_body(result_body, &mut headers)?
+        };
         if let Some(message) = invalid_gemini_provider_success_message(
             &plan,
             report_context.as_ref(),
@@ -3382,7 +3474,7 @@ async fn execute_sync_via_remote_execution_runtime(
                     status: RequestCandidateStatus::Failed,
                     status_code: None,
                     error_type: Some("execution_runtime_unavailable".to_string()),
-                    error_message: Some(format!("{err:?}")),
+                    error_message: Some(persisted_sync_abort_message(&err).to_string()),
                     latency_ms: Some(elapsed_ms_since(candidate_started_at)),
                     started_at_unix_ms: Some(candidate_started_unix_secs),
                     finished_at_unix_ms: Some(terminal_unix_secs),
@@ -3423,9 +3515,15 @@ async fn execute_sync_via_remote_execution_runtime(
     }
 
     let remote_response_observed_at_unix_ms = current_request_candidate_unix_ms();
-    let mut result = response
-        .json::<ExecutionResult>()
-        .await
+    let response_body = aether_http::read_response_bytes_with_limit(
+        response,
+        crate::execution_runtime::transport::execution_result_envelope_limit_bytes(
+            crate::headers::max_internal_buffered_body_bytes(),
+        ),
+    )
+    .await
+    .map_err(|err| GatewayError::Internal(err.to_string()))?;
+    let mut result = serde_json::from_slice::<ExecutionResult>(&response_body)
         .map_err(|err| GatewayError::Internal(err.to_string()))?;
     result
         .response_observation
@@ -3893,6 +3991,20 @@ mod tests {
         assert!(message.is_none());
     }
 
+    #[test]
+    fn forced_sync_abort_message_does_not_copy_internal_error_details() {
+        let secret = "https://user:[email protected]/v1?api_key=should-not-persist";
+        let error = GatewayError::Internal(secret.to_string());
+
+        let message = persisted_sync_abort_message(&error);
+        assert_eq!(
+            message,
+            "local sync attempt aborted before terminal finalization: internal error"
+        );
+        assert!(!message.contains("password"));
+        assert!(!message.contains("api_key"));
+    }
+
     #[tokio::test]
     async fn sync_attempt_terminal_guard_marks_dropped_pending_attempt_cancelled() {
         let usage_repository = Arc::new(InMemoryUsageReadRepository::default());
@@ -4482,4 +4594,22 @@ mod tests {
             Some(&report_context),
         ));
     }
+
+    #[tokio::test]
+    async fn openai_image_sync_json_heartbeat_hides_internal_error_details() {
+        let secret = "https://user:[email protected]/private?api_key=top-secret";
+        let bytes = openai_image_sync_json_heartbeat_final_bytes(Err(GatewayError::Internal(
+            secret.to_string(),
+        )))
+        .await;
+
+        let body: Value = serde_json::from_slice(&bytes).expect("heartbeat error body is JSON");
+        assert_eq!(
+            body.pointer("/error/message").and_then(Value::as_str),
+            Some("sync image execution failed")
+        );
+        let body_text = String::from_utf8(bytes).expect("heartbeat body is UTF-8");
+        assert!(!body_text.contains(secret));
+        assert!(!body_text.contains("top-secret"));
+    }
 }