fix(observability): preserve admin upstream error diagnostics

apps/aether-gateway/src/data/state/catalog.rs

@@ -17,13 +17,13 @@ fn sanitize_request_candidate_rows(
     mut candidates: Vec<StoredRequestCandidate>,
 ) -> Vec<StoredRequestCandidate> {
     for candidate in &mut candidates {
-        candidate.sanitize_sensitive_diagnostics();
+        candidate.sanitize_for_persistence();
     }
     candidates
 }
 
 fn sanitize_request_candidate_row(mut candidate: StoredRequestCandidate) -> StoredRequestCandidate {
-    candidate.sanitize_sensitive_diagnostics();
+    candidate.sanitize_for_persistence();
     candidate
 }
 
@@ -1048,7 +1048,10 @@ mod request_candidate_security_tests {
     fn assert_candidate_is_sanitized(candidate: &StoredRequestCandidate) {
         assert_eq!(candidate.skip_reason.as_deref(), Some("unclassified_skip"));
         assert_eq!(candidate.error_type.as_deref(), Some("unclassified_error"));
-        assert!(candidate.error_message.is_none());
+        assert_eq!(
+            candidate.error_message.as_deref(),
+            Some("Bearer candidate-secret")
+        );
         assert_eq!(
             candidate.extra_data,
             Some(json!({"gateway_execution_runtime": true}))
@@ -1057,13 +1060,15 @@ mod request_candidate_security_tests {
             candidate.required_capabilities,
             Some(json!({"vision": true}))
         );
-        assert!(!serde_json::to_string(candidate)
+        let mut public_candidate = candidate.clone();
+        public_candidate.sanitize_sensitive_diagnostics();
+        assert!(!serde_json::to_string(&public_candidate)
             .expect("candidate should serialize")
             .contains("candidate-secret"));
     }
 
     #[test]
-    fn gateway_candidate_boundary_sanitizes_repository_rows_and_write_results() {
+    fn gateway_candidate_boundary_preserves_admin_errors_and_removes_request_payloads() {
         let candidate = sanitize_request_candidate_row(untrusted_candidate());
         assert_candidate_is_sanitized(&candidate);
 

apps/aether-gateway/src/execution_runtime/stream/execution.rs

@@ -14682,9 +14682,10 @@ mod tests {
             candidate_extra["upstream_response"]["status_code"],
             json!(302)
         );
-        assert!(candidate_extra["upstream_response"]
-            .get("headers")
-            .is_none());
+        assert_eq!(
+            candidate_extra["upstream_response"]["headers"]["location"],
+            "/"
+        );
         assert!(candidate_extra["upstream_response"].get("body").is_none());
         assert!(candidate_extra.get("client_response").is_none());
 

apps/aether-gateway/src/handlers/admin/observability/monitoring/tests/trace.rs

@@ -659,7 +659,7 @@ async fn admin_monitoring_trace_request_exposes_request_path_from_usage_audit()
 }
 
 #[tokio::test]
-async fn admin_monitoring_trace_request_redacts_failed_candidate_response_payloads() {
+async fn admin_monitoring_trace_request_exposes_failed_candidate_response_payloads() {
     let mut candidate = sample_candidate(
         "cand-used",
         "request-1",
@@ -746,14 +746,96 @@ async fn admin_monitoring_trace_request_redacts_failed_candidate_response_payloa
         json!("upstream_response")
     );
     assert!(extra["upstream_response"].get("headers").is_none());
-    assert!(extra["upstream_response"].get("body").is_none());
+    assert_eq!(
+        extra["upstream_response"]["body"]["error"]["message"],
+        "redirect blocked"
+    );
     assert!(extra["upstream_response"].get("body_ref").is_none());
     assert!(extra.get("client_response").is_none());
     assert!(extra.get("provider_response").is_none());
 }
 
 #[tokio::test]
-async fn admin_monitoring_trace_request_does_not_hydrate_ref_backed_usage_response_body() {
+async fn admin_monitoring_trace_request_does_not_replace_attempt_status_with_usage_status() {
+    for (candidate_status, upstream_status, expected_status) in [
+        (Some(400), Some(400), Some(400)),
+        (Some(400), None, Some(400)),
+        (Some(502), Some(200), Some(200)),
+        (None, None, None),
+    ] {
+        let mut candidate = sample_candidate(
+            "cand-used",
+            "request-failover-status",
+            0,
+            RequestCandidateStatus::Failed,
+            Some(101),
+            Some(33),
+            candidate_status,
+        );
+        if let Some(status_code) = upstream_status {
+            candidate.extra_data = Some(json!({
+                "upstream_response": {
+                    "status_code": status_code,
+                    "body": {"error": {"message": "sensitive upstream error"}}
+                }
+            }));
+        }
+        let request_candidates =
+            Arc::new(InMemoryRequestCandidateRepository::seed(vec![candidate]));
+        let mut usage = sample_usage(
+            "request-failover-status",
+            "provider-1",
+            "OpenAI",
+            0,
+            0.0,
+            "failed",
+            Some(503),
+            100,
+        );
+        usage.candidate_id = Some("cand-used".to_string());
+        usage.response_body_state = Some(UsageBodyCaptureState::Reference);
+        let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(vec![usage]));
+        let data_state =
+            crate::data::GatewayDataState::with_request_candidate_and_usage_repository_for_tests(
+                request_candidates,
+                usage_repository,
+            );
+        let state = AppState::new()
+            .expect("state should build")
+            .with_data_state_for_tests(data_state);
+        let context = request_context(
+            http::Method::GET,
+            "/api/admin/monitoring/trace/request-failover-status",
+        );
+
+        let response = local_monitoring_response(&state, &context)
+            .await
+            .expect("handler should not error")
+            .expect("route should be handled locally");
+        assert_eq!(response.status(), http::StatusCode::OK);
+        let body = to_bytes(response.into_body(), usize::MAX)
+            .await
+            .expect("body should read");
+        let payload: serde_json::Value =
+            serde_json::from_slice(&body).expect("json body should parse");
+        let candidate = &payload["candidates"][0];
+        assert_eq!(candidate["status_code"], json!(candidate_status));
+        assert_eq!(
+            candidate["extra_data"]["upstream_response"]["status_code"],
+            json!(expected_status),
+        );
+        if upstream_status.is_some() {
+            assert_eq!(
+                candidate["extra_data"]["upstream_response"]["body"]["error"]["message"],
+                "sensitive upstream error"
+            );
+        }
+        assert!(candidate["error_message"].is_null());
+    }
+}
+
+#[tokio::test]
+async fn admin_monitoring_trace_request_keeps_candidate_errors_without_hydrating_usage_bodies() {
     let mut candidate = sample_candidate(
         "cand-used",
         "request-ref-body",
@@ -771,8 +853,7 @@ async fn admin_monitoring_trace_request_does_not_hydrate_ref_backed_usage_respon
                 "x-request-id": "stale-request-like-body"
             },
             "body": {
-                "model": "gpt-5.6-sol",
-                "input": [{"role": "user", "content": "request prompt"}]
+                "error": {"message": "candidate-specific upstream failure"}
             }
         }
     }));
@@ -837,8 +918,14 @@ async fn admin_monitoring_trace_request_does_not_hydrate_ref_backed_usage_respon
     assert_eq!(upstream_response["status_code"], json!(400));
     assert_eq!(upstream_response["source"], json!("upstream_response"));
     assert_eq!(upstream_response["body_state"], json!("reference"));
-    assert!(upstream_response.get("headers").is_none());
-    assert!(upstream_response.get("body").is_none());
+    assert_eq!(
+        upstream_response["headers"]["content-type"],
+        "text/event-stream"
+    );
+    assert_eq!(
+        upstream_response["body"]["error"]["message"],
+        "candidate-specific upstream failure"
+    );
     assert!(upstream_response.get("body_ref").is_none());
 }
 

apps/aether-gateway/src/handlers/public/support/user_me_usage.rs

@@ -1940,12 +1940,16 @@ mod tests {
 
     #[test]
     fn user_usage_active_override_uses_terminal_candidate_latency() {
-        let candidate = sample_candidate(
+        let mut candidate = sample_candidate(
             RequestCandidateStatus::Success,
             Some(200),
             Some(9_210),
             None,
         );
+        candidate.error_message = Some("private upstream diagnostic".to_string());
+        candidate.extra_data = Some(json!({
+            "upstream_response": {"body": {"error": {"message": "private upstream diagnostic"}}}
+        }));
 
         let payload =
             users_me_usage_terminal_candidate_state_override(&[candidate]).expect("override");
@@ -1953,6 +1957,7 @@ mod tests {
         assert_eq!(payload["status"], "completed");
         assert_eq!(payload["response_time_ms"], 9_210);
         assert_eq!(payload["status_code"], 200);
+        assert!(!payload.to_string().contains("private upstream diagnostic"));
         assert_eq!(
             payload["response_time_updated_at"],
             "1970-01-01T00:00:10.210+00:00"

apps/aether-gateway/src/tests/ai_execute/stream/decision.rs

@@ -2460,15 +2460,21 @@ async fn gateway_retries_next_local_openai_chat_stream_candidate_after_retryable
         failed_candidate.error_type.as_deref(),
         Some("retryable_upstream_status")
     );
-    assert!(failed_candidate.error_message.is_none());
+    assert!(failed_candidate.error_message.is_some());
     let failed_upstream_response = failed_candidate
         .extra_data
         .as_ref()
         .and_then(|value| value.get("upstream_response"))
         .expect("failed stream candidate should keep its upstream response");
     assert_eq!(failed_upstream_response["status_code"], json!(429));
-    assert!(failed_upstream_response.get("headers").is_none());
-    assert!(failed_upstream_response.get("body").is_none());
+    assert_eq!(
+        failed_upstream_response["headers"]["content-type"],
+        "application/json"
+    );
+    assert_eq!(
+        failed_upstream_response["body"]["error"]["message"],
+        "rate limited"
+    );
     assert_eq!(success_candidate.status, RequestCandidateStatus::Success);
     assert_eq!(success_candidate.status_code, Some(200));
     assert!(success_candidate.started_at_unix_ms.is_some());

apps/aether-gateway/src/tests/ai_execute/sync/chat/failover.rs

@@ -1166,15 +1166,21 @@ async fn gateway_retries_next_local_openai_chat_sync_candidate_after_auth_failur
         assert_eq!(failed_candidate.retry_index, retry_index as u32);
         assert_eq!(failed_candidate.status, RequestCandidateStatus::Failed);
         assert_eq!(failed_candidate.status_code, Some(401));
-        assert!(failed_candidate.error_message.is_none());
+        assert!(failed_candidate.error_message.is_some());
         let failed_upstream_response = failed_candidate
             .extra_data
             .as_ref()
             .and_then(|value| value.get("upstream_response"))
             .expect("failed candidate should keep its upstream response");
         assert_eq!(failed_upstream_response["status_code"], json!(401));
-        assert!(failed_upstream_response.get("headers").is_none());
-        assert!(failed_upstream_response.get("body").is_none());
+        assert_eq!(
+            failed_upstream_response["headers"]["content-type"],
+            "application/json"
+        );
+        assert_eq!(
+            failed_upstream_response["body"]["error"]["message"],
+            "invalid auth token"
+        );
     }
     assert_eq!(stored_candidates[2].candidate_index, 1);
     assert_eq!(stored_candidates[2].status, RequestCandidateStatus::Success);

apps/aether-gateway/src/tests/audit.rs

@@ -534,6 +534,24 @@ async fn gateway_exposes_request_audit_bundle_via_internal_audit_endpoint() {
 
 #[tokio::test]
 async fn gateway_exposes_request_candidate_trace_via_internal_audit_endpoint() {
+    let mut failed_candidate = sample_request_candidate(
+        "cand-2",
+        "req-trace-1",
+        1,
+        RequestCandidateStatus::Failed,
+        Some(101),
+        Some(37),
+        Some(502),
+    );
+    failed_candidate.error_message = Some("private upstream diagnostic".to_string());
+    failed_candidate.extra_data = Some(json!({
+        "upstream_response": {
+            "status_code": 502,
+            "headers": {"x-request-id": "private-upstream-id"},
+            "body": {"error": {"message": "private upstream diagnostic"}}
+        },
+        "error_flow": {"status_code": 502, "message": "private upstream diagnostic"}
+    }));
     let repository = Arc::new(InMemoryRequestCandidateRepository::seed(vec![
         sample_request_candidate(
             "cand-1",
@@ -544,15 +562,7 @@ async fn gateway_exposes_request_candidate_trace_via_internal_audit_endpoint() {
             None,
             None,
         ),
-        sample_request_candidate(
-            "cand-2",
-            "req-trace-1",
-            1,
-            RequestCandidateStatus::Failed,
-            Some(101),
-            Some(37),
-            Some(502),
-        ),
+        failed_candidate,
     ]));
 
     let gateway_state = AppState::new()
@@ -582,6 +592,14 @@ async fn gateway_exposes_request_candidate_trace_via_internal_audit_endpoint() {
     );
     assert_eq!(payload["candidates"][0]["id"], "cand-2");
     assert_eq!(payload["candidates"][0]["status"], "failed");
+    assert!(payload["candidates"][0]["error_message"].is_null());
+    assert_eq!(
+        payload["candidates"][0]["extra_data"]["upstream_response"]["status_code"],
+        502
+    );
+    let serialized = payload.to_string();
+    assert!(!serialized.contains("private upstream diagnostic"));
+    assert!(!serialized.contains("private-upstream-id"));
 
     gateway_handle.abort();
 }

apps/aether-gateway/src/tests/control/admin/monitoring.rs

@@ -473,6 +473,23 @@ async fn gateway_handles_admin_monitoring_trace_request_locally_with_trusted_adm
         }),
     );
 
+    let mut failed_candidate = sample_candidate(
+        "cand-used",
+        "request-1",
+        1,
+        RequestCandidateStatus::Failed,
+        Some(101),
+        Some(33),
+        Some(502),
+    );
+    failed_candidate.error_message = Some("private upstream diagnostic".to_string());
+    failed_candidate.extra_data = Some(json!({
+        "upstream_response": {
+            "status_code": 502,
+            "headers": {"x-request-id": "upstream-diagnostic-id"},
+            "body": {"error": {"message": "private upstream diagnostic"}}
+        }
+    }));
     let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![
         sample_candidate(
             "cand-unused",
@@ -483,15 +500,7 @@ async fn gateway_handles_admin_monitoring_trace_request_locally_with_trusted_adm
             None,
             None,
         ),
-        sample_candidate(
-            "cand-used",
-            "request-1",
-            1,
-            RequestCandidateStatus::Failed,
-            Some(101),
-            Some(33),
-            Some(502),
-        ),
+        failed_candidate,
     ]));
     let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
         vec![sample_provider()],
@@ -538,6 +547,37 @@ async fn gateway_handles_admin_monitoring_trace_request_locally_with_trusted_adm
     assert_eq!(payload["candidates"][0]["key_auth_type"], json!("api_key"));
     assert_eq!(payload["candidates"][0]["latency_ms"], json!(33));
     assert_eq!(payload["candidates"][0]["status_code"], json!(502));
+    assert_eq!(
+        payload["candidates"][0]["error_message"],
+        "private upstream diagnostic"
+    );
+    assert_eq!(
+        payload["candidates"][0]["extra_data"]["upstream_response"]["body"]["error"]["message"],
+        "private upstream diagnostic"
+    );
+    for role in [None, Some("user")] {
+        let mut request = reqwest::Client::new().get(format!(
+            "{gateway_url}/api/admin/monitoring/trace/request-1"
+        ));
+        if let Some(role) = role {
+            request = request
+                .header(crate::constants::GATEWAY_HEADER, "rust-phase3b")
+                .header(TRUSTED_ADMIN_USER_ID_HEADER, "regular-user")
+                .header(TRUSTED_ADMIN_USER_ROLE_HEADER, role)
+                .header(TRUSTED_ADMIN_SESSION_ID_HEADER, "regular-session");
+        }
+        let response = request
+            .send()
+            .await
+            .expect("unauthorized probe should complete");
+        assert!(matches!(
+            response.status(),
+            StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN
+        ));
+        let body = response.text().await.expect("denial body should read");
+        assert!(!body.contains("private upstream diagnostic"));
+        assert!(!body.contains("upstream-diagnostic-id"));
+    }
     assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0);
 
     gateway_handle.abort();

apps/aether-gateway/src/tests/control/admin/providers.rs

@@ -85,10 +85,7 @@ async fn admin_provider_summary_health_preserves_redacted_key_summaries() {
     .into_iter()
     .map(|(key_id, auth_type, auth_config, score)| {
         let mut key = sample_key(key_id, "provider-openai", "openai:chat", "test")
-            .with_health_fields(
-                Some(json!({"openai:chat": {"health_score": score}})),
-                None,
-            );
+            .with_health_fields(Some(json!({"openai:chat": {"health_score": score}})), None);
         key.auth_type = auth_type.to_string();
         key.encrypted_api_key = Some("summary".to_string());
         key.encrypted_auth_config = auth_config.map(ToOwned::to_owned);
@@ -115,7 +112,8 @@ async fn admin_provider_summary_health_preserves_redacted_key_summaries() {
         let body = axum::body::to_bytes(response.into_body(), 1024 * 1024)
             .await
             .expect("summary body should read");
-        let payload: serde_json::Value = serde_json::from_slice(&body).expect("summary should parse");
+        let payload: serde_json::Value =
+            serde_json::from_slice(&body).expect("summary should parse");
         let summary = if uri == "/api/admin/providers/summary" {
             &payload["items"][0]
         } else {

apps/aether-gateway/src/usage/reporting/mod.rs

@@ -1308,7 +1308,13 @@ mod tests {
             stored[0].error_type.as_deref(),
             Some("stream_missing_terminal_event")
         );
-        assert!(stored[0].error_message.is_none());
+        assert_eq!(
+            stored[0].error_message.as_deref(),
+            Some(super::STREAM_MISSING_TERMINAL_EVENT_MESSAGE)
+        );
+        let mut public_candidate = stored[0].clone();
+        public_candidate.sanitize_sensitive_diagnostics();
+        assert!(public_candidate.error_message.is_none());
     }
 
     #[tokio::test]