i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-10-04 16:37:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(observability): preserve admin upstream error diagnostics

Browse Source
This commit is contained in:
elky
2026-09-07 10:39:48 +08:00
parent 470c59e197
commit 062e111c03
17 changed files with 690 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/aether-gateway/src/handlers/admin/observability/monitoring/tests/trace.rs
+94 -7
View File
@@ -659,7 +659,7 @@ async fn admin_monitoring_trace_request_exposes_request_path_from_usage_audit()
}
#[tokio::test]
async fn admin_monitoring_trace_request_redacts_failed_candidate_response_payloads() {
async fn admin_monitoring_trace_request_exposes_failed_candidate_response_payloads() {
let mut candidate = sample_candidate(
"cand-used",
"request-1",
@@ -746,14 +746,96 @@ async fn admin_monitoring_trace_request_redacts_failed_candidate_response_payloa
json!("upstream_response")
);
assert!(extra["upstream_response"].get("headers").is_none());
assert!(extra["upstream_response"].get("body").is_none());
assert_eq!(
extra["upstream_response"]["body"]["error"]["message"],
"redirect blocked"
);
assert!(extra["upstream_response"].get("body_ref").is_none());
assert!(extra.get("client_response").is_none());
assert!(extra.get("provider_response").is_none());
}
#[tokio::test]
async fn admin_monitoring_trace_request_does_not_hydrate_ref_backed_usage_response_body() {
async fn admin_monitoring_trace_request_does_not_replace_attempt_status_with_usage_status() {
for (candidate_status, upstream_status, expected_status) in [
(Some(400), Some(400), Some(400)),
(Some(400), None, Some(400)),
(Some(502), Some(200), Some(200)),
(None, None, None),
] {
let mut candidate = sample_candidate(
"cand-used",
"request-failover-status",
0,
RequestCandidateStatus::Failed,
Some(101),
Some(33),
candidate_status,
);
if let Some(status_code) = upstream_status {
candidate.extra_data = Some(json!({
"upstream_response": {
"status_code": status_code,
"body": {"error": {"message": "sensitive upstream error"}}
}
}));
}
let request_candidates =
Arc::new(InMemoryRequestCandidateRepository::seed(vec![candidate]));
let mut usage = sample_usage(
"request-failover-status",
"provider-1",
"OpenAI",
0,
0.0,
"failed",
Some(503),
100,
);
usage.candidate_id = Some("cand-used".to_string());
usage.response_body_state = Some(UsageBodyCaptureState::Reference);
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(vec![usage]));
let data_state =
crate::data::GatewayDataState::with_request_candidate_and_usage_repository_for_tests(
request_candidates,
usage_repository,
);
let state = AppState::new()
.expect("state should build")
.with_data_state_for_tests(data_state);
let context = request_context(
http::Method::GET,
"/api/admin/monitoring/trace/request-failover-status",
);
let response = local_monitoring_response(&state, &context)
.await
.expect("handler should not error")
.expect("route should be handled locally");
assert_eq!(response.status(), http::StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX)
.await
.expect("body should read");
let payload: serde_json::Value =
serde_json::from_slice(&body).expect("json body should parse");
let candidate = &payload["candidates"][0];
assert_eq!(candidate["status_code"], json!(candidate_status));
assert_eq!(
candidate["extra_data"]["upstream_response"]["status_code"],
json!(expected_status),
);
if upstream_status.is_some() {
assert_eq!(
candidate["extra_data"]["upstream_response"]["body"]["error"]["message"],
"sensitive upstream error"
);
}
assert!(candidate["error_message"].is_null());
}
}
#[tokio::test]
async fn admin_monitoring_trace_request_keeps_candidate_errors_without_hydrating_usage_bodies() {
let mut candidate = sample_candidate(
"cand-used",
"request-ref-body",
@@ -771,8 +853,7 @@ async fn admin_monitoring_trace_request_does_not_hydrate_ref_backed_usage_respon
"x-request-id": "stale-request-like-body"
},
"body": {
"model": "gpt-5.6-sol",
"input": [{"role": "user", "content": "request prompt"}]
"error": {"message": "candidate-specific upstream failure"}
}
}
}));
@@ -837,8 +918,14 @@ async fn admin_monitoring_trace_request_does_not_hydrate_ref_backed_usage_respon
assert_eq!(upstream_response["status_code"], json!(400));
assert_eq!(upstream_response["source"], json!("upstream_response"));
assert_eq!(upstream_response["body_state"], json!("reference"));
assert!(upstream_response.get("headers").is_none());
assert!(upstream_response.get("body").is_none());
assert_eq!(
upstream_response["headers"]["content-type"],
"text/event-stream"
);
assert_eq!(
upstream_response["body"]["error"]["message"],
"candidate-specific upstream failure"
);
assert!(upstream_response.get("body_ref").is_none());
}
apps/aether-gateway/src/handlers/public/support/user_me_usage.rs
+6 -1
View File
@@ -1940,12 +1940,16 @@ mod tests {
#[test]
fn user_usage_active_override_uses_terminal_candidate_latency() {
let candidate = sample_candidate(
let mut candidate = sample_candidate(
RequestCandidateStatus::Success,
Some(200),
Some(9_210),
None,
);
candidate.error_message = Some("private upstream diagnostic".to_string());
candidate.extra_data = Some(json!({
"upstream_response": {"body": {"error": {"message": "private upstream diagnostic"}}}
}));
let payload =
users_me_usage_terminal_candidate_state_override(&[candidate]).expect("override");
@@ -1953,6 +1957,7 @@ mod tests {
assert_eq!(payload["status"], "completed");
assert_eq!(payload["response_time_ms"], 9_210);
assert_eq!(payload["status_code"], 200);
assert!(!payload.to_string().contains("private upstream diagnostic"));
assert_eq!(
payload["response_time_updated_at"],
"1970-01-01T00:00:10.210+00:00"