i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-10-07 18:07:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(observability): preserve admin upstream error diagnostics

Browse Source
This commit is contained in:
elky
2026-09-07 10:39:48 +08:00
parent 470c59e197
commit 062e111c03
17 changed files with 690 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/aether-gateway/src/tests/ai_execute/stream/decision.rs
+9 -3
View File
@@ -2460,15 +2460,21 @@ async fn gateway_retries_next_local_openai_chat_stream_candidate_after_retryable
failed_candidate.error_type.as_deref(),
Some("retryable_upstream_status")
);
assert!(failed_candidate.error_message.is_none());
assert!(failed_candidate.error_message.is_some());
let failed_upstream_response = failed_candidate
.extra_data
.as_ref()
.and_then(|value| value.get("upstream_response"))
.expect("failed stream candidate should keep its upstream response");
assert_eq!(failed_upstream_response["status_code"], json!(429));
assert!(failed_upstream_response.get("headers").is_none());
assert!(failed_upstream_response.get("body").is_none());
assert_eq!(
failed_upstream_response["headers"]["content-type"],
"application/json"
);
assert_eq!(
failed_upstream_response["body"]["error"]["message"],
"rate limited"
);
assert_eq!(success_candidate.status, RequestCandidateStatus::Success);
assert_eq!(success_candidate.status_code, Some(200));
assert!(success_candidate.started_at_unix_ms.is_some());
apps/aether-gateway/src/tests/ai_execute/sync/chat/failover.rs
+9 -3
View File
@@ -1166,15 +1166,21 @@ async fn gateway_retries_next_local_openai_chat_sync_candidate_after_auth_failur
assert_eq!(failed_candidate.retry_index, retry_index as u32);
assert_eq!(failed_candidate.status, RequestCandidateStatus::Failed);
assert_eq!(failed_candidate.status_code, Some(401));
assert!(failed_candidate.error_message.is_none());
assert!(failed_candidate.error_message.is_some());
let failed_upstream_response = failed_candidate
.extra_data
.as_ref()
.and_then(|value| value.get("upstream_response"))
.expect("failed candidate should keep its upstream response");
assert_eq!(failed_upstream_response["status_code"], json!(401));
assert!(failed_upstream_response.get("headers").is_none());
assert!(failed_upstream_response.get("body").is_none());
assert_eq!(
failed_upstream_response["headers"]["content-type"],
"application/json"
);
assert_eq!(
failed_upstream_response["body"]["error"]["message"],
"invalid auth token"
);
}
assert_eq!(stored_candidates[2].candidate_index, 1);
assert_eq!(stored_candidates[2].status, RequestCandidateStatus::Success);
apps/aether-gateway/src/tests/audit.rs
+27 -9
View File
@@ -534,6 +534,24 @@ async fn gateway_exposes_request_audit_bundle_via_internal_audit_endpoint() {
#[tokio::test]
async fn gateway_exposes_request_candidate_trace_via_internal_audit_endpoint() {
let mut failed_candidate = sample_request_candidate(
"cand-2",
"req-trace-1",
1,
RequestCandidateStatus::Failed,
Some(101),
Some(37),
Some(502),
);
failed_candidate.error_message = Some("private upstream diagnostic".to_string());
failed_candidate.extra_data = Some(json!({
"upstream_response": {
"status_code": 502,
"headers": {"x-request-id": "private-upstream-id"},
"body": {"error": {"message": "private upstream diagnostic"}}
},
"error_flow": {"status_code": 502, "message": "private upstream diagnostic"}
}));
let repository = Arc::new(InMemoryRequestCandidateRepository::seed(vec![
sample_request_candidate(
"cand-1",
@@ -544,15 +562,7 @@ async fn gateway_exposes_request_candidate_trace_via_internal_audit_endpoint() {
None,
None,
),
sample_request_candidate(
"cand-2",
"req-trace-1",
1,
RequestCandidateStatus::Failed,
Some(101),
Some(37),
Some(502),
),
failed_candidate,
]));
let gateway_state = AppState::new()
@@ -582,6 +592,14 @@ async fn gateway_exposes_request_candidate_trace_via_internal_audit_endpoint() {
);
assert_eq!(payload["candidates"][0]["id"], "cand-2");
assert_eq!(payload["candidates"][0]["status"], "failed");
assert!(payload["candidates"][0]["error_message"].is_null());
assert_eq!(
payload["candidates"][0]["extra_data"]["upstream_response"]["status_code"],
502
);
let serialized = payload.to_string();
assert!(!serialized.contains("private upstream diagnostic"));
assert!(!serialized.contains("private-upstream-id"));
gateway_handle.abort();
}
apps/aether-gateway/src/tests/control/admin/monitoring.rs
+49 -9
View File
@@ -473,6 +473,23 @@ async fn gateway_handles_admin_monitoring_trace_request_locally_with_trusted_adm
}),
);
let mut failed_candidate = sample_candidate(
"cand-used",
"request-1",
1,
RequestCandidateStatus::Failed,
Some(101),
Some(33),
Some(502),
);
failed_candidate.error_message = Some("private upstream diagnostic".to_string());
failed_candidate.extra_data = Some(json!({
"upstream_response": {
"status_code": 502,
"headers": {"x-request-id": "upstream-diagnostic-id"},
"body": {"error": {"message": "private upstream diagnostic"}}
}
}));
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![
sample_candidate(
"cand-unused",
@@ -483,15 +500,7 @@ async fn gateway_handles_admin_monitoring_trace_request_locally_with_trusted_adm
None,
None,
),
sample_candidate(
"cand-used",
"request-1",
1,
RequestCandidateStatus::Failed,
Some(101),
Some(33),
Some(502),
),
failed_candidate,
]));
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider()],
@@ -538,6 +547,37 @@ async fn gateway_handles_admin_monitoring_trace_request_locally_with_trusted_adm
assert_eq!(payload["candidates"][0]["key_auth_type"], json!("api_key"));
assert_eq!(payload["candidates"][0]["latency_ms"], json!(33));
assert_eq!(payload["candidates"][0]["status_code"], json!(502));
assert_eq!(
payload["candidates"][0]["error_message"],
"private upstream diagnostic"
);
assert_eq!(
payload["candidates"][0]["extra_data"]["upstream_response"]["body"]["error"]["message"],
"private upstream diagnostic"
);
for role in [None, Some("user")] {
let mut request = reqwest::Client::new().get(format!(
"{gateway_url}/api/admin/monitoring/trace/request-1"
));
if let Some(role) = role {
request = request
.header(crate::constants::GATEWAY_HEADER, "rust-phase3b")
.header(TRUSTED_ADMIN_USER_ID_HEADER, "regular-user")
.header(TRUSTED_ADMIN_USER_ROLE_HEADER, role)
.header(TRUSTED_ADMIN_SESSION_ID_HEADER, "regular-session");
}
let response = request
.send()
.await
.expect("unauthorized probe should complete");
assert!(matches!(
response.status(),
StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN
));
let body = response.text().await.expect("denial body should read");
assert!(!body.contains("private upstream diagnostic"));
assert!(!body.contains("upstream-diagnostic-id"));
}
assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
apps/aether-gateway/src/tests/control/admin/providers.rs
+3 -5
View File
@@ -85,10 +85,7 @@ async fn admin_provider_summary_health_preserves_redacted_key_summaries() {
.into_iter()
.map(|(key_id, auth_type, auth_config, score)| {
let mut key = sample_key(key_id, "provider-openai", "openai:chat", "test")
.with_health_fields(
Some(json!({"openai:chat": {"health_score": score}})),
None,
);
.with_health_fields(Some(json!({"openai:chat": {"health_score": score}})), None);
key.auth_type = auth_type.to_string();
key.encrypted_api_key = Some("summary".to_string());
key.encrypted_auth_config = auth_config.map(ToOwned::to_owned);
@@ -115,7 +112,8 @@ async fn admin_provider_summary_health_preserves_redacted_key_summaries() {
let body = axum::body::to_bytes(response.into_body(), 1024 * 1024)
.await
.expect("summary body should read");
let payload: serde_json::Value = serde_json::from_slice(&body).expect("summary should parse");
let payload: serde_json::Value =
serde_json::from_slice(&body).expect("summary should parse");
let summary = if uri == "/api/admin/providers/summary" {
&payload["items"][0]
} else {