i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-08 20:20:19 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(observability): preserve admin upstream error diagnostics

Browse Source
This commit is contained in:
elky
2026-09-07 10:39:48 +08:00
parent 470c59e197
commit 062e111c03
17 changed files with 690 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/aether-gateway/src/data/state/catalog.rs
+10 -5
View File
@@ -17,13 +17,13 @@ fn sanitize_request_candidate_rows(
mut candidates: Vec<StoredRequestCandidate>,
) -> Vec<StoredRequestCandidate> {
for candidate in &mut candidates {
candidate.sanitize_sensitive_diagnostics();
candidate.sanitize_for_persistence();
}
candidates
}
fn sanitize_request_candidate_row(mut candidate: StoredRequestCandidate) -> StoredRequestCandidate {
candidate.sanitize_sensitive_diagnostics();
candidate.sanitize_for_persistence();
candidate
}
@@ -1048,7 +1048,10 @@ mod request_candidate_security_tests {
fn assert_candidate_is_sanitized(candidate: &StoredRequestCandidate) {
assert_eq!(candidate.skip_reason.as_deref(), Some("unclassified_skip"));
assert_eq!(candidate.error_type.as_deref(), Some("unclassified_error"));
assert!(candidate.error_message.is_none());
assert_eq!(
candidate.error_message.as_deref(),
Some("Bearer candidate-secret")
);
assert_eq!(
candidate.extra_data,
Some(json!({"gateway_execution_runtime": true}))
@@ -1057,13 +1060,15 @@ mod request_candidate_security_tests {
candidate.required_capabilities,
Some(json!({"vision": true}))
);
assert!(!serde_json::to_string(candidate)
let mut public_candidate = candidate.clone();
public_candidate.sanitize_sensitive_diagnostics();
assert!(!serde_json::to_string(&public_candidate)
.expect("candidate should serialize")
.contains("candidate-secret"));
}
#[test]
fn gateway_candidate_boundary_sanitizes_repository_rows_and_write_results() {
fn gateway_candidate_boundary_preserves_admin_errors_and_removes_request_payloads() {
let candidate = sanitize_request_candidate_row(untrusted_candidate());
assert_candidate_is_sanitized(&candidate);
apps/aether-gateway/src/execution_runtime/stream/execution.rs
+4 -3
View File
@@ -14682,9 +14682,10 @@ mod tests {
candidate_extra["upstream_response"]["status_code"],
json!(302)
);
assert!(candidate_extra["upstream_response"]
.get("headers")
.is_none());
assert_eq!(
candidate_extra["upstream_response"]["headers"]["location"],
"/"
);
assert!(candidate_extra["upstream_response"].get("body").is_none());
assert!(candidate_extra.get("client_response").is_none());
apps/aether-gateway/src/handlers/admin/observability/monitoring/tests/trace.rs
+94 -7
View File
@@ -659,7 +659,7 @@ async fn admin_monitoring_trace_request_exposes_request_path_from_usage_audit()
}
#[tokio::test]
async fn admin_monitoring_trace_request_redacts_failed_candidate_response_payloads() {
async fn admin_monitoring_trace_request_exposes_failed_candidate_response_payloads() {
let mut candidate = sample_candidate(
"cand-used",
"request-1",
@@ -746,14 +746,96 @@ async fn admin_monitoring_trace_request_redacts_failed_candidate_response_payloa
json!("upstream_response")
);
assert!(extra["upstream_response"].get("headers").is_none());
assert!(extra["upstream_response"].get("body").is_none());
assert_eq!(
extra["upstream_response"]["body"]["error"]["message"],
"redirect blocked"
);
assert!(extra["upstream_response"].get("body_ref").is_none());
assert!(extra.get("client_response").is_none());
assert!(extra.get("provider_response").is_none());
}
#[tokio::test]
async fn admin_monitoring_trace_request_does_not_hydrate_ref_backed_usage_response_body() {
async fn admin_monitoring_trace_request_does_not_replace_attempt_status_with_usage_status() {
for (candidate_status, upstream_status, expected_status) in [
(Some(400), Some(400), Some(400)),
(Some(400), None, Some(400)),
(Some(502), Some(200), Some(200)),
(None, None, None),
] {
let mut candidate = sample_candidate(
"cand-used",
"request-failover-status",
0,
RequestCandidateStatus::Failed,
Some(101),
Some(33),
candidate_status,
);
if let Some(status_code) = upstream_status {
candidate.extra_data = Some(json!({
"upstream_response": {
"status_code": status_code,
"body": {"error": {"message": "sensitive upstream error"}}
}
}));
}
let request_candidates =
Arc::new(InMemoryRequestCandidateRepository::seed(vec![candidate]));
let mut usage = sample_usage(
"request-failover-status",
"provider-1",
"OpenAI",
0,
0.0,
"failed",
Some(503),
100,
);
usage.candidate_id = Some("cand-used".to_string());
usage.response_body_state = Some(UsageBodyCaptureState::Reference);
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(vec![usage]));
let data_state =
crate::data::GatewayDataState::with_request_candidate_and_usage_repository_for_tests(
request_candidates,
usage_repository,
);
let state = AppState::new()
.expect("state should build")
.with_data_state_for_tests(data_state);
let context = request_context(
http::Method::GET,
"/api/admin/monitoring/trace/request-failover-status",
);
let response = local_monitoring_response(&state, &context)
.await
.expect("handler should not error")
.expect("route should be handled locally");
assert_eq!(response.status(), http::StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX)
.await
.expect("body should read");
let payload: serde_json::Value =
serde_json::from_slice(&body).expect("json body should parse");
let candidate = &payload["candidates"][0];
assert_eq!(candidate["status_code"], json!(candidate_status));
assert_eq!(
candidate["extra_data"]["upstream_response"]["status_code"],
json!(expected_status),
);
if upstream_status.is_some() {
assert_eq!(
candidate["extra_data"]["upstream_response"]["body"]["error"]["message"],
"sensitive upstream error"
);
}
assert!(candidate["error_message"].is_null());
}
}
#[tokio::test]
async fn admin_monitoring_trace_request_keeps_candidate_errors_without_hydrating_usage_bodies() {
let mut candidate = sample_candidate(
"cand-used",
"request-ref-body",
@@ -771,8 +853,7 @@ async fn admin_monitoring_trace_request_does_not_hydrate_ref_backed_usage_respon
"x-request-id": "stale-request-like-body"
},
"body": {
"model": "gpt-5.6-sol",
"input": [{"role": "user", "content": "request prompt"}]
"error": {"message": "candidate-specific upstream failure"}
}
}
}));
@@ -837,8 +918,14 @@ async fn admin_monitoring_trace_request_does_not_hydrate_ref_backed_usage_respon
assert_eq!(upstream_response["status_code"], json!(400));
assert_eq!(upstream_response["source"], json!("upstream_response"));
assert_eq!(upstream_response["body_state"], json!("reference"));
assert!(upstream_response.get("headers").is_none());
assert!(upstream_response.get("body").is_none());
assert_eq!(
upstream_response["headers"]["content-type"],
"text/event-stream"
);
assert_eq!(
upstream_response["body"]["error"]["message"],
"candidate-specific upstream failure"
);
assert!(upstream_response.get("body_ref").is_none());
}
apps/aether-gateway/src/handlers/public/support/user_me_usage.rs
+6 -1
View File
@@ -1940,12 +1940,16 @@ mod tests {
#[test]
fn user_usage_active_override_uses_terminal_candidate_latency() {
let candidate = sample_candidate(
let mut candidate = sample_candidate(
RequestCandidateStatus::Success,
Some(200),
Some(9_210),
None,
);
candidate.error_message = Some("private upstream diagnostic".to_string());
candidate.extra_data = Some(json!({
"upstream_response": {"body": {"error": {"message": "private upstream diagnostic"}}}
}));
let payload =
users_me_usage_terminal_candidate_state_override(&[candidate]).expect("override");
@@ -1953,6 +1957,7 @@ mod tests {
assert_eq!(payload["status"], "completed");
assert_eq!(payload["response_time_ms"], 9_210);
assert_eq!(payload["status_code"], 200);
assert!(!payload.to_string().contains("private upstream diagnostic"));
assert_eq!(
payload["response_time_updated_at"],
"1970-01-01T00:00:10.210+00:00"
apps/aether-gateway/src/tests/ai_execute/stream/decision.rs
+9 -3
View File
@@ -2460,15 +2460,21 @@ async fn gateway_retries_next_local_openai_chat_stream_candidate_after_retryable
failed_candidate.error_type.as_deref(),
Some("retryable_upstream_status")
);
assert!(failed_candidate.error_message.is_none());
assert!(failed_candidate.error_message.is_some());
let failed_upstream_response = failed_candidate
.extra_data
.as_ref()
.and_then(|value| value.get("upstream_response"))
.expect("failed stream candidate should keep its upstream response");
assert_eq!(failed_upstream_response["status_code"], json!(429));
assert!(failed_upstream_response.get("headers").is_none());
assert!(failed_upstream_response.get("body").is_none());
assert_eq!(
failed_upstream_response["headers"]["content-type"],
"application/json"
);
assert_eq!(
failed_upstream_response["body"]["error"]["message"],
"rate limited"
);
assert_eq!(success_candidate.status, RequestCandidateStatus::Success);
assert_eq!(success_candidate.status_code, Some(200));
assert!(success_candidate.started_at_unix_ms.is_some());
apps/aether-gateway/src/tests/ai_execute/sync/chat/failover.rs
+9 -3
View File
@@ -1166,15 +1166,21 @@ async fn gateway_retries_next_local_openai_chat_sync_candidate_after_auth_failur
assert_eq!(failed_candidate.retry_index, retry_index as u32);
assert_eq!(failed_candidate.status, RequestCandidateStatus::Failed);
assert_eq!(failed_candidate.status_code, Some(401));
assert!(failed_candidate.error_message.is_none());
assert!(failed_candidate.error_message.is_some());
let failed_upstream_response = failed_candidate
.extra_data
.as_ref()
.and_then(|value| value.get("upstream_response"))
.expect("failed candidate should keep its upstream response");
assert_eq!(failed_upstream_response["status_code"], json!(401));
assert!(failed_upstream_response.get("headers").is_none());
assert!(failed_upstream_response.get("body").is_none());
assert_eq!(
failed_upstream_response["headers"]["content-type"],
"application/json"
);
assert_eq!(
failed_upstream_response["body"]["error"]["message"],
"invalid auth token"
);
}
assert_eq!(stored_candidates[2].candidate_index, 1);
assert_eq!(stored_candidates[2].status, RequestCandidateStatus::Success);
apps/aether-gateway/src/tests/audit.rs
+27 -9
View File
@@ -534,6 +534,24 @@ async fn gateway_exposes_request_audit_bundle_via_internal_audit_endpoint() {
#[tokio::test]
async fn gateway_exposes_request_candidate_trace_via_internal_audit_endpoint() {
let mut failed_candidate = sample_request_candidate(
"cand-2",
"req-trace-1",
1,
RequestCandidateStatus::Failed,
Some(101),
Some(37),
Some(502),
);
failed_candidate.error_message = Some("private upstream diagnostic".to_string());
failed_candidate.extra_data = Some(json!({
"upstream_response": {
"status_code": 502,
"headers": {"x-request-id": "private-upstream-id"},
"body": {"error": {"message": "private upstream diagnostic"}}
},
"error_flow": {"status_code": 502, "message": "private upstream diagnostic"}
}));
let repository = Arc::new(InMemoryRequestCandidateRepository::seed(vec![
sample_request_candidate(
"cand-1",
@@ -544,15 +562,7 @@ async fn gateway_exposes_request_candidate_trace_via_internal_audit_endpoint() {
None,
None,
),
sample_request_candidate(
"cand-2",
"req-trace-1",
1,
RequestCandidateStatus::Failed,
Some(101),
Some(37),
Some(502),
),
failed_candidate,
]));
let gateway_state = AppState::new()
@@ -582,6 +592,14 @@ async fn gateway_exposes_request_candidate_trace_via_internal_audit_endpoint() {
);
assert_eq!(payload["candidates"][0]["id"], "cand-2");
assert_eq!(payload["candidates"][0]["status"], "failed");
assert!(payload["candidates"][0]["error_message"].is_null());
assert_eq!(
payload["candidates"][0]["extra_data"]["upstream_response"]["status_code"],
502
);
let serialized = payload.to_string();
assert!(!serialized.contains("private upstream diagnostic"));
assert!(!serialized.contains("private-upstream-id"));
gateway_handle.abort();
}
apps/aether-gateway/src/tests/control/admin/monitoring.rs
+49 -9
View File
@@ -473,6 +473,23 @@ async fn gateway_handles_admin_monitoring_trace_request_locally_with_trusted_adm
}),
);
let mut failed_candidate = sample_candidate(
"cand-used",
"request-1",
1,
RequestCandidateStatus::Failed,
Some(101),
Some(33),
Some(502),
);
failed_candidate.error_message = Some("private upstream diagnostic".to_string());
failed_candidate.extra_data = Some(json!({
"upstream_response": {
"status_code": 502,
"headers": {"x-request-id": "upstream-diagnostic-id"},
"body": {"error": {"message": "private upstream diagnostic"}}
}
}));
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![
sample_candidate(
"cand-unused",
@@ -483,15 +500,7 @@ async fn gateway_handles_admin_monitoring_trace_request_locally_with_trusted_adm
None,
None,
),
sample_candidate(
"cand-used",
"request-1",
1,
RequestCandidateStatus::Failed,
Some(101),
Some(33),
Some(502),
),
failed_candidate,
]));
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider()],
@@ -538,6 +547,37 @@ async fn gateway_handles_admin_monitoring_trace_request_locally_with_trusted_adm
assert_eq!(payload["candidates"][0]["key_auth_type"], json!("api_key"));
assert_eq!(payload["candidates"][0]["latency_ms"], json!(33));
assert_eq!(payload["candidates"][0]["status_code"], json!(502));
assert_eq!(
payload["candidates"][0]["error_message"],
"private upstream diagnostic"
);
assert_eq!(
payload["candidates"][0]["extra_data"]["upstream_response"]["body"]["error"]["message"],
"private upstream diagnostic"
);
for role in [None, Some("user")] {
let mut request = reqwest::Client::new().get(format!(
"{gateway_url}/api/admin/monitoring/trace/request-1"
));
if let Some(role) = role {
request = request
.header(crate::constants::GATEWAY_HEADER, "rust-phase3b")
.header(TRUSTED_ADMIN_USER_ID_HEADER, "regular-user")
.header(TRUSTED_ADMIN_USER_ROLE_HEADER, role)
.header(TRUSTED_ADMIN_SESSION_ID_HEADER, "regular-session");
}
let response = request
.send()
.await
.expect("unauthorized probe should complete");
assert!(matches!(
response.status(),
StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN
));
let body = response.text().await.expect("denial body should read");
assert!(!body.contains("private upstream diagnostic"));
assert!(!body.contains("upstream-diagnostic-id"));
}
assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
apps/aether-gateway/src/tests/control/admin/providers.rs
+3 -5
View File
@@ -85,10 +85,7 @@ async fn admin_provider_summary_health_preserves_redacted_key_summaries() {
.into_iter()
.map(|(key_id, auth_type, auth_config, score)| {
let mut key = sample_key(key_id, "provider-openai", "openai:chat", "test")
.with_health_fields(
Some(json!({"openai:chat": {"health_score": score}})),
None,
);
.with_health_fields(Some(json!({"openai:chat": {"health_score": score}})), None);
key.auth_type = auth_type.to_string();
key.encrypted_api_key = Some("summary".to_string());
key.encrypted_auth_config = auth_config.map(ToOwned::to_owned);
@@ -115,7 +112,8 @@ async fn admin_provider_summary_health_preserves_redacted_key_summaries() {
let body = axum::body::to_bytes(response.into_body(), 1024 * 1024)
.await
.expect("summary body should read");
let payload: serde_json::Value = serde_json::from_slice(&body).expect("summary should parse");
let payload: serde_json::Value =
serde_json::from_slice(&body).expect("summary should parse");
let summary = if uri == "/api/admin/providers/summary" {
&payload["items"][0]
} else {
apps/aether-gateway/src/usage/reporting/mod.rs
+7 -1
View File
@@ -1308,7 +1308,13 @@ mod tests {
stored[0].error_type.as_deref(),
Some("stream_missing_terminal_event")
);
assert!(stored[0].error_message.is_none());
assert_eq!(
stored[0].error_message.as_deref(),
Some(super::STREAM_MISSING_TERMINAL_EVENT_MESSAGE)
);
let mut public_candidate = stored[0].clone();
public_candidate.sanitize_sensitive_diagnostics();
assert!(public_candidate.error_message.is_none());
}
#[tokio::test]
crates/aether-admin/src/observability/monitoring.rs
+18 -10
View File
@@ -1,9 +1,9 @@
use aether_data_contracts::repository::{
candidates::{
sanitize_request_candidate_api_formats, sanitize_request_candidate_error_type,
sanitize_request_candidate_extra_data, sanitize_request_candidate_required_capabilities,
sanitize_request_candidate_skip_reason, DecisionTrace, DecisionTraceCandidate,
RequestCandidateStatus,
sanitize_request_candidate_extra_data_for_persistence,
sanitize_request_candidate_required_capabilities, sanitize_request_candidate_skip_reason,
DecisionTrace, DecisionTraceCandidate, RequestCandidateStatus,
},
provider_catalog::StoredProviderCatalogKey,
usage::StoredRequestUsageAudit,
@@ -334,10 +334,13 @@ pub fn build_admin_monitoring_trace_request_candidate_payload_with_key_accounts(
key_accounts: &BTreeMap<String, AdminMonitoringKeyAccountDisplay>,
) -> Value {
let mut item = item.clone();
item.sanitize_sensitive_diagnostics();
item.sanitize_for_admin();
let candidate = &item.candidate;
let sanitized_extra_data =
build_admin_monitoring_trace_candidate_extra_data(candidate.extra_data.as_ref(), usage);
let sanitized_extra_data = build_admin_monitoring_trace_candidate_extra_data(
candidate.extra_data.as_ref(),
candidate.status_code,
usage,
);
let sanitized_extra_data_ref =
(!sanitized_extra_data.is_null()).then_some(&sanitized_extra_data);
let sanitized_key_api_formats =
@@ -382,7 +385,7 @@ pub fn build_admin_monitoring_trace_request_candidate_payload_with_key_accounts(
"is_cached": candidate.is_cached,
"status_code": candidate.status_code,
"error_type": sanitize_request_candidate_error_type(candidate.error_type.clone()),
"error_message": serde_json::Value::Null,
"error_message": candidate.error_message,
"latency_ms": candidate.latency_ms,
"concurrent_requests": candidate.concurrent_requests,
"ranking": build_admin_monitoring_trace_candidate_ranking(sanitized_extra_data_ref),
@@ -516,9 +519,10 @@ fn build_admin_monitoring_trace_candidate_ranking(existing: Option<&Value>) -> V
fn build_admin_monitoring_trace_candidate_extra_data(
existing: Option<&Value>,
candidate_status_code: Option<u16>,
usage: Option<&StoredRequestUsageAudit>,
) -> Value {
let mut extra_data = sanitize_request_candidate_extra_data(existing.cloned())
let mut extra_data = sanitize_request_candidate_extra_data_for_persistence(existing.cloned())
.and_then(|value| value.as_object().cloned());
if let Some(usage) = usage {
@@ -546,7 +550,7 @@ fn build_admin_monitoring_trace_candidate_extra_data(
if admin_monitoring_usage_is_error_node(usage) {
if let Some(response) = admin_monitoring_trace_response_data(
"upstream_response",
usage.status_code,
candidate_status_code,
usage.response_body_state,
) {
merge_admin_monitoring_trace_response(extra_object, "upstream_response", response);
@@ -573,7 +577,8 @@ fn build_admin_monitoring_trace_candidate_extra_data(
}
}
sanitize_request_candidate_extra_data(extra_data.map(Value::Object)).unwrap_or(Value::Null)
sanitize_request_candidate_extra_data_for_persistence(extra_data.map(Value::Object))
.unwrap_or(Value::Null)
}
fn admin_monitoring_trace_response_data(
@@ -607,6 +612,9 @@ fn merge_admin_monitoring_trace_response(
};
for (field, value) in response_object {
if field == "status_code" && existing_object.get(field).is_some_and(Value::is_number) {
continue;
}
if admin_monitoring_trace_response_value_empty(value)
&& existing_object
.get(field)
crates/aether-data/adapters/postgres/src/candidates.rs
+121 -21
View File
@@ -172,7 +172,7 @@ DO UPDATE SET
THEN __AETHER_SANITIZED_LEGACY_ERROR_TYPE__
ELSE EXCLUDED.error_type
END,
error_message = NULL,
error_message = __AETHER_CANDIDATE_ERROR_MESSAGE__,
latency_ms = CASE
WHEN request_candidates.status IN ('success', 'failed', 'cancelled', 'skipped')
AND EXCLUDED.status <> request_candidates.status
@@ -184,7 +184,7 @@ DO UPDATE SET
ELSE COALESCE(EXCLUDED.latency_ms, request_candidates.latency_ms)
END,
concurrent_requests = COALESCE(EXCLUDED.concurrent_requests, request_candidates.concurrent_requests),
extra_data = EXCLUDED.extra_data,
extra_data = __AETHER_CANDIDATE_EXTRA_DATA__,
required_capabilities = EXCLUDED.required_capabilities,
created_at = CASE
WHEN request_candidates.created_at <= TO_TIMESTAMP(1)
@@ -275,7 +275,7 @@ DO UPDATE SET
THEN __AETHER_SANITIZED_LEGACY_ERROR_TYPE__
ELSE EXCLUDED.error_type
END,
error_message = NULL,
error_message = __AETHER_CANDIDATE_ERROR_MESSAGE__,
latency_ms = CASE
WHEN request_candidates.status IN ('success', 'failed', 'cancelled', 'skipped')
AND EXCLUDED.status <> request_candidates.status
@@ -287,7 +287,7 @@ DO UPDATE SET
ELSE COALESCE(EXCLUDED.latency_ms, request_candidates.latency_ms)
END,
concurrent_requests = COALESCE(EXCLUDED.concurrent_requests, request_candidates.concurrent_requests),
extra_data = EXCLUDED.extra_data,
extra_data = __AETHER_CANDIDATE_EXTRA_DATA__,
required_capabilities = EXCLUDED.required_capabilities,
created_at = CASE
WHEN request_candidates.created_at <= TO_TIMESTAMP(1)
@@ -353,7 +353,7 @@ DO UPDATE SET
THEN __AETHER_SANITIZED_LEGACY_ERROR_TYPE__
ELSE EXCLUDED.error_type
END,
error_message = NULL,
error_message = __AETHER_CANDIDATE_ERROR_MESSAGE__,
latency_ms = CASE
WHEN request_candidates.status IN ('success', 'failed', 'cancelled', 'skipped')
AND EXCLUDED.status <> request_candidates.status
@@ -365,7 +365,7 @@ DO UPDATE SET
ELSE COALESCE(EXCLUDED.latency_ms, request_candidates.latency_ms)
END,
concurrent_requests = COALESCE(EXCLUDED.concurrent_requests, request_candidates.concurrent_requests),
extra_data = EXCLUDED.extra_data,
extra_data = __AETHER_CANDIDATE_EXTRA_DATA__,
required_capabilities = EXCLUDED.required_capabilities,
created_at = CASE
WHEN request_candidates.created_at <= TO_TIMESTAMP(1)
@@ -446,6 +446,23 @@ fn postgres_candidate_upsert_sql(template: &str) -> String {
)
.as_str(),
)
.replace(
"__AETHER_CANDIDATE_ERROR_MESSAGE__",
"CASE WHEN request_candidates.status IN ('success', 'failed', 'cancelled', 'skipped') \
AND EXCLUDED.status <> request_candidates.status \
THEN request_candidates.error_message \
WHEN request_candidates.status = 'pending' AND EXCLUDED.status IN ('available', 'unused') \
THEN request_candidates.error_message \
WHEN request_candidates.status = 'streaming' AND EXCLUDED.status IN ('available', 'unused', 'pending') \
THEN request_candidates.error_message \
ELSE COALESCE(EXCLUDED.error_message, request_candidates.error_message) END",
)
.replace(
"__AETHER_CANDIDATE_EXTRA_DATA__",
"CASE WHEN request_candidates.status IN ('success', 'failed', 'cancelled', 'skipped') \
AND (EXCLUDED.status <> request_candidates.status OR EXCLUDED.extra_data IS NULL) \
THEN request_candidates.extra_data ELSE EXCLUDED.extra_data END",
)
}
fn postgres_sanitized_legacy_diagnostic_sql(
@@ -1381,33 +1398,35 @@ mod tests {
finished_at_unix_ms: Some(2),
};
assert_eq!(sanitize_request_candidate_for_postgres(&mut candidate), 0);
assert_eq!(sanitize_request_candidate_for_postgres(&mut candidate), 1);
assert!(candidate.username.is_none());
assert!(candidate.api_key_name.is_none());
assert_eq!(candidate.skip_reason.as_deref(), Some("unclassified_skip"));
assert_eq!(candidate.error_type.as_deref(), Some("unclassified_error"));
assert!(candidate.error_message.is_none());
assert_eq!(candidate.error_message.as_deref(), Some("bad�message"));
assert!(candidate.extra_data.is_none());
assert!(candidate.required_capabilities.is_none());
}
#[test]
fn every_postgres_candidate_conflict_path_discards_legacy_diagnostics() {
fn every_postgres_candidate_conflict_path_preserves_errors_without_unrelated_legacy_data() {
for sql in [
UPSERT_SQL.as_str(),
UPSERT_CONFLICT_SQL.as_str(),
UPSERT_CONFLICT_INHERIT_IS_CACHED_SQL.as_str(),
] {
assert!(sql.contains("error_message = NULL"));
assert!(sql.contains("extra_data = EXCLUDED.extra_data"));
assert!(
sql.contains("COALESCE(EXCLUDED.error_message, request_candidates.error_message)")
);
assert!(sql.contains("THEN request_candidates.extra_data ELSE EXCLUDED.extra_data END"));
assert!(sql.contains("required_capabilities = EXCLUDED.required_capabilities"));
assert!(!sql.contains("request_candidates.error_message"));
assert!(!sql.contains("request_candidates.extra_data"));
assert!(!sql.contains("COALESCE(request_candidates.extra_data"));
assert!(!sql.contains("request_candidates.required_capabilities"));
assert!(sql.contains("ELSE 'unclassified_skip' END"));
assert!(sql.contains("ELSE 'unclassified_error' END"));
assert!(sql.contains("THEN 'first_byte_timeout'"));
assert!(!sql.contains("__AETHER_SANITIZED_LEGACY_"));
assert!(!sql.contains("__AETHER_CANDIDATE_"));
}
}
@@ -1541,10 +1560,11 @@ VALUES ($1, $2, 0, 0, 'pending', $3, $4, $5::json, $6::json, $7, NOW())
.fetch_one(repository.pool())
.await
.expect("raw candidate diagnostics should load");
assert!(
assert_eq!(
sqlx::Row::try_get::<Option<String>, _>(&raw, "error_message")
.expect("error_message should decode")
.is_none()
.as_deref(),
Some("bad�message")
);
assert_eq!(
sqlx::Row::try_get::<Option<String>, _>(&raw, "skip_reason")
@@ -1576,7 +1596,7 @@ VALUES ($1, $2, 0, 0, 'pending', $3, $4, $5::json, $6::json, $7, NOW())
.expect("sanitized candidate should be readable");
assert_eq!(rows.len(), 1);
assert_eq!(rows[0].status, RequestCandidateStatus::Success);
assert!(rows[0].error_message.is_none());
assert_eq!(rows[0].error_message.as_deref(), Some("bad�message"));
assert!(rows[0].extra_data.is_none());
assert!(rows[0].required_capabilities.is_none());
}
@@ -1589,12 +1609,92 @@ VALUES ($1, $2, 0, 0, 'pending', $3, $4, $5::json, $6::json, $7, NOW())
1
);
let mut cleanup_request_ids = vec![single_request_id, batch_request_id, healthy_request_id];
for write_path in 0..3 {
let request_id = uuid::Uuid::new_v4().to_string();
cleanup_request_ids.push(request_id.clone());
let mut failed = candidate(&request_id, uuid::Uuid::new_v4().to_string());
failed.status = RequestCandidateStatus::Failed;
failed.status_code = Some(400);
failed.error_message = Some("original upstream failure".to_string());
failed.is_cached = (write_path != 2).then_some(false);
failed.extra_data = Some(json!({
"upstream_response": {
"status_code": 400,
"headers": {"x-request-id": "original-upstream-id"},
"body": {"error": {"message": "original upstream failure", "param": "model"}}
},
"error_flow": {"status_code": 400, "message": "original upstream failure"}
}));
let mut pending = failed.clone();
pending.status = RequestCandidateStatus::Pending;
pending.status_code = None;
pending.error_message = None;
pending.extra_data = None;
repository
.upsert(pending.clone())
.await
.expect("pending seed should persist");
if write_path == 0 {
repository
.upsert(failed)
.await
.expect("single failure should persist");
} else {
repository
.upsert_many(vec![failed])
.await
.expect("batch failure should persist");
}
pending.status_code = Some(200);
pending.error_message = Some("late unrelated error".to_string());
pending.extra_data = Some(json!({
"upstream_response": {"status_code": 200, "body": "late unrelated response"}
}));
if write_path == 0 {
repository
.upsert(pending)
.await
.expect("late update should persist");
} else {
repository
.upsert_many(vec![pending])
.await
.expect("late batch should persist");
}
let stored = repository
.list_by_request_id(&request_id)
.await
.expect("failure should read");
assert_eq!(stored[0].status, RequestCandidateStatus::Failed);
assert_eq!(stored[0].status_code, Some(400));
assert_eq!(
stored[0].error_message.as_deref(),
Some("original upstream failure")
);
let extra = stored[0]
.extra_data
.as_ref()
.expect("failure details should remain");
assert_eq!(extra["upstream_response"]["status_code"], 400);
assert_eq!(
extra["upstream_response"]["headers"]["x-request-id"],
"original-upstream-id"
);
assert_eq!(
extra["upstream_response"]["body"]["error"]["param"],
"model"
);
assert_eq!(extra["error_flow"]["message"], "original upstream failure");
let mut public = stored[0].clone();
public.sanitize_sensitive_diagnostics();
assert!(!serde_json::to_string(&public)
.expect("public record should serialize")
.contains("original upstream failure"));
}
sqlx::query("DELETE FROM request_candidates WHERE request_id = ANY($1)")
.bind(vec![
single_request_id,
batch_request_id,
healthy_request_id,
])
.bind(cleanup_request_ids)
.execute(repository.pool())
.await
.expect("candidate NUL test rows should clean up");
crates/aether-data/contracts/src/repository/candidates/mod.rs
+1
View File
@@ -4,6 +4,7 @@ pub use types::{
build_decision_trace, derive_request_candidate_final_status,
request_candidate_lifecycle_would_regress, sanitize_request_candidate_api_formats,
sanitize_request_candidate_error_type, sanitize_request_candidate_extra_data,
sanitize_request_candidate_extra_data_for_persistence,
sanitize_request_candidate_required_capabilities, sanitize_request_candidate_skip_reason,
DecisionTrace, DecisionTraceCandidate, PublicHealthStatusCount, PublicHealthTimelineBucket,
RequestCandidateFinalStatus, RequestCandidateReadRepository, RequestCandidateRepository,
crates/aether-data/contracts/src/repository/candidates/types.rs
+162 -14
View File
@@ -224,6 +224,21 @@ pub struct StoredRequestCandidate {
}
impl StoredRequestCandidate {
pub fn sanitize_for_persistence(&mut self) {
self.username = None;
self.api_key_name = None;
self.skip_reason = sanitize_request_candidate_skip_reason(self.skip_reason.take());
self.error_type = sanitize_request_candidate_error_type(self.error_type.take());
self.error_message = self
.error_message
.take()
.map(limit_candidate_diagnostic_text);
self.extra_data =
sanitize_request_candidate_extra_data_for_persistence(self.extra_data.take());
self.required_capabilities =
sanitize_request_candidate_required_capabilities(self.required_capabilities.take());
}
pub fn sanitize_sensitive_diagnostics(&mut self) {
self.username = None;
self.api_key_name = None;
@@ -349,7 +364,7 @@ impl StoredRequestCandidate {
started_at_unix_ms,
finished_at_unix_ms,
};
candidate.sanitize_sensitive_diagnostics();
candidate.sanitize_for_persistence();
Ok(candidate)
}
}
@@ -386,7 +401,7 @@ impl RequestCandidateTrace {
attempted_only: bool,
) -> Option<Self> {
for candidate in &mut all_candidates {
candidate.sanitize_sensitive_diagnostics();
candidate.sanitize_for_persistence();
}
if all_candidates.is_empty() {
return None;
@@ -510,8 +525,17 @@ pub struct DecisionTraceCandidate {
}
impl DecisionTraceCandidate {
pub fn sanitize_for_admin(&mut self) {
self.candidate.sanitize_for_persistence();
self.sanitize_catalog_metadata();
}
pub fn sanitize_sensitive_diagnostics(&mut self) {
self.candidate.sanitize_sensitive_diagnostics();
self.sanitize_catalog_metadata();
}
fn sanitize_catalog_metadata(&mut self) {
self.provider_website = self
.provider_website
.take()
@@ -574,7 +598,9 @@ pub fn build_decision_trace(
})
.collect(),
};
trace.sanitize_sensitive_diagnostics();
for item in &mut trace.candidates {
item.sanitize_for_admin();
}
trace
}
@@ -727,8 +753,12 @@ impl UpsertRequestCandidateRecord {
self.api_key_name = None;
self.skip_reason = sanitize_request_candidate_skip_reason(self.skip_reason.take());
self.error_type = sanitize_request_candidate_error_type(self.error_type.take());
self.error_message = None;
self.extra_data = sanitize_request_candidate_extra_data(self.extra_data.take());
self.error_message = self
.error_message
.take()
.map(limit_candidate_diagnostic_text);
self.extra_data =
sanitize_request_candidate_extra_data_for_persistence(self.extra_data.take());
self.required_capabilities =
sanitize_request_candidate_required_capabilities(self.required_capabilities.take());
}
@@ -788,6 +818,78 @@ pub fn sanitize_request_candidate_error_type(value: Option<String>) -> Option<St
Some(safe)
}
const MAX_CANDIDATE_DIAGNOSTIC_BYTES: usize = 65_536;
fn limit_candidate_diagnostic_text(mut text: String) -> String {
const SUFFIX: &str = "...[truncated]";
if text.len() > MAX_CANDIDATE_DIAGNOSTIC_BYTES {
let mut boundary = MAX_CANDIDATE_DIAGNOSTIC_BYTES - SUFFIX.len();
while !text.is_char_boundary(boundary) {
boundary -= 1;
}
text.truncate(boundary);
text.push_str(SUFFIX);
}
text
}
fn limit_candidate_diagnostic_value(value: &serde_json::Value) -> serde_json::Value {
if let Some(text) = value.as_str() {
return serde_json::Value::String(limit_candidate_diagnostic_text(text.to_string()));
}
let serialized = value.to_string();
if serialized.len() > MAX_CANDIDATE_DIAGNOSTIC_BYTES {
serde_json::Value::String(limit_candidate_diagnostic_text(serialized))
} else {
value.clone()
}
}
pub fn sanitize_request_candidate_extra_data_for_persistence(
extra_data: Option<serde_json::Value>,
) -> Option<serde_json::Value> {
let object = extra_data.as_ref()?.as_object()?;
let mut sanitized = sanitize_request_candidate_extra_data(extra_data.clone())
.and_then(|value| value.as_object().cloned())
.unwrap_or_default();
for (key, fields) in [
("upstream_response", &["headers", "body"][..]),
("error_flow", &["message"][..]),
(
"failure_diagnostic",
&["path", "field_path", "message", "type", "reason"][..],
),
(
"request_conversion_error",
&["path", "field_path", "message", "type", "reason"][..],
),
(
"request_body_build_error",
&["path", "field_path", "message", "type", "reason"][..],
),
] {
let Some(diagnostic) = object.get(key).and_then(serde_json::Value::as_object) else {
continue;
};
let mut summary = sanitized
.remove(key)
.and_then(|value| value.as_object().cloned())
.unwrap_or_default();
for field in fields {
if let Some(value) = diagnostic.get(*field).filter(|value| !value.is_null()) {
summary.insert(
(*field).to_string(),
limit_candidate_diagnostic_value(value),
);
}
}
if !summary.is_empty() {
sanitized.insert(key.to_string(), serde_json::Value::Object(summary));
}
}
(!sanitized.is_empty()).then_some(serde_json::Value::Object(sanitized))
}
pub fn sanitize_request_candidate_extra_data(
extra_data: Option<serde_json::Value>,
) -> Option<serde_json::Value> {
@@ -1949,7 +2051,7 @@ mod tests {
}
#[test]
fn candidate_persistence_removes_credentials_and_raw_payloads() {
fn candidate_persistence_keeps_admin_errors_but_removes_request_credentials() {
let mut record = UpsertRequestCandidateRecord {
id: "candidate-1".to_string(),
request_id: "request-1".to_string(),
@@ -2079,7 +2181,7 @@ mod tests {
);
assert!(record.username.is_none());
assert!(record.api_key_name.is_none());
assert!(record.error_message.is_none());
assert_eq!(record.error_message.as_deref(), Some("unauthorized"));
let extra = record
.extra_data
.as_ref()
@@ -2112,7 +2214,10 @@ mod tests {
assert_eq!(extra["error_flow"]["stage"], "upstream");
assert_eq!(extra["error_flow"]["retryable"], true);
assert_eq!(extra["error_flow"]["status_code"], 401);
assert!(extra["error_flow"].get("message").is_none());
assert_eq!(
extra["error_flow"]["message"],
"token vertex-secret rejected"
);
assert_eq!(extra["gateway_execution_runtime"], true);
assert_eq!(extra["client_api_format"], "openai:responses");
assert_eq!(extra["provider_api_format"], "claude:messages");
@@ -2127,8 +2232,14 @@ mod tests {
assert_eq!(extra["upstream_response"]["source"], "upstream_response");
assert_eq!(extra["upstream_response"]["status_code"], 401);
assert_eq!(extra["upstream_response"]["body_state"], "inline");
assert!(extra["upstream_response"].get("headers").is_none());
assert!(extra["upstream_response"].get("body").is_none());
assert_eq!(
extra["upstream_response"]["headers"]["set-cookie"],
"session=secret"
);
assert_eq!(
extra["upstream_response"]["body"]["error"]["message"],
"token vertex-secret rejected"
);
assert_eq!(
extra["image_progress"]["last_client_visible_event"],
"image_generation.partial_image"
@@ -2178,7 +2289,6 @@ mod tests {
let serialized = serde_json::to_string(&record).expect("candidate should serialize");
for sensitive in [
"vertex-secret",
"client-secret",
"credential-label-secret",
"header-rule-secret",
@@ -2199,6 +2309,11 @@ mod tests {
"candidate must not retain {sensitive}"
);
}
let public_extra = super::sanitize_request_candidate_extra_data(record.extra_data);
let serialized =
serde_json::to_string(&public_extra).expect("public data should serialize");
assert!(!serialized.contains("vertex-secret"));
assert!(!serialized.contains("session=secret"));
}
#[test]
@@ -2278,8 +2393,8 @@ mod tests {
}
#[test]
fn candidate_database_read_sanitizes_legacy_diagnostic_text() {
let candidate = StoredRequestCandidate::new(
fn candidate_database_read_preserves_errors_until_public_projection() {
let mut candidate = StoredRequestCandidate::new(
"candidate-1".to_string(),
"request-1".to_string(),
None,
@@ -2315,9 +2430,42 @@ mod tests {
candidate.error_type.as_deref(),
Some(UNCLASSIFIED_CANDIDATE_ERROR_TYPE)
);
assert!(candidate.error_message.is_none());
assert_eq!(
candidate.error_message.as_deref(),
Some("legacy secret message")
);
assert!(candidate.username.is_none());
assert!(candidate.api_key_name.is_none());
candidate.sanitize_sensitive_diagnostics();
assert!(candidate.error_message.is_none());
}
#[test]
fn admin_diagnostics_are_bounded_and_public_projection_removes_them() {
let raw = json!({
"upstream_response": {"status_code": 400, "body": "错误内容".repeat(20_000)},
"error_flow": {"status_code": 400, "message": "private upstream failure"},
"failure_diagnostic": {"path": "$.input", "message": "private conversion failure"},
"request_body": {"input": "private prompt"}
});
let admin = super::sanitize_request_candidate_extra_data_for_persistence(Some(raw))
.expect("admin diagnostics should remain");
let body = admin["upstream_response"]["body"]
.as_str()
.expect("body should be text");
assert!(body.len() <= 65_536);
assert!(body.ends_with("...[truncated]"));
assert!(admin.get("request_body").is_none());
assert_eq!(
super::sanitize_request_candidate_extra_data_for_persistence(Some(admin.clone())),
Some(admin.clone()),
);
let public = super::sanitize_request_candidate_extra_data(Some(admin))
.expect("public status should remain");
assert_eq!(public["upstream_response"]["status_code"], 400);
assert!(public["upstream_response"].get("body").is_none());
assert!(public["error_flow"].get("message").is_none());
assert!(public.get("failure_diagnostic").is_none());
}
#[test]
crates/aether-data/runtime/src/repository/candidates/memory.rs
+51 -10
View File
@@ -10,19 +10,31 @@ use crate::DataLayerError;
use async_trait::async_trait;
fn sanitize_stored_candidate(mut candidate: StoredRequestCandidate) -> StoredRequestCandidate {
candidate.sanitize_sensitive_diagnostics();
candidate.sanitize_for_persistence();
candidate
}
fn merge_extra_data(
existing: Option<serde_json::Value>,
overlay: Option<serde_json::Value>,
preserve_error_details: bool,
) -> Option<serde_json::Value> {
match (existing, overlay) {
(
Some(serde_json::Value::Object(mut existing_object)),
Some(serde_json::Value::Object(overlay_object)),
Some(serde_json::Value::Object(mut overlay_object)),
) => {
if preserve_error_details {
for key in [
"upstream_response",
"error_flow",
"failure_diagnostic",
"request_conversion_error",
"request_body_build_error",
] {
overlay_object.remove(key);
}
}
existing_object.extend(overlay_object);
Some(serde_json::Value::Object(existing_object))
}
@@ -397,7 +409,13 @@ impl RequestCandidateWriteRepository for InMemoryRequestCandidateRepository {
.error_type
.or_else(|| existing.as_ref().and_then(|row| row.error_type.clone()))
},
error_message: None,
error_message: if preserve_existing_lifecycle {
existing.as_ref().and_then(|row| row.error_message.clone())
} else {
candidate
.error_message
.or_else(|| existing.as_ref().and_then(|row| row.error_message.clone()))
},
latency_ms: if preserve_existing_lifecycle {
existing.as_ref().and_then(|row| row.latency_ms)
} else {
@@ -411,6 +429,7 @@ impl RequestCandidateWriteRepository for InMemoryRequestCandidateRepository {
extra_data: merge_extra_data(
existing.as_ref().and_then(|row| row.extra_data.clone()),
candidate.extra_data,
preserve_existing_lifecycle,
),
required_capabilities: candidate.required_capabilities.or_else(|| {
existing
@@ -589,7 +608,10 @@ mod tests {
let candidate = stored
.get("cand-raw")
.expect("seeded candidate should exist");
assert!(candidate.error_message.is_none());
assert_eq!(
candidate.error_message.as_deref(),
Some("Bearer secret-token")
);
assert_eq!(candidate.skip_reason.as_deref(), Some("unclassified_skip"));
assert_eq!(candidate.error_type.as_deref(), Some("unclassified_error"));
assert_eq!(
@@ -619,7 +641,10 @@ mod tests {
.iter()
.find(|candidate| candidate.id == "cand-bypassed")
.expect("bypassed candidate should be returned");
assert!(candidate.error_message.is_none());
assert_eq!(
candidate.error_message.as_deref(),
Some("Bearer secret-token")
);
assert_eq!(
candidate.extra_data,
Some(json!({"gateway_execution_runtime": true}))
@@ -665,7 +690,7 @@ mod tests {
.await
.expect("candidate merge should succeed");
assert_eq!(merged.id, "cand-bypassed");
assert!(merged.error_message.is_none());
assert_eq!(merged.error_message.as_deref(), Some("Bearer secret-token"));
assert_eq!(
merged.extra_data,
Some(json!({
@@ -839,7 +864,13 @@ mod tests {
Some("retryable upstream failure".to_string()),
Some(45),
Some(1),
Some(json!({"stream_completed": true})),
Some(json!({
"stream_completed": true,
"upstream_response": {
"status_code": 503,
"body": {"error": {"message": "original upstream failure"}}
}
})),
None,
100,
Some(101),
@@ -869,7 +900,10 @@ mod tests {
error_message: None,
latency_ms: Some(9_999),
concurrent_requests: Some(2),
extra_data: Some(json!({"gateway_execution_runtime": true})),
extra_data: Some(json!({
"gateway_execution_runtime": true,
"upstream_response": {"status_code": 200, "body": "late unrelated response"}
})),
required_capabilities: None,
created_at_unix_ms: None,
started_at_unix_ms: Some(102),
@@ -882,7 +916,10 @@ mod tests {
assert_eq!(updated.status, RequestCandidateStatus::Failed);
assert_eq!(updated.status_code, Some(503));
assert_eq!(updated.error_type.as_deref(), Some("upstream_error"));
assert!(updated.error_message.is_none());
assert_eq!(
updated.error_message.as_deref(),
Some("retryable upstream failure")
);
assert_eq!(updated.latency_ms, Some(45));
assert_eq!(updated.concurrent_requests, Some(2));
assert_eq!(updated.finished_at_unix_ms, Some(145));
@@ -890,7 +927,11 @@ mod tests {
updated.extra_data,
Some(json!({
"gateway_execution_runtime": true,
"stream_completed": true
"stream_completed": true,
"upstream_response": {
"status_code": 503,
"body": {"error": {"message": "original upstream failure"}}
}
}))
);
}
frontend/src/features/usage/components/HorizontalRequestTimeline.vue
+27 -15
View File
@@ -477,6 +477,18 @@
HTTP {{ currentAttemptRequestError.statusCode }}
</span>
</div>
<div
v-if="currentAttemptRequestError.upstreamStatusCode != null && currentAttemptRequestError.upstreamStatusCode !== currentAttemptRequestError.statusCode"
class="error-upstream-status text-xs text-muted-foreground mb-1"
>
上游响应状态：HTTP {{ currentAttemptRequestError.upstreamStatusCode }}（本次尝试状态见上方）
</div>
<div
v-if="currentAttemptRequestError.finalStatusCode != null"
class="error-final-status text-xs text-muted-foreground mb-1"
>
请求最终状态：HTTP {{ currentAttemptRequestError.finalStatusCode }}（与本次尝试状态不同）
</div>
<div
v-if="currentAttemptRequestError.message"
class="error-msg"
@@ -1630,21 +1642,19 @@ const shouldShowAttemptMessageWithUpstreamResponse = (
rawMessage: string,
upstreamResponse: Record<string, unknown> | null,
): boolean => {
if (!upstreamResponse) return true
if (!upstreamResponse || !hasRenderableValue(upstreamResponse.body)) return true
const normalized = rawMessage.trim()
if (!normalized) return false
if (isLocalSyncFinalizeDiagnostic(normalized)) return true
if (isConversionDiagnosticMessage(normalized)) return true
if (isGenericExecutionRuntimeStatusMessage(normalized)) return false
const hasBody = hasRenderableValue(upstreamResponse.body)
const bodyState = (readStringField(upstreamResponse, 'body_state') ?? '').toLowerCase()
return !hasBody && bodyState === 'disabled'
return false
}
const currentAttemptRequestError = computed<{
message: string
statusCode?: number
upstreamStatusCode?: number
finalStatusCode?: number
upstreamResponse: Record<string, unknown> | null
diagnostic: Record<string, unknown> | null
} | null>(() => {
@@ -1654,11 +1664,16 @@ const currentAttemptRequestError = computed<{
const extra = extractObject(attempt.extra_data)
const upstreamResponse = extractObject(extra?.upstream_response)
const errorFlow = extractObject(extra?.error_flow)
const statusCode = readNumberField(upstreamResponse ?? {}, 'status_code')
const upstreamStatusCode = readNumberField(upstreamResponse ?? {}, 'status_code')
?? readNumberField(upstreamResponse ?? {}, 'statusCode')
const statusCode = attempt.status_code
?? upstreamStatusCode
?? readNumberField(errorFlow ?? {}, 'status_code')
?? readNumberField(errorFlow ?? {}, 'statusCode')
?? attempt.status_code
const finalStatusCode = !['pending', 'streaming'].includes(computedFinalStatus.value)
&& props.overrideStatusCode != null && props.overrideStatusCode !== statusCode
? props.overrideStatusCode
: undefined
const flowMessage = errorFlow
? readStringField(errorFlow, 'message')
: ''
@@ -1671,6 +1686,7 @@ const currentAttemptRequestError = computed<{
const diagnosticMessage = extractVisibleDiagnosticMessage(extra)
const rawMessage = chooseAttemptRawErrorMessage(flowMessage || '', fallbackMessage, diagnosticMessage)
const message = formatAttemptErrorMessage(rawMessage, statusCode) || fallbackType
|| '本次尝试失败，链路追踪未包含详细错误内容。'
const upstreamResponseDisplay = normalizeUpstreamResponseDisplay(extra?.upstream_response)
const visibleDiagnosticObjects = extractVisibleDiagnosticObjects(extra)
const shouldAttachDiagnostic = Boolean(
@@ -1695,20 +1711,16 @@ const currentAttemptRequestError = computed<{
const response = Object.keys(upstreamResponseData).length > 0
? upstreamResponseData
: null
if (
!message
&& statusCode == null
&& !response
&& !diagnostic
) return null
const showMessage = shouldShowAttemptMessageWithUpstreamResponse(
rawMessage || fallbackType,
upstreamResponseDisplay,
)
return {
message: showMessage ? (message || '未知错误') : '',
message: showMessage ? message : '',
statusCode,
upstreamStatusCode,
finalStatusCode,
upstreamResponse: response,
diagnostic,
}
frontend/src/features/usage/components/__tests__/HorizontalRequestTimeline.spec.ts
+92
View File
@@ -659,6 +659,98 @@ describe('HorizontalRequestTimeline', () => {
expect(root.textContent).not.toContain('该错误被标记为敏感上游错误')
})
it.each(['inline', 'reference', 'disabled', 'unavailable', 'none', undefined])(
'shows a fallback for redacted errors with body state %s',
async (bodyState) => {
const trace = buildTrace([
buildCandidate({
status_code: 400,
extra_data: {
upstream_response: {
status_code: 400,
body_state: bodyState,
},
error_flow: {
source: 'upstream_response',
status_code: 400,
decision: 'retry_next_candidate',
},
},
}),
])
trace.final_status = 'failed'
const root = mountTimeline(trace, {
overrideStatusCode: 503,
requestStatus: 'failed',
})
await nextTick()
expect(root.querySelector('.status-tag')?.textContent?.trim()).toBe('400')
expect(root.querySelector('.error-status-badge')?.textContent?.trim()).toBe('HTTP 400')
expect(root.querySelector('.error-msg')?.textContent).toContain('链路追踪未包含详细错误内容')
expect(root.querySelector('.error-final-status')?.textContent).toContain('请求最终状态：HTTP 503')
expect(root.querySelector('.error-upstream-response-json')).toBeNull()
},
)
it('keeps the attempt status separate from the upstream transport status', async () => {
const root = mountTimeline(buildTrace([
buildCandidate({
status_code: 502,
error_type: 'stream_error',
extra_data: {
upstream_response: { status_code: 200, body_state: 'disabled' },
},
}),
]))
await nextTick()
expect(root.querySelector('.status-tag')?.textContent?.trim()).toBe('502')
expect(root.querySelector('.error-status-badge')?.textContent?.trim()).toBe('HTTP 502')
expect(root.querySelector('.error-upstream-status')?.textContent).toContain('上游响应状态：HTTP 200')
expect(root.querySelector('.error-final-status')).toBeNull()
})
it('does not label an active request status as final', async () => {
const root = mountTimeline(buildTrace([
buildCandidate({ status_code: 400 }),
]), {
overrideStatusCode: 200,
requestStatus: 'streaming',
})
await nextTick()
expect(root.querySelector('.error-final-status')).toBeNull()
})
it('shows failure details even when no status or error message was retained', async () => {
const root = mountTimeline(buildTrace([buildCandidate()]))
await nextTick()
expect(root.querySelector('.error-msg')?.textContent).toContain('链路追踪未包含详细错误内容')
})
it('keeps a generic error visible when only upstream headers are available', async () => {
const root = mountTimeline(buildTrace([
buildCandidate({
status_code: 400,
error_message: 'execution runtime stream returned non-success status 400',
extra_data: {
upstream_response: {
status_code: 400,
headers: { 'content-type': 'application/json' },
body_state: 'reference',
},
},
}),
]))
await nextTick()
expect(root.querySelector('.error-msg')?.textContent).toContain('上游返回非成功状态 400')
expect(root.querySelector('.error-upstream-response-json')?.textContent).toContain('application/json')
})
it('keeps local sync diagnostics visible when upstream response body capture is disabled', async () => {
const trace = buildTrace([
buildCandidate({