fix monitoring trace lookup fallback

This commit is contained in:
fawney19
2026-05-23 01:22:24 +08:00
parent c7641dad0a
commit ab048b8a03
4 changed files with 298 additions and 17 deletions

View File

@@ -81,6 +81,133 @@ async fn admin_monitoring_trace_request_returns_local_payload() {
assert_eq!(payload["candidates"][0]["status_code"], json!(502)); assert_eq!(payload["candidates"][0]["status_code"], json!(502));
} }
#[tokio::test]
async fn admin_monitoring_trace_request_resolves_usage_id_to_header_trace_id() {
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![
sample_candidate(
"cand-used",
"trace-1",
0,
RequestCandidateStatus::Success,
Some(101),
Some(33),
Some(200),
),
]));
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider()],
vec![sample_endpoint()],
vec![sample_key()],
));
let mut usage = sample_usage(
"usage-request-1",
"provider-1",
"OpenAI",
40,
0.02,
"completed",
Some(200),
100,
);
usage.id = "usage-row-1".to_string();
usage.candidate_id = Some("cand-used".to_string());
usage.request_headers = Some(json!({
"x-trace-id": "trace-1"
}));
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(vec![usage]));
let data_state =
crate::data::GatewayDataState::with_request_candidate_and_usage_repository_for_tests(
request_candidates,
usage_repository,
)
.with_provider_catalog_reader(provider_catalog);
let state = AppState::new()
.expect("state should build")
.with_data_state_for_tests(data_state);
let context = request_context(
http::Method::GET,
"/api/admin/monitoring/trace/usage-row-1?attempted_only=true",
);
let response = local_monitoring_response(&state, &context)
.await
.expect("handler should not error")
.expect("route should be handled locally");
assert_eq!(response.status(), http::StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX)
.await
.expect("body should read");
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
assert_eq!(payload["request_id"], json!("trace-1"));
assert_eq!(payload["candidates"][0]["id"], json!("cand-used"));
assert_eq!(
payload["candidates"][0]["extra_data"]["first_byte_time_ms"],
json!(30)
);
}
#[tokio::test]
async fn admin_monitoring_trace_request_resolves_usage_request_id_to_metadata_trace_id() {
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![
sample_candidate(
"cand-used",
"trace-2",
0,
RequestCandidateStatus::Success,
Some(101),
Some(33),
Some(200),
),
]));
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider()],
vec![sample_endpoint()],
vec![sample_key()],
));
let mut usage = sample_usage(
"usage-request-2",
"provider-1",
"OpenAI",
40,
0.02,
"completed",
Some(200),
100,
);
usage.candidate_id = Some("cand-used".to_string());
usage.request_metadata = Some(json!({
"trace_id": "trace-2"
}));
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(vec![usage]));
let data_state =
crate::data::GatewayDataState::with_request_candidate_and_usage_repository_for_tests(
request_candidates,
usage_repository,
)
.with_provider_catalog_reader(provider_catalog);
let state = AppState::new()
.expect("state should build")
.with_data_state_for_tests(data_state);
let context = request_context(
http::Method::GET,
"/api/admin/monitoring/trace/usage-request-2",
);
let response = local_monitoring_response(&state, &context)
.await
.expect("handler should not error")
.expect("route should be handled locally");
assert_eq!(response.status(), http::StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX)
.await
.expect("body should read");
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
assert_eq!(payload["request_id"], json!("trace-2"));
assert_eq!(payload["candidates"][0]["id"], json!("cand-used"));
}
#[tokio::test] #[tokio::test]
async fn admin_monitoring_trace_request_returns_oauth_account_label_from_auth_config() { async fn admin_monitoring_trace_request_returns_oauth_account_label_from_auth_config() {
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![ let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![

View File

@@ -12,6 +12,7 @@ use aether_admin::observability::monitoring::{
use aether_data_contracts::repository::{ use aether_data_contracts::repository::{
candidates::{DecisionTrace, RequestCandidateStatus}, candidates::{DecisionTrace, RequestCandidateStatus},
provider_catalog::StoredProviderCatalogKey, provider_catalog::StoredProviderCatalogKey,
usage::StoredRequestUsageAudit,
}; };
use axum::{ use axum::{
body::Body, body::Body,
@@ -21,12 +22,16 @@ use serde_json::{Map, Value};
use std::collections::BTreeMap; use std::collections::BTreeMap;
use tracing::debug; use tracing::debug;
struct ResolvedAdminMonitoringTrace {
trace: DecisionTrace,
usage: Option<StoredRequestUsageAudit>,
}
pub(super) async fn build_admin_monitoring_trace_request_response( pub(super) async fn build_admin_monitoring_trace_request_response(
state: &AdminAppState<'_>, state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>, request_context: &AdminRequestContext<'_>,
) -> Result<Response<Body>, GatewayError> { ) -> Result<Response<Body>, GatewayError> {
let admin_state = state; let admin_state = state;
let state = state.as_ref();
let Some(request_id) = let Some(request_id) =
admin_monitoring_trace_request_id_from_path(&request_context.request_path) admin_monitoring_trace_request_id_from_path(&request_context.request_path)
else { else {
@@ -39,11 +44,8 @@ pub(super) async fn build_admin_monitoring_trace_request_response(
Err(detail) => return Ok(admin_monitoring_bad_request_response(detail)), Err(detail) => return Ok(admin_monitoring_bad_request_response(detail)),
}; };
let Some(trace) = state let Some(resolved) =
.data resolve_admin_monitoring_trace(admin_state, &request_id, attempted_only).await?
.read_decision_trace(&request_id, attempted_only)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?
else { else {
debug!( debug!(
event_name = "admin_monitoring_request_trace_not_found", event_name = "admin_monitoring_request_trace_not_found",
@@ -58,22 +60,113 @@ pub(super) async fn build_admin_monitoring_trace_request_response(
attempted_only, attempted_only,
)); ));
}; };
let usage = state let key_accounts =
.data build_admin_monitoring_key_account_display_map(admin_state, &resolved.trace).await?;
.read_request_usage_audit(&request_id)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?;
let key_accounts = build_admin_monitoring_key_account_display_map(admin_state, &trace).await?;
Ok( Ok(
build_admin_monitoring_trace_request_payload_response_with_key_accounts( build_admin_monitoring_trace_request_payload_response_with_key_accounts(
&trace, &resolved.trace,
usage.as_ref(), resolved.usage.as_ref(),
&key_accounts, &key_accounts,
), ),
) )
} }
async fn resolve_admin_monitoring_trace(
state: &AdminAppState<'_>,
request_id: &str,
attempted_only: bool,
) -> Result<Option<ResolvedAdminMonitoringTrace>, GatewayError> {
let app = state.as_ref();
if let Some(trace) = app
.data
.read_decision_trace(request_id, attempted_only)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?
{
let usage = app
.data
.read_request_usage_audit(request_id)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?;
return Ok(Some(ResolvedAdminMonitoringTrace { trace, usage }));
}
let mut usage_candidates = Vec::new();
if let Some(usage) = app
.data
.read_request_usage_audit(request_id)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?
{
usage_candidates.push(usage);
}
if let Some(usage) = state.find_request_usage_by_id(request_id).await? {
if !usage_candidates.iter().any(|item| item.id == usage.id) {
usage_candidates.push(usage);
}
}
for usage in usage_candidates {
for trace_request_id in admin_monitoring_usage_trace_request_ids(&usage) {
if trace_request_id == request_id {
continue;
}
if let Some(trace) = app
.data
.read_decision_trace(&trace_request_id, attempted_only)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?
{
return Ok(Some(ResolvedAdminMonitoringTrace {
trace,
usage: Some(usage),
}));
}
}
}
Ok(None)
}
fn admin_monitoring_usage_trace_request_ids(usage: &StoredRequestUsageAudit) -> Vec<String> {
let mut ids = Vec::new();
push_non_empty_unique(&mut ids, usage.request_id.as_str());
if let Some(trace_id) = usage.trace_id() {
push_non_empty_unique(&mut ids, trace_id);
}
if let Some(trace_id) = usage_trace_id_from_headers(usage.request_headers.as_ref()) {
push_non_empty_unique(&mut ids, trace_id.as_str());
}
if let Some(trace_id) = usage_trace_id_from_headers(usage.provider_request_headers.as_ref()) {
push_non_empty_unique(&mut ids, trace_id.as_str());
}
ids
}
fn usage_trace_id_from_headers(headers: Option<&Value>) -> Option<String> {
let object = headers?.as_object()?;
object.iter().find_map(|(key, value)| {
key.eq_ignore_ascii_case(crate::constants::TRACE_ID_HEADER)
.then(|| {
value
.as_str()
.map(str::trim)
.filter(|value| !value.is_empty())
})
.flatten()
.map(ToOwned::to_owned)
})
}
fn push_non_empty_unique(values: &mut Vec<String>, value: &str) {
let value = value.trim();
if value.is_empty() || values.iter().any(|existing| existing == value) {
return;
}
values.push(value.to_string());
}
async fn build_admin_monitoring_key_account_display_map( async fn build_admin_monitoring_key_account_display_map(
state: &AdminAppState<'_>, state: &AdminAppState<'_>,
trace: &DecisionTrace, trace: &DecisionTrace,

View File

@@ -385,7 +385,7 @@ fn resolve_admin_monitoring_usage_candidate_id(
trace: &DecisionTrace, trace: &DecisionTrace,
usage: &StoredRequestUsageAudit, usage: &StoredRequestUsageAudit,
) -> Option<String> { ) -> Option<String> {
if usage.request_id.trim() != trace.request_id { if !admin_monitoring_usage_matches_trace(usage, trace.request_id.as_str()) {
return None; return None;
} }
@@ -413,6 +413,43 @@ fn resolve_admin_monitoring_usage_candidate_id(
.map(|item| item.candidate.id.clone()) .map(|item| item.candidate.id.clone())
} }
fn admin_monitoring_usage_matches_trace(
usage: &StoredRequestUsageAudit,
trace_request_id: &str,
) -> bool {
let trace_request_id = trace_request_id.trim();
if trace_request_id.is_empty() {
return false;
}
usage.request_id.trim() == trace_request_id
|| usage
.trace_id()
.is_some_and(|value| value.trim() == trace_request_id)
|| admin_monitoring_headers_contain_trace_id(
usage.request_headers.as_ref(),
trace_request_id,
)
|| admin_monitoring_headers_contain_trace_id(
usage.provider_request_headers.as_ref(),
trace_request_id,
)
}
fn admin_monitoring_headers_contain_trace_id(
headers: Option<&Value>,
trace_request_id: &str,
) -> bool {
headers.and_then(Value::as_object).is_some_and(|object| {
object.iter().any(|(key, value)| {
key.eq_ignore_ascii_case("x-trace-id")
&& value
.as_str()
.map(str::trim)
.is_some_and(|value| value == trace_request_id)
})
})
}
fn admin_monitoring_candidate_status_rank(status: RequestCandidateStatus) -> u8 { fn admin_monitoring_candidate_status_rank(status: RequestCandidateStatus) -> u8 {
match status { match status {
RequestCandidateStatus::Success => 7, RequestCandidateStatus::Success => 7,

View File

@@ -537,9 +537,9 @@
<!-- 请求链路追踪卡片 --> <!-- 请求链路追踪卡片 -->
<div> <div>
<HorizontalRequestTimeline <HorizontalRequestTimeline
v-if="showTimeline && (detail.request_id || detail.id)" v-if="showTimeline && traceTimelineRequestId"
ref="timelineRef" ref="timelineRef"
:request-id="detail.request_id || detail.id" :request-id="traceTimelineRequestId"
:override-status-code="detail.status_code" :override-status-code="detail.status_code"
:request-status="detail.status" :request-status="detail.status"
:request-api-format="detail.api_format || null" :request-api-format="detail.api_format || null"
@@ -996,6 +996,17 @@ function getNestedString(record: JsonRecord | null, ...path: string[]): string |
return typeof value === 'string' && value.trim() ? value.trim() : null return typeof value === 'string' && value.trim() ? value.trim() : null
} }
function getCaseInsensitiveString(record: JsonRecord | null, key: string): string | null {
if (!record) return null
const normalizedKey = key.toLowerCase()
for (const [name, value] of Object.entries(record)) {
if (name.toLowerCase() === normalizedKey && typeof value === 'string' && value.trim()) {
return value.trim()
}
}
return null
}
function normalizeCacheTtlPricing(value: unknown): CacheTTLPriceEntry[] { function normalizeCacheTtlPricing(value: unknown): CacheTTLPriceEntry[] {
if (!Array.isArray(value)) return [] if (!Array.isArray(value)) return []
return value return value
@@ -1095,6 +1106,19 @@ const traceRequestMetadata = computed<Record<string, unknown> | null>(() => {
return meta as Record<string, unknown> return meta as Record<string, unknown>
}) })
const traceRecord = computed<Record<string, unknown> | null>(() =>
asRecord(detail.value?.trace ?? null),
)
const traceTimelineRequestId = computed(() =>
getNestedString(traceRecord.value, 'trace_id')
?? getNestedString(traceRequestMetadata.value, 'trace_id')
?? getCaseInsensitiveString(asRecord(detail.value?.request_headers ?? null), 'x-trace-id')
?? detail.value?.request_id
?? detail.value?.id
?? null,
)
const metadataPanelData = computed<Record<string, unknown> | null>(() => { const metadataPanelData = computed<Record<string, unknown> | null>(() => {
if (!detail.value) return null if (!detail.value) return null