mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 17:30:23 +08:00
fix monitoring trace lookup fallback
This commit is contained in:
@@ -81,6 +81,133 @@ async fn admin_monitoring_trace_request_returns_local_payload() {
|
|||||||
assert_eq!(payload["candidates"][0]["status_code"], json!(502));
|
assert_eq!(payload["candidates"][0]["status_code"], json!(502));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn admin_monitoring_trace_request_resolves_usage_id_to_header_trace_id() {
|
||||||
|
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![
|
||||||
|
sample_candidate(
|
||||||
|
"cand-used",
|
||||||
|
"trace-1",
|
||||||
|
0,
|
||||||
|
RequestCandidateStatus::Success,
|
||||||
|
Some(101),
|
||||||
|
Some(33),
|
||||||
|
Some(200),
|
||||||
|
),
|
||||||
|
]));
|
||||||
|
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||||
|
vec![sample_provider()],
|
||||||
|
vec![sample_endpoint()],
|
||||||
|
vec![sample_key()],
|
||||||
|
));
|
||||||
|
let mut usage = sample_usage(
|
||||||
|
"usage-request-1",
|
||||||
|
"provider-1",
|
||||||
|
"OpenAI",
|
||||||
|
40,
|
||||||
|
0.02,
|
||||||
|
"completed",
|
||||||
|
Some(200),
|
||||||
|
100,
|
||||||
|
);
|
||||||
|
usage.id = "usage-row-1".to_string();
|
||||||
|
usage.candidate_id = Some("cand-used".to_string());
|
||||||
|
usage.request_headers = Some(json!({
|
||||||
|
"x-trace-id": "trace-1"
|
||||||
|
}));
|
||||||
|
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(vec![usage]));
|
||||||
|
let data_state =
|
||||||
|
crate::data::GatewayDataState::with_request_candidate_and_usage_repository_for_tests(
|
||||||
|
request_candidates,
|
||||||
|
usage_repository,
|
||||||
|
)
|
||||||
|
.with_provider_catalog_reader(provider_catalog);
|
||||||
|
let state = AppState::new()
|
||||||
|
.expect("state should build")
|
||||||
|
.with_data_state_for_tests(data_state);
|
||||||
|
let context = request_context(
|
||||||
|
http::Method::GET,
|
||||||
|
"/api/admin/monitoring/trace/usage-row-1?attempted_only=true",
|
||||||
|
);
|
||||||
|
|
||||||
|
let response = local_monitoring_response(&state, &context)
|
||||||
|
.await
|
||||||
|
.expect("handler should not error")
|
||||||
|
.expect("route should be handled locally");
|
||||||
|
|
||||||
|
assert_eq!(response.status(), http::StatusCode::OK);
|
||||||
|
let body = to_bytes(response.into_body(), usize::MAX)
|
||||||
|
.await
|
||||||
|
.expect("body should read");
|
||||||
|
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||||
|
assert_eq!(payload["request_id"], json!("trace-1"));
|
||||||
|
assert_eq!(payload["candidates"][0]["id"], json!("cand-used"));
|
||||||
|
assert_eq!(
|
||||||
|
payload["candidates"][0]["extra_data"]["first_byte_time_ms"],
|
||||||
|
json!(30)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn admin_monitoring_trace_request_resolves_usage_request_id_to_metadata_trace_id() {
|
||||||
|
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![
|
||||||
|
sample_candidate(
|
||||||
|
"cand-used",
|
||||||
|
"trace-2",
|
||||||
|
0,
|
||||||
|
RequestCandidateStatus::Success,
|
||||||
|
Some(101),
|
||||||
|
Some(33),
|
||||||
|
Some(200),
|
||||||
|
),
|
||||||
|
]));
|
||||||
|
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||||
|
vec![sample_provider()],
|
||||||
|
vec![sample_endpoint()],
|
||||||
|
vec![sample_key()],
|
||||||
|
));
|
||||||
|
let mut usage = sample_usage(
|
||||||
|
"usage-request-2",
|
||||||
|
"provider-1",
|
||||||
|
"OpenAI",
|
||||||
|
40,
|
||||||
|
0.02,
|
||||||
|
"completed",
|
||||||
|
Some(200),
|
||||||
|
100,
|
||||||
|
);
|
||||||
|
usage.candidate_id = Some("cand-used".to_string());
|
||||||
|
usage.request_metadata = Some(json!({
|
||||||
|
"trace_id": "trace-2"
|
||||||
|
}));
|
||||||
|
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(vec![usage]));
|
||||||
|
let data_state =
|
||||||
|
crate::data::GatewayDataState::with_request_candidate_and_usage_repository_for_tests(
|
||||||
|
request_candidates,
|
||||||
|
usage_repository,
|
||||||
|
)
|
||||||
|
.with_provider_catalog_reader(provider_catalog);
|
||||||
|
let state = AppState::new()
|
||||||
|
.expect("state should build")
|
||||||
|
.with_data_state_for_tests(data_state);
|
||||||
|
let context = request_context(
|
||||||
|
http::Method::GET,
|
||||||
|
"/api/admin/monitoring/trace/usage-request-2",
|
||||||
|
);
|
||||||
|
|
||||||
|
let response = local_monitoring_response(&state, &context)
|
||||||
|
.await
|
||||||
|
.expect("handler should not error")
|
||||||
|
.expect("route should be handled locally");
|
||||||
|
|
||||||
|
assert_eq!(response.status(), http::StatusCode::OK);
|
||||||
|
let body = to_bytes(response.into_body(), usize::MAX)
|
||||||
|
.await
|
||||||
|
.expect("body should read");
|
||||||
|
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||||
|
assert_eq!(payload["request_id"], json!("trace-2"));
|
||||||
|
assert_eq!(payload["candidates"][0]["id"], json!("cand-used"));
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn admin_monitoring_trace_request_returns_oauth_account_label_from_auth_config() {
|
async fn admin_monitoring_trace_request_returns_oauth_account_label_from_auth_config() {
|
||||||
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![
|
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ use aether_admin::observability::monitoring::{
|
|||||||
use aether_data_contracts::repository::{
|
use aether_data_contracts::repository::{
|
||||||
candidates::{DecisionTrace, RequestCandidateStatus},
|
candidates::{DecisionTrace, RequestCandidateStatus},
|
||||||
provider_catalog::StoredProviderCatalogKey,
|
provider_catalog::StoredProviderCatalogKey,
|
||||||
|
usage::StoredRequestUsageAudit,
|
||||||
};
|
};
|
||||||
use axum::{
|
use axum::{
|
||||||
body::Body,
|
body::Body,
|
||||||
@@ -21,12 +22,16 @@ use serde_json::{Map, Value};
|
|||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
use tracing::debug;
|
use tracing::debug;
|
||||||
|
|
||||||
|
struct ResolvedAdminMonitoringTrace {
|
||||||
|
trace: DecisionTrace,
|
||||||
|
usage: Option<StoredRequestUsageAudit>,
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) async fn build_admin_monitoring_trace_request_response(
|
pub(super) async fn build_admin_monitoring_trace_request_response(
|
||||||
state: &AdminAppState<'_>,
|
state: &AdminAppState<'_>,
|
||||||
request_context: &AdminRequestContext<'_>,
|
request_context: &AdminRequestContext<'_>,
|
||||||
) -> Result<Response<Body>, GatewayError> {
|
) -> Result<Response<Body>, GatewayError> {
|
||||||
let admin_state = state;
|
let admin_state = state;
|
||||||
let state = state.as_ref();
|
|
||||||
let Some(request_id) =
|
let Some(request_id) =
|
||||||
admin_monitoring_trace_request_id_from_path(&request_context.request_path)
|
admin_monitoring_trace_request_id_from_path(&request_context.request_path)
|
||||||
else {
|
else {
|
||||||
@@ -39,11 +44,8 @@ pub(super) async fn build_admin_monitoring_trace_request_response(
|
|||||||
Err(detail) => return Ok(admin_monitoring_bad_request_response(detail)),
|
Err(detail) => return Ok(admin_monitoring_bad_request_response(detail)),
|
||||||
};
|
};
|
||||||
|
|
||||||
let Some(trace) = state
|
let Some(resolved) =
|
||||||
.data
|
resolve_admin_monitoring_trace(admin_state, &request_id, attempted_only).await?
|
||||||
.read_decision_trace(&request_id, attempted_only)
|
|
||||||
.await
|
|
||||||
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
|
||||||
else {
|
else {
|
||||||
debug!(
|
debug!(
|
||||||
event_name = "admin_monitoring_request_trace_not_found",
|
event_name = "admin_monitoring_request_trace_not_found",
|
||||||
@@ -58,22 +60,113 @@ pub(super) async fn build_admin_monitoring_trace_request_response(
|
|||||||
attempted_only,
|
attempted_only,
|
||||||
));
|
));
|
||||||
};
|
};
|
||||||
let usage = state
|
let key_accounts =
|
||||||
.data
|
build_admin_monitoring_key_account_display_map(admin_state, &resolved.trace).await?;
|
||||||
.read_request_usage_audit(&request_id)
|
|
||||||
.await
|
|
||||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
|
||||||
let key_accounts = build_admin_monitoring_key_account_display_map(admin_state, &trace).await?;
|
|
||||||
|
|
||||||
Ok(
|
Ok(
|
||||||
build_admin_monitoring_trace_request_payload_response_with_key_accounts(
|
build_admin_monitoring_trace_request_payload_response_with_key_accounts(
|
||||||
&trace,
|
&resolved.trace,
|
||||||
usage.as_ref(),
|
resolved.usage.as_ref(),
|
||||||
&key_accounts,
|
&key_accounts,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn resolve_admin_monitoring_trace(
|
||||||
|
state: &AdminAppState<'_>,
|
||||||
|
request_id: &str,
|
||||||
|
attempted_only: bool,
|
||||||
|
) -> Result<Option<ResolvedAdminMonitoringTrace>, GatewayError> {
|
||||||
|
let app = state.as_ref();
|
||||||
|
if let Some(trace) = app
|
||||||
|
.data
|
||||||
|
.read_decision_trace(request_id, attempted_only)
|
||||||
|
.await
|
||||||
|
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||||
|
{
|
||||||
|
let usage = app
|
||||||
|
.data
|
||||||
|
.read_request_usage_audit(request_id)
|
||||||
|
.await
|
||||||
|
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||||
|
return Ok(Some(ResolvedAdminMonitoringTrace { trace, usage }));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut usage_candidates = Vec::new();
|
||||||
|
if let Some(usage) = app
|
||||||
|
.data
|
||||||
|
.read_request_usage_audit(request_id)
|
||||||
|
.await
|
||||||
|
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||||
|
{
|
||||||
|
usage_candidates.push(usage);
|
||||||
|
}
|
||||||
|
if let Some(usage) = state.find_request_usage_by_id(request_id).await? {
|
||||||
|
if !usage_candidates.iter().any(|item| item.id == usage.id) {
|
||||||
|
usage_candidates.push(usage);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for usage in usage_candidates {
|
||||||
|
for trace_request_id in admin_monitoring_usage_trace_request_ids(&usage) {
|
||||||
|
if trace_request_id == request_id {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if let Some(trace) = app
|
||||||
|
.data
|
||||||
|
.read_decision_trace(&trace_request_id, attempted_only)
|
||||||
|
.await
|
||||||
|
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||||
|
{
|
||||||
|
return Ok(Some(ResolvedAdminMonitoringTrace {
|
||||||
|
trace,
|
||||||
|
usage: Some(usage),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(None)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn admin_monitoring_usage_trace_request_ids(usage: &StoredRequestUsageAudit) -> Vec<String> {
|
||||||
|
let mut ids = Vec::new();
|
||||||
|
push_non_empty_unique(&mut ids, usage.request_id.as_str());
|
||||||
|
if let Some(trace_id) = usage.trace_id() {
|
||||||
|
push_non_empty_unique(&mut ids, trace_id);
|
||||||
|
}
|
||||||
|
if let Some(trace_id) = usage_trace_id_from_headers(usage.request_headers.as_ref()) {
|
||||||
|
push_non_empty_unique(&mut ids, trace_id.as_str());
|
||||||
|
}
|
||||||
|
if let Some(trace_id) = usage_trace_id_from_headers(usage.provider_request_headers.as_ref()) {
|
||||||
|
push_non_empty_unique(&mut ids, trace_id.as_str());
|
||||||
|
}
|
||||||
|
ids
|
||||||
|
}
|
||||||
|
|
||||||
|
fn usage_trace_id_from_headers(headers: Option<&Value>) -> Option<String> {
|
||||||
|
let object = headers?.as_object()?;
|
||||||
|
object.iter().find_map(|(key, value)| {
|
||||||
|
key.eq_ignore_ascii_case(crate::constants::TRACE_ID_HEADER)
|
||||||
|
.then(|| {
|
||||||
|
value
|
||||||
|
.as_str()
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
})
|
||||||
|
.flatten()
|
||||||
|
.map(ToOwned::to_owned)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_non_empty_unique(values: &mut Vec<String>, value: &str) {
|
||||||
|
let value = value.trim();
|
||||||
|
if value.is_empty() || values.iter().any(|existing| existing == value) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
values.push(value.to_string());
|
||||||
|
}
|
||||||
|
|
||||||
async fn build_admin_monitoring_key_account_display_map(
|
async fn build_admin_monitoring_key_account_display_map(
|
||||||
state: &AdminAppState<'_>,
|
state: &AdminAppState<'_>,
|
||||||
trace: &DecisionTrace,
|
trace: &DecisionTrace,
|
||||||
|
|||||||
@@ -385,7 +385,7 @@ fn resolve_admin_monitoring_usage_candidate_id(
|
|||||||
trace: &DecisionTrace,
|
trace: &DecisionTrace,
|
||||||
usage: &StoredRequestUsageAudit,
|
usage: &StoredRequestUsageAudit,
|
||||||
) -> Option<String> {
|
) -> Option<String> {
|
||||||
if usage.request_id.trim() != trace.request_id {
|
if !admin_monitoring_usage_matches_trace(usage, trace.request_id.as_str()) {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -413,6 +413,43 @@ fn resolve_admin_monitoring_usage_candidate_id(
|
|||||||
.map(|item| item.candidate.id.clone())
|
.map(|item| item.candidate.id.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn admin_monitoring_usage_matches_trace(
|
||||||
|
usage: &StoredRequestUsageAudit,
|
||||||
|
trace_request_id: &str,
|
||||||
|
) -> bool {
|
||||||
|
let trace_request_id = trace_request_id.trim();
|
||||||
|
if trace_request_id.is_empty() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
usage.request_id.trim() == trace_request_id
|
||||||
|
|| usage
|
||||||
|
.trace_id()
|
||||||
|
.is_some_and(|value| value.trim() == trace_request_id)
|
||||||
|
|| admin_monitoring_headers_contain_trace_id(
|
||||||
|
usage.request_headers.as_ref(),
|
||||||
|
trace_request_id,
|
||||||
|
)
|
||||||
|
|| admin_monitoring_headers_contain_trace_id(
|
||||||
|
usage.provider_request_headers.as_ref(),
|
||||||
|
trace_request_id,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn admin_monitoring_headers_contain_trace_id(
|
||||||
|
headers: Option<&Value>,
|
||||||
|
trace_request_id: &str,
|
||||||
|
) -> bool {
|
||||||
|
headers.and_then(Value::as_object).is_some_and(|object| {
|
||||||
|
object.iter().any(|(key, value)| {
|
||||||
|
key.eq_ignore_ascii_case("x-trace-id")
|
||||||
|
&& value
|
||||||
|
.as_str()
|
||||||
|
.map(str::trim)
|
||||||
|
.is_some_and(|value| value == trace_request_id)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn admin_monitoring_candidate_status_rank(status: RequestCandidateStatus) -> u8 {
|
fn admin_monitoring_candidate_status_rank(status: RequestCandidateStatus) -> u8 {
|
||||||
match status {
|
match status {
|
||||||
RequestCandidateStatus::Success => 7,
|
RequestCandidateStatus::Success => 7,
|
||||||
|
|||||||
@@ -537,9 +537,9 @@
|
|||||||
<!-- 请求链路追踪卡片 -->
|
<!-- 请求链路追踪卡片 -->
|
||||||
<div>
|
<div>
|
||||||
<HorizontalRequestTimeline
|
<HorizontalRequestTimeline
|
||||||
v-if="showTimeline && (detail.request_id || detail.id)"
|
v-if="showTimeline && traceTimelineRequestId"
|
||||||
ref="timelineRef"
|
ref="timelineRef"
|
||||||
:request-id="detail.request_id || detail.id"
|
:request-id="traceTimelineRequestId"
|
||||||
:override-status-code="detail.status_code"
|
:override-status-code="detail.status_code"
|
||||||
:request-status="detail.status"
|
:request-status="detail.status"
|
||||||
:request-api-format="detail.api_format || null"
|
:request-api-format="detail.api_format || null"
|
||||||
@@ -996,6 +996,17 @@ function getNestedString(record: JsonRecord | null, ...path: string[]): string |
|
|||||||
return typeof value === 'string' && value.trim() ? value.trim() : null
|
return typeof value === 'string' && value.trim() ? value.trim() : null
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getCaseInsensitiveString(record: JsonRecord | null, key: string): string | null {
|
||||||
|
if (!record) return null
|
||||||
|
const normalizedKey = key.toLowerCase()
|
||||||
|
for (const [name, value] of Object.entries(record)) {
|
||||||
|
if (name.toLowerCase() === normalizedKey && typeof value === 'string' && value.trim()) {
|
||||||
|
return value.trim()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
function normalizeCacheTtlPricing(value: unknown): CacheTTLPriceEntry[] {
|
function normalizeCacheTtlPricing(value: unknown): CacheTTLPriceEntry[] {
|
||||||
if (!Array.isArray(value)) return []
|
if (!Array.isArray(value)) return []
|
||||||
return value
|
return value
|
||||||
@@ -1095,6 +1106,19 @@ const traceRequestMetadata = computed<Record<string, unknown> | null>(() => {
|
|||||||
return meta as Record<string, unknown>
|
return meta as Record<string, unknown>
|
||||||
})
|
})
|
||||||
|
|
||||||
|
const traceRecord = computed<Record<string, unknown> | null>(() =>
|
||||||
|
asRecord(detail.value?.trace ?? null),
|
||||||
|
)
|
||||||
|
|
||||||
|
const traceTimelineRequestId = computed(() =>
|
||||||
|
getNestedString(traceRecord.value, 'trace_id')
|
||||||
|
?? getNestedString(traceRequestMetadata.value, 'trace_id')
|
||||||
|
?? getCaseInsensitiveString(asRecord(detail.value?.request_headers ?? null), 'x-trace-id')
|
||||||
|
?? detail.value?.request_id
|
||||||
|
?? detail.value?.id
|
||||||
|
?? null,
|
||||||
|
)
|
||||||
|
|
||||||
const metadataPanelData = computed<Record<string, unknown> | null>(() => {
|
const metadataPanelData = computed<Record<string, unknown> | null>(() => {
|
||||||
if (!detail.value) return null
|
if (!detail.value) return null
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user