feat(observability): 引入错误链路 error_flow 元数据并区分上游/客户端错误

- 网关在本地 failover 时构建 error_flow 元数据(分类/决策/传播策略),写入 report_context
- scheduler-core 解析并透传 error_flow 至候选 extra_data
- admin usage 详情拆分 request/upstream/client/failure_summary 错误域,敏感上游错误标记为 suppressed
- 前端 RequestDetailDrawer 拆出"返回客户端"与"上游响应"双错误卡片
- HorizontalRequestTimeline 节点详情展示真实请求错误及 error_flow 标签
This commit is contained in:
fawney19
2026-04-25 17:01:07 +08:00
parent bc97e383d3
commit 00744c0ce5
9 changed files with 997 additions and 44 deletions
@@ -70,10 +70,10 @@ use crate::execution_runtime::{
use crate::execution_runtime::{MAX_STREAM_PREFETCH_BYTES, MAX_STREAM_PREFETCH_FRAMES};
use crate::log_ids::short_request_id;
use crate::orchestration::{
apply_local_execution_effect, LocalAdaptiveRateLimitEffect, LocalAdaptiveSuccessEffect,
LocalAttemptFailureEffect, LocalExecutionEffect, LocalExecutionEffectContext,
LocalHealthFailureEffect, LocalHealthSuccessEffect, LocalOAuthInvalidationEffect,
LocalPoolErrorEffect,
apply_local_execution_effect, build_local_error_flow_metadata, with_error_flow_report_context,
LocalAdaptiveRateLimitEffect, LocalAdaptiveSuccessEffect, LocalAttemptFailureEffect,
LocalExecutionEffect, LocalExecutionEffectContext, LocalHealthFailureEffect,
LocalHealthSuccessEffect, LocalOAuthInvalidationEffect, LocalPoolErrorEffect,
};
use crate::request_candidate_runtime::{
ensure_execution_request_candidate_slot, record_local_request_candidate_status,
@@ -893,10 +893,20 @@ async fn execute_stream_from_frame_stream(
);
if matches!(failover_decision, LocalFailoverDecision::RetryNextCandidate) {
let terminal_unix_secs = current_request_candidate_unix_ms();
let error_flow_report_context = with_error_flow_report_context(
report_context.as_ref(),
build_local_error_flow_metadata(
status_code,
error_response_text.as_deref(),
failover_analysis,
),
);
record_local_request_candidate_status(
state,
&plan,
report_context.as_ref(),
error_flow_report_context
.as_ref()
.or(report_context.as_ref()),
SchedulerRequestCandidateStatusUpdate {
status: RequestCandidateStatus::Failed,
status_code: Some(status_code),
@@ -934,10 +944,20 @@ async fn execute_stream_from_frame_stream(
)
{
let terminal_unix_secs = current_request_candidate_unix_ms();
let error_flow_report_context = with_error_flow_report_context(
report_context.as_ref(),
build_local_error_flow_metadata(
status_code,
error_response_text.as_deref(),
failover_analysis,
),
);
record_local_request_candidate_status(
state,
&plan,
report_context.as_ref(),
error_flow_report_context
.as_ref()
.or(report_context.as_ref()),
SchedulerRequestCandidateStatusUpdate {
status: RequestCandidateStatus::Failed,
status_code: Some(status_code),
@@ -970,10 +990,20 @@ async fn execute_stream_from_frame_stream(
);
record_sync_terminal_usage(state, &plan, payload.report_context.as_ref(), &payload);
let terminal_unix_secs = current_request_candidate_unix_ms();
let error_flow_report_context = with_error_flow_report_context(
payload.report_context.as_ref(),
build_local_error_flow_metadata(
status_code,
error_response_text.as_deref(),
failover_analysis,
),
);
record_local_request_candidate_status(
state,
&plan,
payload.report_context.as_ref(),
error_flow_report_context
.as_ref()
.or(payload.report_context.as_ref()),
SchedulerRequestCandidateStatusUpdate {
status: RequestCandidateStatus::Failed,
status_code: Some(status_code),
@@ -35,10 +35,10 @@ use crate::execution_runtime::{
};
use crate::log_ids::short_request_id;
use crate::orchestration::{
apply_local_execution_effect, LocalAdaptiveRateLimitEffect, LocalAdaptiveSuccessEffect,
LocalAttemptFailureEffect, LocalExecutionEffect, LocalExecutionEffectContext,
LocalHealthFailureEffect, LocalHealthSuccessEffect, LocalOAuthInvalidationEffect,
LocalPoolErrorEffect,
apply_local_execution_effect, build_local_error_flow_metadata, with_error_flow_report_context,
LocalAdaptiveRateLimitEffect, LocalAdaptiveSuccessEffect, LocalAttemptFailureEffect,
LocalExecutionEffect, LocalExecutionEffectContext, LocalHealthFailureEffect,
LocalHealthSuccessEffect, LocalOAuthInvalidationEffect, LocalPoolErrorEffect,
};
use crate::request_candidate_runtime::{
ensure_execution_request_candidate_slot, record_local_request_candidate_status,
@@ -426,10 +426,20 @@ pub(crate) async fn execute_execution_runtime_sync(
LocalFailoverDecision::RetryNextCandidate
) {
let terminal_unix_secs = current_request_candidate_unix_ms();
let error_flow_report_context = with_error_flow_report_context(
report_context.as_ref(),
build_local_error_flow_metadata(
result.status_code,
local_failover_response_text.as_deref(),
local_failover_analysis,
),
);
record_local_request_candidate_status(
state,
&plan,
report_context.as_ref(),
error_flow_report_context
.as_ref()
.or(report_context.as_ref()),
SchedulerRequestCandidateStatusUpdate {
status: RequestCandidateStatus::Failed,
status_code: Some(result.status_code),
@@ -488,10 +498,20 @@ pub(crate) async fn execute_execution_runtime_sync(
mapped_error_finalize_kind.is_some(),
) {
let terminal_unix_secs = current_request_candidate_unix_ms();
let error_flow_report_context = with_error_flow_report_context(
report_context.as_ref(),
build_local_error_flow_metadata(
result.status_code,
local_failover_response_text.as_deref(),
local_failover_analysis,
),
);
record_local_request_candidate_status(
state,
&plan,
report_context.as_ref(),
error_flow_report_context
.as_ref()
.or(report_context.as_ref()),
SchedulerRequestCandidateStatusUpdate {
status: RequestCandidateStatus::Failed,
status_code: Some(result.status_code),
@@ -507,10 +527,24 @@ pub(crate) async fn execute_execution_runtime_sync(
}
let terminal_unix_secs = current_request_candidate_unix_ms();
let error_flow_report_context = (result.status_code >= 400)
.then(|| {
with_error_flow_report_context(
report_context.as_ref(),
build_local_error_flow_metadata(
result.status_code,
local_failover_response_text.as_deref(),
local_failover_analysis,
),
)
})
.flatten();
record_local_request_candidate_status(
state,
&plan,
report_context.as_ref(),
error_flow_report_context
.as_ref()
.or(report_context.as_ref()),
SchedulerRequestCandidateStatusUpdate {
status: if result.status_code >= 400 {
RequestCandidateStatus::Failed
@@ -145,6 +145,23 @@ pub(crate) enum LocalFailoverClassification {
RetryUpstreamFailure,
}
impl LocalFailoverClassification {
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::UseDefault => "use_default",
Self::StopStatusCode => "stop_status_code",
Self::StopErrorPattern => "stop_error_pattern",
Self::StopSemanticClientError => "stop_semantic_client_error",
Self::RetrySuccessPattern => "retry_success_pattern",
Self::RetrySemanticCompatibilityError => "retry_semantic_compatibility_error",
Self::RetrySemanticRateLimit => "retry_semantic_rate_limit",
Self::RetrySemanticThinkingError => "retry_semantic_thinking_error",
Self::RetryStatusCode => "retry_status_code",
Self::RetryUpstreamFailure => "retry_upstream_failure",
}
}
}
pub(crate) fn classify_local_failover(
policy: &LocalFailoverPolicy,
input: LocalFailoverInput<'_>,
@@ -1,4 +1,5 @@
use aether_contracts::ExecutionPlan;
use serde_json::{json, Value};
use crate::AppState;
@@ -78,3 +79,43 @@ pub(crate) async fn resolve_local_failover_decision_for_attempt(
.await
.decision
}
pub(crate) fn build_local_error_flow_metadata(
status_code: u16,
response_text: Option<&str>,
analysis: LocalFailoverAnalysis,
) -> Value {
let safe_to_expose = matches!(
analysis.classification,
LocalFailoverClassification::StopSemanticClientError
| LocalFailoverClassification::StopStatusCode
| LocalFailoverClassification::StopErrorPattern
);
let propagation = match analysis.decision {
LocalFailoverDecision::RetryNextCandidate => "suppressed",
LocalFailoverDecision::StopLocalFailover if safe_to_expose => "converted",
LocalFailoverDecision::StopLocalFailover => "suppressed",
LocalFailoverDecision::UseDefault if status_code >= 400 => "passthrough",
LocalFailoverDecision::UseDefault => "none",
};
json!({
"stage": "candidate",
"source": "upstream_response",
"status_code": status_code,
"classification": analysis.classification.as_str(),
"decision": analysis.decision.as_str(),
"retryable": matches!(analysis.decision, LocalFailoverDecision::RetryNextCandidate),
"safe_to_expose": safe_to_expose,
"propagation": propagation,
"message": local_failover_error_message(response_text),
})
}
pub(crate) fn with_error_flow_report_context(
report_context: Option<&Value>,
error_flow: Value,
) -> Option<Value> {
let mut object = report_context?.as_object()?.clone();
object.insert("error_flow".to_string(), error_flow);
Some(Value::Object(object))
}
@@ -315,6 +315,307 @@ fn admin_usage_strip_trace_metadata(metadata: &mut serde_json::Map<String, Value
metadata.remove("trace_id");
}
fn admin_usage_string_field<'a>(value: &'a Value, field: &str) -> Option<&'a str> {
value
.get(field)
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
}
fn admin_usage_error_field<'a>(body: &'a Value, field: &str) -> Option<&'a str> {
body.get("error")
.and_then(|error| match error {
Value::Object(object) => object.get(field).and_then(Value::as_str),
_ => None,
})
.map(str::trim)
.filter(|value| !value.is_empty())
.or_else(|| admin_usage_string_field(body, field))
}
fn admin_usage_error_message_from_body(body: &Value) -> Option<String> {
body.get("error")
.and_then(|error| match error {
Value::Object(object) => object.get("message").and_then(Value::as_str),
Value::String(message) => Some(message.as_str()),
_ => None,
})
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
.or_else(|| {
admin_usage_string_field(body, "message")
.or_else(|| admin_usage_string_field(body, "detail"))
.map(ToOwned::to_owned)
})
}
fn admin_usage_error_type_from_body(body: &Value) -> Option<String> {
admin_usage_error_field(body, "type")
.or_else(|| admin_usage_error_field(body, "status"))
.or_else(|| admin_usage_error_field(body, "kind"))
.map(ToOwned::to_owned)
}
fn admin_usage_error_code_from_body(body: &Value) -> Option<Value> {
body.get("error")
.and_then(|error| match error {
Value::Object(object) => object.get("code").cloned(),
_ => None,
})
.or_else(|| body.get("code").cloned())
}
fn admin_usage_header_content_type(headers: Option<&Value>) -> Option<String> {
let object = headers?.as_object()?;
for (key, value) in object {
if key.eq_ignore_ascii_case("content-type") {
return value
.as_str()
.map(ToOwned::to_owned)
.or_else(|| (!value.is_null()).then(|| value.to_string()));
}
}
None
}
fn admin_usage_error_domain_json(
source: &str,
status_code: Option<u16>,
headers: Option<&Value>,
body: Option<&Value>,
fallback_type: Option<&str>,
fallback_message: Option<&str>,
) -> Value {
let error_type = body
.and_then(admin_usage_error_type_from_body)
.or_else(|| fallback_type.map(ToOwned::to_owned));
let message = body
.and_then(admin_usage_error_message_from_body)
.or_else(|| {
fallback_message
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
});
let code = body.and_then(admin_usage_error_code_from_body);
let content_type = admin_usage_header_content_type(headers);
if status_code.is_none()
&& error_type.is_none()
&& message.is_none()
&& code.is_none()
&& body.is_none()
{
return Value::Null;
}
json!({
"source": source,
"status_code": status_code,
"type": error_type,
"message": message,
"code": code,
"content_type": content_type,
"body": body.cloned().unwrap_or(Value::Null),
})
}
fn admin_usage_error_domain_message(domain: &Value) -> Option<String> {
domain
.get("message")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
}
fn admin_usage_error_domain_type(domain: &Value) -> Option<String> {
domain
.get("type")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
}
fn admin_usage_error_domain_source(domain: &Value) -> Option<String> {
domain
.get("source")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
}
fn admin_usage_failure_summary_json(
item: &StoredRequestUsageAudit,
request_error: &Value,
upstream_error: &Value,
client_error: &Value,
) -> Value {
let selected = [client_error, upstream_error, request_error]
.into_iter()
.find(|domain| !domain.is_null() && admin_usage_error_domain_message(domain).is_some());
let message = selected
.and_then(admin_usage_error_domain_message)
.or_else(|| {
item.error_message
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
});
let Some(message) = message else {
return Value::Null;
};
let error_type = selected
.and_then(admin_usage_error_domain_type)
.or_else(|| item.error_category.clone());
let source = selected
.and_then(admin_usage_error_domain_source)
.unwrap_or_else(|| "usage_summary".to_string());
json!({
"source": source,
"status_code": item.status_code,
"type": error_type,
"message": message,
"category": item.error_category,
})
}
fn admin_usage_error_domains_json(item: &StoredRequestUsageAudit) -> Value {
let has_upstream_attempt = item.candidate_id.is_some()
|| item.provider_api_key_id.is_some()
|| item.provider_request_headers.is_some()
|| item.provider_request_body.is_some()
|| item.provider_request_body_ref.is_some();
let upstream_error = if has_upstream_attempt {
admin_usage_error_domain_json(
"upstream_response",
item.status_code,
item.response_headers.as_ref(),
item.response_body.as_ref(),
item.error_category.as_deref(),
item.error_message.as_deref(),
)
} else {
Value::Null
};
let client_error = admin_usage_error_domain_json(
"client_response",
item.status_code,
item.client_response_headers.as_ref(),
item.client_response_body.as_ref(),
item.error_category.as_deref(),
item.error_message.as_deref(),
);
let request_error = Value::Null;
let failure_summary =
admin_usage_failure_summary_json(item, &request_error, &upstream_error, &client_error);
json!({
"request_error": request_error,
"upstream_error": upstream_error,
"client_error": client_error,
"failure_summary": failure_summary,
})
}
fn admin_usage_error_domain_search_text(domain: &Value) -> String {
[
domain.get("type").and_then(Value::as_str),
domain.get("message").and_then(Value::as_str),
domain.get("code").and_then(Value::as_str),
]
.into_iter()
.flatten()
.map(str::to_ascii_lowercase)
.collect::<Vec<_>>()
.join(" ")
}
fn admin_usage_upstream_error_is_sensitive(domain: &Value) -> bool {
let text = admin_usage_error_domain_search_text(domain);
[
"insufficient_quota",
"insufficient quota",
"quota exhausted",
"credits exhausted",
"credit balance",
"credit limit",
"payment_required",
"payment required",
"account disabled",
"account_deactivated",
"subscription inactive",
"verification required",
]
.iter()
.any(|pattern| text.contains(pattern))
}
fn admin_usage_error_flow_json(item: &StoredRequestUsageAudit, error_domains: &Value) -> Value {
let request_error = &error_domains["request_error"];
let upstream_error = &error_domains["upstream_error"];
let client_error = &error_domains["client_error"];
let failure_summary = &error_domains["failure_summary"];
if request_error.is_null()
&& upstream_error.is_null()
&& client_error.is_null()
&& failure_summary.is_null()
{
return Value::Null;
}
let upstream_sensitive =
!upstream_error.is_null() && admin_usage_upstream_error_is_sensitive(upstream_error);
let propagation = if upstream_sensitive {
"suppressed"
} else if !client_error.is_null() && !upstream_error.is_null() {
let upstream_message = admin_usage_error_domain_message(upstream_error);
let client_message = admin_usage_error_domain_message(client_error);
if upstream_message.is_some() && upstream_message == client_message {
"passthrough"
} else {
"converted"
}
} else if !client_error.is_null() {
"local"
} else if !upstream_error.is_null() {
"captured"
} else {
"none"
};
let source = if !request_error.is_null() {
"request"
} else if !upstream_error.is_null() {
"upstream"
} else if !client_error.is_null() {
"gateway"
} else {
"summary"
};
let client_response_source = if client_error.is_null() {
Value::Null
} else if !upstream_error.is_null() {
Value::String("converted_or_sanitized_upstream".to_string())
} else {
Value::String("gateway_generated".to_string())
};
json!({
"source": source,
"status_code": item.status_code,
"propagation": propagation,
"client_response_source": client_response_source,
"safe_to_expose_upstream": !upstream_sensitive,
"summary_source": failure_summary.get("source").cloned().unwrap_or(Value::Null),
})
}
fn maybe_insert_number_field(
object: &mut serde_json::Map<String, Value>,
key: &str,
@@ -1790,6 +2091,14 @@ pub fn build_admin_usage_detail_payload(
payload["body_capture"] = admin_usage_body_capture_json(item);
payload["settlement"] = admin_usage_settlement_json(item);
payload["trace"] = admin_usage_trace_json(item);
let error_domains = admin_usage_error_domains_json(item);
let error_flow = admin_usage_error_flow_json(item, &error_domains);
payload["errors"] = error_domains.clone();
payload["request_error"] = error_domains["request_error"].clone();
payload["upstream_error"] = error_domains["upstream_error"].clone();
payload["client_error"] = error_domains["client_error"].clone();
payload["failure_summary"] = error_domains["failure_summary"].clone();
payload["error_flow"] = error_flow;
payload["has_request_body"] = json!(admin_usage_has_body_value(
item,
item.request_body.as_ref(),
@@ -2261,6 +2570,158 @@ mod tests {
assert_eq!(payload["has_client_response_body"], true);
}
#[test]
fn detail_payload_separates_upstream_client_and_summary_errors() {
let item = StoredRequestUsageAudit {
error_message: Some(
"execution runtime stream returned retryable status 400".to_string(),
),
error_category: Some("server_error".to_string()),
response_headers: Some(json!({
"content-type": "application/json"
})),
response_body: Some(json!({
"error": {
"type": "retryable_upstream_status",
"message": "execution runtime stream returned retryable status 400",
"code": 400
}
})),
client_response_headers: Some(json!({
"content-type": "application/json"
})),
client_response_body: Some(json!({
"error": {
"type": "http_error",
"message": "local execution runtime exhausted"
}
})),
..sample_usage(
"failed",
Some(503),
Some("execution runtime stream returned retryable status 400"),
)
};
let payload = build_admin_usage_detail_payload(
&item,
&BTreeMap::new(),
&BTreeMap::new(),
false,
false,
None,
true,
Some(json!({"model": "gpt-5.4"})),
&BTreeMap::new(),
);
assert_eq!(payload["upstream_error"]["source"], "upstream_response");
assert_eq!(
payload["errors"]["upstream_error"],
payload["upstream_error"]
);
assert_eq!(
payload["upstream_error"]["message"],
"execution runtime stream returned retryable status 400"
);
assert_eq!(payload["client_error"]["source"], "client_response");
assert_eq!(
payload["client_error"]["message"],
"local execution runtime exhausted"
);
assert_eq!(payload["failure_summary"]["source"], "client_response");
assert_eq!(
payload["failure_summary"]["message"],
"local execution runtime exhausted"
);
assert_eq!(payload["error_flow"]["propagation"], "converted");
assert!(payload["request_error"].is_null());
}
#[test]
fn detail_payload_marks_sensitive_upstream_account_errors_suppressed() {
let item = StoredRequestUsageAudit {
error_message: Some("credit balance exhausted".to_string()),
error_category: Some("server_error".to_string()),
response_body: Some(json!({
"error": {
"type": "insufficient_quota",
"message": "credit balance exhausted"
}
})),
client_response_body: Some(json!({
"error": {
"type": "http_error",
"message": "upstream provider unavailable"
}
})),
..sample_usage("failed", Some(503), Some("credit balance exhausted"))
};
let payload = build_admin_usage_detail_payload(
&item,
&BTreeMap::new(),
&BTreeMap::new(),
false,
false,
None,
true,
Some(json!({"model": "gpt-5.4"})),
&BTreeMap::new(),
);
assert_eq!(payload["error_flow"]["propagation"], "suppressed");
assert_eq!(payload["error_flow"]["safe_to_expose_upstream"], false);
assert_eq!(payload["upstream_error"]["type"], "insufficient_quota");
assert_eq!(
payload["client_error"]["message"],
"upstream provider unavailable"
);
}
#[test]
fn detail_payload_does_not_promote_local_client_error_to_upstream_error() {
let message = "没有可用提供商支持模型 gpt-5.4 的同步请求。请检查模型映射、端点启用状态和 API Key 权限(原因代码: candidate_list_empty";
let item = StoredRequestUsageAudit {
provider_api_key_id: None,
provider_request_headers: None,
provider_request_body: None,
provider_request_body_ref: None,
candidate_id: None,
error_category: Some("http_error".to_string()),
client_response_body: Some(json!({
"error": {
"type": "http_error",
"message": message
}
})),
response_body: Some(json!({
"error": {
"type": "http_error",
"message": message
}
})),
..sample_usage("failed", Some(503), Some(message))
};
let payload = build_admin_usage_detail_payload(
&item,
&BTreeMap::new(),
&BTreeMap::new(),
false,
false,
None,
true,
Some(json!({"model": "gpt-5.4"})),
&BTreeMap::new(),
);
assert!(payload["upstream_error"].is_null());
assert_eq!(payload["client_error"]["message"], message);
assert_eq!(payload["error_flow"]["source"], "gateway");
assert_eq!(payload["error_flow"]["propagation"], "local");
}
#[test]
fn detail_payload_preserves_legacy_body_capture_metadata_keys() {
let item = StoredRequestUsageAudit {
@@ -23,6 +23,7 @@ pub struct SchedulerRequestCandidateReportContext {
pub header_rules: Option<Value>,
pub body_rules: Option<Value>,
pub proxy: Option<Value>,
pub error_flow: Option<Value>,
}
#[derive(Debug, Clone, PartialEq)]
@@ -47,6 +48,19 @@ pub struct SchedulerExecutionRequestCandidateSeed {
pub report_context: Value,
}
#[derive(Debug, Clone, Default)]
struct ReportCandidateExtraDataInput {
client_api_format: Option<String>,
provider_api_format: Option<String>,
upstream_url: Option<String>,
mapped_model: Option<String>,
key_name: Option<String>,
header_rules: Option<Value>,
body_rules: Option<Value>,
proxy: Option<Value>,
error_flow: Option<Value>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SchedulerRequestCandidateStatusUpdate {
pub status: RequestCandidateStatus,
@@ -123,6 +137,10 @@ pub fn parse_request_candidate_report_context(
.get("proxy")
.cloned()
.filter(|value| !value.is_null()),
error_flow: report_context
.get("error_flow")
.cloned()
.filter(|value| !value.is_null()),
})
}
@@ -151,6 +169,7 @@ pub fn resolve_report_request_candidate_slot(
header_rules,
body_rules,
proxy,
error_flow,
} = metadata;
let request_id = request_id?;
let synthesized_extra_data = build_report_candidate_extra_data(ReportCandidateExtraDataInput {
@@ -162,6 +181,7 @@ pub fn resolve_report_request_candidate_slot(
header_rules,
body_rules,
proxy,
error_flow,
});
let created_at_unix_ms = matched_candidate
.as_ref()
@@ -325,6 +345,7 @@ pub fn build_local_request_candidate_status_record(
header_rules: metadata.header_rules.clone(),
body_rules: metadata.body_rules.clone(),
proxy: metadata.proxy.clone(),
error_flow: metadata.error_flow.clone(),
});
let created_at_unix_ms = started_at_unix_ms.or(finished_at_unix_ms);
@@ -526,17 +547,6 @@ fn next_candidate_index(candidates: &[StoredRequestCandidate]) -> u32 {
.unwrap_or_default()
}
struct ReportCandidateExtraDataInput {
client_api_format: Option<String>,
provider_api_format: Option<String>,
upstream_url: Option<String>,
mapped_model: Option<String>,
key_name: Option<String>,
header_rules: Option<Value>,
body_rules: Option<Value>,
proxy: Option<Value>,
}
fn build_report_candidate_extra_data(input: ReportCandidateExtraDataInput) -> Option<Value> {
let ReportCandidateExtraDataInput {
client_api_format,
@@ -547,6 +557,7 @@ fn build_report_candidate_extra_data(input: ReportCandidateExtraDataInput) -> Op
header_rules,
body_rules,
proxy,
error_flow,
} = input;
let mut extra_data = Map::with_capacity(8);
extra_data.insert("gateway_execution_runtime".to_string(), Value::Bool(true));
@@ -581,6 +592,9 @@ fn build_report_candidate_extra_data(input: ReportCandidateExtraDataInput) -> Op
if let Some(proxy) = proxy {
extra_data.insert("proxy".to_string(), proxy);
}
if let Some(error_flow) = error_flow {
extra_data.insert("error_flow".to_string(), error_flow);
}
(!extra_data.is_empty()).then_some(Value::Object(extra_data))
}
@@ -729,6 +743,11 @@ mod tests {
"node_id": "proxy-node-1",
"node_name": "edge-1",
"source": "provider"
},
"error_flow": {
"classification": "retry_upstream_failure",
"decision": "retry_next_candidate",
"propagation": "suppressed"
}
})))
.expect("metadata");
@@ -777,6 +796,13 @@ mod tests {
.map(Vec::len),
Some(1)
);
assert_eq!(
slot.extra_data
.as_ref()
.and_then(|value| value.get("error_flow"))
.and_then(|value| value.get("propagation")),
Some(&json!("suppressed"))
);
}
#[test]
+33
View File
@@ -115,6 +115,33 @@ export interface VideoBilling {
status?: string // 计费状态
}
export interface RequestErrorDomain {
source?: string | null
status_code?: number | null
type?: string | null
message?: string | null
code?: string | number | null
content_type?: string | null
body?: unknown
category?: string | null
}
export interface RequestErrorDomains {
request_error?: RequestErrorDomain | null
upstream_error?: RequestErrorDomain | null
client_error?: RequestErrorDomain | null
failure_summary?: RequestErrorDomain | null
}
export interface RequestErrorFlow {
source?: string | null
status_code?: number | null
propagation?: string | null
client_response_source?: string | null
safe_to_expose_upstream?: boolean | null
summary_source?: string | null
}
export interface RequestDetail {
id: string // UUID
request_id: string
@@ -173,6 +200,12 @@ export interface RequestDetail {
status_code: number
status?: string // pending, streaming, completed, failed, cancelled
error_message?: string
request_error?: RequestErrorDomain | null
upstream_error?: RequestErrorDomain | null
client_error?: RequestErrorDomain | null
failure_summary?: RequestErrorDomain | null
errors?: RequestErrorDomains | null
error_flow?: RequestErrorFlow | null
response_time_ms: number
created_at: string
request_headers?: Record<string, unknown>
@@ -439,16 +439,32 @@
<span class="reason-value">{{ currentAttemptSkipReasonDisplay }}</span>
</div>
<!-- 错误信息 -->
<!-- 真实请求错误节点级调试原因和对客户端返回的摘要分开 -->
<div
v-if="currentAttempt.status === 'failed' && (currentAttempt.error_message || currentAttempt.error_type)"
v-if="currentAttempt.status === 'failed' && currentAttemptRequestError"
class="error-block"
>
<div class="error-type">
{{ currentAttempt.error_type || '错误' }}
真实请求错误
</div>
<div class="error-msg">
{{ currentAttempt.error_message || '未知错误' }}
{{ currentAttemptRequestError.message }}
</div>
<div
v-if="currentAttemptRequestError.meta.length > 0"
class="error-flow-meta"
>
<span
v-for="item in currentAttemptRequestError.meta"
:key="item"
class="error-flow-chip"
>{{ item }}</span>
</div>
<div
v-if="currentAttemptRequestError.safetyHint"
class="error-flow-safety"
>
{{ currentAttemptRequestError.safetyHint }}
</div>
</div>
@@ -551,6 +567,17 @@ interface UsageData {
}
}
interface AttemptErrorFlow {
source?: string
statusCode?: number
classification?: string
decision?: string
retryable?: boolean
safeToExpose?: boolean
propagation?: string
message?: string
}
const props = defineProps<{
requestId?: string | null
/** 外部传入的状态码,用于覆盖 trace.final_status 的判断 */
@@ -569,6 +596,7 @@ const props = defineProps<{
const emit = defineEmits<{
selectAttempt: [attempt: CandidateRecord | null]
traceState: [state: { loaded: boolean, hasTrace: boolean }]
}>()
// 用量数据(从 props 获取)
@@ -626,6 +654,19 @@ const trace = computed(() => props.traceData ?? internalTrace.value)
const selectedGroupIndex = ref(0)
const selectedAttemptIndex = ref(0)
const hoveredGroupIndex = ref<number | null>(null)
const traceLoadStarted = ref(false)
watch(
[trace, loading],
([value, isLoading]) => {
const waitingForInternalTrace = Boolean(props.requestId && !props.traceData && !traceLoadStarted.value && !value)
emit('traceState', {
loaded: !isLoading && !waitingForInternalTrace,
hasTrace: Boolean(value?.candidates?.length),
})
},
{ immediate: true },
)
// 格式化延迟(自动调整单位)
const formatLatency = (ms: number | undefined | null): string => {
@@ -1022,6 +1063,80 @@ const extractObject = (value: unknown): Record<string, unknown> | null => {
return value as Record<string, unknown>
}
const readStringField = (obj: Record<string, unknown>, key: string): string | undefined => {
const value = obj[key]
return typeof value === 'string' && value.trim() ? value.trim() : undefined
}
const readNumberField = (obj: Record<string, unknown>, key: string): number | undefined => {
const value = obj[key]
if (typeof value === 'number' && Number.isFinite(value)) return value
if (typeof value === 'string' && value.trim()) {
const parsed = Number(value)
if (Number.isFinite(parsed)) return parsed
}
return undefined
}
const readBooleanField = (obj: Record<string, unknown>, key: string): boolean | undefined => {
const value = obj[key]
return typeof value === 'boolean' ? value : undefined
}
const normalizeAttemptErrorFlow = (value: unknown): AttemptErrorFlow | null => {
const raw = extractObject(value)
if (!raw) return null
const flow: AttemptErrorFlow = {
source: readStringField(raw, 'source'),
statusCode: readNumberField(raw, 'status_code') ?? readNumberField(raw, 'statusCode'),
classification: readStringField(raw, 'classification'),
decision: readStringField(raw, 'decision'),
retryable: readBooleanField(raw, 'retryable'),
safeToExpose: readBooleanField(raw, 'safe_to_expose') ?? readBooleanField(raw, 'safeToExpose'),
propagation: readStringField(raw, 'propagation'),
message: readStringField(raw, 'message'),
}
return Object.values(flow).some(value => value !== undefined) ? flow : null
}
const labelFromMap = (value: string | undefined, labels: Record<string, string>): string | undefined => {
if (!value) return undefined
return labels[value] || value
}
const formatErrorFlowSource = (value?: string): string | undefined => labelFromMap(value, {
upstream_response: '上游响应',
request_validation: '请求校验',
gateway: '网关处理',
transport: '传输层',
scheduler: '调度层',
})
const formatErrorFlowDecision = (value?: string): string | undefined => labelFromMap(value, {
retry_next_candidate: '重试下一个候选',
stop_local_failover: '停止本地转移',
use_default: '默认处理',
return_to_client: '返回客户端',
})
const formatErrorFlowPropagation = (value?: string): string | undefined => labelFromMap(value, {
suppressed: '已抑制',
converted: '已转换',
passthrough: '直接透传',
local: '本地生成',
captured: '仅采集',
})
const formatErrorFlowClassification = (value?: string): string | undefined => labelFromMap(value, {
retryable: '可重试',
terminal: '终止',
provider_auth: '上游认证',
provider_quota: '上游额度',
invalid_request: '请求无效',
})
const extractStringList = (value: unknown): string[] => {
if (Array.isArray(value)) {
return value
@@ -1256,6 +1371,45 @@ const currentAttemptSkipReasonDisplay = computed(() => {
return detailedReason || attempt.skip_reason
})
const currentAttemptRequestError = computed<{
message: string
meta: string[]
safetyHint: string
} | null>(() => {
const attempt = currentAttempt.value
if (!attempt || attempt.status !== 'failed') return null
const extra = extractObject(attempt.extra_data)
const flow = normalizeAttemptErrorFlow(extra?.error_flow)
const fallbackMessage = typeof attempt.error_message === 'string' && attempt.error_message.trim()
? attempt.error_message.trim()
: ''
const fallbackType = typeof attempt.error_type === 'string' && attempt.error_type.trim()
? attempt.error_type.trim()
: ''
const message = flow?.message || fallbackMessage
if (!message && !fallbackType && !flow) return null
const meta = [
flow?.statusCode != null ? `HTTP ${flow.statusCode}` : (attempt.status_code ? `HTTP ${attempt.status_code}` : ''),
formatErrorFlowSource(flow?.source),
formatErrorFlowClassification(flow?.classification) || fallbackType,
formatErrorFlowDecision(flow?.decision),
formatErrorFlowPropagation(flow?.propagation),
flow?.retryable != null ? (flow.retryable ? '会继续重试' : '不再重试') : '',
].filter((item): item is string => Boolean(item))
const safetyHint = flow?.safeToExpose === false
? '该错误被标记为敏感上游错误:仅在链路节点展示,不应完整返回给客户端。'
: ''
return {
message: message || fallbackType || '未知错误',
meta,
safetyHint,
}
})
// 计算当前尝试启用的能力标签(请求需要的能力)
const activeCapabilities = computed(() => {
if (!currentAttempt.value?.required_capabilities) return []
@@ -1390,6 +1544,7 @@ const loadTrace = async (silent = false) => {
if (!props.requestId || props.traceData) return
isSilentRefresh.value = silent
traceLoadStarted.value = true
if (!silent) {
loading.value = true
@@ -1481,6 +1636,7 @@ watch(
() => {
selectedGroupIndex.value = 0
selectedAttemptIndex.value = 0
traceLoadStarted.value = false
if (props.traceData) {
internalTrace.value = null
@@ -2371,6 +2527,38 @@ const getDisplayStatus = (attempt: CandidateRecord | null | undefined): string =
word-break: break-word;
}
.error-flow-meta {
display: flex;
flex-wrap: wrap;
gap: 0.375rem;
margin-top: 0.625rem;
}
.error-flow-chip {
padding: 0.125rem 0.45rem;
border-radius: 999px;
background: #ef444414;
border: 1px solid #ef44442e;
color: #991b1b;
font-size: 0.72rem;
line-height: 1.35;
}
.error-flow-safety {
margin-top: 0.625rem;
color: #991b1b;
font-size: 0.78rem;
line-height: 1.5;
}
.dark .error-flow-chip {
color: #fecaca;
}
.dark .error-flow-safety {
color: #fecaca;
}
/* 额外信息 */
.extra-block {
margin-top: 1rem;
@@ -414,25 +414,58 @@
:request-status="detail.status"
:request-api-format="detail.api_format || null"
:request-metadata="traceRequestMetadata"
@trace-state="handleTraceState"
/>
</div>
<!-- 响应客户端错误卡片 -->
<Card
v-if="detail.error_message"
class="border-red-200 dark:border-red-800"
<!-- 错误域卡片保持上游响应客户端响应两个边界可对照 -->
<div
v-if="hasVisibleErrorCards"
class="space-y-3"
>
<div class="p-4">
<h4 class="text-sm font-semibold text-red-600 dark:text-red-400 mb-2">
响应客户端错误
</h4>
<div class="bg-red-50 dark:bg-red-900/20 rounded-lg p-3">
<p class="text-sm text-red-800 dark:text-red-300">
{{ detail.error_message }}
</p>
</div>
<div
class="grid gap-3"
:class="visibleErrorCardCount > 1 ? 'lg:grid-cols-2' : 'grid-cols-1'"
>
<Card
v-if="displayClientErrorMessage"
class="border-amber-200 dark:border-amber-800"
>
<div class="p-4">
<h4 class="text-sm font-semibold text-amber-700 dark:text-amber-300 mb-2">
返回客户端错误
</h4>
<div class="bg-amber-50 dark:bg-amber-900/20 rounded-lg p-3 space-y-1">
<p class="text-sm text-amber-900 dark:text-amber-200">
{{ displayClientErrorMessage }}
</p>
</div>
</div>
</Card>
<Card
v-if="normalizedUpstreamError"
class="border-orange-200 dark:border-orange-800"
>
<div class="p-4">
<h4 class="text-sm font-semibold text-orange-700 dark:text-orange-300 mb-2">
上游响应错误
</h4>
<div class="bg-orange-50 dark:bg-orange-900/20 rounded-lg p-3 space-y-1">
<p class="text-sm text-orange-900 dark:text-orange-200">
{{ normalizedUpstreamError.message }}
</p>
<p
v-if="formatErrorDomainMeta(normalizedUpstreamError)"
class="text-xs text-orange-800/70 dark:text-orange-200/70 font-mono"
>
{{ formatErrorDomainMeta(normalizedUpstreamError) }}
</p>
</div>
</div>
</Card>
</div>
</Card>
</div>
<!-- Tabs 区域 -->
<Card>
@@ -676,7 +709,7 @@ import Skeleton from '@/components/ui/skeleton.vue'
import Tabs from '@/components/ui/tabs.vue'
import TabsContent from '@/components/ui/tabs-content.vue'
import { Check, Columns2, RefreshCw, X, Monitor, Server, MessageSquareText, Code2, Terminal, Play } from 'lucide-vue-next'
import { dashboardApi, type RequestDetail } from '@/api/dashboard'
import { dashboardApi, type RequestDetail, type RequestErrorDomain } from '@/api/dashboard'
import { formatApiFormat } from '@/api/endpoints/types/api-format'
import { formatShortRequestId } from '@/utils/format'
import { log } from '@/utils/logger'
@@ -717,6 +750,8 @@ const loading = ref(false)
const error = ref<string | null>(null)
const detail = ref<RequestDetail | null>(null)
const timelineRef = ref<InstanceType<typeof HorizontalRequestTimeline> | null>(null)
const timelineLoaded = ref(false)
const timelineHasTrace = ref(false)
const activeTab = ref('request-body')
const copiedStates = ref<Record<string, boolean>>({})
const viewMode = ref<'compare' | 'formatted' | 'raw'>('formatted')
@@ -749,11 +784,64 @@ type PricingTierLike = {
type JsonRecord = Record<string, unknown>
type NormalizedErrorDomain = {
source?: string | null
status_code?: number | null
type?: string | null
message: string
code?: string | number | null
category?: string | null
}
function asRecord(value: unknown): JsonRecord | null {
if (!value || typeof value !== 'object' || Array.isArray(value)) return null
return value as JsonRecord
}
function normalizeErrorDomain(domain: RequestErrorDomain | null | undefined): NormalizedErrorDomain | null {
if (!domain || typeof domain !== 'object') return null
const message = typeof domain.message === 'string' ? domain.message.trim() : ''
if (!message) return null
return {
source: domain.source ?? null,
status_code: domain.status_code ?? null,
type: domain.type ?? null,
message,
code: domain.code ?? null,
category: domain.category ?? null,
}
}
function formatErrorDomainMeta(domain: NormalizedErrorDomain): string {
const parts: string[] = []
if (domain.status_code != null) parts.push(`HTTP ${domain.status_code}`)
if (domain.type) parts.push(domain.type)
if (domain.source) parts.push(`source=${domain.source}`)
return parts.join(' · ')
}
function simplifyClientErrorMessage(message: string): string {
let simplified = message.trim()
if (!simplified) return ''
simplified = simplified
.replace(/[(]\s*原因代码\s*[:][^)]*[)]/gi, '')
.replace(/\s+/g, ' ')
.trim()
const advisoryIndex = simplified.search(/[。.!?]\s*(请检查|请确认|原因代码|Reason|Code)/i)
if (advisoryIndex > 0) {
simplified = simplified.slice(0, advisoryIndex)
}
return simplified.replace(/[。.!?;,:\s]+$/u, '').trim()
}
function handleTraceState(state: { loaded: boolean, hasTrace: boolean }) {
timelineLoaded.value = state.loaded
timelineHasTrace.value = state.hasTrace
}
function toNumber(value: unknown): number | null {
if (typeof value === 'number' && Number.isFinite(value)) return value
if (typeof value === 'string') {
@@ -871,6 +959,27 @@ const metadataPanelData = computed<Record<string, unknown> | null>(() => {
return Object.keys(merged).length > 0 ? merged : null
})
const normalizedClientError = computed(() =>
normalizeErrorDomain(detail.value?.errors?.client_error ?? detail.value?.client_error),
)
const normalizedUpstreamError = computed(() =>
normalizeErrorDomain(detail.value?.errors?.upstream_error ?? detail.value?.upstream_error),
)
const displayClientErrorMessage = computed(() =>
normalizedClientError.value ? simplifyClientErrorMessage(normalizedClientError.value.message) : '',
)
const hasVisibleErrorCards = computed(() =>
Boolean(displayClientErrorMessage.value || normalizedUpstreamError.value),
)
const visibleErrorCardCount = computed(() =>
(displayClientErrorMessage.value ? 1 : 0)
+ (normalizedUpstreamError.value ? 1 : 0),
)
const settlementInfo = computed<JsonRecord | null>(() =>
asRecord(detail.value?.settlement ?? null),
)
@@ -1652,6 +1761,12 @@ async function ensureBodyContentLoaded() {
has_provider_request_body: response.has_provider_request_body,
has_response_body: response.has_response_body,
has_client_response_body: response.has_client_response_body,
request_error: response.request_error,
upstream_error: response.upstream_error,
client_error: response.client_error,
failure_summary: response.failure_summary,
errors: response.errors,
error_flow: response.error_flow,
}
bodiesLoadedForRequestId.value = cacheKey
} catch (err) {
@@ -1673,6 +1788,8 @@ async function loadDetail(id: string, silent = false) {
if (!silent) {
loading.value = true
historicalPricing.value = null
timelineLoaded.value = false
timelineHasTrace.value = false
showTimeline.value = false
clearTimelineMountTimer()
++bodyLoadRequestId
@@ -1696,6 +1813,12 @@ async function loadDetail(id: string, silent = false) {
provider_request_body: sameRequest ? previousDetail?.provider_request_body : undefined,
response_body: sameRequest ? previousDetail?.response_body : undefined,
client_response_body: sameRequest ? previousDetail?.client_response_body : undefined,
request_error: sameRequest ? (previousDetail?.request_error ?? response.request_error) : response.request_error,
upstream_error: sameRequest ? (previousDetail?.upstream_error ?? response.upstream_error) : response.upstream_error,
client_error: sameRequest ? (previousDetail?.client_error ?? response.client_error) : response.client_error,
failure_summary: sameRequest ? (previousDetail?.failure_summary ?? response.failure_summary) : response.failure_summary,
errors: sameRequest ? (previousDetail?.errors ?? response.errors) : response.errors,
error_flow: sameRequest ? (previousDetail?.error_flow ?? response.error_flow) : response.error_flow,
}
bodiesLoadedForRequestId.value = sameRequest ? bodiesLoadedForRequestId.value : null