feat(observability): 引入错误链路 error_flow 元数据并区分上游/客户端错误

- 网关在本地 failover 时构建 error_flow 元数据(分类/决策/传播策略),写入 report_context
- scheduler-core 解析并透传 error_flow 至候选 extra_data
- admin usage 详情拆分 request/upstream/client/failure_summary 错误域,敏感上游错误标记为 suppressed
- 前端 RequestDetailDrawer 拆出"返回客户端"与"上游响应"双错误卡片
- HorizontalRequestTimeline 节点详情展示真实请求错误及 error_flow 标签
This commit is contained in:
fawney19
2026-04-25 16:58:17 +08:00
parent bc97e383d3
commit 00744c0ce5
9 changed files with 997 additions and 44 deletions

View File

@@ -70,10 +70,10 @@ use crate::execution_runtime::{
use crate::execution_runtime::{MAX_STREAM_PREFETCH_BYTES, MAX_STREAM_PREFETCH_FRAMES};
use crate::log_ids::short_request_id;
use crate::orchestration::{
apply_local_execution_effect, LocalAdaptiveRateLimitEffect, LocalAdaptiveSuccessEffect,
LocalAttemptFailureEffect, LocalExecutionEffect, LocalExecutionEffectContext,
LocalHealthFailureEffect, LocalHealthSuccessEffect, LocalOAuthInvalidationEffect,
LocalPoolErrorEffect,
apply_local_execution_effect, build_local_error_flow_metadata, with_error_flow_report_context,
LocalAdaptiveRateLimitEffect, LocalAdaptiveSuccessEffect, LocalAttemptFailureEffect,
LocalExecutionEffect, LocalExecutionEffectContext, LocalHealthFailureEffect,
LocalHealthSuccessEffect, LocalOAuthInvalidationEffect, LocalPoolErrorEffect,
};
use crate::request_candidate_runtime::{
ensure_execution_request_candidate_slot, record_local_request_candidate_status,
@@ -893,10 +893,20 @@ async fn execute_stream_from_frame_stream(
);
if matches!(failover_decision, LocalFailoverDecision::RetryNextCandidate) {
let terminal_unix_secs = current_request_candidate_unix_ms();
let error_flow_report_context = with_error_flow_report_context(
report_context.as_ref(),
build_local_error_flow_metadata(
status_code,
error_response_text.as_deref(),
failover_analysis,
),
);
record_local_request_candidate_status(
state,
&plan,
report_context.as_ref(),
error_flow_report_context
.as_ref()
.or(report_context.as_ref()),
SchedulerRequestCandidateStatusUpdate {
status: RequestCandidateStatus::Failed,
status_code: Some(status_code),
@@ -934,10 +944,20 @@ async fn execute_stream_from_frame_stream(
)
{
let terminal_unix_secs = current_request_candidate_unix_ms();
let error_flow_report_context = with_error_flow_report_context(
report_context.as_ref(),
build_local_error_flow_metadata(
status_code,
error_response_text.as_deref(),
failover_analysis,
),
);
record_local_request_candidate_status(
state,
&plan,
report_context.as_ref(),
error_flow_report_context
.as_ref()
.or(report_context.as_ref()),
SchedulerRequestCandidateStatusUpdate {
status: RequestCandidateStatus::Failed,
status_code: Some(status_code),
@@ -970,10 +990,20 @@ async fn execute_stream_from_frame_stream(
);
record_sync_terminal_usage(state, &plan, payload.report_context.as_ref(), &payload);
let terminal_unix_secs = current_request_candidate_unix_ms();
let error_flow_report_context = with_error_flow_report_context(
payload.report_context.as_ref(),
build_local_error_flow_metadata(
status_code,
error_response_text.as_deref(),
failover_analysis,
),
);
record_local_request_candidate_status(
state,
&plan,
payload.report_context.as_ref(),
error_flow_report_context
.as_ref()
.or(payload.report_context.as_ref()),
SchedulerRequestCandidateStatusUpdate {
status: RequestCandidateStatus::Failed,
status_code: Some(status_code),

View File

@@ -35,10 +35,10 @@ use crate::execution_runtime::{
};
use crate::log_ids::short_request_id;
use crate::orchestration::{
apply_local_execution_effect, LocalAdaptiveRateLimitEffect, LocalAdaptiveSuccessEffect,
LocalAttemptFailureEffect, LocalExecutionEffect, LocalExecutionEffectContext,
LocalHealthFailureEffect, LocalHealthSuccessEffect, LocalOAuthInvalidationEffect,
LocalPoolErrorEffect,
apply_local_execution_effect, build_local_error_flow_metadata, with_error_flow_report_context,
LocalAdaptiveRateLimitEffect, LocalAdaptiveSuccessEffect, LocalAttemptFailureEffect,
LocalExecutionEffect, LocalExecutionEffectContext, LocalHealthFailureEffect,
LocalHealthSuccessEffect, LocalOAuthInvalidationEffect, LocalPoolErrorEffect,
};
use crate::request_candidate_runtime::{
ensure_execution_request_candidate_slot, record_local_request_candidate_status,
@@ -426,10 +426,20 @@ pub(crate) async fn execute_execution_runtime_sync(
LocalFailoverDecision::RetryNextCandidate
) {
let terminal_unix_secs = current_request_candidate_unix_ms();
let error_flow_report_context = with_error_flow_report_context(
report_context.as_ref(),
build_local_error_flow_metadata(
result.status_code,
local_failover_response_text.as_deref(),
local_failover_analysis,
),
);
record_local_request_candidate_status(
state,
&plan,
report_context.as_ref(),
error_flow_report_context
.as_ref()
.or(report_context.as_ref()),
SchedulerRequestCandidateStatusUpdate {
status: RequestCandidateStatus::Failed,
status_code: Some(result.status_code),
@@ -488,10 +498,20 @@ pub(crate) async fn execute_execution_runtime_sync(
mapped_error_finalize_kind.is_some(),
) {
let terminal_unix_secs = current_request_candidate_unix_ms();
let error_flow_report_context = with_error_flow_report_context(
report_context.as_ref(),
build_local_error_flow_metadata(
result.status_code,
local_failover_response_text.as_deref(),
local_failover_analysis,
),
);
record_local_request_candidate_status(
state,
&plan,
report_context.as_ref(),
error_flow_report_context
.as_ref()
.or(report_context.as_ref()),
SchedulerRequestCandidateStatusUpdate {
status: RequestCandidateStatus::Failed,
status_code: Some(result.status_code),
@@ -507,10 +527,24 @@ pub(crate) async fn execute_execution_runtime_sync(
}
let terminal_unix_secs = current_request_candidate_unix_ms();
let error_flow_report_context = (result.status_code >= 400)
.then(|| {
with_error_flow_report_context(
report_context.as_ref(),
build_local_error_flow_metadata(
result.status_code,
local_failover_response_text.as_deref(),
local_failover_analysis,
),
)
})
.flatten();
record_local_request_candidate_status(
state,
&plan,
report_context.as_ref(),
error_flow_report_context
.as_ref()
.or(report_context.as_ref()),
SchedulerRequestCandidateStatusUpdate {
status: if result.status_code >= 400 {
RequestCandidateStatus::Failed

View File

@@ -145,6 +145,23 @@ pub(crate) enum LocalFailoverClassification {
RetryUpstreamFailure,
}
impl LocalFailoverClassification {
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::UseDefault => "use_default",
Self::StopStatusCode => "stop_status_code",
Self::StopErrorPattern => "stop_error_pattern",
Self::StopSemanticClientError => "stop_semantic_client_error",
Self::RetrySuccessPattern => "retry_success_pattern",
Self::RetrySemanticCompatibilityError => "retry_semantic_compatibility_error",
Self::RetrySemanticRateLimit => "retry_semantic_rate_limit",
Self::RetrySemanticThinkingError => "retry_semantic_thinking_error",
Self::RetryStatusCode => "retry_status_code",
Self::RetryUpstreamFailure => "retry_upstream_failure",
}
}
}
pub(crate) fn classify_local_failover(
policy: &LocalFailoverPolicy,
input: LocalFailoverInput<'_>,

View File

@@ -1,4 +1,5 @@
use aether_contracts::ExecutionPlan;
use serde_json::{json, Value};
use crate::AppState;
@@ -78,3 +79,43 @@ pub(crate) async fn resolve_local_failover_decision_for_attempt(
.await
.decision
}
pub(crate) fn build_local_error_flow_metadata(
status_code: u16,
response_text: Option<&str>,
analysis: LocalFailoverAnalysis,
) -> Value {
let safe_to_expose = matches!(
analysis.classification,
LocalFailoverClassification::StopSemanticClientError
| LocalFailoverClassification::StopStatusCode
| LocalFailoverClassification::StopErrorPattern
);
let propagation = match analysis.decision {
LocalFailoverDecision::RetryNextCandidate => "suppressed",
LocalFailoverDecision::StopLocalFailover if safe_to_expose => "converted",
LocalFailoverDecision::StopLocalFailover => "suppressed",
LocalFailoverDecision::UseDefault if status_code >= 400 => "passthrough",
LocalFailoverDecision::UseDefault => "none",
};
json!({
"stage": "candidate",
"source": "upstream_response",
"status_code": status_code,
"classification": analysis.classification.as_str(),
"decision": analysis.decision.as_str(),
"retryable": matches!(analysis.decision, LocalFailoverDecision::RetryNextCandidate),
"safe_to_expose": safe_to_expose,
"propagation": propagation,
"message": local_failover_error_message(response_text),
})
}
pub(crate) fn with_error_flow_report_context(
report_context: Option<&Value>,
error_flow: Value,
) -> Option<Value> {
let mut object = report_context?.as_object()?.clone();
object.insert("error_flow".to_string(), error_flow);
Some(Value::Object(object))
}