mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 01:10:23 +08:00
feat(observability): 引入错误链路 error_flow 元数据并区分上游/客户端错误
- 网关在本地 failover 时构建 error_flow 元数据(分类/决策/传播策略),写入 report_context - scheduler-core 解析并透传 error_flow 至候选 extra_data - admin usage 详情拆分 request/upstream/client/failure_summary 错误域,敏感上游错误标记为 suppressed - 前端 RequestDetailDrawer 拆出"返回客户端"与"上游响应"双错误卡片 - HorizontalRequestTimeline 节点详情展示真实请求错误及 error_flow 标签
This commit is contained in:
@@ -70,10 +70,10 @@ use crate::execution_runtime::{
|
||||
use crate::execution_runtime::{MAX_STREAM_PREFETCH_BYTES, MAX_STREAM_PREFETCH_FRAMES};
|
||||
use crate::log_ids::short_request_id;
|
||||
use crate::orchestration::{
|
||||
apply_local_execution_effect, LocalAdaptiveRateLimitEffect, LocalAdaptiveSuccessEffect,
|
||||
LocalAttemptFailureEffect, LocalExecutionEffect, LocalExecutionEffectContext,
|
||||
LocalHealthFailureEffect, LocalHealthSuccessEffect, LocalOAuthInvalidationEffect,
|
||||
LocalPoolErrorEffect,
|
||||
apply_local_execution_effect, build_local_error_flow_metadata, with_error_flow_report_context,
|
||||
LocalAdaptiveRateLimitEffect, LocalAdaptiveSuccessEffect, LocalAttemptFailureEffect,
|
||||
LocalExecutionEffect, LocalExecutionEffectContext, LocalHealthFailureEffect,
|
||||
LocalHealthSuccessEffect, LocalOAuthInvalidationEffect, LocalPoolErrorEffect,
|
||||
};
|
||||
use crate::request_candidate_runtime::{
|
||||
ensure_execution_request_candidate_slot, record_local_request_candidate_status,
|
||||
@@ -893,10 +893,20 @@ async fn execute_stream_from_frame_stream(
|
||||
);
|
||||
if matches!(failover_decision, LocalFailoverDecision::RetryNextCandidate) {
|
||||
let terminal_unix_secs = current_request_candidate_unix_ms();
|
||||
let error_flow_report_context = with_error_flow_report_context(
|
||||
report_context.as_ref(),
|
||||
build_local_error_flow_metadata(
|
||||
status_code,
|
||||
error_response_text.as_deref(),
|
||||
failover_analysis,
|
||||
),
|
||||
);
|
||||
record_local_request_candidate_status(
|
||||
state,
|
||||
&plan,
|
||||
report_context.as_ref(),
|
||||
error_flow_report_context
|
||||
.as_ref()
|
||||
.or(report_context.as_ref()),
|
||||
SchedulerRequestCandidateStatusUpdate {
|
||||
status: RequestCandidateStatus::Failed,
|
||||
status_code: Some(status_code),
|
||||
@@ -934,10 +944,20 @@ async fn execute_stream_from_frame_stream(
|
||||
)
|
||||
{
|
||||
let terminal_unix_secs = current_request_candidate_unix_ms();
|
||||
let error_flow_report_context = with_error_flow_report_context(
|
||||
report_context.as_ref(),
|
||||
build_local_error_flow_metadata(
|
||||
status_code,
|
||||
error_response_text.as_deref(),
|
||||
failover_analysis,
|
||||
),
|
||||
);
|
||||
record_local_request_candidate_status(
|
||||
state,
|
||||
&plan,
|
||||
report_context.as_ref(),
|
||||
error_flow_report_context
|
||||
.as_ref()
|
||||
.or(report_context.as_ref()),
|
||||
SchedulerRequestCandidateStatusUpdate {
|
||||
status: RequestCandidateStatus::Failed,
|
||||
status_code: Some(status_code),
|
||||
@@ -970,10 +990,20 @@ async fn execute_stream_from_frame_stream(
|
||||
);
|
||||
record_sync_terminal_usage(state, &plan, payload.report_context.as_ref(), &payload);
|
||||
let terminal_unix_secs = current_request_candidate_unix_ms();
|
||||
let error_flow_report_context = with_error_flow_report_context(
|
||||
payload.report_context.as_ref(),
|
||||
build_local_error_flow_metadata(
|
||||
status_code,
|
||||
error_response_text.as_deref(),
|
||||
failover_analysis,
|
||||
),
|
||||
);
|
||||
record_local_request_candidate_status(
|
||||
state,
|
||||
&plan,
|
||||
payload.report_context.as_ref(),
|
||||
error_flow_report_context
|
||||
.as_ref()
|
||||
.or(payload.report_context.as_ref()),
|
||||
SchedulerRequestCandidateStatusUpdate {
|
||||
status: RequestCandidateStatus::Failed,
|
||||
status_code: Some(status_code),
|
||||
|
||||
@@ -35,10 +35,10 @@ use crate::execution_runtime::{
|
||||
};
|
||||
use crate::log_ids::short_request_id;
|
||||
use crate::orchestration::{
|
||||
apply_local_execution_effect, LocalAdaptiveRateLimitEffect, LocalAdaptiveSuccessEffect,
|
||||
LocalAttemptFailureEffect, LocalExecutionEffect, LocalExecutionEffectContext,
|
||||
LocalHealthFailureEffect, LocalHealthSuccessEffect, LocalOAuthInvalidationEffect,
|
||||
LocalPoolErrorEffect,
|
||||
apply_local_execution_effect, build_local_error_flow_metadata, with_error_flow_report_context,
|
||||
LocalAdaptiveRateLimitEffect, LocalAdaptiveSuccessEffect, LocalAttemptFailureEffect,
|
||||
LocalExecutionEffect, LocalExecutionEffectContext, LocalHealthFailureEffect,
|
||||
LocalHealthSuccessEffect, LocalOAuthInvalidationEffect, LocalPoolErrorEffect,
|
||||
};
|
||||
use crate::request_candidate_runtime::{
|
||||
ensure_execution_request_candidate_slot, record_local_request_candidate_status,
|
||||
@@ -426,10 +426,20 @@ pub(crate) async fn execute_execution_runtime_sync(
|
||||
LocalFailoverDecision::RetryNextCandidate
|
||||
) {
|
||||
let terminal_unix_secs = current_request_candidate_unix_ms();
|
||||
let error_flow_report_context = with_error_flow_report_context(
|
||||
report_context.as_ref(),
|
||||
build_local_error_flow_metadata(
|
||||
result.status_code,
|
||||
local_failover_response_text.as_deref(),
|
||||
local_failover_analysis,
|
||||
),
|
||||
);
|
||||
record_local_request_candidate_status(
|
||||
state,
|
||||
&plan,
|
||||
report_context.as_ref(),
|
||||
error_flow_report_context
|
||||
.as_ref()
|
||||
.or(report_context.as_ref()),
|
||||
SchedulerRequestCandidateStatusUpdate {
|
||||
status: RequestCandidateStatus::Failed,
|
||||
status_code: Some(result.status_code),
|
||||
@@ -488,10 +498,20 @@ pub(crate) async fn execute_execution_runtime_sync(
|
||||
mapped_error_finalize_kind.is_some(),
|
||||
) {
|
||||
let terminal_unix_secs = current_request_candidate_unix_ms();
|
||||
let error_flow_report_context = with_error_flow_report_context(
|
||||
report_context.as_ref(),
|
||||
build_local_error_flow_metadata(
|
||||
result.status_code,
|
||||
local_failover_response_text.as_deref(),
|
||||
local_failover_analysis,
|
||||
),
|
||||
);
|
||||
record_local_request_candidate_status(
|
||||
state,
|
||||
&plan,
|
||||
report_context.as_ref(),
|
||||
error_flow_report_context
|
||||
.as_ref()
|
||||
.or(report_context.as_ref()),
|
||||
SchedulerRequestCandidateStatusUpdate {
|
||||
status: RequestCandidateStatus::Failed,
|
||||
status_code: Some(result.status_code),
|
||||
@@ -507,10 +527,24 @@ pub(crate) async fn execute_execution_runtime_sync(
|
||||
}
|
||||
|
||||
let terminal_unix_secs = current_request_candidate_unix_ms();
|
||||
let error_flow_report_context = (result.status_code >= 400)
|
||||
.then(|| {
|
||||
with_error_flow_report_context(
|
||||
report_context.as_ref(),
|
||||
build_local_error_flow_metadata(
|
||||
result.status_code,
|
||||
local_failover_response_text.as_deref(),
|
||||
local_failover_analysis,
|
||||
),
|
||||
)
|
||||
})
|
||||
.flatten();
|
||||
record_local_request_candidate_status(
|
||||
state,
|
||||
&plan,
|
||||
report_context.as_ref(),
|
||||
error_flow_report_context
|
||||
.as_ref()
|
||||
.or(report_context.as_ref()),
|
||||
SchedulerRequestCandidateStatusUpdate {
|
||||
status: if result.status_code >= 400 {
|
||||
RequestCandidateStatus::Failed
|
||||
|
||||
@@ -145,6 +145,23 @@ pub(crate) enum LocalFailoverClassification {
|
||||
RetryUpstreamFailure,
|
||||
}
|
||||
|
||||
impl LocalFailoverClassification {
|
||||
pub(crate) const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::UseDefault => "use_default",
|
||||
Self::StopStatusCode => "stop_status_code",
|
||||
Self::StopErrorPattern => "stop_error_pattern",
|
||||
Self::StopSemanticClientError => "stop_semantic_client_error",
|
||||
Self::RetrySuccessPattern => "retry_success_pattern",
|
||||
Self::RetrySemanticCompatibilityError => "retry_semantic_compatibility_error",
|
||||
Self::RetrySemanticRateLimit => "retry_semantic_rate_limit",
|
||||
Self::RetrySemanticThinkingError => "retry_semantic_thinking_error",
|
||||
Self::RetryStatusCode => "retry_status_code",
|
||||
Self::RetryUpstreamFailure => "retry_upstream_failure",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn classify_local_failover(
|
||||
policy: &LocalFailoverPolicy,
|
||||
input: LocalFailoverInput<'_>,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use aether_contracts::ExecutionPlan;
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::AppState;
|
||||
|
||||
@@ -78,3 +79,43 @@ pub(crate) async fn resolve_local_failover_decision_for_attempt(
|
||||
.await
|
||||
.decision
|
||||
}
|
||||
|
||||
pub(crate) fn build_local_error_flow_metadata(
|
||||
status_code: u16,
|
||||
response_text: Option<&str>,
|
||||
analysis: LocalFailoverAnalysis,
|
||||
) -> Value {
|
||||
let safe_to_expose = matches!(
|
||||
analysis.classification,
|
||||
LocalFailoverClassification::StopSemanticClientError
|
||||
| LocalFailoverClassification::StopStatusCode
|
||||
| LocalFailoverClassification::StopErrorPattern
|
||||
);
|
||||
let propagation = match analysis.decision {
|
||||
LocalFailoverDecision::RetryNextCandidate => "suppressed",
|
||||
LocalFailoverDecision::StopLocalFailover if safe_to_expose => "converted",
|
||||
LocalFailoverDecision::StopLocalFailover => "suppressed",
|
||||
LocalFailoverDecision::UseDefault if status_code >= 400 => "passthrough",
|
||||
LocalFailoverDecision::UseDefault => "none",
|
||||
};
|
||||
json!({
|
||||
"stage": "candidate",
|
||||
"source": "upstream_response",
|
||||
"status_code": status_code,
|
||||
"classification": analysis.classification.as_str(),
|
||||
"decision": analysis.decision.as_str(),
|
||||
"retryable": matches!(analysis.decision, LocalFailoverDecision::RetryNextCandidate),
|
||||
"safe_to_expose": safe_to_expose,
|
||||
"propagation": propagation,
|
||||
"message": local_failover_error_message(response_text),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn with_error_flow_report_context(
|
||||
report_context: Option<&Value>,
|
||||
error_flow: Value,
|
||||
) -> Option<Value> {
|
||||
let mut object = report_context?.as_object()?.clone();
|
||||
object.insert("error_flow".to_string(), error_flow);
|
||||
Some(Value::Object(object))
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user