i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 01:10:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

Improve request trace upstream diagnostics

Browse Source
This commit is contained in:
fawney19
2026-05-09 10:46:04 +08:00
parent 4cf0de681a
commit 79d07ac79a
33 changed files with 2315 additions and 427 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.lock generated
View File

@@ -53,6 +53,7 @@ dependencies = [
"serde_json",
"sha1",
"sha2",
"url",
"uuid",
]

2
apps/aether-gateway/src/ai_serving/planner/passthrough/provider/family/payload.rs
View File

@@ -112,6 +112,8 @@ pub(crate) async fn maybe_build_local_same_format_provider_decision_payload_for_
provider_request_method: Some(serde_json::Value::Null),
provider_request_headers: Some(&resolved.provider_request_headers),
original_headers: &parts.headers,
request_path: Some(parts.uri.path()),
request_query_string: parts.uri.query(),
request_origin: Some(crate::ai_serving::request_origin_from_parts(parts)),
original_request_body_json: Some(body_json),
original_request_body_base64: None,

118
apps/aether-gateway/src/ai_serving/planner/report_context.rs
View File

@@ -9,7 +9,11 @@ use aether_ai_serving::{
use aether_scheduler_core::{ClientSessionAffinity, SchedulerRankingOutcome};
use serde_json::{Map, Value};
use crate::ai_serving::{request_origin_from_headers, ExecutionRuntimeAuthContext, RequestOrigin};
use crate::ai_serving::{
request_origin_from_headers, request_path_implies_stream_request, sanitize_request_path,
sanitize_request_path_and_query, sanitize_request_query_string, ExecutionRuntimeAuthContext,
RequestOrigin,
};
use crate::client_session_affinity::{
client_session_affinity_report_context_value, CLIENT_SESSION_AFFINITY_REPORT_CONTEXT_FIELD,
};
@@ -40,6 +44,8 @@ pub(crate) struct LocalExecutionReportContextParts<'a> {
pub(crate) provider_request_method: Option<Value>,
pub(crate) provider_request_headers: Option<&'a BTreeMap<String, String>>,
pub(crate) original_headers: &'a http::HeaderMap,
pub(crate) request_path: Option<&'a str>,
pub(crate) request_query_string: Option<&'a str>,
pub(crate) request_origin: Option<RequestOrigin>,
pub(crate) original_request_body_json: Option<&'a Value>,
pub(crate) original_request_body_base64: Option<&'a str>,
@@ -80,6 +86,15 @@ pub(crate) fn build_local_execution_report_context(
{
merge_incoming_tls_fingerprint(&mut extra_fields, incoming_tls);
}
insert_request_path_fields(
&mut extra_fields,
parts.request_path,
parts.request_query_string,
);
let client_requested_stream = parts.client_requested_stream
|| parts
.request_path
.is_some_and(request_path_implies_stream_request);
build_ai_execution_report_context(AiExecutionReportContextParts {
auth_context: parts.auth_context,
@@ -113,7 +128,7 @@ pub(crate) fn build_local_execution_report_context(
client_ip,
user_agent,
},
client_requested_stream: parts.client_requested_stream,
client_requested_stream,
upstream_is_stream: parts.upstream_is_stream,
has_envelope: parts.has_envelope,
needs_conversion: parts.needs_conversion,
@@ -121,6 +136,30 @@ pub(crate) fn build_local_execution_report_context(
})
}
fn insert_request_path_fields(
extra_fields: &mut Map<String, Value>,
request_path: Option<&str>,
request_query_string: Option<&str>,
) {
let Some(path) = request_path.and_then(sanitize_request_path) else {
return;
};
let query = request_query_string.and_then(sanitize_request_query_string);
let path_and_query = sanitize_request_path_and_query(path.as_str(), query.as_deref())
.unwrap_or_else(|| path.clone());
extra_fields
.entry("request_path".to_string())
.or_insert_with(|| Value::String(path.clone()));
if let Some(query) = query.clone() {
extra_fields
.entry("request_query_string".to_string())
.or_insert_with(|| Value::String(query.to_string()));
}
extra_fields
.entry("request_path_and_query".to_string())
.or_insert_with(|| Value::String(path_and_query));
}
pub(crate) fn provider_stream_event_api_format_for_provider_type(
provider_type: &str,
) -> Option<&'static str> {
@@ -226,6 +265,8 @@ mod tests {
provider_request_method: None,
provider_request_headers: Some(&provider_request_headers),
original_headers: &original_headers,
request_path: Some("/v1/chat/completions"),
request_query_string: Some("debug=true&limit=10"),
request_origin: Some(RequestOrigin {
client_ip: Some("203.0.113.8".to_string()),
user_agent: Some("Claude-Code/1.0".to_string()),
@@ -255,6 +296,77 @@ mod tests {
"session_key": "account=account-1;session=session-1"
})
);
assert_eq!(report_context["request_path"], "/v1/chat/completions");
assert_eq!(report_context["request_query_string"], "limit=10");
assert_eq!(
report_context["request_path_and_query"],
"/v1/chat/completions?limit=10"
);
}
#[test]
fn local_execution_report_context_treats_stream_generate_content_path_as_client_stream() {
let auth_context = ExecutionRuntimeAuthContext {
user_id: "user-1".to_string(),
api_key_id: "api-key-1".to_string(),
username: None,
api_key_name: None,
balance_remaining: None,
access_allowed: true,
api_key_is_standalone: false,
};
let original_headers = http::HeaderMap::new();
let provider_request_headers = BTreeMap::new();
let report_context =
build_local_execution_report_context(LocalExecutionReportContextParts {
auth_context: &auth_context,
request_id: "trace-1",
candidate_id: "candidate-1",
attempt_identity: ExecutionAttemptIdentity::new(0, 0),
model: "gemini-3.1-flash-image-preview",
provider_name: "Gemini",
provider_id: "provider-1",
endpoint_id: "endpoint-1",
key_id: "key-1",
key_name: None,
model_id: None,
global_model_id: None,
global_model_name: None,
provider_api_format: "gemini:generate_content",
client_api_format: "gemini:generate_content",
mapped_model: None,
candidate_group_id: None,
ranking: None,
upstream_url: None,
header_rules: None,
body_rules: None,
provider_request_method: None,
provider_request_headers: Some(&provider_request_headers),
original_headers: &original_headers,
request_path: Some(
"/v1beta/models/gemini-3.1-flash-image-preview:streamGenerateContent",
),
request_query_string: Some("key=secret&alt=sse"),
request_origin: None,
original_request_body_json: Some(&json!({
"contents": [{"role": "user", "parts": [{"text": "hi"}]}]
})),
original_request_body_base64: None,
client_session_affinity: None,
client_requested_stream: false,
upstream_is_stream: true,
has_envelope: false,
needs_conversion: false,
extra_fields: Map::new(),
});
assert_eq!(report_context["client_requested_stream"], true);
assert_eq!(report_context["request_query_string"], "alt=sse");
assert_eq!(
report_context["request_path_and_query"],
"/v1beta/models/gemini-3.1-flash-image-preview:streamGenerateContent?alt=sse"
);
}
#[test]
@@ -300,6 +412,8 @@ mod tests {
provider_request_method: None,
provider_request_headers: Some(&provider_request_headers),
original_headers: &original_headers,
request_path: None,
request_query_string: None,
request_origin: None,
original_request_body_json: Some(&json!({"model": "gpt-5"})),
original_request_body_base64: None,

2
apps/aether-gateway/src/ai_serving/planner/specialized/files/decision.rs
View File

@@ -94,6 +94,8 @@ pub(super) async fn maybe_build_local_gemini_files_decision_payload_for_candidat
provider_request_method: None,
provider_request_headers: None,
original_headers: &parts.headers,
request_path: Some(parts.uri.path()),
request_query_string: parts.uri.query(),
request_origin: Some(crate::ai_serving::request_origin_from_parts(parts)),
original_request_body_json: Some(body_json),
original_request_body_base64: resolved.provider_request_body_base64.as_deref(),

2
apps/aether-gateway/src/ai_serving/planner/specialized/image/decision.rs
View File

@@ -114,6 +114,8 @@ pub(super) async fn maybe_build_local_openai_image_decision_payload_for_candidat
provider_request_method: Some(serde_json::Value::String(parts.method.to_string())),
provider_request_headers: Some(&resolved.provider_request_headers),
original_headers: &parts.headers,
request_path: Some(parts.uri.path()),
request_query_string: parts.uri.query(),
request_origin: Some(crate::ai_serving::request_origin_from_parts(parts)),
original_request_body_json: Some(body_json),
original_request_body_base64: body_base64,

2
apps/aether-gateway/src/ai_serving/planner/specialized/video/decision.rs
View File

@@ -75,6 +75,8 @@ pub(super) async fn maybe_build_local_video_create_decision_payload_for_candidat
provider_request_method: None,
provider_request_headers: None,
original_headers: &parts.headers,
request_path: Some(parts.uri.path()),
request_query_string: parts.uri.query(),
request_origin: Some(crate::ai_serving::request_origin_from_parts(parts)),
original_request_body_json: Some(body_json),
original_request_body_base64: None,

2
apps/aether-gateway/src/ai_serving/planner/standard/family/payload.rs
View File

@@ -120,6 +120,8 @@ pub(super) async fn maybe_build_local_standard_decision_payload_for_candidate(
provider_request_method: Some(serde_json::Value::Null),
provider_request_headers: Some(&resolved.provider_request_headers),
original_headers: &parts.headers,
request_path: Some(parts.uri.path()),
request_query_string: parts.uri.query(),
request_origin: Some(crate::ai_serving::request_origin_from_parts(parts)),
original_request_body_json: Some(body_json),
original_request_body_base64: None,

2
apps/aether-gateway/src/ai_serving/planner/standard/openai/chat/decision/payload.rs
View File

@@ -107,6 +107,8 @@ pub(crate) async fn maybe_build_local_openai_chat_decision_payload_for_candidate
provider_request_method: Some(serde_json::Value::Null),
provider_request_headers: Some(&resolved.provider_request_headers),
original_headers: &parts.headers,
request_path: Some(parts.uri.path()),
request_query_string: parts.uri.query(),
request_origin: Some(crate::ai_serving::request_origin_from_parts(parts)),
original_request_body_json: Some(body_json),
original_request_body_base64: None,

2
apps/aether-gateway/src/ai_serving/planner/standard/openai/responses/decision/payload.rs
View File

@@ -105,6 +105,8 @@ pub(crate) async fn maybe_build_local_openai_responses_decision_payload_for_cand
provider_request_method: Some(serde_json::Value::Null),
provider_request_headers: Some(&resolved.provider_request_headers),
original_headers: &parts.headers,
request_path: Some(parts.uri.path()),
request_query_string: parts.uri.query(),
request_origin: Some(crate::ai_serving::request_origin_from_parts(parts)),
original_request_body_json: Some(body_json),
original_request_body_base64: None,

14
apps/aether-gateway/src/ai_serving/pure/mod.rs
View File

@@ -70,7 +70,8 @@ pub(crate) use aether_ai_formats::api::{
provider_adaptation_should_unwrap_stream_envelope,
provider_private_response_allows_sync_finalize, request_candidate_api_format_preference,
request_candidate_api_formats, request_conversion_kind,
request_conversion_requires_enable_flag, resolve_claude_stream_spec, resolve_claude_sync_spec,
request_conversion_requires_enable_flag, request_path_implies_stream_request,
resolve_claude_stream_spec, resolve_claude_sync_spec,
resolve_execution_runtime_stream_plan_kind, resolve_execution_runtime_sync_plan_kind,
resolve_finalize_stream_rewrite_mode, resolve_gemini_files_stream_spec,
resolve_gemini_files_sync_spec, resolve_gemini_stream_spec, resolve_gemini_sync_spec,
@@ -79,11 +80,12 @@ pub(crate) use aether_ai_formats::api::{
resolve_local_video_sync_spec, resolve_openai_chat_max_tokens,
resolve_openai_responses_stream_spec, resolve_openai_responses_sync_spec,
resolve_requested_gemini_image_model_for_request,
resolve_requested_openai_image_model_for_request, stream_body_contains_error_event,
supports_stream_execution_decision_kind, supports_sync_execution_decision_kind,
sync_chat_response_conversion_kind, sync_cli_response_conversion_kind,
transform_provider_private_stream_line, value_as_u64, AiControlPlanRequest,
AiSurfaceFinalizeError, AiSurfaceStreamRewriter, CanonicalStreamFrame,
resolve_requested_openai_image_model_for_request, sanitize_request_path,
sanitize_request_path_and_query, sanitize_request_query_string,
stream_body_contains_error_event, supports_stream_execution_decision_kind,
supports_sync_execution_decision_kind, sync_chat_response_conversion_kind,
sync_cli_response_conversion_kind, transform_provider_private_stream_line, value_as_u64,
AiControlPlanRequest, AiSurfaceFinalizeError, AiSurfaceStreamRewriter, CanonicalStreamFrame,
ChatGptWebImageRequestError, ClaudeClientEmitter, ClaudeProviderState,
ExecutionRuntimeAuthContext, FinalizeStreamRewriteMode, FormatContext, GeminiClientEmitter,
GeminiImageRequestForOpenAi, GeminiProviderState, KiroToClaudeCliStreamState,

153
apps/aether-gateway/src/execution_runtime/stream/error.rs
View File

@@ -1,23 +1,17 @@
use std::collections::BTreeMap;
use aether_contracts::{StreamFrame, StreamFramePayload};
use axum::body::Body;
use axum::http::Response;
use axum::http::StatusCode;
use base64::Engine as _;
use futures_util::StreamExt;
use serde_json::json;
use serde_json::{json, Map, Value};
use tokio_util::codec::{FramedRead, LinesCodec};
use tracing::warn;
use crate::api::response::build_client_response_from_parts;
use crate::control::GatewayControlDecision;
use crate::execution_runtime::ndjson::decode_stream_frame_ndjson;
use crate::execution_runtime::submission::{has_nested_error, strip_utf8_bom_and_ws};
use crate::GatewayError;
use crate::{
GEMINI_FILES_DOWNLOAD_PLAN_KIND, MAX_ERROR_BODY_BYTES, MAX_STREAM_PREFETCH_FRAMES,
OPENAI_VIDEO_CONTENT_PLAN_KIND,
};
use crate::{MAX_ERROR_BODY_BYTES, MAX_STREAM_PREFETCH_FRAMES};
#[derive(Debug)]
pub(super) enum StreamPrefetchInspection {
@@ -51,6 +45,88 @@ pub(super) fn decode_stream_error_body(
)
}
fn header_value_case_insensitive<'a>(
headers: &'a BTreeMap<String, String>,
name: &str,
) -> Option<&'a str> {
headers
.iter()
.find(|(key, _)| key.eq_ignore_ascii_case(name))
.map(|(_, value)| value.as_str())
.map(str::trim)
.filter(|value| !value.is_empty())
}
fn remove_header_case_insensitive(headers: &mut BTreeMap<String, String>, name: &str) {
let keys = headers
.keys()
.filter(|key| key.eq_ignore_ascii_case(name))
.cloned()
.collect::<Vec<_>>();
for key in keys {
headers.remove(&key);
}
}
pub(super) fn should_synthesize_non_success_stream_error_body(
status_code: u16,
error_body: &[u8],
) -> bool {
!(200..300).contains(&status_code)
&& ((300..400).contains(&status_code) || error_body.is_empty())
}
pub(super) fn build_synthetic_non_success_stream_error_body(
status_code: u16,
headers: &BTreeMap<String, String>,
) -> Value {
let mut error = Map::from_iter([
(
"type".to_string(),
Value::String("execution_runtime_non_success_status".to_string()),
),
(
"message".to_string(),
Value::String(format!(
"execution runtime stream returned non-success status {status_code}"
)),
),
("code".to_string(), Value::from(status_code)),
("upstream_status".to_string(), Value::from(status_code)),
]);
if let Some(location) = header_value_case_insensitive(headers, "location") {
error.insert("location".to_string(), Value::String(location.to_string()));
}
Value::Object(Map::from_iter([(
"error".to_string(),
Value::Object(error),
)]))
}
pub(super) fn synthetic_error_response_headers(
mut headers: BTreeMap<String, String>,
) -> BTreeMap<String, String> {
remove_header_case_insensitive(&mut headers, "content-encoding");
remove_header_case_insensitive(&mut headers, "content-length");
remove_header_case_insensitive(&mut headers, "content-type");
remove_header_case_insensitive(&mut headers, "location");
headers.insert("content-type".to_string(), "application/json".to_string());
headers
}
fn client_error_status_code_for_upstream_status(status_code: u16) -> u16 {
if (300..400).contains(&status_code) || status_code < 200 {
StatusCode::BAD_GATEWAY.as_u16()
} else {
status_code
}
}
pub(super) fn stream_client_error_status_code_for_upstream_status(status_code: u16) -> u16 {
client_error_status_code_for_upstream_status(status_code)
}
pub(super) fn inspect_prefetched_stream_body(
headers: &BTreeMap<String, String>,
body: &[u8],
@@ -166,62 +242,3 @@ where
}
Ok(None)
}
pub(super) fn build_execution_runtime_error_response(
trace_id: &str,
decision: &GatewayControlDecision,
plan_kind: &str,
status_code: u16,
headers: BTreeMap<String, String>,
error_body: Vec<u8>,
) -> Result<Response<Body>, GatewayError> {
let content_type = headers
.get("content-type")
.map(|value| value.to_ascii_lowercase())
.unwrap_or_default();
if plan_kind == GEMINI_FILES_DOWNLOAD_PLAN_KIND && !content_type.starts_with("application/json")
{
let wrapped = serde_json::to_vec(&json!({
"error": String::from_utf8_lossy(&error_body).to_string(),
}))
.map_err(|err| GatewayError::Internal(err.to_string()))?;
let wrapped_headers =
BTreeMap::from([("content-type".to_string(), "application/json".to_string())]);
return build_client_response_from_parts(
status_code,
&wrapped_headers,
Body::from(wrapped),
trace_id,
Some(decision),
);
}
if plan_kind == OPENAI_VIDEO_CONTENT_PLAN_KIND && !content_type.starts_with("application/json")
{
let wrapped = serde_json::to_vec(&json!({
"error": {
"type": "upstream_error",
"message": "Video not available",
}
}))
.map_err(|err| GatewayError::Internal(err.to_string()))?;
let wrapped_headers =
BTreeMap::from([("content-type".to_string(), "application/json".to_string())]);
return build_client_response_from_parts(
status_code,
&wrapped_headers,
Body::from(wrapped),
trace_id,
Some(decision),
);
}
build_client_response_from_parts(
status_code,
&headers,
Body::from(error_body),
trace_id,
Some(decision),
)
}

449
apps/aether-gateway/src/execution_runtime/stream/execution.rs
View File

@@ -23,15 +23,18 @@ use axum::http::Response;
use base64::Engine as _;
use futures_util::stream::BoxStream;
use futures_util::{StreamExt, TryStreamExt};
use serde_json::Value;
use serde_json::{json, Value};
use tokio::sync::mpsc;
use tokio_util::codec::{FramedRead, LinesCodec};
use tokio_util::io::StreamReader;
use tracing::{debug, info, warn};
use super::error::{
build_execution_runtime_error_response, collect_error_body, decode_stream_error_body,
inspect_prefetched_stream_body, read_next_frame, StreamPrefetchInspection,
build_synthetic_non_success_stream_error_body, collect_error_body, decode_stream_error_body,
inspect_prefetched_stream_body, read_next_frame,
should_synthesize_non_success_stream_error_body,
stream_client_error_status_code_for_upstream_status, synthetic_error_response_headers,
StreamPrefetchInspection,
};
#[path = "execution_failures.rs"]
mod execution_failures;
@@ -76,9 +79,10 @@ use crate::execution_runtime::{MAX_STREAM_PREFETCH_BYTES, MAX_STREAM_PREFETCH_FR
use crate::log_ids::short_request_id;
use crate::orchestration::{
apply_local_execution_effect, build_local_error_flow_metadata, with_error_flow_report_context,
LocalAdaptiveRateLimitEffect, LocalAdaptiveSuccessEffect, LocalAttemptFailureEffect,
LocalExecutionEffect, LocalExecutionEffectContext, LocalHealthFailureEffect,
LocalHealthSuccessEffect, LocalOAuthInvalidationEffect, LocalPoolErrorEffect,
with_upstream_response_report_context, LocalAdaptiveRateLimitEffect,
LocalAdaptiveSuccessEffect, LocalAttemptFailureEffect, LocalExecutionEffect,
LocalExecutionEffectContext, LocalHealthFailureEffect, LocalHealthSuccessEffect,
LocalOAuthInvalidationEffect, LocalPoolErrorEffect,
};
use crate::request_candidate_runtime::{
ensure_execution_request_candidate_slot, record_local_request_candidate_status,
@@ -86,7 +90,9 @@ use crate::request_candidate_runtime::{
};
use crate::usage::submit_stream_report;
use crate::usage::{GatewayStreamReportRequest, GatewaySyncReportRequest};
use crate::{AppState, GatewayError};
use crate::{
AppState, GatewayError, GEMINI_FILES_DOWNLOAD_PLAN_KIND, OPENAI_VIDEO_CONTENT_PLAN_KIND,
};
fn record_sync_terminal_usage(
state: &AppState,
@@ -124,6 +130,52 @@ fn build_stream_sync_payload(
}
}
#[allow(clippy::too_many_arguments)]
fn build_stream_error_sync_payload(
trace_id: &str,
report_kind: String,
report_context: Option<Value>,
upstream_status_code: u16,
provider_headers: BTreeMap<String, String>,
provider_body_json: Option<Value>,
provider_body_base64: Option<String>,
client_headers: BTreeMap<String, String>,
client_body_json: Option<Value>,
telemetry: Option<ExecutionTelemetry>,
) -> GatewaySyncReportRequest {
let client_status_code =
stream_client_error_status_code_for_upstream_status(upstream_status_code);
let mut report_context = report_context;
if client_status_code != upstream_status_code || client_headers != provider_headers {
let mut object = match report_context {
Some(Value::Object(object)) => object,
Some(other) => serde_json::Map::from_iter([("seed".to_string(), other)]),
None => serde_json::Map::new(),
};
object.insert(
"client_response_status_code".to_string(),
Value::from(client_status_code),
);
object.insert(
"client_response_headers".to_string(),
serde_json::to_value(client_headers).unwrap_or(Value::Null),
);
report_context = Some(Value::Object(object));
}
GatewaySyncReportRequest {
trace_id: trace_id.to_string(),
report_kind,
report_context,
status_code: upstream_status_code,
headers: provider_headers,
body_json: provider_body_json,
client_body_json,
body_base64: provider_body_base64,
telemetry,
}
}
fn record_stream_terminal_usage(
state: &AppState,
plan: &ExecutionPlan,
@@ -157,6 +209,55 @@ fn build_stream_body_capture(
(body_base64, body_state)
}
fn wrap_non_json_binary_stream_error_for_client(
plan_kind: &str,
headers: &BTreeMap<String, String>,
error_body: &[u8],
) -> Result<Option<Value>, GatewayError> {
let content_type = headers
.get("content-type")
.map(|value| value.to_ascii_lowercase())
.unwrap_or_default();
if content_type.starts_with("application/json") {
return Ok(None);
}
let body = match plan_kind {
GEMINI_FILES_DOWNLOAD_PLAN_KIND => json!({
"error": String::from_utf8_lossy(error_body).to_string(),
}),
OPENAI_VIDEO_CONTENT_PLAN_KIND => json!({
"error": {
"type": "upstream_error",
"message": "Video not available",
}
}),
_ => return Ok(None),
};
Ok(Some(body))
}
fn with_stream_error_trace_context(
report_context: Option<&Value>,
status_code: u16,
headers: &BTreeMap<String, String>,
response_text: Option<&str>,
local_failover_analysis: crate::orchestration::LocalFailoverAnalysis,
) -> Option<Value> {
let upstream_context = with_upstream_response_report_context(
report_context,
status_code,
Some(headers),
None,
None,
None,
);
with_error_flow_report_context(
upstream_context.as_ref().or(report_context),
build_local_error_flow_metadata(status_code, response_text, local_failover_analysis),
)
}
#[allow(clippy::too_many_arguments)] // stream report payload assembly mirrors runtime state
fn build_stream_usage_payload(
trace_id: String,
@@ -948,11 +1049,29 @@ async fn execute_stream_from_frame_stream(
let stream_error_finalize_kind =
resolve_core_stream_error_finalize_report_kind(plan_kind, status_code);
if status_code >= 400 {
let error_body = collect_error_body(&mut lines).await?;
let (body_json, body_base64) = decode_stream_error_body(&headers, &error_body);
if !(200..300).contains(&status_code) {
let provider_error_body = collect_error_body(&mut lines).await?;
let synthetic_body_json =
should_synthesize_non_success_stream_error_body(status_code, &provider_error_body)
.then(|| build_synthetic_non_success_stream_error_body(status_code, &headers));
let (provider_body_json, provider_body_base64) =
decode_stream_error_body(&headers, &provider_error_body);
let client_status_code = stream_client_error_status_code_for_upstream_status(status_code);
let wrapped_binary_body_json = wrap_non_json_binary_stream_error_for_client(
plan_kind,
&headers,
&provider_error_body,
)?;
let (client_body_json, client_error_body) =
if let Some(body_json) = synthetic_body_json.or(wrapped_binary_body_json) {
let body_bytes = serde_json::to_vec(&body_json)
.map_err(|err| GatewayError::Internal(err.to_string()))?;
(Some(body_json), body_bytes)
} else {
(provider_body_json.clone(), provider_error_body.clone())
};
let error_response_text =
local_failover_response_text(body_json.as_ref(), &error_body, None);
local_failover_response_text(client_body_json.as_ref(), &client_error_body, None);
let failover_analysis = resolve_local_candidate_failover_analysis_stream(
state,
&plan,
@@ -1043,18 +1162,17 @@ async fn execute_stream_from_frame_stream(
);
if matches!(failover_decision, LocalFailoverDecision::RetryNextCandidate) {
let terminal_unix_secs = current_request_candidate_unix_ms();
let error_flow_report_context = with_error_flow_report_context(
let error_trace_report_context = with_stream_error_trace_context(
report_context.as_ref(),
build_local_error_flow_metadata(
status_code,
error_response_text.as_deref(),
failover_analysis,
),
status_code,
&headers,
error_response_text.as_deref(),
failover_analysis,
);
record_local_request_candidate_status(
state,
&plan,
error_flow_report_context
error_trace_report_context
.as_ref()
.or(report_context.as_ref()),
SchedulerRequestCandidateStatusUpdate {
@@ -1094,18 +1212,17 @@ async fn execute_stream_from_frame_stream(
)
{
let terminal_unix_secs = current_request_candidate_unix_ms();
let error_flow_report_context = with_error_flow_report_context(
let error_trace_report_context = with_stream_error_trace_context(
report_context.as_ref(),
build_local_error_flow_metadata(
status_code,
error_response_text.as_deref(),
failover_analysis,
),
status_code,
&headers,
error_response_text.as_deref(),
failover_analysis,
);
record_local_request_candidate_status(
state,
&plan,
error_flow_report_context
error_trace_report_context
.as_ref()
.or(report_context.as_ref()),
SchedulerRequestCandidateStatusUpdate {
@@ -1124,46 +1241,60 @@ async fn execute_stream_from_frame_stream(
return Ok(None);
}
let mut client_headers = headers.clone();
apply_endpoint_response_header_rules(state, &plan, &mut client_headers, body_json.as_ref())
.await?;
let mut client_headers = if (300..400).contains(&status_code) {
let mut headers = synthetic_error_response_headers(headers.clone());
headers.insert(
"x-aether-upstream-status".to_string(),
status_code.to_string(),
);
headers
} else {
headers.clone()
};
apply_endpoint_response_header_rules(
state,
&plan,
&mut client_headers,
client_body_json.as_ref(),
)
.await?;
let payload = build_stream_sync_payload(
let client_response_headers = client_headers.clone();
let error_trace_report_context = with_stream_error_trace_context(
report_context.as_ref(),
status_code,
&headers,
error_response_text.as_deref(),
failover_analysis,
);
let payload = build_stream_error_sync_payload(
trace_id,
stream_error_finalize_kind
.as_deref()
.or(report_kind.as_deref())
.unwrap_or_default()
.to_string(),
report_context,
error_trace_report_context.or(report_context),
status_code,
headers.clone(),
provider_body_json,
provider_body_base64,
client_headers,
body_json,
body_base64,
client_body_json,
None,
);
record_sync_terminal_usage(state, &plan, payload.report_context.as_ref(), &payload);
let terminal_unix_secs = current_request_candidate_unix_ms();
let error_flow_report_context = with_error_flow_report_context(
payload.report_context.as_ref(),
build_local_error_flow_metadata(
status_code,
error_response_text.as_deref(),
failover_analysis,
),
);
record_local_request_candidate_status(
state,
&plan,
error_flow_report_context
.as_ref()
.or(payload.report_context.as_ref()),
payload.report_context.as_ref(),
SchedulerRequestCandidateStatusUpdate {
status: RequestCandidateStatus::Failed,
status_code: Some(status_code),
error_type: Some("execution_runtime_stream_error".to_string()),
error_type: Some("execution_runtime_stream_non_success_status".to_string()),
error_message: Some(format!(
"execution runtime stream returned error status {status_code}"
"execution runtime stream returned non-success status {status_code}"
)),
latency_ms: None,
started_at_unix_ms: Some(candidate_started_unix_secs),
@@ -1182,13 +1313,12 @@ async fn execute_stream_from_frame_stream(
)?));
}
return Ok(Some(attach_control_metadata_headers(
build_execution_runtime_error_response(
build_client_response_from_parts(
client_status_code,
&client_response_headers,
Body::from(client_error_body),
trace_id,
decision,
plan_kind,
status_code,
payload.headers,
error_body,
Some(decision),
)?,
Some(request_id),
candidate_id,
@@ -2343,7 +2473,9 @@ mod tests {
ExecutionPlan, ExecutionStreamTerminalSummary, ExecutionTimeouts, RequestBody,
StandardizedUsage,
};
use aether_data::repository::candidates::InMemoryRequestCandidateRepository;
use aether_data::repository::usage::InMemoryUsageReadRepository;
use aether_data_contracts::repository::candidates::RequestCandidateReadRepository;
use aether_data_contracts::repository::usage::UsageReadRepository;
use aether_usage_runtime::UsageRuntimeConfig;
use async_stream::stream;
@@ -2532,9 +2664,15 @@ mod tests {
});
let usage_repository = Arc::new(InMemoryUsageReadRepository::default());
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
let state = AppState::new()
.expect("app state should build")
.with_usage_data_repository_for_tests(Arc::clone(&usage_repository))
.with_data_state_for_tests(
crate::data::GatewayDataState::with_request_candidate_and_usage_repository_for_tests(
Arc::clone(&request_candidate_repository),
Arc::clone(&usage_repository),
),
)
.with_usage_runtime_for_tests(UsageRuntimeConfig {
enabled: true,
..UsageRuntimeConfig::default()
@@ -2736,6 +2874,213 @@ mod tests {
server.abort();
}
#[tokio::test]
async fn execute_execution_runtime_stream_rewrites_redirect_to_structured_failure() {
let listener = crate::test_support::bind_loopback_listener()
.await
.expect("listener should bind");
let addr = listener.local_addr().expect("local addr should resolve");
let server = tokio::spawn(async move {
let app = Router::new().route(
"/v1/execute/stream",
any(|_request: Request| async move {
let frames = concat!(
"{\"type\":\"headers\",\"payload\":{\"kind\":\"headers\",\"status_code\":302,\"headers\":{\"location\":\"/\",\"content-type\":\"text/html\",\"content-length\":\"0\"}}}\n",
"{\"type\":\"eof\",\"payload\":{\"kind\":\"eof\"}}\n"
);
let mut response = axum::http::Response::new(Body::from(frames));
response.headers_mut().insert(
header::CONTENT_TYPE,
HeaderValue::from_static("application/x-ndjson"),
);
response
}),
);
axum::serve(listener, app)
.await
.expect("server should start");
});
let usage_repository = Arc::new(InMemoryUsageReadRepository::default());
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
let state = AppState::new()
.expect("app state should build")
.with_data_state_for_tests(
crate::data::GatewayDataState::with_request_candidate_and_usage_repository_for_tests(
Arc::clone(&request_candidate_repository),
Arc::clone(&usage_repository),
),
)
.with_usage_runtime_for_tests(UsageRuntimeConfig {
enabled: true,
..UsageRuntimeConfig::default()
})
.with_execution_runtime_override_base_url(format!("http://{addr}"));
let plan = ExecutionPlan {
request_id: "req-remote-runtime-stream-redirect".into(),
candidate_id: Some("cand-remote-runtime-stream-redirect".into()),
provider_name: Some("ChatGPTWeb".into()),
provider_id: "prov-redirect".into(),
endpoint_id: "ep-redirect".into(),
key_id: "key-redirect".into(),
method: "POST".into(),
url: "https://chatgpt.com/backend-api/codex/responses".into(),
headers: BTreeMap::from([
("content-type".into(), "application/json".into()),
("accept".into(), "text/event-stream".into()),
]),
content_type: Some("application/json".into()),
content_encoding: None,
body: RequestBody::from_json(json!({
"model": "gpt-5.4",
"input": "hello",
"stream": true
})),
stream: true,
client_api_format: "gemini:generate_content".into(),
provider_api_format: "openai:responses".into(),
model_name: Some("gemini-3.1-flash-image-preview".into()),
proxy: None,
transport_profile: None,
timeouts: Some(ExecutionTimeouts {
connect_ms: Some(5_000),
total_ms: Some(5_000),
..ExecutionTimeouts::default()
}),
};
let decision = GatewayControlDecision::synthetic(
"/v1beta/models/gemini-3.1-flash-image-preview:streamGenerateContent",
Some("ai_public".to_string()),
Some("gemini".to_string()),
Some("generate_content".to_string()),
Some("gemini:generate_content".to_string()),
)
.with_execution_runtime_candidate(true);
let response = execute_execution_runtime_stream(
&state,
plan,
"trace-remote-runtime-stream-redirect",
&decision,
"gemini_chat_stream",
None,
Some(json!({
"request_id": "req-remote-runtime-stream-redirect",
"candidate_id": "cand-remote-runtime-stream-redirect",
"candidate_index": 0,
"retry_index": 0,
"provider_api_format": "openai:responses",
"client_api_format": "gemini:generate_content",
"needs_conversion": true
})),
)
.await
.expect("execution should succeed")
.expect("execution should return a client response");
assert_eq!(response.status(), axum::http::StatusCode::BAD_GATEWAY);
assert_eq!(
response
.headers()
.get(header::CONTENT_TYPE)
.and_then(|value| value.to_str().ok()),
Some("application/json")
);
assert_eq!(
response
.headers()
.get("x-aether-upstream-status")
.and_then(|value| value.to_str().ok()),
Some("302")
);
assert!(
response.headers().get(header::LOCATION).is_none(),
"redirect location should not be forwarded to AI clients"
);
let body = to_bytes(response.into_body(), usize::MAX)
.await
.expect("response body should read");
let body_json: Value =
serde_json::from_slice(&body).expect("response body should decode as json");
assert_eq!(
body_json["error"]["type"],
json!("execution_runtime_non_success_status")
);
assert_eq!(body_json["error"]["upstream_status"], json!(302));
assert_eq!(body_json["error"]["location"], json!("/"));
assert!(body_json["error"]["message"]
.as_str()
.is_some_and(|value| value.contains("non-success status 302")));
let usage = tokio::time::timeout(Duration::from_secs(2), async {
loop {
if let Some(usage) = usage_repository
.find_by_request_id("req-remote-runtime-stream-redirect")
.await
.expect("usage should read")
.filter(|usage| usage.status == "failed")
{
break usage;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("usage should be written");
assert_eq!(usage.status_code, Some(302));
assert_eq!(usage.error_category.as_deref(), Some("redirect"));
assert!(usage
.error_message
.as_deref()
.is_some_and(|value| value.contains("non-success status 302")));
assert_eq!(
usage
.client_response_headers
.as_ref()
.and_then(|headers| headers.get("x-aether-upstream-status")),
Some(&json!("302"))
);
assert_eq!(
usage
.response_headers
.as_ref()
.and_then(|headers| headers.get("location")),
Some(&json!("/"))
);
assert!(
usage.response_body.is_none(),
"upstream redirect did not include a body"
);
assert_eq!(
usage
.client_response_body
.as_ref()
.and_then(|body| body.pointer("/error/upstream_status")),
Some(&json!(302))
);
let candidates = request_candidate_repository
.list_by_request_id("req-remote-runtime-stream-redirect")
.await
.expect("candidate trace should read");
let candidate_extra = candidates
.first()
.and_then(|candidate| candidate.extra_data.as_ref())
.expect("failed candidate extra_data should exist");
assert_eq!(
candidate_extra["upstream_response"]["status_code"],
json!(302)
);
assert_eq!(
candidate_extra["upstream_response"]["headers"]["location"],
json!("/")
);
assert!(candidate_extra["upstream_response"].get("body").is_none());
assert!(candidate_extra.get("client_response").is_none());
server.abort();
}
#[tokio::test]
async fn execute_execution_runtime_stream_bridges_openai_image_sync_json_from_remote_runtime_to_image_sse(
) {

15
apps/aether-gateway/src/execution_runtime/stream/execution_failures.rs
View File

@@ -20,9 +20,9 @@ use crate::execution_runtime::submission::{
use crate::log_ids::short_request_id;
use crate::orchestration::{
apply_local_execution_effect, resolve_local_failover_analysis_for_attempt,
LocalAdaptiveRateLimitEffect, LocalAttemptFailureEffect, LocalExecutionEffect,
LocalExecutionEffectContext, LocalHealthFailureEffect, LocalOAuthInvalidationEffect,
LocalPoolErrorEffect,
with_upstream_response_report_context, LocalAdaptiveRateLimitEffect, LocalAttemptFailureEffect,
LocalExecutionEffect, LocalExecutionEffectContext, LocalHealthFailureEffect,
LocalOAuthInvalidationEffect, LocalPoolErrorEffect,
};
use crate::request_candidate_runtime::record_report_request_candidate_status;
use crate::usage::submit_sync_report;
@@ -135,6 +135,15 @@ fn build_stream_failure_sync_payload(
failure: StreamFailureReport,
) -> GatewaySyncReportRequest {
let status_code = failure.status_code;
let report_context = with_upstream_response_report_context(
report_context.as_ref(),
status_code,
Some(&headers),
None,
None,
None,
)
.or(report_context);
headers.remove("content-encoding");
headers.remove("content-length");
headers.insert("content-type".to_string(), "application/json".to_string());

65
apps/aether-gateway/src/execution_runtime/sync/execution.rs
View File

@@ -39,9 +39,10 @@ use crate::execution_runtime::{
use crate::log_ids::short_request_id;
use crate::orchestration::{
apply_local_execution_effect, build_local_error_flow_metadata, with_error_flow_report_context,
LocalAdaptiveRateLimitEffect, LocalAdaptiveSuccessEffect, LocalAttemptFailureEffect,
LocalExecutionEffect, LocalExecutionEffectContext, LocalHealthFailureEffect,
LocalHealthSuccessEffect, LocalOAuthInvalidationEffect, LocalPoolErrorEffect,
with_upstream_response_report_context, LocalAdaptiveRateLimitEffect,
LocalAdaptiveSuccessEffect, LocalAttemptFailureEffect, LocalExecutionEffect,
LocalExecutionEffectContext, LocalHealthFailureEffect, LocalHealthSuccessEffect,
LocalOAuthInvalidationEffect, LocalPoolErrorEffect,
};
use crate::request_candidate_runtime::{
ensure_execution_request_candidate_slot, record_local_request_candidate_status,
@@ -81,6 +82,27 @@ fn record_sync_terminal_usage(
.record_sync_terminal(state.data.as_ref(), context_seed, payload_seed);
}
fn with_sync_error_trace_context(
report_context: Option<&serde_json::Value>,
status_code: u16,
headers: &BTreeMap<String, String>,
response_text: Option<&str>,
local_failover_analysis: crate::orchestration::LocalFailoverAnalysis,
) -> Option<serde_json::Value> {
let upstream_context = with_upstream_response_report_context(
report_context,
status_code,
Some(headers),
None,
None,
None,
);
with_error_flow_report_context(
upstream_context.as_ref().or(report_context),
build_local_error_flow_metadata(status_code, response_text, local_failover_analysis),
)
}
fn build_sync_report_payload(
trace_id: &str,
report_kind: String,
@@ -572,18 +594,17 @@ pub(crate) async fn execute_execution_runtime_sync(
LocalFailoverDecision::RetryNextCandidate
) {
let terminal_unix_secs = current_request_candidate_unix_ms();
let error_flow_report_context = with_error_flow_report_context(
let error_trace_report_context = with_sync_error_trace_context(
report_context.as_ref(),
build_local_error_flow_metadata(
result.status_code,
local_failover_response_text.as_deref(),
local_failover_analysis,
),
result.status_code,
&headers,
local_failover_response_text.as_deref(),
local_failover_analysis,
);
record_local_request_candidate_status(
state,
&plan,
error_flow_report_context
error_trace_report_context
.as_ref()
.or(report_context.as_ref()),
SchedulerRequestCandidateStatusUpdate {
@@ -649,18 +670,17 @@ pub(crate) async fn execute_execution_runtime_sync(
mapped_error_finalize_kind.is_some(),
) {
let terminal_unix_secs = current_request_candidate_unix_ms();
let error_flow_report_context = with_error_flow_report_context(
let error_trace_report_context = with_sync_error_trace_context(
report_context.as_ref(),
build_local_error_flow_metadata(
result.status_code,
local_failover_response_text.as_deref(),
local_failover_analysis,
),
result.status_code,
&headers,
local_failover_response_text.as_deref(),
local_failover_analysis,
);
record_local_request_candidate_status(
state,
&plan,
error_flow_report_context
error_trace_report_context
.as_ref()
.or(report_context.as_ref()),
SchedulerRequestCandidateStatusUpdate {
@@ -680,13 +700,12 @@ pub(crate) async fn execute_execution_runtime_sync(
let terminal_unix_secs = current_request_candidate_unix_ms();
let error_flow_report_context = (result.status_code >= 400)
.then(|| {
with_error_flow_report_context(
with_sync_error_trace_context(
report_context.as_ref(),
build_local_error_flow_metadata(
result.status_code,
local_failover_response_text.as_deref(),
local_failover_analysis,
),
result.status_code,
&headers,
local_failover_response_text.as_deref(),
local_failover_analysis,
)
})
.flatten();

158
apps/aether-gateway/src/handlers/admin/observability/monitoring/tests/trace.rs
View File

@@ -356,6 +356,164 @@ async fn admin_monitoring_trace_request_enriches_proxy_timing_from_usage_audit()
payload["candidates"][0]["extra_data"]["proxy"]["timing"]["response_wait_ms"],
json!(475)
);
assert!(payload["candidates"][0]["extra_data"]
.get("upstream_response")
.is_none());
}
#[tokio::test]
async fn admin_monitoring_trace_request_exposes_request_path_from_usage_audit() {
let mut candidate = sample_candidate(
"cand-used",
"request-1",
0,
RequestCandidateStatus::Failed,
Some(101),
Some(33),
Some(403),
);
candidate.extra_data = Some(json!({
"client_api_format": "gemini:generate_content"
}));
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![candidate]));
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider()],
vec![sample_endpoint()],
vec![sample_key()],
));
let mut usage = sample_usage(
"request-1",
"provider-1",
"OpenAI",
40,
0.02,
"failed",
Some(403),
100,
);
usage.candidate_id = Some("cand-used".to_string());
usage.request_metadata = Some(json!({
"request_path": "/v1beta/models/gemini-2.5-pro:generateContent",
"request_query_string": "alt=sse"
}));
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(vec![usage]));
let data_state =
crate::data::GatewayDataState::with_request_candidate_and_usage_repository_for_tests(
request_candidates,
usage_repository,
)
.with_provider_catalog_reader(provider_catalog);
let state = AppState::new()
.expect("state should build")
.with_data_state_for_tests(data_state);
let context = request_context(http::Method::GET, "/api/admin/monitoring/trace/request-1");
let response = local_monitoring_response(&state, &context)
.await
.expect("handler should not error")
.expect("route should be handled locally");
assert_eq!(response.status(), http::StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX)
.await
.expect("body should read");
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
assert_eq!(
payload["request_path"],
json!("/v1beta/models/gemini-2.5-pro:generateContent")
);
assert_eq!(payload["request_query_string"], json!("alt=sse"));
assert_eq!(
payload["request_path_and_query"],
json!("/v1beta/models/gemini-2.5-pro:generateContent?alt=sse")
);
assert_eq!(
payload["candidates"][0]["extra_data"]["request_path_and_query"],
json!("/v1beta/models/gemini-2.5-pro:generateContent?alt=sse")
);
}
#[tokio::test]
async fn admin_monitoring_trace_request_exposes_failed_candidate_upstream_response_boundary() {
let candidate = sample_candidate(
"cand-used",
"request-1",
0,
RequestCandidateStatus::Failed,
Some(101),
Some(33),
Some(302),
);
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![candidate]));
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider()],
vec![sample_endpoint()],
vec![sample_key()],
));
let mut usage = sample_usage(
"request-1",
"provider-1",
"OpenAI",
40,
0.02,
"failed",
Some(302),
100,
);
usage.candidate_id = Some("cand-used".to_string());
usage.response_headers = Some(json!({
"location": "/",
"content-type": "text/html"
}));
usage.client_response_headers = Some(json!({
"content-type": "application/json",
"x-aether-upstream-status": "302"
}));
usage.client_response_body = Some(json!({
"error": {
"type": "execution_runtime_non_success_status",
"message": "execution runtime stream returned non-success status 302",
"upstream_status": 302,
"location": "/"
}
}));
usage.request_metadata = Some(json!({
"client_response_status_code": 502
}));
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(vec![usage]));
let data_state =
crate::data::GatewayDataState::with_request_candidate_and_usage_repository_for_tests(
request_candidates,
usage_repository,
)
.with_provider_catalog_reader(provider_catalog);
let state = AppState::new()
.expect("state should build")
.with_data_state_for_tests(data_state);
let context = request_context(http::Method::GET, "/api/admin/monitoring/trace/request-1");
let response = local_monitoring_response(&state, &context)
.await
.expect("handler should not error")
.expect("route should be handled locally");
assert_eq!(response.status(), http::StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX)
.await
.expect("body should read");
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
let extra = &payload["candidates"][0]["extra_data"];
assert_eq!(extra["upstream_response"]["status_code"], json!(302));
assert_eq!(
extra["upstream_response"]["headers"]["location"],
json!("/")
);
assert!(extra["upstream_response"]["body"].is_null());
assert!(extra.get("client_response").is_none());
assert!(extra.get("provider_response").is_none());
}
#[tokio::test]

65
apps/aether-gateway/src/orchestration/mod.rs
View File

@@ -1,5 +1,5 @@
use aether_contracts::ExecutionPlan;
use serde_json::{json, Value};
use serde_json::{json, Map, Value};
use crate::AppState;
@@ -120,3 +120,66 @@ pub(crate) fn with_error_flow_report_context(
object.insert("error_flow".to_string(), error_flow);
Some(Value::Object(object))
}
pub(crate) fn with_upstream_response_report_context(
report_context: Option<&Value>,
status_code: u16,
headers: Option<&std::collections::BTreeMap<String, String>>,
body: Option<&Value>,
body_ref: Option<&str>,
body_state: Option<&str>,
) -> Option<Value> {
let mut object = report_context?.as_object()?.clone();
let mut upstream_response = serde_json::Map::new();
upstream_response.insert("status_code".to_string(), json!(status_code));
if let Some(headers) = headers {
upstream_response.insert("headers".to_string(), trace_headers_to_json(headers));
}
if let Some(body) = body {
upstream_response.insert("body".to_string(), body.clone());
}
if let Some(body_ref) = body_ref {
upstream_response.insert("body_ref".to_string(), json!(body_ref));
}
if let Some(body_state) = body_state {
upstream_response.insert("body_state".to_string(), json!(body_state));
}
object.insert(
"upstream_response".to_string(),
Value::Object(upstream_response),
);
Some(Value::Object(object))
}
fn trace_headers_to_json(headers: &std::collections::BTreeMap<String, String>) -> Value {
Value::Object(Map::from_iter(headers.iter().map(|(key, value)| {
(
key.clone(),
Value::String(mask_trace_header_value(key, value)),
)
})))
}
fn mask_trace_header_value(name: &str, value: &str) -> String {
if !trace_header_is_sensitive(name) {
return value.to_string();
}
if value.len() <= 8 {
return "****".to_string();
}
format!("{}****{}", &value[..4], &value[value.len() - 4..])
}
fn trace_header_is_sensitive(name: &str) -> bool {
[
"authorization",
"x-api-key",
"api-key",
"x-goog-api-key",
"cookie",
"set-cookie",
"proxy-authorization",
]
.iter()
.any(|candidate| name.trim().eq_ignore_ascii_case(candidate))
}

422
apps/aether-gateway/src/tests/ai_execute/stream/decision.rs
View File

@@ -928,6 +928,428 @@ async fn gateway_executes_openai_chat_stream_via_local_openai_responses_cross_fo
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_executes_openai_chat_stream_via_local_cross_format_gemini_candidate_with_stream_path_rewrite(
) {
#[derive(Debug, Clone)]
struct SeenExecutionRuntimeStreamRequest {
trace_id: String,
url: String,
provider_model: String,
auth_header_value: String,
accept: String,
endpoint_tag: String,
}
fn hash_api_key(value: &str) -> String {
let mut hasher = Sha256::new();
hasher.update(value.as_bytes());
format!("{:x}", hasher.finalize())
}
fn sample_auth_snapshot(api_key_id: &str, user_id: &str) -> StoredAuthApiKeySnapshot {
StoredAuthApiKeySnapshot::new(
user_id.to_string(),
"alice".to_string(),
Some("alice@example.com".to_string()),
"user".to_string(),
"local".to_string(),
true,
false,
Some(serde_json::json!(["openai", "gemini"])),
Some(serde_json::json!(["openai:chat"])),
Some(serde_json::json!(["gpt-5"])),
api_key_id.to_string(),
Some("default".to_string()),
true,
false,
false,
Some(60),
Some(5),
Some(4_102_444_800),
Some(serde_json::json!(["openai", "gemini"])),
Some(serde_json::json!(["openai:chat"])),
Some(serde_json::json!(["gpt-5"])),
)
.expect("auth snapshot should build")
}
fn sample_candidate_row() -> StoredMinimalCandidateSelectionRow {
StoredMinimalCandidateSelectionRow {
provider_id: "provider-openai-chat-gemini-stream-local-1".to_string(),
provider_name: "gemini".to_string(),
provider_type: "custom".to_string(),
provider_priority: 10,
provider_is_active: true,
endpoint_id: "endpoint-openai-chat-gemini-stream-local-1".to_string(),
endpoint_api_format: "gemini:generate_content".to_string(),
endpoint_api_family: Some("gemini".to_string()),
endpoint_kind: Some("chat".to_string()),
endpoint_is_active: true,
key_id: "key-openai-chat-gemini-stream-local-1".to_string(),
key_name: "prod".to_string(),
key_auth_type: "api_key".to_string(),
key_is_active: true,
key_api_formats: Some(vec!["gemini:generate_content".to_string()]),
key_allowed_models: None,
key_capabilities: None,
key_internal_priority: 5,
key_global_priority_by_format: Some(serde_json::json!({"gemini:generate_content": 1})),
model_id: "model-openai-chat-gemini-stream-local-1".to_string(),
global_model_id: "global-model-openai-chat-gemini-stream-local-1".to_string(),
global_model_name: "gpt-5".to_string(),
global_model_mappings: None,
global_model_supports_streaming: Some(true),
model_provider_model_name: "gemini-2.5-pro-upstream".to_string(),
model_provider_model_mappings: Some(vec![StoredProviderModelMapping {
name: "gemini-2.5-pro-upstream".to_string(),
priority: 1,
api_formats: Some(vec!["gemini:generate_content".to_string()]),
endpoint_ids: None,
}]),
model_supports_streaming: Some(true),
model_is_active: true,
model_is_available: true,
}
}
fn sample_provider_catalog_provider() -> StoredProviderCatalogProvider {
StoredProviderCatalogProvider::new(
"provider-openai-chat-gemini-stream-local-1".to_string(),
"gemini".to_string(),
Some("https://example.com".to_string()),
"custom".to_string(),
)
.expect("provider should build")
.with_transport_fields(
true,
false,
true,
None,
Some(2),
None,
Some(20.0),
None,
None,
)
}
fn sample_provider_catalog_endpoint() -> StoredProviderCatalogEndpoint {
StoredProviderCatalogEndpoint::new(
"endpoint-openai-chat-gemini-stream-local-1".to_string(),
"provider-openai-chat-gemini-stream-local-1".to_string(),
"gemini:generate_content".to_string(),
Some("gemini".to_string()),
Some("chat".to_string()),
true,
)
.expect("endpoint should build")
.with_transport_fields(
"https://generativelanguage.googleapis.com".to_string(),
Some(serde_json::json!([
{"action":"set","key":"x-endpoint-tag","value":"openai-chat-gemini-cross-format-stream"}
])),
None,
Some(2),
Some("/custom/v1beta/models/gemini-2.5-pro-upstream:generateContent".to_string()),
None,
None,
None,
)
.expect("endpoint transport should build")
}
fn sample_provider_catalog_key() -> StoredProviderCatalogKey {
StoredProviderCatalogKey::new(
"key-openai-chat-gemini-stream-local-1".to_string(),
"provider-openai-chat-gemini-stream-local-1".to_string(),
"prod".to_string(),
"api_key".to_string(),
None,
true,
)
.expect("key should build")
.with_transport_fields(
Some(serde_json::json!(["gemini:generate_content"])),
encrypt_python_fernet_plaintext(
DEVELOPMENT_ENCRYPTION_KEY,
"sk-upstream-openai-chat-gemini-stream",
)
.expect("api key should encrypt"),
None,
None,
Some(serde_json::json!({"gemini:generate_content": 1})),
None,
None,
None,
None,
)
.expect("key transport should build")
}
let seen_execution_runtime = Arc::new(Mutex::new(None::<SeenExecutionRuntimeStreamRequest>));
let seen_execution_runtime_clone = Arc::clone(&seen_execution_runtime);
let seen_report = Arc::new(Mutex::new(false));
let seen_report_clone = Arc::clone(&seen_report);
let decision_hits = Arc::new(Mutex::new(0usize));
let decision_hits_clone = Arc::clone(&decision_hits);
let plan_hits = Arc::new(Mutex::new(0usize));
let plan_hits_clone = Arc::clone(&plan_hits);
let public_hits = Arc::new(Mutex::new(0usize));
let public_hits_clone = Arc::clone(&public_hits);
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
let upstream = Router::new()
.route(
"/api/internal/gateway/resolve",
any(|_request: Request| async move {
Json(json!({
"action": "proxy_public",
"route_class": "ai_public",
"route_family": "openai",
"route_kind": "chat",
"auth_endpoint_signature": "openai:chat",
"execution_runtime_candidate": true,
"auth_context": {
"user_id": "user-openai-chat-gemini-stream-local-1",
"api_key_id": "api-key-openai-chat-gemini-stream-local-1",
"access_allowed": true
},
"public_path": "/v1/chat/completions"
}))
}),
)
.route(
"/api/internal/gateway/decision-stream",
any(move |_request: Request| {
let decision_hits_inner = Arc::clone(&decision_hits_clone);
async move {
*decision_hits_inner.lock().expect("mutex should lock") += 1;
Json(json!({"action": "proxy_public"}))
}
}),
)
.route(
"/api/internal/gateway/plan-stream",
any(move |_request: Request| {
let plan_hits_inner = Arc::clone(&plan_hits_clone);
async move {
*plan_hits_inner.lock().expect("mutex should lock") += 1;
Json(json!({"action": "proxy_public"}))
}
}),
)
.route(
"/api/internal/gateway/report-stream",
any(move |request: Request| {
let seen_report_inner = Arc::clone(&seen_report_clone);
async move {
let (_parts, body) = request.into_parts();
let _raw_body = to_bytes(body, usize::MAX).await.expect("body should read");
*seen_report_inner.lock().expect("mutex should lock") = true;
Json(json!({"ok": true}))
}
}),
)
.route(
"/v1/chat/completions",
any(move |_request: Request| {
let public_hits_inner = Arc::clone(&public_hits_clone);
async move {
*public_hits_inner.lock().expect("mutex should lock") += 1;
(StatusCode::IM_A_TEAPOT, Body::from("public-route-hit"))
}
}),
);
let execution_runtime = Router::new().route(
"/v1/execute/stream",
any(move |request: Request| {
let seen_execution_runtime_inner = Arc::clone(&seen_execution_runtime_clone);
async move {
let (parts, body) = request.into_parts();
let raw_body = to_bytes(body, usize::MAX).await.expect("body should read");
let payload: serde_json::Value =
serde_json::from_slice(&raw_body).expect("execution runtime payload should parse");
*seen_execution_runtime_inner
.lock()
.expect("mutex should lock") = Some(SeenExecutionRuntimeStreamRequest {
trace_id: parts
.headers
.get(TRACE_ID_HEADER)
.and_then(|value| value.to_str().ok())
.unwrap_or_default()
.to_string(),
url: payload
.get("url")
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
provider_model: payload
.get("body")
.and_then(|value| value.get("json_body"))
.and_then(|value| value.get("model"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
auth_header_value: payload
.get("headers")
.and_then(|value| value.get("x-goog-api-key"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
accept: payload
.get("headers")
.and_then(|value| value.get("accept"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
endpoint_tag: payload
.get("headers")
.and_then(|value| value.get("x-endpoint-tag"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
});
let frames = concat!(
"{\"type\":\"headers\",\"payload\":{\"kind\":\"headers\",\"status_code\":200,\"headers\":{\"content-type\":\"text/event-stream\"}}}\n",
"{\"type\":\"data\",\"payload\":{\"kind\":\"data\",\"text\":\"data: {\\\"responseId\\\":\\\"resp_openai_chat_gemini_stream_123\\\",\\\"candidates\\\":[{\\\"content\\\":{\\\"parts\\\":[{\\\"text\\\":\\\"Hello \\\"}],\\\"role\\\":\\\"model\\\"},\\\"index\\\":0}],\\\"modelVersion\\\":\\\"gemini-2.5-pro-upstream\\\"}\\n\\n\"}}\n",
"{\"type\":\"data\",\"payload\":{\"kind\":\"data\",\"text\":\"data: {\\\"responseId\\\":\\\"resp_openai_chat_gemini_stream_123\\\",\\\"candidates\\\":[{\\\"content\\\":{\\\"parts\\\":[{\\\"text\\\":\\\"Hello Gemini stream\\\"}],\\\"role\\\":\\\"model\\\"},\\\"finishReason\\\":\\\"STOP\\\",\\\"index\\\":0}],\\\"modelVersion\\\":\\\"gemini-2.5-pro-upstream\\\",\\\"usageMetadata\\\":{\\\"promptTokenCount\\\":1,\\\"candidatesTokenCount\\\":2,\\\"totalTokenCount\\\":3}}\\n\\n\"}}\n",
"{\"type\":\"telemetry\",\"payload\":{\"kind\":\"telemetry\",\"telemetry\":{\"elapsed_ms\":31,\"ttfb_ms\":11,\"upstream_bytes\":37}}}\n",
"{\"type\":\"eof\",\"payload\":{\"kind\":\"eof\"}}\n"
);
let mut response = Response::builder()
.status(StatusCode::OK)
.body(Body::from(frames))
.expect("response should build");
response.headers_mut().insert(
http::header::CONTENT_TYPE,
HeaderValue::from_static("application/x-ndjson"),
);
response
}
}),
);
let auth_repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
Some(hash_api_key("sk-client-openai-chat-gemini-stream")),
sample_auth_snapshot(
"api-key-openai-chat-gemini-stream-local-1",
"user-openai-chat-gemini-stream-local-1",
),
)]));
let candidate_selection_repository =
Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(vec![
sample_candidate_row(),
]));
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider_catalog_provider()],
vec![sample_provider_catalog_endpoint()],
vec![sample_provider_catalog_key()],
));
let (upstream_url, upstream_handle) = start_server(upstream).await;
let (execution_runtime_url, execution_runtime_handle) = start_server(execution_runtime).await;
let gateway_state =
build_state_with_execution_runtime_override(execution_runtime_url.clone())
.with_data_state_for_tests(
crate::data::GatewayDataState::with_auth_candidate_selection_provider_catalog_and_request_candidate_repository_for_tests(
auth_repository,
candidate_selection_repository,
provider_catalog_repository,
Arc::clone(&request_candidate_repository),
DEVELOPMENT_ENCRYPTION_KEY,
)
.with_system_config_values_for_tests(vec![(
"provider_priority_mode".to_string(),
json!("global_key"),
)]),
);
let gateway = build_router_with_state(gateway_state);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.post(format!("{gateway_url}/v1/chat/completions"))
.header(http::header::CONTENT_TYPE, "application/json")
.header(
http::header::AUTHORIZATION,
"Bearer sk-client-openai-chat-gemini-stream",
)
.header(TRACE_ID_HEADER, "trace-openai-chat-gemini-stream-123")
.body(
"{\"model\":\"gpt-5\",\"messages\":[{\"role\":\"system\",\"content\":\"You are terse.\"},{\"role\":\"user\",\"content\":\"Say hello\"}],\"stream\":true}",
)
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response
.headers()
.get(EXECUTION_PATH_HEADER)
.and_then(|value| value.to_str().ok()),
Some(EXECUTION_PATH_EXECUTION_RUNTIME_STREAM)
);
let response_text = response.text().await.expect("body should read");
assert!(response_text.contains("\"object\":\"chat.completion.chunk\""));
assert!(response_text.contains("data: [DONE]"));
let seen_execution_runtime_request = seen_execution_runtime
.lock()
.expect("mutex should lock")
.clone()
.expect("execution runtime stream should be captured");
assert_eq!(
seen_execution_runtime_request.trace_id,
"trace-openai-chat-gemini-stream-123"
);
assert_eq!(
seen_execution_runtime_request.url,
"https://generativelanguage.googleapis.com/custom/v1beta/models/gemini-2.5-pro-upstream:streamGenerateContent?alt=sse"
);
assert_eq!(
seen_execution_runtime_request.provider_model,
"gemini-2.5-pro-upstream"
);
assert_eq!(
seen_execution_runtime_request.auth_header_value,
"sk-upstream-openai-chat-gemini-stream"
);
assert_eq!(seen_execution_runtime_request.accept, "text/event-stream");
assert_eq!(
seen_execution_runtime_request.endpoint_tag,
"openai-chat-gemini-cross-format-stream"
);
let stored_candidates = request_candidate_repository
.list_by_request_id("trace-openai-chat-gemini-stream-123")
.await
.expect("request candidate trace should read");
assert_eq!(stored_candidates.len(), 1);
assert_eq!(stored_candidates[0].status, RequestCandidateStatus::Success);
let extra_data = stored_candidates[0]
.extra_data
.as_ref()
.expect("request candidate extra_data should exist");
assert_eq!(extra_data["client_api_format"], "openai:chat");
assert_eq!(extra_data["provider_api_format"], "gemini:generate_content");
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
assert!(
!*seen_report.lock().expect("mutex should lock"),
"report-stream should stay local when request candidate persistence is available"
);
assert_eq!(*decision_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*plan_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
execution_runtime_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_executes_openai_chat_stream_with_custom_path_via_local_decision_gate_with_local_stream_decision(
) {

120
crates/aether-admin/src/observability/monitoring.rs
View File

@@ -301,6 +301,9 @@ pub fn build_admin_monitoring_trace_request_payload_response_with_key_accounts(
.collect::<Vec<_>>();
Json(json!({
"request_id": trace.request_id,
"request_path": admin_monitoring_trace_request_path(usage),
"request_query_string": admin_monitoring_trace_request_query_string(usage),
"request_path_and_query": admin_monitoring_trace_request_path_and_query(usage),
"total_candidates": trace.total_candidates,
"final_status": trace.final_status,
"total_latency_ms": trace.total_latency_ms,
@@ -470,6 +473,33 @@ fn build_admin_monitoring_trace_candidate_extra_data(
.entry("first_byte_time_ms".to_string())
.or_insert_with(|| json!(first_byte_time_ms));
}
if let Some(request_path) = admin_monitoring_usage_request_path(usage) {
extra_object
.entry("request_path".to_string())
.or_insert_with(|| json!(request_path));
}
if let Some(request_query_string) = admin_monitoring_usage_request_query_string(usage) {
extra_object
.entry("request_query_string".to_string())
.or_insert_with(|| json!(request_query_string));
}
if let Some(request_path_and_query) = admin_monitoring_usage_request_path_and_query(usage) {
extra_object
.entry("request_path_and_query".to_string())
.or_insert_with(|| json!(request_path_and_query));
}
if admin_monitoring_usage_is_error_node(usage) {
if let Some(response) = admin_monitoring_trace_response_data(
"upstream_response",
usage.status_code,
usage.response_headers.as_ref(),
usage.response_body.as_ref(),
usage.response_body_ref.as_deref(),
usage.response_body_state,
) {
extra_object.insert("upstream_response".to_string(), response);
}
}
if let Some(proxy_value) = extra_object.get_mut("proxy") {
if let Some(proxy_object) = proxy_value.as_object_mut() {
@@ -497,6 +527,96 @@ fn build_admin_monitoring_trace_candidate_extra_data(
}
}
fn admin_monitoring_trace_response_data(
source: &str,
status_code: Option<u16>,
headers: Option<&Value>,
body: Option<&Value>,
body_ref: Option<&str>,
body_state: Option<aether_data_contracts::repository::usage::UsageBodyCaptureState>,
) -> Option<Value> {
if status_code.is_none()
&& headers.is_none()
&& body.is_none()
&& body_ref.is_none()
&& body_state.is_none()
{
return None;
}
Some(json!({
"source": source,
"status_code": status_code,
"headers": headers.cloned().unwrap_or(Value::Null),
"body": body.cloned().unwrap_or(Value::Null),
"body_ref": body_ref,
"body_state": body_state.map(|state| state.as_str()),
}))
}
fn admin_monitoring_usage_is_error_node(usage: &StoredRequestUsageAudit) -> bool {
!usage.status.eq_ignore_ascii_case("completed")
|| usage
.status_code
.is_some_and(|status| !(200..300).contains(&status))
}
fn admin_monitoring_trace_request_path(usage: Option<&StoredRequestUsageAudit>) -> Option<String> {
usage.and_then(admin_monitoring_usage_request_path)
}
fn admin_monitoring_trace_request_query_string(
usage: Option<&StoredRequestUsageAudit>,
) -> Option<String> {
usage.and_then(admin_monitoring_usage_request_query_string)
}
fn admin_monitoring_trace_request_path_and_query(
usage: Option<&StoredRequestUsageAudit>,
) -> Option<String> {
usage.and_then(admin_monitoring_usage_request_path_and_query)
}
fn admin_monitoring_usage_request_path(usage: &StoredRequestUsageAudit) -> Option<String> {
admin_monitoring_usage_metadata_string(usage, "request_path")
}
fn admin_monitoring_usage_request_query_string(usage: &StoredRequestUsageAudit) -> Option<String> {
admin_monitoring_usage_metadata_string(usage, "request_query_string")
.map(|value| value.trim_start_matches('?').to_string())
.filter(|value| !value.is_empty())
}
fn admin_monitoring_usage_request_path_and_query(
usage: &StoredRequestUsageAudit,
) -> Option<String> {
admin_monitoring_usage_metadata_string(usage, "request_path_and_query").or_else(|| {
let path = admin_monitoring_usage_metadata_string(usage, "request_path")?;
let query = admin_monitoring_usage_metadata_string(usage, "request_query_string")
.map(|value| value.trim_start_matches('?').to_string())
.filter(|value| !value.is_empty());
Some(match query {
Some(query) if !path.contains('?') => format!("{path}?{query}"),
_ => path,
})
})
}
fn admin_monitoring_usage_metadata_string(
usage: &StoredRequestUsageAudit,
key: &str,
) -> Option<String> {
usage
.request_metadata
.as_ref()
.and_then(Value::as_object)
.and_then(|metadata| metadata.get(key))
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
}
fn json_string_field(object: &serde_json::Map<String, Value>, key: &str) -> Option<String> {
object
.get(key)

96
crates/aether-admin/src/observability/usage.rs
View File

@@ -1,4 +1,5 @@
use crate::observability::stats::{aggregate_usage_stats, parse_bounded_u32, round_to};
use aether_ai_formats::api::request_path_implies_stream_request;
use aether_billing::{
normalize_input_tokens_for_billing, normalize_total_input_context_for_cache_hit_rate,
};
@@ -227,7 +228,9 @@ pub fn admin_usage_matches_api_format(
}
pub fn admin_usage_is_failed(item: &StoredRequestUsageAudit) -> bool {
let has_failure_signal = item.status_code.is_some_and(|value| value >= 400)
let has_failure_signal = item
.status_code
.is_some_and(|value| !(200..300).contains(&value))
|| item
.error_message
.as_deref()
@@ -256,7 +259,9 @@ pub fn admin_usage_matches_status(item: &StoredRequestUsageAudit, status: Option
"stream" => item.is_stream,
"standard" => !item.is_stream,
"error" => {
item.status_code.is_some_and(|value| value >= 400) || item.error_message.is_some()
item.status_code
.is_some_and(|value| !(200..300).contains(&value))
|| item.error_message.is_some()
}
"pending" | "streaming" | "completed" | "cancelled" => item.status == status,
"failed" => admin_usage_is_failed(item),
@@ -955,12 +960,26 @@ fn admin_usage_infer_upstream_stream_from_captured_bodies(
}
}
fn admin_usage_request_path_implies_client_stream(item: &StoredRequestUsageAudit) -> bool {
let Some(metadata) = item.request_metadata.as_ref().and_then(Value::as_object) else {
return false;
};
["request_path", "request_path_and_query"]
.into_iter()
.filter_map(|field| metadata.get(field).and_then(Value::as_str))
.any(request_path_implies_stream_request)
}
pub fn admin_usage_client_is_stream(item: &StoredRequestUsageAudit) -> bool {
item.request_metadata
.as_ref()
.and_then(Value::as_object)
.and_then(|metadata| metadata.get("client_requested_stream"))
.and_then(Value::as_bool)
admin_usage_request_path_implies_client_stream(item)
.then_some(true)
.or_else(|| {
item.request_metadata
.as_ref()
.and_then(Value::as_object)
.and_then(|metadata| metadata.get("client_requested_stream"))
.and_then(Value::as_bool)
})
.or_else(|| admin_usage_request_body_stream_flag(item))
.or_else(|| admin_usage_headers_stream_flag(item.client_response_headers.as_ref()))
.or_else(|| admin_usage_request_body_implies_default_non_stream(item).then_some(false))
@@ -1665,7 +1684,9 @@ pub fn admin_usage_is_success(item: &StoredRequestUsageAudit) -> bool {
matches!(
item.status.as_str(),
"completed" | "success" | "ok" | "billed" | "settled"
) && item.status_code.is_none_or(|code| code < 400)
) && item
.status_code
.is_none_or(|code| (200..300).contains(&code))
}
pub fn admin_usage_matches_optional_id(value: Option<&str>, expected: Option<&str>) -> bool {
@@ -2258,10 +2279,10 @@ mod tests {
use super::{
admin_usage_active_request_json, admin_usage_client_is_stream, admin_usage_has_body_value,
admin_usage_has_fallback, admin_usage_is_failed, admin_usage_matches_search,
admin_usage_matches_status, admin_usage_matches_username, admin_usage_record_json,
admin_usage_resolve_request_capture_body, admin_usage_upstream_is_stream,
build_admin_usage_detail_payload,
admin_usage_has_fallback, admin_usage_is_failed, admin_usage_is_success,
admin_usage_matches_search, admin_usage_matches_status, admin_usage_matches_username,
admin_usage_record_json, admin_usage_resolve_request_capture_body,
admin_usage_upstream_is_stream, build_admin_usage_detail_payload,
};
use aether_data_contracts::repository::usage::{StoredRequestUsageAudit, UsageBodyField};
@@ -2349,6 +2370,33 @@ mod tests {
assert_eq!(record["client_is_stream"], false);
}
#[test]
fn client_requested_stream_uses_stream_generate_content_path_over_stale_metadata_flag() {
let item = StoredRequestUsageAudit {
is_stream: true,
request_metadata: Some(json!({
"client_requested_stream": false,
"request_path": "/v1beta/models/gemini-3.1-flash-image-preview:streamGenerateContent",
"request_path_and_query": "/v1beta/models/gemini-3.1-flash-image-preview:streamGenerateContent?alt=sse"
})),
..sample_usage("completed", Some(200), None)
};
assert!(admin_usage_client_is_stream(&item));
let record = admin_usage_record_json(
&item,
&BTreeMap::new(),
&BTreeMap::new(),
false,
false,
None,
);
assert_eq!(record["upstream_is_stream"], true);
assert_eq!(record["client_requested_stream"], true);
assert_eq!(record["client_is_stream"], true);
}
#[test]
fn client_requested_stream_falls_back_to_request_body_stream_flag() {
let item = StoredRequestUsageAudit {
@@ -2528,6 +2576,30 @@ mod tests {
assert!(admin_usage_matches_status(&item, Some("failed")));
}
#[test]
fn redirect_status_is_not_admin_usage_success() {
let item = sample_usage("completed", Some(302), None);
assert!(!admin_usage_is_success(&item));
assert!(!admin_usage_is_failed(&item));
assert!(admin_usage_matches_status(&item, Some("error")));
assert!(admin_usage_matches_status(&item, Some("completed")));
}
#[test]
fn failed_redirect_status_counts_as_admin_usage_failed() {
let item = sample_usage(
"failed",
Some(302),
Some("execution runtime stream returned non-success status 302"),
);
assert!(admin_usage_is_failed(&item));
assert!(admin_usage_matches_status(&item, Some("failed")));
assert!(admin_usage_matches_status(&item, Some("error")));
assert!(!admin_usage_is_success(&item));
}
#[test]
fn active_status_with_failure_signal_counts_as_failed() {
let item = sample_usage("pending", Some(503), Some("upstream failed"));

1
crates/aether-ai-formats/Cargo.toml
View File

@@ -15,4 +15,5 @@ serde.workspace = true
serde_json.workspace = true
sha1 = "0.10"
sha2.workspace = true
url.workspace = true
uuid.workspace = true

4
crates/aether-ai-formats/src/api.rs
View File

@@ -96,7 +96,9 @@ pub use crate::formats::shared::response::{
};
pub use crate::formats::shared::routing::{
is_matching_stream_http_request, is_matching_stream_request,
resolve_execution_runtime_stream_plan_kind, resolve_execution_runtime_sync_plan_kind,
request_path_implies_stream_request, resolve_execution_runtime_stream_plan_kind,
resolve_execution_runtime_sync_plan_kind, sanitize_request_path,
sanitize_request_path_and_query, sanitize_request_query_string,
supports_stream_execution_decision_kind, supports_sync_execution_decision_kind,
};
pub use crate::formats::shared::sse::{encode_done_sse, encode_json_sse, map_claude_stop_reason};

103
crates/aether-ai-formats/src/formats/shared/routing.rs
View File

@@ -1,4 +1,5 @@
use http::Method;
use url::form_urlencoded;
use crate::contracts::{
CLAUDE_CHAT_STREAM_PLAN_KIND, CLAUDE_CHAT_SYNC_PLAN_KIND, CLAUDE_CLI_STREAM_PLAN_KIND,
@@ -304,6 +305,67 @@ fn resolve_gemini_generate_content_plan_kind(
}
}
pub fn request_path_implies_stream_request(path: &str) -> bool {
let trimmed = path.trim();
let path = trimmed
.split_once('?')
.map(|(path, _)| path)
.unwrap_or(trimmed);
path.ends_with(":streamGenerateContent")
}
pub fn sanitize_request_path(path: &str) -> Option<String> {
let path = path
.trim()
.split_once('?')
.map(|(path, _)| path)
.unwrap_or_else(|| path.trim())
.trim();
(!path.is_empty()).then(|| path.to_string())
}
pub fn sanitize_request_query_string(query: &str) -> Option<String> {
let query = query.trim().trim_start_matches('?').trim();
if query.is_empty() {
return None;
}
let mut serializer = form_urlencoded::Serializer::new(String::new());
for (key, value) in form_urlencoded::parse(query.as_bytes()) {
if request_query_key_is_safe_to_trace(key.as_ref()) {
serializer.append_pair(key.as_ref(), value.as_ref());
}
}
let sanitized = serializer.finish();
(!sanitized.is_empty()).then_some(sanitized)
}
pub fn sanitize_request_path_and_query(path: &str, query: Option<&str>) -> Option<String> {
let trimmed = path.trim();
let (path, embedded_query) = trimmed
.split_once('?')
.map(|(path, query)| (path.trim(), Some(query)))
.unwrap_or((trimmed, None));
if path.is_empty() {
return None;
}
let sanitized_query = query
.and_then(sanitize_request_query_string)
.or_else(|| embedded_query.and_then(sanitize_request_query_string));
Some(match sanitized_query {
Some(query) => format!("{path}?{query}"),
None => path.to_string(),
})
}
fn request_query_key_is_safe_to_trace(key: &str) -> bool {
matches!(
key.to_ascii_lowercase().as_str(),
"alt" | "view" | "pagesize" | "page_size" | "limit" | "offset"
)
}
pub fn is_matching_stream_request(
plan_kind: &str,
path: &str,
@@ -320,7 +382,7 @@ pub fn is_matching_stream_request(
.and_then(|value| value.as_bool())
.unwrap_or(false),
GEMINI_CHAT_STREAM_PLAN_KIND | GEMINI_CLI_STREAM_PLAN_KIND => {
path.ends_with(":streamGenerateContent")
request_path_implies_stream_request(path)
}
_ => true,
}
@@ -388,7 +450,9 @@ mod tests {
use super::{
is_matching_stream_http_request, is_matching_stream_request,
resolve_execution_runtime_stream_plan_kind, resolve_execution_runtime_sync_plan_kind,
request_path_implies_stream_request, resolve_execution_runtime_stream_plan_kind,
resolve_execution_runtime_sync_plan_kind, sanitize_request_path,
sanitize_request_path_and_query, sanitize_request_query_string,
supports_stream_execution_decision_kind, supports_sync_execution_decision_kind,
};
use crate::contracts::{
@@ -609,6 +673,41 @@ mod tests {
);
}
#[test]
fn stream_path_detection_handles_gemini_method_paths_with_query() {
assert!(request_path_implies_stream_request(
"/v1beta/models/gemini-2.5-pro:streamGenerateContent?alt=sse"
));
assert!(request_path_implies_stream_request(
" /v1internal:streamGenerateContent?alt=sse "
));
assert!(!request_path_implies_stream_request(
"/v1beta/models/gemini-2.5-pro:generateContent?alt=sse"
));
}
#[test]
fn request_path_metadata_sanitizer_drops_sensitive_query_parameters() {
assert_eq!(
sanitize_request_path("/v1beta/models/gemini-2.5-pro:generateContent?key=secret")
.as_deref(),
Some("/v1beta/models/gemini-2.5-pro:generateContent")
);
assert_eq!(
sanitize_request_query_string("?key=secret&alt=sse&pageSize=10&token=hidden")
.as_deref(),
Some("alt=sse&pageSize=10")
);
assert_eq!(
sanitize_request_path_and_query(
"/v1beta/models/gemini-2.5-pro:streamGenerateContent?key=secret&alt=sse",
None
)
.as_deref(),
Some("/v1beta/models/gemini-2.5-pro:streamGenerateContent?alt=sse")
);
}
#[test]
fn stream_matching_requires_openai_stream_flag() {
assert!(!is_matching_stream_request(

90
crates/aether-provider-transport/src/request_url/mod.rs
View File

@@ -12,7 +12,7 @@ use crate::claude_code::build_claude_code_messages_url;
use crate::snapshot::GatewayProviderTransportSnapshot;
use crate::url::{
build_claude_messages_url, build_gemini_content_url, build_openai_chat_url,
build_openai_responses_url, build_passthrough_path_url,
build_openai_responses_url, build_passthrough_path_url, normalize_gemini_content_action_path,
};
use crate::vertex::{
build_vertex_api_key_gemini_content_url, resolve_local_vertex_api_key_query_auth,
@@ -36,6 +36,8 @@ pub fn build_transport_request_url(
}
let provider_api_format = params.provider_api_format.trim().to_ascii_lowercase();
let normalized_provider_api_format =
aether_ai_formats::normalize_api_format_alias(&provider_api_format);
let custom_path = transport
.endpoint
.custom_path
@@ -45,14 +47,19 @@ pub fn build_transport_request_url(
.map(|path| expand_custom_path_template(path, build_path_params(params)));
if let Some(path) = custom_path.as_deref() {
let blocked_keys = if provider_api_format.starts_with("gemini:") {
let blocked_keys = if normalized_provider_api_format.starts_with("gemini:") {
&["key"][..]
} else {
&[][..]
};
let normalized_path = if normalized_provider_api_format == "gemini:generate_content" {
normalize_gemini_content_action_path(path, params.upstream_is_stream)
} else {
path.to_string()
};
let url = build_passthrough_path_url(
&transport.endpoint.base_url,
path,
normalized_path.as_str(),
params.request_query,
blocked_keys,
)?;
@@ -63,7 +70,7 @@ pub fn build_transport_request_url(
));
}
let url = match aether_ai_formats::normalize_api_format_alias(&provider_api_format).as_str() {
let url = match normalized_provider_api_format.as_str() {
"openai:chat" => Some(build_openai_chat_url(
&transport.endpoint.base_url,
params.request_query,
@@ -570,6 +577,81 @@ mod tests {
);
}
#[test]
fn rewrites_hardcoded_gemini_custom_path_action_to_match_stream_mode() {
let stream_transport = sample_transport(
"custom",
"gemini:generate_content",
"https://generativelanguage.googleapis.com",
Some("/v1beta/models/{model}:generateContent"),
);
let stream_url = build_transport_request_url(
&stream_transport,
TransportRequestUrlParams {
provider_api_format: "gemini:generate_content",
mapped_model: Some("gemini-2.5-pro"),
upstream_is_stream: true,
request_query: Some("key=client-key&foo=bar"),
kiro_api_region: None,
},
)
.expect("stream custom path url");
assert_eq!(
stream_url,
"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:streamGenerateContent?foo=bar&alt=sse"
);
let sync_transport = sample_transport(
"custom",
"gemini:generate_content",
"https://generativelanguage.googleapis.com",
Some("/v1beta/models/{model}:streamGenerateContent"),
);
let sync_url = build_transport_request_url(
&sync_transport,
TransportRequestUrlParams {
provider_api_format: "gemini:generate_content",
mapped_model: Some("gemini-2.5-pro"),
upstream_is_stream: false,
request_query: Some("foo=bar"),
kiro_api_region: None,
},
)
.expect("sync custom path url");
assert_eq!(
sync_url,
"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:generateContent?foo=bar"
);
let v1_transport = sample_transport(
"custom",
"gemini:generate_content",
"https://generativelanguage.googleapis.com",
Some("/v1/models/{model}:generateContent"),
);
let v1_stream_url = build_transport_request_url(
&v1_transport,
TransportRequestUrlParams {
provider_api_format: "gemini:generate_content",
mapped_model: Some("gemini-2.5-pro"),
upstream_is_stream: true,
request_query: None,
kiro_api_region: None,
},
)
.expect("v1 stream custom path url");
assert_eq!(
v1_stream_url,
"https://generativelanguage.googleapis.com/v1/models/gemini-2.5-pro:streamGenerateContent?alt=sse"
);
}
#[test]
fn keeps_original_custom_path_when_template_params_are_missing() {
let transport = sample_transport(

99
crates/aether-provider-transport/src/url.rs
View File

@@ -69,9 +69,10 @@ pub fn build_gemini_content_url(
} else {
"generateContent"
};
let mut url = if trimmed_base_url.ends_with("/v1beta") {
let mut url = if trimmed_base_url.ends_with("/v1") || trimmed_base_url.ends_with("/v1beta") {
format!("{trimmed_base_url}/models/{trimmed_model}:{operation}")
} else if trimmed_base_url.contains("/v1beta/models/") {
} else if gemini_content_base_url_contains_model_path(trimmed_base_url) {
let trimmed_base_url = strip_gemini_content_action(trimmed_base_url);
format!("{trimmed_base_url}:{operation}")
} else {
format!("{trimmed_base_url}/v1beta/models/{trimmed_model}:{operation}")
@@ -80,6 +81,37 @@ pub fn build_gemini_content_url(
Some(url)
}
pub fn normalize_gemini_content_action_path(path: &str, stream: bool) -> String {
let trimmed = path.trim();
let (path, query) = split_path_query(trimmed);
let action = if stream {
"streamGenerateContent"
} else {
"generateContent"
};
let normalized = strip_gemini_content_action(path);
let normalized = if normalized.len() == path.len() {
path.to_string()
} else {
format!("{normalized}:{action}")
};
match query {
Some(query) => format!("{normalized}?{query}"),
None => normalized,
}
}
fn strip_gemini_content_action(value: &str) -> &str {
value
.strip_suffix(":streamGenerateContent")
.or_else(|| value.strip_suffix(":generateContent"))
.unwrap_or(value)
}
fn gemini_content_base_url_contains_model_path(value: &str) -> bool {
value.contains("/v1/models/") || value.contains("/v1beta/models/")
}
pub fn build_gemini_video_predict_long_running_url(
upstream_base_url: &str,
model: &str,
@@ -92,9 +124,9 @@ pub fn build_gemini_video_predict_long_running_url(
return None;
}
let mut url = if trimmed_base_url.ends_with("/v1beta") {
let mut url = if trimmed_base_url.ends_with("/v1") || trimmed_base_url.ends_with("/v1beta") {
format!("{trimmed_base_url}/models/{trimmed_model}:predictLongRunning")
} else if trimmed_base_url.contains("/v1beta/models/") {
} else if gemini_content_base_url_contains_model_path(trimmed_base_url) {
format!("{trimmed_base_url}:predictLongRunning")
} else {
format!("{trimmed_base_url}/v1beta/models/{trimmed_model}:predictLongRunning")
@@ -248,6 +280,7 @@ mod tests {
build_gemini_content_url, build_gemini_files_passthrough_url,
build_gemini_video_predict_long_running_url, build_openai_chat_url,
build_openai_responses_url, build_passthrough_path_url,
normalize_gemini_content_action_path,
};
#[test]
@@ -289,6 +322,64 @@ mod tests {
);
}
#[test]
fn gemini_content_urls_rewrite_existing_base_action_for_stream_mode() {
assert_eq!(
build_gemini_content_url(
"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:generateContent",
"ignored-model",
true,
Some("foo=bar")
)
.as_deref(),
Some(
"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:streamGenerateContent?foo=bar"
)
);
assert_eq!(
build_gemini_content_url(
"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:streamGenerateContent",
"ignored-model",
false,
Some("foo=bar")
)
.as_deref(),
Some(
"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:generateContent?foo=bar"
)
);
assert_eq!(
build_gemini_content_url(
"https://generativelanguage.googleapis.com/v1/models/gemini-2.5-pro:generateContent",
"ignored-model",
true,
Some("foo=bar")
)
.as_deref(),
Some(
"https://generativelanguage.googleapis.com/v1/models/gemini-2.5-pro:streamGenerateContent?foo=bar"
)
);
}
#[test]
fn normalizes_gemini_content_action_in_custom_paths() {
assert_eq!(
normalize_gemini_content_action_path(
"/v1beta/models/gemini-2.5-pro:generateContent?alt=sse",
true
),
"/v1beta/models/gemini-2.5-pro:streamGenerateContent?alt=sse"
);
assert_eq!(
normalize_gemini_content_action_path(
"/v1beta/models/gemini-2.5-pro:streamGenerateContent",
false
),
"/v1beta/models/gemini-2.5-pro:generateContent"
);
}
#[test]
fn merges_base_path_and_request_query_for_passthrough_paths() {
assert_eq!(

81
crates/aether-scheduler-core/src/request_candidate.rs
View File

@@ -17,11 +17,15 @@ pub struct SchedulerRequestCandidateReportContext {
pub key_id: Option<String>,
pub client_api_format: Option<String>,
pub provider_api_format: Option<String>,
pub request_path: Option<String>,
pub request_query_string: Option<String>,
pub request_path_and_query: Option<String>,
pub upstream_url: Option<String>,
pub mapped_model: Option<String>,
pub key_name: Option<String>,
pub header_rules: Option<Value>,
pub body_rules: Option<Value>,
pub upstream_response: Option<Value>,
pub proxy: Option<Value>,
pub error_flow: Option<Value>,
pub candidate_group_id: Option<String>,
@@ -60,11 +64,15 @@ pub struct SchedulerExecutionRequestCandidateSeed {
struct ReportCandidateExtraDataInput {
client_api_format: Option<String>,
provider_api_format: Option<String>,
request_path: Option<String>,
request_query_string: Option<String>,
request_path_and_query: Option<String>,
upstream_url: Option<String>,
mapped_model: Option<String>,
key_name: Option<String>,
header_rules: Option<Value>,
body_rules: Option<Value>,
upstream_response: Option<Value>,
proxy: Option<Value>,
error_flow: Option<Value>,
candidate_group_id: Option<String>,
@@ -138,6 +146,9 @@ pub fn parse_request_candidate_report_context(
key_id: string_field(report_context, "key_id"),
client_api_format: string_field(report_context, "client_api_format"),
provider_api_format: string_field(report_context, "provider_api_format"),
request_path: string_field(report_context, "request_path"),
request_query_string: string_field(report_context, "request_query_string"),
request_path_and_query: string_field(report_context, "request_path_and_query"),
upstream_url: string_field(report_context, "upstream_url"),
mapped_model: string_field(report_context, "mapped_model"),
key_name: string_field(report_context, "key_name"),
@@ -149,6 +160,10 @@ pub fn parse_request_candidate_report_context(
.get("body_rules")
.cloned()
.filter(|value| !value.is_null()),
upstream_response: report_context
.get("upstream_response")
.cloned()
.filter(|value| !value.is_null()),
proxy: report_context
.get("proxy")
.cloned()
@@ -187,11 +202,15 @@ pub fn resolve_report_request_candidate_slot(
key_id,
client_api_format,
provider_api_format,
request_path,
request_query_string,
request_path_and_query,
upstream_url,
mapped_model,
key_name,
header_rules,
body_rules,
upstream_response,
proxy,
error_flow,
candidate_group_id,
@@ -207,11 +226,15 @@ pub fn resolve_report_request_candidate_slot(
let synthesized_extra_data = build_report_candidate_extra_data(ReportCandidateExtraDataInput {
client_api_format,
provider_api_format,
request_path,
request_query_string,
request_path_and_query,
upstream_url,
mapped_model,
key_name,
header_rules,
body_rules,
upstream_response,
proxy,
error_flow,
candidate_group_id,
@@ -326,11 +349,15 @@ pub fn build_execution_request_candidate_seed(
build_report_candidate_extra_data(ReportCandidateExtraDataInput {
client_api_format: metadata.client_api_format,
provider_api_format: metadata.provider_api_format,
request_path: metadata.request_path,
request_query_string: metadata.request_query_string,
request_path_and_query: metadata.request_path_and_query,
upstream_url: metadata.upstream_url,
mapped_model: metadata.mapped_model,
key_name: metadata.key_name,
header_rules: metadata.header_rules,
body_rules: metadata.body_rules,
upstream_response: metadata.upstream_response,
proxy: metadata.proxy,
error_flow: metadata.error_flow,
candidate_group_id: metadata.candidate_group_id,
@@ -431,11 +458,15 @@ pub fn build_local_request_candidate_status_record(
let extra_data = build_report_candidate_extra_data(ReportCandidateExtraDataInput {
client_api_format: metadata.client_api_format.clone(),
provider_api_format: metadata.provider_api_format.clone(),
request_path: metadata.request_path.clone(),
request_query_string: metadata.request_query_string.clone(),
request_path_and_query: metadata.request_path_and_query.clone(),
upstream_url: metadata.upstream_url.clone(),
mapped_model: metadata.mapped_model.clone(),
key_name: metadata.key_name.clone(),
header_rules: metadata.header_rules.clone(),
body_rules: metadata.body_rules.clone(),
upstream_response: metadata.upstream_response.clone(),
proxy: metadata.proxy.clone(),
error_flow: metadata.error_flow.clone(),
candidate_group_id: metadata.candidate_group_id.clone(),
@@ -667,11 +698,15 @@ fn build_report_candidate_extra_data(input: ReportCandidateExtraDataInput) -> Op
let ReportCandidateExtraDataInput {
client_api_format,
provider_api_format,
request_path,
request_query_string,
request_path_and_query,
upstream_url,
mapped_model,
key_name,
header_rules,
body_rules,
upstream_response,
proxy,
error_flow,
candidate_group_id,
@@ -698,6 +733,21 @@ fn build_report_candidate_extra_data(input: ReportCandidateExtraDataInput) -> Op
Value::String(provider_api_format),
);
}
if let Some(request_path) = request_path {
extra_data.insert("request_path".to_string(), Value::String(request_path));
}
if let Some(request_query_string) = request_query_string {
extra_data.insert(
"request_query_string".to_string(),
Value::String(request_query_string),
);
}
if let Some(request_path_and_query) = request_path_and_query {
extra_data.insert(
"request_path_and_query".to_string(),
Value::String(request_path_and_query),
);
}
if let Some(upstream_url) = upstream_url {
extra_data.insert("upstream_url".to_string(), Value::String(upstream_url));
}
@@ -713,6 +763,9 @@ fn build_report_candidate_extra_data(input: ReportCandidateExtraDataInput) -> Op
if let Some(body_rules) = body_rules {
extra_data.insert("body_rules".to_string(), body_rules);
}
if let Some(upstream_response) = upstream_response {
extra_data.insert("upstream_response".to_string(), upstream_response);
}
if let Some(proxy) = proxy {
extra_data.insert("proxy".to_string(), proxy);
}
@@ -903,6 +956,11 @@ mod tests {
"body_rules": [
{"op": "remove", "path": "/store"}
],
"upstream_response": {
"status_code": 503,
"headers": {"retry-after": "2"},
"body": {"error": {"message": "overloaded"}}
},
"proxy": {
"node_id": "proxy-node-1",
"node_name": "edge-1",
@@ -962,6 +1020,13 @@ mod tests {
.map(Vec::len),
Some(1)
);
assert_eq!(
slot.extra_data
.as_ref()
.and_then(|value| value.get("upstream_response"))
.and_then(|value| value.get("status_code")),
Some(&json!(503))
);
assert_eq!(
slot.extra_data
.as_ref()
@@ -1063,6 +1128,8 @@ mod tests {
"api_key_id": "api-key-1",
"client_api_format": "openai:chat",
"provider_api_format": "openai:responses",
"request_path": "/v1/responses",
"request_query_string": "debug=true",
"upstream_url": "https://example.com/v1/responses",
"mapped_model": "gpt-5-upstream",
"key_name": "primary",
@@ -1098,6 +1165,20 @@ mod tests {
.and_then(|value| value.get("provider_api_format")),
Some(&json!("openai:responses"))
);
assert_eq!(
record
.extra_data
.as_ref()
.and_then(|value| value.get("request_path")),
Some(&json!("/v1/responses"))
);
assert_eq!(
record
.extra_data
.as_ref()
.and_then(|value| value.get("request_query_string")),
Some(&json!("debug=true"))
);
assert_eq!(
record
.extra_data

63
crates/aether-usage-runtime/src/request_metadata.rs
View File

@@ -1,3 +1,6 @@
use aether_ai_formats::api::{
sanitize_request_path, sanitize_request_path_and_query, sanitize_request_query_string,
};
use aether_contracts::ExecutionPlan;
use serde_json::{json, Map, Value};
@@ -72,9 +75,13 @@ fn copy_allowed_metadata_fields(source: &Map<String, Value>, target: &mut Map<St
copy_bool(source, target, "client_requested_stream");
copy_bool(source, target, "upstream_is_stream");
copy_bool(source, target, "api_key_is_standalone");
copy_non_empty_string(source, target, "request_path");
copy_non_empty_string(source, target, "request_query_string");
copy_non_empty_string(source, target, "request_path_and_query");
copy_number(source, target, "provider_request_body_base64_bytes");
copy_number(source, target, "provider_response_body_base64_bytes");
copy_number(source, target, "client_response_body_base64_bytes");
copy_number(source, target, "client_response_status_code");
copy_non_null_value(source, target, "billing_snapshot");
copy_non_empty_string(source, target, "billing_snapshot_schema_version");
copy_non_empty_string(source, target, "billing_snapshot_status");
@@ -96,6 +103,7 @@ fn copy_allowed_metadata_fields(source: &Map<String, Value>, target: &mut Map<St
copy_number(source, target, "cache_read_price_per_1m");
copy_number(source, target, "price_per_request");
copy_non_null_value(source, target, "proxy");
sanitize_request_path_metadata_fields(target);
}
fn move_allowed_metadata_fields(mut source: Map<String, Value>, target: &mut Map<String, Value>) {
@@ -105,9 +113,13 @@ fn move_allowed_metadata_fields(mut source: Map<String, Value>, target: &mut Map
remove_bool(&mut source, target, "client_requested_stream");
remove_bool(&mut source, target, "upstream_is_stream");
remove_bool(&mut source, target, "api_key_is_standalone");
remove_non_empty_string(&mut source, target, "request_path");
remove_non_empty_string(&mut source, target, "request_query_string");
remove_non_empty_string(&mut source, target, "request_path_and_query");
remove_number(&mut source, target, "provider_request_body_base64_bytes");
remove_number(&mut source, target, "provider_response_body_base64_bytes");
remove_number(&mut source, target, "client_response_body_base64_bytes");
remove_number(&mut source, target, "client_response_status_code");
remove_non_null_value(&mut source, target, "billing_snapshot");
remove_non_empty_string(&mut source, target, "billing_snapshot_schema_version");
remove_non_empty_string(&mut source, target, "billing_snapshot_status");
@@ -129,6 +141,38 @@ fn move_allowed_metadata_fields(mut source: Map<String, Value>, target: &mut Map
remove_number(&mut source, target, "cache_read_price_per_1m");
remove_number(&mut source, target, "price_per_request");
remove_non_null_value(&mut source, target, "proxy");
sanitize_request_path_metadata_fields(target);
}
fn sanitize_request_path_metadata_fields(target: &mut Map<String, Value>) {
let path = target
.get("request_path")
.and_then(Value::as_str)
.and_then(sanitize_request_path);
let query = target
.get("request_query_string")
.and_then(Value::as_str)
.and_then(sanitize_request_query_string);
let path_and_query = target
.get("request_path_and_query")
.and_then(Value::as_str)
.and_then(|value| sanitize_request_path_and_query(value, None))
.or_else(|| {
path.as_deref()
.and_then(|path| sanitize_request_path_and_query(path, query.as_deref()))
});
apply_optional_string_field(target, "request_path", path.as_deref());
apply_optional_string_field(target, "request_query_string", query.as_deref());
apply_optional_string_field(target, "request_path_and_query", path_and_query.as_deref());
}
fn apply_optional_string_field(target: &mut Map<String, Value>, key: &str, value: Option<&str>) {
if let Some(value) = value {
target.insert(key.to_string(), Value::String(value.to_string()));
} else {
target.remove(key);
}
}
fn copy_non_empty_string(source: &Map<String, Value>, target: &mut Map<String, Value>, key: &str) {
@@ -451,6 +495,25 @@ mod tests {
);
}
#[test]
fn sanitizes_request_path_query_metadata() {
let metadata = sanitize_usage_request_metadata(Some(json!({
"request_path": "/v1beta/models/gemini-2.5-pro:streamGenerateContent?key=secret",
"request_query_string": "key=secret&alt=sse&pageSize=10&token=hidden",
"request_path_and_query": "/v1beta/models/gemini-2.5-pro:streamGenerateContent?key=secret&alt=sse&pageSize=10&token=hidden",
})))
.expect("metadata should remain");
assert_eq!(
metadata,
json!({
"request_path": "/v1beta/models/gemini-2.5-pro:streamGenerateContent",
"request_query_string": "alt=sse&pageSize=10",
"request_path_and_query": "/v1beta/models/gemini-2.5-pro:streamGenerateContent?alt=sse&pageSize=10",
})
);
}
#[test]
fn sanitizes_large_allowed_metadata_values_to_bounded_representations() {
let metadata = sanitize_usage_request_metadata(Some(json!({

89
crates/aether-usage-runtime/src/write.rs
View File

@@ -529,7 +529,8 @@ fn build_terminal_usage_event_from_seed_impl(
};
let routing = merge_routing_seed_with_metadata_owned(routing, request_metadata.as_ref());
let body_refs = merge_body_refs_seed_with_metadata_owned(body_refs, request_metadata.as_ref());
let error_message = resolve_error_message(status_code, provider_response.as_ref(), None);
let error_message = resolve_error_message(status_code, provider_response.as_ref(), None)
.or_else(|| resolve_error_message(status_code, client_response.as_ref(), None));
let api_family = infer_api_family(&client_contract).map(ToOwned::to_owned);
let endpoint_kind = infer_endpoint_kind(&client_contract).map(ToOwned::to_owned);
let provider_api_family = infer_api_family(&provider_contract).map(ToOwned::to_owned);
@@ -734,7 +735,8 @@ pub fn build_sync_terminal_usage_payload_seed(
let context = payload.report_context.as_ref().and_then(Value::as_object);
let provider_response_headers = context_usage_value(context, "provider_response_headers")
.or_else(|| headers_to_json(&payload.headers));
let client_response_headers = headers_to_json(&payload.headers);
let client_response_headers = context_usage_value(context, "client_response_headers")
.or_else(|| headers_to_json(&payload.headers));
SyncTerminalUsagePayloadSeed {
report_kind: payload.report_kind.clone(),
status_code: payload.status_code,
@@ -941,7 +943,7 @@ fn infer_sync_terminal_state(
) -> UsageTerminalState {
if status_code == 499 || report_kind.contains("cancel") {
UsageTerminalState::Cancelled
} else if status_code >= 400
} else if !(200..300).contains(&status_code)
|| provider_response
.and_then(|value| value.get("error"))
.is_some_and(|value| !value.is_null())
@@ -959,7 +961,7 @@ fn infer_stream_terminal_state(
) -> UsageTerminalState {
if cancelled || status_code == 499 || report_kind.contains("cancel") {
UsageTerminalState::Cancelled
} else if status_code >= 400 {
} else if !(200..300).contains(&status_code) {
UsageTerminalState::Failed
} else {
UsageTerminalState::Completed
@@ -1982,6 +1984,8 @@ fn resolve_error_category(status_code: u16, event_type: UsageEventType) -> Optio
UsageEventType::Cancelled => Some("cancelled".to_string()),
UsageEventType::Failed if status_code >= 500 => Some("server_error".to_string()),
UsageEventType::Failed if status_code >= 400 => Some("client_error".to_string()),
UsageEventType::Failed if status_code >= 300 => Some("redirect".to_string()),
UsageEventType::Failed => Some("non_success_status".to_string()),
_ => None,
}
}
@@ -2000,7 +2004,7 @@ fn resolve_error_message(
if explicit_error_message.is_some() {
return explicit_error_message;
}
if status_code < 400 {
if (200..300).contains(&status_code) {
return None;
}
@@ -2798,6 +2802,81 @@ mod tests {
);
}
#[test]
fn stream_terminal_usage_marks_redirect_status_as_failed() {
let plan = ExecutionPlan {
request_id: "req-stream-redirect-usage".to_string(),
candidate_id: Some("cand-stream-redirect-usage".to_string()),
provider_name: Some("ChatGPTWeb".to_string()),
provider_id: "provider-redirect".to_string(),
endpoint_id: "endpoint-redirect".to_string(),
key_id: "key-redirect".to_string(),
method: "POST".to_string(),
url: "https://example.com/v1beta/models/gemini:streamGenerateContent".to_string(),
headers: BTreeMap::new(),
content_type: None,
content_encoding: None,
body: RequestBody {
json_body: None,
body_bytes_b64: None,
body_ref: None,
},
stream: true,
client_api_format: "gemini:generate_content".to_string(),
provider_api_format: "gemini:generate_content".to_string(),
model_name: Some("gemini".to_string()),
proxy: None,
transport_profile: None,
timeouts: None,
};
let client_body = json!({
"error": {
"type": "execution_runtime_non_success_status",
"message": "execution runtime stream returned non-success status 302",
"code": 302,
"upstream_status": 302,
"location": "/"
}
});
let payload = GatewayStreamReportRequest {
trace_id: "trace-stream-redirect-usage".to_string(),
report_kind: "gemini_chat_stream_success".to_string(),
report_context: Some(json!({
"client_api_format": "gemini:generate_content",
"provider_api_format": "gemini:generate_content"
})),
status_code: 302,
headers: BTreeMap::from([
("content-type".to_string(), "application/json".to_string()),
("x-aether-upstream-status".to_string(), "302".to_string()),
]),
provider_body_base64: Some(
base64::engine::general_purpose::STANDARD
.encode(br#"{"error":{"message":"raw redirect body"}}"#),
),
provider_body_state: Some(UsageBodyCaptureState::Inline),
client_body_base64: Some(
base64::engine::general_purpose::STANDARD
.encode(serde_json::to_vec(&client_body).expect("body should encode")),
),
client_body_state: Some(UsageBodyCaptureState::Inline),
terminal_summary: None,
telemetry: None,
};
let event =
build_stream_terminal_usage_event(&plan, payload.report_context.as_ref(), &payload)
.expect("usage event should build");
assert_eq!(event.event_type, UsageEventType::Failed);
assert_eq!(event.data.status_code, Some(302));
assert_eq!(event.data.error_category.as_deref(), Some("redirect"));
assert_eq!(
event.data.error_message.as_deref(),
Some("raw redirect body")
);
}
#[test]
fn builds_stream_terminal_usage_from_terminal_summary_usage_without_decoding_bodies() {
let plan = ExecutionPlan {

16
frontend/src/api/requestTrace.ts
View File

@@ -9,6 +9,15 @@ export interface CandidateRankingMetadata {
demoted_by?: string
}
export interface CandidateResponseBoundary {
source?: string
status_code?: number | null
headers?: Record<string, unknown> | null
body?: unknown
body_ref?: string | null
body_state?: string | null
}
export interface CandidateRecord {
id: string
request_id: string
@@ -46,7 +55,9 @@ export interface CandidateRecord {
latency_ms?: number
concurrent_requests?: number
ranking?: CandidateRankingMetadata | null
extra_data?: Record<string, unknown>
extra_data?: Record<string, unknown> & {
upstream_response?: CandidateResponseBoundary
}
created_at: string
started_at?: string
finished_at?: string
@@ -54,6 +65,9 @@ export interface CandidateRecord {
export interface RequestTrace {
request_id: string
request_path?: string
request_query_string?: string
request_path_and_query?: string
total_candidates: number
final_status: 'success' | 'failed' | 'streaming' | 'pending' | 'cancelled'
total_latency_ms: number

46
frontend/src/composables/useDarkMode.ts
View File

@@ -25,12 +25,42 @@ const applyDarkMode = (value: boolean) => {
}
const getSystemPreference = (): boolean => {
if (typeof window === 'undefined') {
if (typeof window === 'undefined' || typeof window.matchMedia !== 'function') {
return false
}
return window.matchMedia('(prefers-color-scheme: dark)').matches
}
const getThemeStorage = (): Storage | null => {
if (typeof window === 'undefined') {
return null
}
const storage = window.localStorage
if (!storage || typeof storage.getItem !== 'function' || typeof storage.setItem !== 'function') {
return null
}
return storage
}
const readStoredTheme = (): ThemeMode | null => {
try {
const value = getThemeStorage()?.getItem(THEME_STORAGE_KEY)
return value === 'dark' || value === 'light' || value === 'system' ? value : null
} catch {
return null
}
}
const writeStoredTheme = (value: ThemeMode) => {
try {
getThemeStorage()?.setItem(THEME_STORAGE_KEY, value)
} catch {
// Ignore storage failures in restricted or test-like environments.
}
}
const updateDarkMode = () => {
if (themeMode.value === 'system') {
isDark.value = getSystemPreference()
@@ -59,9 +89,7 @@ const ensureWatcher = () => {
(value) => {
updateDarkMode()
if (typeof window !== 'undefined') {
localStorage.setItem(THEME_STORAGE_KEY, value)
}
writeStoredTheme(value)
},
{ flush: 'post' }
)
@@ -77,9 +105,9 @@ const initialize = () => {
ensureWatcher()
if (typeof window !== 'undefined') {
const storedTheme = localStorage.getItem(THEME_STORAGE_KEY) as ThemeMode | null
const storedTheme = readStoredTheme()
if (storedTheme === 'dark' || storedTheme === 'light' || storedTheme === 'system') {
if (storedTheme) {
themeMode.value = storedTheme
} else {
// 兼容旧版本存储格式，旧版本直接存储 'dark' 或 'light'
@@ -87,8 +115,10 @@ const initialize = () => {
}
// 监听系统主题变化
mediaQuery = window.matchMedia('(prefers-color-scheme: dark)')
mediaQuery.addEventListener('change', handleSystemChange)
if (typeof window.matchMedia === 'function') {
mediaQuery = window.matchMedia('(prefers-color-scheme: dark)')
mediaQuery.addEventListener('change', handleSystemChange)
}
}
updateDarkMode()

3
frontend/src/features/providers/components/provider-tabs/ModelTestDialog.vue
View File

@@ -673,6 +673,7 @@ import type { CandidateRecord, RequestTrace } from '@/api/requestTrace'
import HorizontalRequestTimeline from '@/features/usage/components/HorizontalRequestTimeline.vue'
import JsonContent from '@/features/usage/components/RequestDetailDrawer/JsonContent.vue'
import { useClipboard } from '@/composables/useClipboard'
import { useDarkMode } from '@/composables/useDarkMode'
type TestEndpointOption = {
id: string
@@ -716,7 +717,7 @@ const traceCandidates = computed(() => props.trace?.candidates ?? [])
const showSetup = computed(() => props.open && !props.testing && !props.result)
const showResult = computed(() => !!props.result)
const showTraceTimeline = computed(() => Boolean(props.requestId) && traceCandidates.value.length > 0)
const isDark = computed(() => typeof document !== 'undefined' && document.documentElement.classList.contains('dark'))
const { isDark } = useDarkMode()
const { copyToClipboard } = useClipboard()
const dialogTitle = computed(() => {

261
frontend/src/features/usage/components/HorizontalRequestTimeline.vue
View File

@@ -431,38 +431,42 @@
</span>
</div>
<!-- 真实请求错误：节点级调试原因，和对客户端返回的摘要分开 -->
<!-- 错误信息：真实上游响应合并在此处展示 -->
<div
v-if="currentAttempt.status === 'failed' && currentAttemptRequestError"
class="error-block"
>
<div class="error-type">
真实请求错误
<div class="error-heading">
<span class="error-type">错误信息</span>
<span
v-if="currentAttemptRequestError.statusCode != null"
class="error-status-badge"
:class="currentAttemptRequestError.statusCode >= 400 ? 'is-error' : currentAttemptRequestError.statusCode >= 300 ? 'is-warning' : 'is-success'"
>
HTTP {{ currentAttemptRequestError.statusCode }}
</span>
</div>
<div class="error-msg">
<div
v-if="currentAttemptRequestError.message"
class="error-msg"
>
{{ currentAttemptRequestError.message }}
</div>
<div
v-if="currentAttemptRequestError.meta.length > 0"
class="error-flow-meta"
v-if="currentAttemptRequestError.upstreamResponse"
class="error-json"
>
<span
v-for="item in currentAttemptRequestError.meta"
:key="item"
class="error-flow-chip"
>{{ item }}</span>
</div>
<div
v-if="currentAttemptRequestError.safetyHint"
class="error-flow-safety"
>
{{ currentAttemptRequestError.safetyHint }}
<JsonContentPanel
:data="currentAttemptRequestError.upstreamResponse"
:is-dark="isDark"
empty-message="无上游响应信息"
/>
</div>
</div>
<!-- 额外数据 -->
<details
v-if="currentAttempt.extra_data && Object.keys(currentAttempt.extra_data).length > 0"
v-if="currentAttemptExtraDataDisplay"
class="extra-block"
>
<summary class="extra-toggle">
@@ -470,7 +474,7 @@
</summary>
<JsonContentPanel
class="extra-json-panel"
:data="currentAttempt.extra_data"
:data="currentAttemptExtraDataDisplay"
:is-dark="isDark"
empty-message="无额外信息"
/>
@@ -508,6 +512,7 @@ import { requestTraceApi, type RequestTrace, type CandidateRecord } from '@/api/
import { log } from '@/utils/logger'
import { parseApiError } from '@/utils/errorParser'
import { formatApiFormat } from '@/api/endpoints/types/api-format'
import { useDarkMode } from '@/composables/useDarkMode'
import { resolveTimelineFinalStatus } from '../utils/status'
import {
buildPoolGroupVisibleAttempts,
@@ -559,17 +564,6 @@ interface UsageData {
}
}
interface AttemptErrorFlow {
source?: string
statusCode?: number
classification?: string
decision?: string
retryable?: boolean
safeToExpose?: boolean
propagation?: string
message?: string
}
const props = defineProps<{
requestId?: string | null
/** 外部传入的状态码，用于覆盖 trace.final_status 的判断 */
@@ -641,7 +635,7 @@ const getFinalStatusBadgeVariant = (status: string): BadgeVariant => {
const loading = ref(false)
const error = ref<string | null>(null)
const internalTrace = ref<RequestTrace | null>(null)
const isDark = computed(() => document.documentElement.classList.contains('dark'))
const { isDark } = useDarkMode()
const trace = computed(() => props.traceData ?? internalTrace.value)
const selectedGroupIndex = ref(0)
const selectedAttemptIndex = ref(0)
@@ -1073,65 +1067,42 @@ const readNumberField = (obj: Record<string, unknown>, key: string): number | un
return undefined
}
const readBooleanField = (obj: Record<string, unknown>, key: string): boolean | undefined => {
const value = obj[key]
return typeof value === 'boolean' ? value : undefined
const hasRenderableValue = (value: unknown): boolean => {
if (value == null) return false
if (typeof value === 'string') return value.trim().length > 0
if (typeof value === 'object') return Object.keys(value as Record<string, unknown>).length > 0
return true
}
const normalizeAttemptErrorFlow = (value: unknown): AttemptErrorFlow | null => {
const normalizeUpstreamResponseDisplay = (value: unknown): Record<string, unknown> | null => {
const raw = extractObject(value)
if (!raw) return null
const statusCode = readNumberField(raw, 'status_code') ?? readNumberField(raw, 'statusCode')
const headers = raw.headers
const body = raw.body
const bodyRef = readStringField(raw, 'body_ref') ?? readStringField(raw, 'bodyRef')
const bodyState = readStringField(raw, 'body_state') ?? readStringField(raw, 'bodyState')
const flow: AttemptErrorFlow = {
source: readStringField(raw, 'source'),
statusCode: readNumberField(raw, 'status_code') ?? readNumberField(raw, 'statusCode'),
classification: readStringField(raw, 'classification'),
decision: readStringField(raw, 'decision'),
retryable: readBooleanField(raw, 'retryable'),
safeToExpose: readBooleanField(raw, 'safe_to_expose') ?? readBooleanField(raw, 'safeToExpose'),
propagation: readStringField(raw, 'propagation'),
message: readStringField(raw, 'message'),
if (
statusCode == null &&
!hasRenderableValue(headers) &&
!hasRenderableValue(body) &&
!bodyRef &&
!bodyState
) {
return null
}
return Object.values(flow).some(value => value !== undefined) ? flow : null
const data: Record<string, unknown> = {}
if (statusCode != null) data.status_code = statusCode
if (hasRenderableValue(headers)) data.headers = headers
if (hasRenderableValue(body)) data.body = body
if (bodyRef) data.body_ref = bodyRef
if (bodyState) data.body_state = bodyState
return data
}
const labelFromMap = (value: string | undefined, labels: Record<string, string>): string | undefined => {
if (!value) return undefined
return labels[value] || value
}
const formatErrorFlowSource = (value?: string): string | undefined => labelFromMap(value, {
upstream_response: '上游响应',
request_validation: '请求校验',
gateway: '网关处理',
transport: '传输层',
scheduler: '调度层',
})
const formatErrorFlowDecision = (value?: string): string | undefined => labelFromMap(value, {
retry_next_candidate: '重试下一个候选',
stop_local_failover: '停止本地转移',
use_default: '默认处理',
return_to_client: '返回客户端',
})
const formatErrorFlowPropagation = (value?: string): string | undefined => labelFromMap(value, {
suppressed: '已抑制',
converted: '已转换',
passthrough: '直接透传',
local: '本地生成',
captured: '仅采集',
})
const formatErrorFlowClassification = (value?: string): string | undefined => labelFromMap(value, {
retryable: '可重试',
terminal: '终止',
provider_auth: '上游认证',
provider_quota: '上游额度',
invalid_request: '请求无效',
})
const extractStringList = (value: unknown): string[] => {
if (Array.isArray(value)) {
return value
@@ -1224,6 +1195,9 @@ const currentAttemptRequestPathDisplay = computed(() => {
const fromAttempt = resolveRequestPathFromObject(attempt?.extra_data)
if (fromAttempt) return fromAttempt
const fromTrace = resolveRequestPathFromObject(trace.value)
if (fromTrace) return fromTrace
const fromRequestMetadata = resolveRequestPathFromObject(props.requestMetadata)
if (fromRequestMetadata) return fromRequestMetadata
@@ -1290,45 +1264,64 @@ const currentAttemptFailureDiagnostic = computed<{
}
})
const formatAttemptErrorMessage = (message: string, statusCode?: number): string => {
const normalized = message.trim()
if (!normalized) return ''
if (/execution runtime (stream )?returned non-success status \d+/i.test(normalized)) {
return statusCode != null ? `上游返回非成功状态 ${statusCode}` : '上游返回非成功状态'
}
return normalized
}
const currentAttemptRequestError = computed<{
message: string
meta: string[]
safetyHint: string
statusCode?: number
upstreamResponse: Record<string, unknown> | null
} | null>(() => {
const attempt = currentAttempt.value
if (!attempt || attempt.status !== 'failed') return null
const extra = extractObject(attempt.extra_data)
const flow = normalizeAttemptErrorFlow(extra?.error_flow)
const upstreamResponse = extractObject(extra?.upstream_response)
const errorFlow = extractObject(extra?.error_flow)
const statusCode = readNumberField(upstreamResponse ?? {}, 'status_code')
?? readNumberField(upstreamResponse ?? {}, 'statusCode')
?? readNumberField(errorFlow ?? {}, 'status_code')
?? readNumberField(errorFlow ?? {}, 'statusCode')
?? attempt.status_code
const flowMessage = errorFlow
? readStringField(errorFlow, 'message')
: ''
const fallbackMessage = typeof attempt.error_message === 'string' && attempt.error_message.trim()
? attempt.error_message.trim()
: ''
const fallbackType = typeof attempt.error_type === 'string' && attempt.error_type.trim()
? attempt.error_type.trim()
: ''
const message = flow?.message || fallbackMessage
if (!message && !fallbackType && !flow) return null
const meta = [
flow?.statusCode != null ? `HTTP ${flow.statusCode}` : (attempt.status_code ? `HTTP ${attempt.status_code}` : ''),
formatErrorFlowSource(flow?.source),
formatErrorFlowClassification(flow?.classification) || fallbackType,
formatErrorFlowDecision(flow?.decision),
formatErrorFlowPropagation(flow?.propagation),
flow?.retryable != null ? (flow.retryable ? '会继续重试' : '不再重试') : '',
].filter((item): item is string => Boolean(item))
const safetyHint = flow?.safeToExpose === false
? '该错误被标记为敏感上游错误：仅在链路节点展示，不应完整返回给客户端。'
: ''
const message = formatAttemptErrorMessage(flowMessage || fallbackMessage, statusCode) || fallbackType
const upstreamResponseDisplay = normalizeUpstreamResponseDisplay(extra?.upstream_response)
if (!message && statusCode == null && !upstreamResponseDisplay) return null
return {
message: message || fallbackType || '未知错误',
meta,
safetyHint,
message: upstreamResponseDisplay ? '' : (message || '未知错误'),
statusCode,
upstreamResponse: upstreamResponseDisplay,
}
})
const currentAttemptExtraDataDisplay = computed<Record<string, unknown> | null>(() => {
const extra = extractObject(currentAttempt.value?.extra_data)
if (!extra) return null
const display = { ...extra }
delete display.upstream_response
delete display.error_flow
delete display.client_response
delete display.provider_response
return Object.keys(display).length > 0 ? display : null
})
// 计算当前尝试启用的能力标签（请求需要的能力）
const activeCapabilities = computed(() => {
if (!currentAttempt.value?.required_capabilities) return []
@@ -2495,50 +2488,66 @@ function getDisplayStatus(attempt: CandidateRecord | null | undefined): string {
border-radius: 8px;
}
.error-heading {
display: flex;
align-items: center;
justify-content: space-between;
gap: 0.75rem;
margin-bottom: 0.25rem;
}
.error-type {
font-size: 0.75rem;
font-weight: 600;
color: #ef4444;
margin-bottom: 0.25rem;
text-transform: uppercase;
letter-spacing: 0.025em;
}
.error-status-badge {
flex-shrink: 0;
padding: 0.125rem 0.45rem;
border-radius: 999px;
font-size: 0.72rem;
font-family: ui-monospace, monospace;
background: hsl(var(--muted));
color: hsl(var(--muted-foreground));
}
.error-status-badge.is-success {
color: #166534;
background: #22c55e18;
}
.error-status-badge.is-warning {
color: #92400e;
background: #f59e0b1f;
}
.error-status-badge.is-error {
color: #991b1b;
background: #ef44441f;
}
.error-msg {
font-size: 0.85rem;
color: #dc2626;
word-break: break-word;
}
.error-flow-meta {
display: flex;
flex-wrap: wrap;
gap: 0.375rem;
margin-top: 0.625rem;
.error-json {
margin-top: 0.75rem;
}
.error-flow-chip {
padding: 0.125rem 0.45rem;
border-radius: 999px;
background: #ef444414;
border: 1px solid #ef44442e;
color: #991b1b;
font-size: 0.72rem;
line-height: 1.35;
.dark .error-status-badge.is-success {
color: #bbf7d0;
}
.error-flow-safety {
margin-top: 0.625rem;
color: #991b1b;
font-size: 0.78rem;
line-height: 1.5;
.dark .error-status-badge.is-warning {
color: #fde68a;
}
.dark .error-flow-chip {
color: #fecaca;
}
.dark .error-flow-safety {
.dark .error-status-badge.is-error {
color: #fecaca;
}

109
frontend/src/features/usage/components/RequestDetailDrawer.vue
View File

@@ -459,55 +459,6 @@
/>
</div>
<!-- 错误域卡片：保持上游响应与客户端响应两个边界可对照 -->
<div
v-if="hasVisibleErrorCards"
class="space-y-3"
>
<div
class="grid gap-3"
:class="visibleErrorCardCount > 1 ? 'lg:grid-cols-2' : 'grid-cols-1'"
>
<Card
v-if="displayClientErrorMessage"
class="border-amber-200 dark:border-amber-800"
>
<div class="p-4">
<h4 class="text-sm font-semibold text-amber-700 dark:text-amber-300 mb-2">
返回客户端错误
</h4>
<div class="bg-amber-50 dark:bg-amber-900/20 rounded-lg p-3 space-y-1">
<p class="text-sm text-amber-900 dark:text-amber-200">
{{ displayClientErrorMessage }}
</p>
</div>
</div>
</Card>
<Card
v-if="normalizedUpstreamError"
class="border-orange-200 dark:border-orange-800"
>
<div class="p-4">
<h4 class="text-sm font-semibold text-orange-700 dark:text-orange-300 mb-2">
上游响应错误
</h4>
<div class="bg-orange-50 dark:bg-orange-900/20 rounded-lg p-3 space-y-1">
<p class="text-sm text-orange-900 dark:text-orange-200">
{{ normalizedUpstreamError.message }}
</p>
<p
v-if="formatErrorDomainMeta(normalizedUpstreamError)"
class="text-xs text-orange-800/70 dark:text-orange-200/70 font-mono"
>
{{ formatErrorDomainMeta(normalizedUpstreamError) }}
</p>
</div>
</div>
</Card>
</div>
</div>
<!-- Tabs 区域 -->
<Card>
<div class="p-3 sm:p-4">
@@ -743,6 +694,7 @@ import { ref, watch, computed, onMounted, onBeforeUnmount } from 'vue'
import Button from '@/components/ui/button.vue'
import { useEscapeKey } from '@/composables/useEscapeKey'
import { useClipboard } from '@/composables/useClipboard'
import { useDarkMode } from '@/composables/useDarkMode'
import Card from '@/components/ui/card.vue'
import Badge from '@/components/ui/badge.vue'
import Separator from '@/components/ui/separator.vue'
@@ -750,7 +702,7 @@ import Skeleton from '@/components/ui/skeleton.vue'
import Tabs from '@/components/ui/tabs.vue'
import TabsContent from '@/components/ui/tabs-content.vue'
import { Check, Columns2, RefreshCw, X, Monitor, Server, MessageSquareText, Code2, Terminal, Play } from 'lucide-vue-next'
import { dashboardApi, type RequestDetail, type RequestErrorDomain } from '@/api/dashboard'
import { dashboardApi, type RequestDetail } from '@/api/dashboard'
import { formatApiFormat } from '@/api/endpoints/types/api-format'
import { formatShortRequestId } from '@/utils/format'
import { log } from '@/utils/logger'
@@ -832,42 +784,11 @@ type PricingTierLike = {
type JsonRecord = Record<string, unknown>
type NormalizedErrorDomain = {
source?: string | null
status_code?: number | null
type?: string | null
message: string
code?: string | number | null
category?: string | null
}
function asRecord(value: unknown): JsonRecord | null {
if (!value || typeof value !== 'object' || Array.isArray(value)) return null
return value as JsonRecord
}
function normalizeErrorDomain(domain: RequestErrorDomain | null | undefined): NormalizedErrorDomain | null {
if (!domain || typeof domain !== 'object') return null
const message = typeof domain.message === 'string' ? domain.message.trim() : ''
if (!message) return null
return {
source: domain.source ?? null,
status_code: domain.status_code ?? null,
type: domain.type ?? null,
message,
code: domain.code ?? null,
category: domain.category ?? null,
}
}
function formatErrorDomainMeta(domain: NormalizedErrorDomain): string {
const parts: string[] = []
if (domain.status_code != null) parts.push(`HTTP ${domain.status_code}`)
if (domain.type) parts.push(domain.type)
if (domain.source) parts.push(`source=${domain.source}`)
return parts.join(' · ')
}
function handleTraceState(state: { loaded: boolean, hasTrace: boolean }) {
timelineLoaded.value = state.loaded
timelineHasTrace.value = state.hasTrace
@@ -987,10 +908,7 @@ watch(activeTab, (newTab) => {
}
})
// 检测暗色模式
const isDark = computed(() => {
return document.documentElement.classList.contains('dark')
})
const { isDark } = useDarkMode()
const traceRequestMetadata = computed<Record<string, unknown> | null>(() => {
const meta = detail.value?.metadata
@@ -1021,27 +939,6 @@ const metadataPanelData = computed<Record<string, unknown> | null>(() => {
return Object.keys(merged).length > 0 ? merged : null
})
const normalizedClientError = computed(() =>
normalizeErrorDomain(detail.value?.errors?.client_error ?? detail.value?.client_error),
)
const normalizedUpstreamError = computed(() =>
normalizeErrorDomain(detail.value?.errors?.upstream_error ?? detail.value?.upstream_error),
)
const displayClientErrorMessage = computed(() =>
normalizedClientError.value?.message ?? '',
)
const hasVisibleErrorCards = computed(() =>
Boolean(displayClientErrorMessage.value || normalizedUpstreamError.value),
)
const visibleErrorCardCount = computed(() =>
(displayClientErrorMessage.value ? 1 : 0)
+ (normalizedUpstreamError.value ? 1 : 0),
)
const settlementInfo = computed<JsonRecord | null>(() =>
asRecord(detail.value?.settlement ?? null),
)

87
frontend/src/features/usage/components/__tests__/HorizontalRequestTimeline.spec.ts
View File

@@ -45,8 +45,14 @@ vi.mock('../JsonContentPanel.vue', async () => {
return {
default: defineComponent({
name: 'JsonContentPanelStub',
setup() {
return () => h('div')
props: {
data: {
type: null,
default: null,
},
},
setup(props) {
return () => h('pre', JSON.stringify(props.data))
},
}),
}
@@ -321,4 +327,81 @@ describe('HorizontalRequestTimeline', () => {
const requestPathCode = root.querySelector<HTMLElement>('.request-path-code')
expect(requestPathCode?.textContent).toContain('/v1beta/models/gemini-2.5-pro:generateContent?alt=sse')
})
it('shows request path from trace payload', async () => {
const trace: RequestTrace = {
...buildTrace([
buildCandidate({
id: 'cand-trace-path',
provider_id: 'provider-path',
provider_name: 'Provider Path',
key_id: 'key-path',
key_name: 'Path Key',
candidate_index: 0,
status: 'failed',
}),
]),
request_path: '/v1/images/generations',
}
const root = mountTimeline(trace)
await nextTick()
expect(root.textContent).toContain('请求路径')
const requestPathCode = root.querySelector<HTMLElement>('.request-path-code')
expect(requestPathCode?.textContent).toContain('/v1/images/generations')
})
it('shows upstream response JSON inside the error block on trace nodes', async () => {
const trace = buildTrace([
buildCandidate({
id: 'cand-upstream-response',
provider_id: 'provider-upstream',
provider_name: 'Provider Upstream',
key_id: 'key-upstream',
key_name: 'Upstream Key',
candidate_index: 0,
status: 'failed',
error_message: 'execution runtime stream returned non-success status 302',
extra_data: {
upstream_response: {
status_code: 302,
headers: { location: '/' },
},
error_flow: {
source: 'upstream_response',
status_code: 302,
classification: 'use_default',
decision: 'use_default',
propagation: 'none',
retryable: false,
safe_to_expose: false,
message: 'execution runtime stream returned non-success status 302',
},
client_response: {
status_code: 502,
headers: { 'content-type': 'application/json' },
},
},
}),
])
const root = mountTimeline(trace)
await nextTick()
expect(root.textContent).toContain('错误信息')
expect(root.textContent).toContain('HTTP 302')
expect(root.textContent).not.toContain('上游返回非成功状态 302')
expect(root.querySelector('.error-block .error-json')?.textContent).toContain('"status_code":302')
expect(root.querySelector('.error-block .error-json')?.textContent).toContain('"headers"')
expect(root.textContent).not.toContain('上游真实响应')
expect(root.textContent).not.toContain('execution runtime stream returned non-success status 302')
expect(root.textContent).not.toContain('真实请求错误')
expect(root.textContent).not.toContain('返回客户端响应')
expect(root.textContent).not.toContain('上游响应')
expect(root.textContent).not.toContain('默认处理')
expect(root.textContent).not.toContain('none')
expect(root.textContent).not.toContain('不再重试')
expect(root.textContent).not.toContain('该错误被标记为敏感上游错误')
})
})