mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 01:10:23 +08:00
Improve request trace upstream diagnostics
This commit is contained in:
1
Cargo.lock
generated
1
Cargo.lock
generated
@@ -53,6 +53,7 @@ dependencies = [
|
|||||||
"serde_json",
|
"serde_json",
|
||||||
"sha1",
|
"sha1",
|
||||||
"sha2",
|
"sha2",
|
||||||
|
"url",
|
||||||
"uuid",
|
"uuid",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|||||||
@@ -112,6 +112,8 @@ pub(crate) async fn maybe_build_local_same_format_provider_decision_payload_for_
|
|||||||
provider_request_method: Some(serde_json::Value::Null),
|
provider_request_method: Some(serde_json::Value::Null),
|
||||||
provider_request_headers: Some(&resolved.provider_request_headers),
|
provider_request_headers: Some(&resolved.provider_request_headers),
|
||||||
original_headers: &parts.headers,
|
original_headers: &parts.headers,
|
||||||
|
request_path: Some(parts.uri.path()),
|
||||||
|
request_query_string: parts.uri.query(),
|
||||||
request_origin: Some(crate::ai_serving::request_origin_from_parts(parts)),
|
request_origin: Some(crate::ai_serving::request_origin_from_parts(parts)),
|
||||||
original_request_body_json: Some(body_json),
|
original_request_body_json: Some(body_json),
|
||||||
original_request_body_base64: None,
|
original_request_body_base64: None,
|
||||||
|
|||||||
@@ -9,7 +9,11 @@ use aether_ai_serving::{
|
|||||||
use aether_scheduler_core::{ClientSessionAffinity, SchedulerRankingOutcome};
|
use aether_scheduler_core::{ClientSessionAffinity, SchedulerRankingOutcome};
|
||||||
use serde_json::{Map, Value};
|
use serde_json::{Map, Value};
|
||||||
|
|
||||||
use crate::ai_serving::{request_origin_from_headers, ExecutionRuntimeAuthContext, RequestOrigin};
|
use crate::ai_serving::{
|
||||||
|
request_origin_from_headers, request_path_implies_stream_request, sanitize_request_path,
|
||||||
|
sanitize_request_path_and_query, sanitize_request_query_string, ExecutionRuntimeAuthContext,
|
||||||
|
RequestOrigin,
|
||||||
|
};
|
||||||
use crate::client_session_affinity::{
|
use crate::client_session_affinity::{
|
||||||
client_session_affinity_report_context_value, CLIENT_SESSION_AFFINITY_REPORT_CONTEXT_FIELD,
|
client_session_affinity_report_context_value, CLIENT_SESSION_AFFINITY_REPORT_CONTEXT_FIELD,
|
||||||
};
|
};
|
||||||
@@ -40,6 +44,8 @@ pub(crate) struct LocalExecutionReportContextParts<'a> {
|
|||||||
pub(crate) provider_request_method: Option<Value>,
|
pub(crate) provider_request_method: Option<Value>,
|
||||||
pub(crate) provider_request_headers: Option<&'a BTreeMap<String, String>>,
|
pub(crate) provider_request_headers: Option<&'a BTreeMap<String, String>>,
|
||||||
pub(crate) original_headers: &'a http::HeaderMap,
|
pub(crate) original_headers: &'a http::HeaderMap,
|
||||||
|
pub(crate) request_path: Option<&'a str>,
|
||||||
|
pub(crate) request_query_string: Option<&'a str>,
|
||||||
pub(crate) request_origin: Option<RequestOrigin>,
|
pub(crate) request_origin: Option<RequestOrigin>,
|
||||||
pub(crate) original_request_body_json: Option<&'a Value>,
|
pub(crate) original_request_body_json: Option<&'a Value>,
|
||||||
pub(crate) original_request_body_base64: Option<&'a str>,
|
pub(crate) original_request_body_base64: Option<&'a str>,
|
||||||
@@ -80,6 +86,15 @@ pub(crate) fn build_local_execution_report_context(
|
|||||||
{
|
{
|
||||||
merge_incoming_tls_fingerprint(&mut extra_fields, incoming_tls);
|
merge_incoming_tls_fingerprint(&mut extra_fields, incoming_tls);
|
||||||
}
|
}
|
||||||
|
insert_request_path_fields(
|
||||||
|
&mut extra_fields,
|
||||||
|
parts.request_path,
|
||||||
|
parts.request_query_string,
|
||||||
|
);
|
||||||
|
let client_requested_stream = parts.client_requested_stream
|
||||||
|
|| parts
|
||||||
|
.request_path
|
||||||
|
.is_some_and(request_path_implies_stream_request);
|
||||||
|
|
||||||
build_ai_execution_report_context(AiExecutionReportContextParts {
|
build_ai_execution_report_context(AiExecutionReportContextParts {
|
||||||
auth_context: parts.auth_context,
|
auth_context: parts.auth_context,
|
||||||
@@ -113,7 +128,7 @@ pub(crate) fn build_local_execution_report_context(
|
|||||||
client_ip,
|
client_ip,
|
||||||
user_agent,
|
user_agent,
|
||||||
},
|
},
|
||||||
client_requested_stream: parts.client_requested_stream,
|
client_requested_stream,
|
||||||
upstream_is_stream: parts.upstream_is_stream,
|
upstream_is_stream: parts.upstream_is_stream,
|
||||||
has_envelope: parts.has_envelope,
|
has_envelope: parts.has_envelope,
|
||||||
needs_conversion: parts.needs_conversion,
|
needs_conversion: parts.needs_conversion,
|
||||||
@@ -121,6 +136,30 @@ pub(crate) fn build_local_execution_report_context(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn insert_request_path_fields(
|
||||||
|
extra_fields: &mut Map<String, Value>,
|
||||||
|
request_path: Option<&str>,
|
||||||
|
request_query_string: Option<&str>,
|
||||||
|
) {
|
||||||
|
let Some(path) = request_path.and_then(sanitize_request_path) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let query = request_query_string.and_then(sanitize_request_query_string);
|
||||||
|
let path_and_query = sanitize_request_path_and_query(path.as_str(), query.as_deref())
|
||||||
|
.unwrap_or_else(|| path.clone());
|
||||||
|
extra_fields
|
||||||
|
.entry("request_path".to_string())
|
||||||
|
.or_insert_with(|| Value::String(path.clone()));
|
||||||
|
if let Some(query) = query.clone() {
|
||||||
|
extra_fields
|
||||||
|
.entry("request_query_string".to_string())
|
||||||
|
.or_insert_with(|| Value::String(query.to_string()));
|
||||||
|
}
|
||||||
|
extra_fields
|
||||||
|
.entry("request_path_and_query".to_string())
|
||||||
|
.or_insert_with(|| Value::String(path_and_query));
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn provider_stream_event_api_format_for_provider_type(
|
pub(crate) fn provider_stream_event_api_format_for_provider_type(
|
||||||
provider_type: &str,
|
provider_type: &str,
|
||||||
) -> Option<&'static str> {
|
) -> Option<&'static str> {
|
||||||
@@ -226,6 +265,8 @@ mod tests {
|
|||||||
provider_request_method: None,
|
provider_request_method: None,
|
||||||
provider_request_headers: Some(&provider_request_headers),
|
provider_request_headers: Some(&provider_request_headers),
|
||||||
original_headers: &original_headers,
|
original_headers: &original_headers,
|
||||||
|
request_path: Some("/v1/chat/completions"),
|
||||||
|
request_query_string: Some("debug=true&limit=10"),
|
||||||
request_origin: Some(RequestOrigin {
|
request_origin: Some(RequestOrigin {
|
||||||
client_ip: Some("203.0.113.8".to_string()),
|
client_ip: Some("203.0.113.8".to_string()),
|
||||||
user_agent: Some("Claude-Code/1.0".to_string()),
|
user_agent: Some("Claude-Code/1.0".to_string()),
|
||||||
@@ -255,6 +296,77 @@ mod tests {
|
|||||||
"session_key": "account=account-1;session=session-1"
|
"session_key": "account=account-1;session=session-1"
|
||||||
})
|
})
|
||||||
);
|
);
|
||||||
|
assert_eq!(report_context["request_path"], "/v1/chat/completions");
|
||||||
|
assert_eq!(report_context["request_query_string"], "limit=10");
|
||||||
|
assert_eq!(
|
||||||
|
report_context["request_path_and_query"],
|
||||||
|
"/v1/chat/completions?limit=10"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn local_execution_report_context_treats_stream_generate_content_path_as_client_stream() {
|
||||||
|
let auth_context = ExecutionRuntimeAuthContext {
|
||||||
|
user_id: "user-1".to_string(),
|
||||||
|
api_key_id: "api-key-1".to_string(),
|
||||||
|
username: None,
|
||||||
|
api_key_name: None,
|
||||||
|
balance_remaining: None,
|
||||||
|
access_allowed: true,
|
||||||
|
api_key_is_standalone: false,
|
||||||
|
};
|
||||||
|
let original_headers = http::HeaderMap::new();
|
||||||
|
let provider_request_headers = BTreeMap::new();
|
||||||
|
|
||||||
|
let report_context =
|
||||||
|
build_local_execution_report_context(LocalExecutionReportContextParts {
|
||||||
|
auth_context: &auth_context,
|
||||||
|
request_id: "trace-1",
|
||||||
|
candidate_id: "candidate-1",
|
||||||
|
attempt_identity: ExecutionAttemptIdentity::new(0, 0),
|
||||||
|
model: "gemini-3.1-flash-image-preview",
|
||||||
|
provider_name: "Gemini",
|
||||||
|
provider_id: "provider-1",
|
||||||
|
endpoint_id: "endpoint-1",
|
||||||
|
key_id: "key-1",
|
||||||
|
key_name: None,
|
||||||
|
model_id: None,
|
||||||
|
global_model_id: None,
|
||||||
|
global_model_name: None,
|
||||||
|
provider_api_format: "gemini:generate_content",
|
||||||
|
client_api_format: "gemini:generate_content",
|
||||||
|
mapped_model: None,
|
||||||
|
candidate_group_id: None,
|
||||||
|
ranking: None,
|
||||||
|
upstream_url: None,
|
||||||
|
header_rules: None,
|
||||||
|
body_rules: None,
|
||||||
|
provider_request_method: None,
|
||||||
|
provider_request_headers: Some(&provider_request_headers),
|
||||||
|
original_headers: &original_headers,
|
||||||
|
request_path: Some(
|
||||||
|
"/v1beta/models/gemini-3.1-flash-image-preview:streamGenerateContent",
|
||||||
|
),
|
||||||
|
request_query_string: Some("key=secret&alt=sse"),
|
||||||
|
request_origin: None,
|
||||||
|
original_request_body_json: Some(&json!({
|
||||||
|
"contents": [{"role": "user", "parts": [{"text": "hi"}]}]
|
||||||
|
})),
|
||||||
|
original_request_body_base64: None,
|
||||||
|
client_session_affinity: None,
|
||||||
|
client_requested_stream: false,
|
||||||
|
upstream_is_stream: true,
|
||||||
|
has_envelope: false,
|
||||||
|
needs_conversion: false,
|
||||||
|
extra_fields: Map::new(),
|
||||||
|
});
|
||||||
|
|
||||||
|
assert_eq!(report_context["client_requested_stream"], true);
|
||||||
|
assert_eq!(report_context["request_query_string"], "alt=sse");
|
||||||
|
assert_eq!(
|
||||||
|
report_context["request_path_and_query"],
|
||||||
|
"/v1beta/models/gemini-3.1-flash-image-preview:streamGenerateContent?alt=sse"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -300,6 +412,8 @@ mod tests {
|
|||||||
provider_request_method: None,
|
provider_request_method: None,
|
||||||
provider_request_headers: Some(&provider_request_headers),
|
provider_request_headers: Some(&provider_request_headers),
|
||||||
original_headers: &original_headers,
|
original_headers: &original_headers,
|
||||||
|
request_path: None,
|
||||||
|
request_query_string: None,
|
||||||
request_origin: None,
|
request_origin: None,
|
||||||
original_request_body_json: Some(&json!({"model": "gpt-5"})),
|
original_request_body_json: Some(&json!({"model": "gpt-5"})),
|
||||||
original_request_body_base64: None,
|
original_request_body_base64: None,
|
||||||
|
|||||||
@@ -94,6 +94,8 @@ pub(super) async fn maybe_build_local_gemini_files_decision_payload_for_candidat
|
|||||||
provider_request_method: None,
|
provider_request_method: None,
|
||||||
provider_request_headers: None,
|
provider_request_headers: None,
|
||||||
original_headers: &parts.headers,
|
original_headers: &parts.headers,
|
||||||
|
request_path: Some(parts.uri.path()),
|
||||||
|
request_query_string: parts.uri.query(),
|
||||||
request_origin: Some(crate::ai_serving::request_origin_from_parts(parts)),
|
request_origin: Some(crate::ai_serving::request_origin_from_parts(parts)),
|
||||||
original_request_body_json: Some(body_json),
|
original_request_body_json: Some(body_json),
|
||||||
original_request_body_base64: resolved.provider_request_body_base64.as_deref(),
|
original_request_body_base64: resolved.provider_request_body_base64.as_deref(),
|
||||||
|
|||||||
@@ -114,6 +114,8 @@ pub(super) async fn maybe_build_local_openai_image_decision_payload_for_candidat
|
|||||||
provider_request_method: Some(serde_json::Value::String(parts.method.to_string())),
|
provider_request_method: Some(serde_json::Value::String(parts.method.to_string())),
|
||||||
provider_request_headers: Some(&resolved.provider_request_headers),
|
provider_request_headers: Some(&resolved.provider_request_headers),
|
||||||
original_headers: &parts.headers,
|
original_headers: &parts.headers,
|
||||||
|
request_path: Some(parts.uri.path()),
|
||||||
|
request_query_string: parts.uri.query(),
|
||||||
request_origin: Some(crate::ai_serving::request_origin_from_parts(parts)),
|
request_origin: Some(crate::ai_serving::request_origin_from_parts(parts)),
|
||||||
original_request_body_json: Some(body_json),
|
original_request_body_json: Some(body_json),
|
||||||
original_request_body_base64: body_base64,
|
original_request_body_base64: body_base64,
|
||||||
|
|||||||
@@ -75,6 +75,8 @@ pub(super) async fn maybe_build_local_video_create_decision_payload_for_candidat
|
|||||||
provider_request_method: None,
|
provider_request_method: None,
|
||||||
provider_request_headers: None,
|
provider_request_headers: None,
|
||||||
original_headers: &parts.headers,
|
original_headers: &parts.headers,
|
||||||
|
request_path: Some(parts.uri.path()),
|
||||||
|
request_query_string: parts.uri.query(),
|
||||||
request_origin: Some(crate::ai_serving::request_origin_from_parts(parts)),
|
request_origin: Some(crate::ai_serving::request_origin_from_parts(parts)),
|
||||||
original_request_body_json: Some(body_json),
|
original_request_body_json: Some(body_json),
|
||||||
original_request_body_base64: None,
|
original_request_body_base64: None,
|
||||||
|
|||||||
@@ -120,6 +120,8 @@ pub(super) async fn maybe_build_local_standard_decision_payload_for_candidate(
|
|||||||
provider_request_method: Some(serde_json::Value::Null),
|
provider_request_method: Some(serde_json::Value::Null),
|
||||||
provider_request_headers: Some(&resolved.provider_request_headers),
|
provider_request_headers: Some(&resolved.provider_request_headers),
|
||||||
original_headers: &parts.headers,
|
original_headers: &parts.headers,
|
||||||
|
request_path: Some(parts.uri.path()),
|
||||||
|
request_query_string: parts.uri.query(),
|
||||||
request_origin: Some(crate::ai_serving::request_origin_from_parts(parts)),
|
request_origin: Some(crate::ai_serving::request_origin_from_parts(parts)),
|
||||||
original_request_body_json: Some(body_json),
|
original_request_body_json: Some(body_json),
|
||||||
original_request_body_base64: None,
|
original_request_body_base64: None,
|
||||||
|
|||||||
@@ -107,6 +107,8 @@ pub(crate) async fn maybe_build_local_openai_chat_decision_payload_for_candidate
|
|||||||
provider_request_method: Some(serde_json::Value::Null),
|
provider_request_method: Some(serde_json::Value::Null),
|
||||||
provider_request_headers: Some(&resolved.provider_request_headers),
|
provider_request_headers: Some(&resolved.provider_request_headers),
|
||||||
original_headers: &parts.headers,
|
original_headers: &parts.headers,
|
||||||
|
request_path: Some(parts.uri.path()),
|
||||||
|
request_query_string: parts.uri.query(),
|
||||||
request_origin: Some(crate::ai_serving::request_origin_from_parts(parts)),
|
request_origin: Some(crate::ai_serving::request_origin_from_parts(parts)),
|
||||||
original_request_body_json: Some(body_json),
|
original_request_body_json: Some(body_json),
|
||||||
original_request_body_base64: None,
|
original_request_body_base64: None,
|
||||||
|
|||||||
@@ -105,6 +105,8 @@ pub(crate) async fn maybe_build_local_openai_responses_decision_payload_for_cand
|
|||||||
provider_request_method: Some(serde_json::Value::Null),
|
provider_request_method: Some(serde_json::Value::Null),
|
||||||
provider_request_headers: Some(&resolved.provider_request_headers),
|
provider_request_headers: Some(&resolved.provider_request_headers),
|
||||||
original_headers: &parts.headers,
|
original_headers: &parts.headers,
|
||||||
|
request_path: Some(parts.uri.path()),
|
||||||
|
request_query_string: parts.uri.query(),
|
||||||
request_origin: Some(crate::ai_serving::request_origin_from_parts(parts)),
|
request_origin: Some(crate::ai_serving::request_origin_from_parts(parts)),
|
||||||
original_request_body_json: Some(body_json),
|
original_request_body_json: Some(body_json),
|
||||||
original_request_body_base64: None,
|
original_request_body_base64: None,
|
||||||
|
|||||||
@@ -70,7 +70,8 @@ pub(crate) use aether_ai_formats::api::{
|
|||||||
provider_adaptation_should_unwrap_stream_envelope,
|
provider_adaptation_should_unwrap_stream_envelope,
|
||||||
provider_private_response_allows_sync_finalize, request_candidate_api_format_preference,
|
provider_private_response_allows_sync_finalize, request_candidate_api_format_preference,
|
||||||
request_candidate_api_formats, request_conversion_kind,
|
request_candidate_api_formats, request_conversion_kind,
|
||||||
request_conversion_requires_enable_flag, resolve_claude_stream_spec, resolve_claude_sync_spec,
|
request_conversion_requires_enable_flag, request_path_implies_stream_request,
|
||||||
|
resolve_claude_stream_spec, resolve_claude_sync_spec,
|
||||||
resolve_execution_runtime_stream_plan_kind, resolve_execution_runtime_sync_plan_kind,
|
resolve_execution_runtime_stream_plan_kind, resolve_execution_runtime_sync_plan_kind,
|
||||||
resolve_finalize_stream_rewrite_mode, resolve_gemini_files_stream_spec,
|
resolve_finalize_stream_rewrite_mode, resolve_gemini_files_stream_spec,
|
||||||
resolve_gemini_files_sync_spec, resolve_gemini_stream_spec, resolve_gemini_sync_spec,
|
resolve_gemini_files_sync_spec, resolve_gemini_stream_spec, resolve_gemini_sync_spec,
|
||||||
@@ -79,11 +80,12 @@ pub(crate) use aether_ai_formats::api::{
|
|||||||
resolve_local_video_sync_spec, resolve_openai_chat_max_tokens,
|
resolve_local_video_sync_spec, resolve_openai_chat_max_tokens,
|
||||||
resolve_openai_responses_stream_spec, resolve_openai_responses_sync_spec,
|
resolve_openai_responses_stream_spec, resolve_openai_responses_sync_spec,
|
||||||
resolve_requested_gemini_image_model_for_request,
|
resolve_requested_gemini_image_model_for_request,
|
||||||
resolve_requested_openai_image_model_for_request, stream_body_contains_error_event,
|
resolve_requested_openai_image_model_for_request, sanitize_request_path,
|
||||||
supports_stream_execution_decision_kind, supports_sync_execution_decision_kind,
|
sanitize_request_path_and_query, sanitize_request_query_string,
|
||||||
sync_chat_response_conversion_kind, sync_cli_response_conversion_kind,
|
stream_body_contains_error_event, supports_stream_execution_decision_kind,
|
||||||
transform_provider_private_stream_line, value_as_u64, AiControlPlanRequest,
|
supports_sync_execution_decision_kind, sync_chat_response_conversion_kind,
|
||||||
AiSurfaceFinalizeError, AiSurfaceStreamRewriter, CanonicalStreamFrame,
|
sync_cli_response_conversion_kind, transform_provider_private_stream_line, value_as_u64,
|
||||||
|
AiControlPlanRequest, AiSurfaceFinalizeError, AiSurfaceStreamRewriter, CanonicalStreamFrame,
|
||||||
ChatGptWebImageRequestError, ClaudeClientEmitter, ClaudeProviderState,
|
ChatGptWebImageRequestError, ClaudeClientEmitter, ClaudeProviderState,
|
||||||
ExecutionRuntimeAuthContext, FinalizeStreamRewriteMode, FormatContext, GeminiClientEmitter,
|
ExecutionRuntimeAuthContext, FinalizeStreamRewriteMode, FormatContext, GeminiClientEmitter,
|
||||||
GeminiImageRequestForOpenAi, GeminiProviderState, KiroToClaudeCliStreamState,
|
GeminiImageRequestForOpenAi, GeminiProviderState, KiroToClaudeCliStreamState,
|
||||||
|
|||||||
@@ -1,23 +1,17 @@
|
|||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
use aether_contracts::{StreamFrame, StreamFramePayload};
|
use aether_contracts::{StreamFrame, StreamFramePayload};
|
||||||
use axum::body::Body;
|
use axum::http::StatusCode;
|
||||||
use axum::http::Response;
|
|
||||||
use base64::Engine as _;
|
use base64::Engine as _;
|
||||||
use futures_util::StreamExt;
|
use futures_util::StreamExt;
|
||||||
use serde_json::json;
|
use serde_json::{json, Map, Value};
|
||||||
use tokio_util::codec::{FramedRead, LinesCodec};
|
use tokio_util::codec::{FramedRead, LinesCodec};
|
||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
|
|
||||||
use crate::api::response::build_client_response_from_parts;
|
|
||||||
use crate::control::GatewayControlDecision;
|
|
||||||
use crate::execution_runtime::ndjson::decode_stream_frame_ndjson;
|
use crate::execution_runtime::ndjson::decode_stream_frame_ndjson;
|
||||||
use crate::execution_runtime::submission::{has_nested_error, strip_utf8_bom_and_ws};
|
use crate::execution_runtime::submission::{has_nested_error, strip_utf8_bom_and_ws};
|
||||||
use crate::GatewayError;
|
use crate::GatewayError;
|
||||||
use crate::{
|
use crate::{MAX_ERROR_BODY_BYTES, MAX_STREAM_PREFETCH_FRAMES};
|
||||||
GEMINI_FILES_DOWNLOAD_PLAN_KIND, MAX_ERROR_BODY_BYTES, MAX_STREAM_PREFETCH_FRAMES,
|
|
||||||
OPENAI_VIDEO_CONTENT_PLAN_KIND,
|
|
||||||
};
|
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub(super) enum StreamPrefetchInspection {
|
pub(super) enum StreamPrefetchInspection {
|
||||||
@@ -51,6 +45,88 @@ pub(super) fn decode_stream_error_body(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn header_value_case_insensitive<'a>(
|
||||||
|
headers: &'a BTreeMap<String, String>,
|
||||||
|
name: &str,
|
||||||
|
) -> Option<&'a str> {
|
||||||
|
headers
|
||||||
|
.iter()
|
||||||
|
.find(|(key, _)| key.eq_ignore_ascii_case(name))
|
||||||
|
.map(|(_, value)| value.as_str())
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remove_header_case_insensitive(headers: &mut BTreeMap<String, String>, name: &str) {
|
||||||
|
let keys = headers
|
||||||
|
.keys()
|
||||||
|
.filter(|key| key.eq_ignore_ascii_case(name))
|
||||||
|
.cloned()
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
for key in keys {
|
||||||
|
headers.remove(&key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn should_synthesize_non_success_stream_error_body(
|
||||||
|
status_code: u16,
|
||||||
|
error_body: &[u8],
|
||||||
|
) -> bool {
|
||||||
|
!(200..300).contains(&status_code)
|
||||||
|
&& ((300..400).contains(&status_code) || error_body.is_empty())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn build_synthetic_non_success_stream_error_body(
|
||||||
|
status_code: u16,
|
||||||
|
headers: &BTreeMap<String, String>,
|
||||||
|
) -> Value {
|
||||||
|
let mut error = Map::from_iter([
|
||||||
|
(
|
||||||
|
"type".to_string(),
|
||||||
|
Value::String("execution_runtime_non_success_status".to_string()),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"message".to_string(),
|
||||||
|
Value::String(format!(
|
||||||
|
"execution runtime stream returned non-success status {status_code}"
|
||||||
|
)),
|
||||||
|
),
|
||||||
|
("code".to_string(), Value::from(status_code)),
|
||||||
|
("upstream_status".to_string(), Value::from(status_code)),
|
||||||
|
]);
|
||||||
|
if let Some(location) = header_value_case_insensitive(headers, "location") {
|
||||||
|
error.insert("location".to_string(), Value::String(location.to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
Value::Object(Map::from_iter([(
|
||||||
|
"error".to_string(),
|
||||||
|
Value::Object(error),
|
||||||
|
)]))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn synthetic_error_response_headers(
|
||||||
|
mut headers: BTreeMap<String, String>,
|
||||||
|
) -> BTreeMap<String, String> {
|
||||||
|
remove_header_case_insensitive(&mut headers, "content-encoding");
|
||||||
|
remove_header_case_insensitive(&mut headers, "content-length");
|
||||||
|
remove_header_case_insensitive(&mut headers, "content-type");
|
||||||
|
remove_header_case_insensitive(&mut headers, "location");
|
||||||
|
headers.insert("content-type".to_string(), "application/json".to_string());
|
||||||
|
headers
|
||||||
|
}
|
||||||
|
|
||||||
|
fn client_error_status_code_for_upstream_status(status_code: u16) -> u16 {
|
||||||
|
if (300..400).contains(&status_code) || status_code < 200 {
|
||||||
|
StatusCode::BAD_GATEWAY.as_u16()
|
||||||
|
} else {
|
||||||
|
status_code
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn stream_client_error_status_code_for_upstream_status(status_code: u16) -> u16 {
|
||||||
|
client_error_status_code_for_upstream_status(status_code)
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) fn inspect_prefetched_stream_body(
|
pub(super) fn inspect_prefetched_stream_body(
|
||||||
headers: &BTreeMap<String, String>,
|
headers: &BTreeMap<String, String>,
|
||||||
body: &[u8],
|
body: &[u8],
|
||||||
@@ -166,62 +242,3 @@ where
|
|||||||
}
|
}
|
||||||
Ok(None)
|
Ok(None)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn build_execution_runtime_error_response(
|
|
||||||
trace_id: &str,
|
|
||||||
decision: &GatewayControlDecision,
|
|
||||||
plan_kind: &str,
|
|
||||||
status_code: u16,
|
|
||||||
headers: BTreeMap<String, String>,
|
|
||||||
error_body: Vec<u8>,
|
|
||||||
) -> Result<Response<Body>, GatewayError> {
|
|
||||||
let content_type = headers
|
|
||||||
.get("content-type")
|
|
||||||
.map(|value| value.to_ascii_lowercase())
|
|
||||||
.unwrap_or_default();
|
|
||||||
|
|
||||||
if plan_kind == GEMINI_FILES_DOWNLOAD_PLAN_KIND && !content_type.starts_with("application/json")
|
|
||||||
{
|
|
||||||
let wrapped = serde_json::to_vec(&json!({
|
|
||||||
"error": String::from_utf8_lossy(&error_body).to_string(),
|
|
||||||
}))
|
|
||||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
|
||||||
let wrapped_headers =
|
|
||||||
BTreeMap::from([("content-type".to_string(), "application/json".to_string())]);
|
|
||||||
return build_client_response_from_parts(
|
|
||||||
status_code,
|
|
||||||
&wrapped_headers,
|
|
||||||
Body::from(wrapped),
|
|
||||||
trace_id,
|
|
||||||
Some(decision),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
if plan_kind == OPENAI_VIDEO_CONTENT_PLAN_KIND && !content_type.starts_with("application/json")
|
|
||||||
{
|
|
||||||
let wrapped = serde_json::to_vec(&json!({
|
|
||||||
"error": {
|
|
||||||
"type": "upstream_error",
|
|
||||||
"message": "Video not available",
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
|
||||||
let wrapped_headers =
|
|
||||||
BTreeMap::from([("content-type".to_string(), "application/json".to_string())]);
|
|
||||||
return build_client_response_from_parts(
|
|
||||||
status_code,
|
|
||||||
&wrapped_headers,
|
|
||||||
Body::from(wrapped),
|
|
||||||
trace_id,
|
|
||||||
Some(decision),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
build_client_response_from_parts(
|
|
||||||
status_code,
|
|
||||||
&headers,
|
|
||||||
Body::from(error_body),
|
|
||||||
trace_id,
|
|
||||||
Some(decision),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -23,15 +23,18 @@ use axum::http::Response;
|
|||||||
use base64::Engine as _;
|
use base64::Engine as _;
|
||||||
use futures_util::stream::BoxStream;
|
use futures_util::stream::BoxStream;
|
||||||
use futures_util::{StreamExt, TryStreamExt};
|
use futures_util::{StreamExt, TryStreamExt};
|
||||||
use serde_json::Value;
|
use serde_json::{json, Value};
|
||||||
use tokio::sync::mpsc;
|
use tokio::sync::mpsc;
|
||||||
use tokio_util::codec::{FramedRead, LinesCodec};
|
use tokio_util::codec::{FramedRead, LinesCodec};
|
||||||
use tokio_util::io::StreamReader;
|
use tokio_util::io::StreamReader;
|
||||||
use tracing::{debug, info, warn};
|
use tracing::{debug, info, warn};
|
||||||
|
|
||||||
use super::error::{
|
use super::error::{
|
||||||
build_execution_runtime_error_response, collect_error_body, decode_stream_error_body,
|
build_synthetic_non_success_stream_error_body, collect_error_body, decode_stream_error_body,
|
||||||
inspect_prefetched_stream_body, read_next_frame, StreamPrefetchInspection,
|
inspect_prefetched_stream_body, read_next_frame,
|
||||||
|
should_synthesize_non_success_stream_error_body,
|
||||||
|
stream_client_error_status_code_for_upstream_status, synthetic_error_response_headers,
|
||||||
|
StreamPrefetchInspection,
|
||||||
};
|
};
|
||||||
#[path = "execution_failures.rs"]
|
#[path = "execution_failures.rs"]
|
||||||
mod execution_failures;
|
mod execution_failures;
|
||||||
@@ -76,9 +79,10 @@ use crate::execution_runtime::{MAX_STREAM_PREFETCH_BYTES, MAX_STREAM_PREFETCH_FR
|
|||||||
use crate::log_ids::short_request_id;
|
use crate::log_ids::short_request_id;
|
||||||
use crate::orchestration::{
|
use crate::orchestration::{
|
||||||
apply_local_execution_effect, build_local_error_flow_metadata, with_error_flow_report_context,
|
apply_local_execution_effect, build_local_error_flow_metadata, with_error_flow_report_context,
|
||||||
LocalAdaptiveRateLimitEffect, LocalAdaptiveSuccessEffect, LocalAttemptFailureEffect,
|
with_upstream_response_report_context, LocalAdaptiveRateLimitEffect,
|
||||||
LocalExecutionEffect, LocalExecutionEffectContext, LocalHealthFailureEffect,
|
LocalAdaptiveSuccessEffect, LocalAttemptFailureEffect, LocalExecutionEffect,
|
||||||
LocalHealthSuccessEffect, LocalOAuthInvalidationEffect, LocalPoolErrorEffect,
|
LocalExecutionEffectContext, LocalHealthFailureEffect, LocalHealthSuccessEffect,
|
||||||
|
LocalOAuthInvalidationEffect, LocalPoolErrorEffect,
|
||||||
};
|
};
|
||||||
use crate::request_candidate_runtime::{
|
use crate::request_candidate_runtime::{
|
||||||
ensure_execution_request_candidate_slot, record_local_request_candidate_status,
|
ensure_execution_request_candidate_slot, record_local_request_candidate_status,
|
||||||
@@ -86,7 +90,9 @@ use crate::request_candidate_runtime::{
|
|||||||
};
|
};
|
||||||
use crate::usage::submit_stream_report;
|
use crate::usage::submit_stream_report;
|
||||||
use crate::usage::{GatewayStreamReportRequest, GatewaySyncReportRequest};
|
use crate::usage::{GatewayStreamReportRequest, GatewaySyncReportRequest};
|
||||||
use crate::{AppState, GatewayError};
|
use crate::{
|
||||||
|
AppState, GatewayError, GEMINI_FILES_DOWNLOAD_PLAN_KIND, OPENAI_VIDEO_CONTENT_PLAN_KIND,
|
||||||
|
};
|
||||||
|
|
||||||
fn record_sync_terminal_usage(
|
fn record_sync_terminal_usage(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
@@ -124,6 +130,52 @@ fn build_stream_sync_payload(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
fn build_stream_error_sync_payload(
|
||||||
|
trace_id: &str,
|
||||||
|
report_kind: String,
|
||||||
|
report_context: Option<Value>,
|
||||||
|
upstream_status_code: u16,
|
||||||
|
provider_headers: BTreeMap<String, String>,
|
||||||
|
provider_body_json: Option<Value>,
|
||||||
|
provider_body_base64: Option<String>,
|
||||||
|
client_headers: BTreeMap<String, String>,
|
||||||
|
client_body_json: Option<Value>,
|
||||||
|
telemetry: Option<ExecutionTelemetry>,
|
||||||
|
) -> GatewaySyncReportRequest {
|
||||||
|
let client_status_code =
|
||||||
|
stream_client_error_status_code_for_upstream_status(upstream_status_code);
|
||||||
|
let mut report_context = report_context;
|
||||||
|
if client_status_code != upstream_status_code || client_headers != provider_headers {
|
||||||
|
let mut object = match report_context {
|
||||||
|
Some(Value::Object(object)) => object,
|
||||||
|
Some(other) => serde_json::Map::from_iter([("seed".to_string(), other)]),
|
||||||
|
None => serde_json::Map::new(),
|
||||||
|
};
|
||||||
|
object.insert(
|
||||||
|
"client_response_status_code".to_string(),
|
||||||
|
Value::from(client_status_code),
|
||||||
|
);
|
||||||
|
object.insert(
|
||||||
|
"client_response_headers".to_string(),
|
||||||
|
serde_json::to_value(client_headers).unwrap_or(Value::Null),
|
||||||
|
);
|
||||||
|
report_context = Some(Value::Object(object));
|
||||||
|
}
|
||||||
|
|
||||||
|
GatewaySyncReportRequest {
|
||||||
|
trace_id: trace_id.to_string(),
|
||||||
|
report_kind,
|
||||||
|
report_context,
|
||||||
|
status_code: upstream_status_code,
|
||||||
|
headers: provider_headers,
|
||||||
|
body_json: provider_body_json,
|
||||||
|
client_body_json,
|
||||||
|
body_base64: provider_body_base64,
|
||||||
|
telemetry,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn record_stream_terminal_usage(
|
fn record_stream_terminal_usage(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
plan: &ExecutionPlan,
|
plan: &ExecutionPlan,
|
||||||
@@ -157,6 +209,55 @@ fn build_stream_body_capture(
|
|||||||
(body_base64, body_state)
|
(body_base64, body_state)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn wrap_non_json_binary_stream_error_for_client(
|
||||||
|
plan_kind: &str,
|
||||||
|
headers: &BTreeMap<String, String>,
|
||||||
|
error_body: &[u8],
|
||||||
|
) -> Result<Option<Value>, GatewayError> {
|
||||||
|
let content_type = headers
|
||||||
|
.get("content-type")
|
||||||
|
.map(|value| value.to_ascii_lowercase())
|
||||||
|
.unwrap_or_default();
|
||||||
|
if content_type.starts_with("application/json") {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
|
||||||
|
let body = match plan_kind {
|
||||||
|
GEMINI_FILES_DOWNLOAD_PLAN_KIND => json!({
|
||||||
|
"error": String::from_utf8_lossy(error_body).to_string(),
|
||||||
|
}),
|
||||||
|
OPENAI_VIDEO_CONTENT_PLAN_KIND => json!({
|
||||||
|
"error": {
|
||||||
|
"type": "upstream_error",
|
||||||
|
"message": "Video not available",
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
_ => return Ok(None),
|
||||||
|
};
|
||||||
|
Ok(Some(body))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn with_stream_error_trace_context(
|
||||||
|
report_context: Option<&Value>,
|
||||||
|
status_code: u16,
|
||||||
|
headers: &BTreeMap<String, String>,
|
||||||
|
response_text: Option<&str>,
|
||||||
|
local_failover_analysis: crate::orchestration::LocalFailoverAnalysis,
|
||||||
|
) -> Option<Value> {
|
||||||
|
let upstream_context = with_upstream_response_report_context(
|
||||||
|
report_context,
|
||||||
|
status_code,
|
||||||
|
Some(headers),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
with_error_flow_report_context(
|
||||||
|
upstream_context.as_ref().or(report_context),
|
||||||
|
build_local_error_flow_metadata(status_code, response_text, local_failover_analysis),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)] // stream report payload assembly mirrors runtime state
|
#[allow(clippy::too_many_arguments)] // stream report payload assembly mirrors runtime state
|
||||||
fn build_stream_usage_payload(
|
fn build_stream_usage_payload(
|
||||||
trace_id: String,
|
trace_id: String,
|
||||||
@@ -948,11 +1049,29 @@ async fn execute_stream_from_frame_stream(
|
|||||||
let stream_error_finalize_kind =
|
let stream_error_finalize_kind =
|
||||||
resolve_core_stream_error_finalize_report_kind(plan_kind, status_code);
|
resolve_core_stream_error_finalize_report_kind(plan_kind, status_code);
|
||||||
|
|
||||||
if status_code >= 400 {
|
if !(200..300).contains(&status_code) {
|
||||||
let error_body = collect_error_body(&mut lines).await?;
|
let provider_error_body = collect_error_body(&mut lines).await?;
|
||||||
let (body_json, body_base64) = decode_stream_error_body(&headers, &error_body);
|
let synthetic_body_json =
|
||||||
|
should_synthesize_non_success_stream_error_body(status_code, &provider_error_body)
|
||||||
|
.then(|| build_synthetic_non_success_stream_error_body(status_code, &headers));
|
||||||
|
let (provider_body_json, provider_body_base64) =
|
||||||
|
decode_stream_error_body(&headers, &provider_error_body);
|
||||||
|
let client_status_code = stream_client_error_status_code_for_upstream_status(status_code);
|
||||||
|
let wrapped_binary_body_json = wrap_non_json_binary_stream_error_for_client(
|
||||||
|
plan_kind,
|
||||||
|
&headers,
|
||||||
|
&provider_error_body,
|
||||||
|
)?;
|
||||||
|
let (client_body_json, client_error_body) =
|
||||||
|
if let Some(body_json) = synthetic_body_json.or(wrapped_binary_body_json) {
|
||||||
|
let body_bytes = serde_json::to_vec(&body_json)
|
||||||
|
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||||
|
(Some(body_json), body_bytes)
|
||||||
|
} else {
|
||||||
|
(provider_body_json.clone(), provider_error_body.clone())
|
||||||
|
};
|
||||||
let error_response_text =
|
let error_response_text =
|
||||||
local_failover_response_text(body_json.as_ref(), &error_body, None);
|
local_failover_response_text(client_body_json.as_ref(), &client_error_body, None);
|
||||||
let failover_analysis = resolve_local_candidate_failover_analysis_stream(
|
let failover_analysis = resolve_local_candidate_failover_analysis_stream(
|
||||||
state,
|
state,
|
||||||
&plan,
|
&plan,
|
||||||
@@ -1043,18 +1162,17 @@ async fn execute_stream_from_frame_stream(
|
|||||||
);
|
);
|
||||||
if matches!(failover_decision, LocalFailoverDecision::RetryNextCandidate) {
|
if matches!(failover_decision, LocalFailoverDecision::RetryNextCandidate) {
|
||||||
let terminal_unix_secs = current_request_candidate_unix_ms();
|
let terminal_unix_secs = current_request_candidate_unix_ms();
|
||||||
let error_flow_report_context = with_error_flow_report_context(
|
let error_trace_report_context = with_stream_error_trace_context(
|
||||||
report_context.as_ref(),
|
report_context.as_ref(),
|
||||||
build_local_error_flow_metadata(
|
status_code,
|
||||||
status_code,
|
&headers,
|
||||||
error_response_text.as_deref(),
|
error_response_text.as_deref(),
|
||||||
failover_analysis,
|
failover_analysis,
|
||||||
),
|
|
||||||
);
|
);
|
||||||
record_local_request_candidate_status(
|
record_local_request_candidate_status(
|
||||||
state,
|
state,
|
||||||
&plan,
|
&plan,
|
||||||
error_flow_report_context
|
error_trace_report_context
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.or(report_context.as_ref()),
|
.or(report_context.as_ref()),
|
||||||
SchedulerRequestCandidateStatusUpdate {
|
SchedulerRequestCandidateStatusUpdate {
|
||||||
@@ -1094,18 +1212,17 @@ async fn execute_stream_from_frame_stream(
|
|||||||
)
|
)
|
||||||
{
|
{
|
||||||
let terminal_unix_secs = current_request_candidate_unix_ms();
|
let terminal_unix_secs = current_request_candidate_unix_ms();
|
||||||
let error_flow_report_context = with_error_flow_report_context(
|
let error_trace_report_context = with_stream_error_trace_context(
|
||||||
report_context.as_ref(),
|
report_context.as_ref(),
|
||||||
build_local_error_flow_metadata(
|
status_code,
|
||||||
status_code,
|
&headers,
|
||||||
error_response_text.as_deref(),
|
error_response_text.as_deref(),
|
||||||
failover_analysis,
|
failover_analysis,
|
||||||
),
|
|
||||||
);
|
);
|
||||||
record_local_request_candidate_status(
|
record_local_request_candidate_status(
|
||||||
state,
|
state,
|
||||||
&plan,
|
&plan,
|
||||||
error_flow_report_context
|
error_trace_report_context
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.or(report_context.as_ref()),
|
.or(report_context.as_ref()),
|
||||||
SchedulerRequestCandidateStatusUpdate {
|
SchedulerRequestCandidateStatusUpdate {
|
||||||
@@ -1124,46 +1241,60 @@ async fn execute_stream_from_frame_stream(
|
|||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut client_headers = headers.clone();
|
let mut client_headers = if (300..400).contains(&status_code) {
|
||||||
apply_endpoint_response_header_rules(state, &plan, &mut client_headers, body_json.as_ref())
|
let mut headers = synthetic_error_response_headers(headers.clone());
|
||||||
.await?;
|
headers.insert(
|
||||||
|
"x-aether-upstream-status".to_string(),
|
||||||
|
status_code.to_string(),
|
||||||
|
);
|
||||||
|
headers
|
||||||
|
} else {
|
||||||
|
headers.clone()
|
||||||
|
};
|
||||||
|
apply_endpoint_response_header_rules(
|
||||||
|
state,
|
||||||
|
&plan,
|
||||||
|
&mut client_headers,
|
||||||
|
client_body_json.as_ref(),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
let payload = build_stream_sync_payload(
|
let client_response_headers = client_headers.clone();
|
||||||
|
let error_trace_report_context = with_stream_error_trace_context(
|
||||||
|
report_context.as_ref(),
|
||||||
|
status_code,
|
||||||
|
&headers,
|
||||||
|
error_response_text.as_deref(),
|
||||||
|
failover_analysis,
|
||||||
|
);
|
||||||
|
let payload = build_stream_error_sync_payload(
|
||||||
trace_id,
|
trace_id,
|
||||||
stream_error_finalize_kind
|
stream_error_finalize_kind
|
||||||
.as_deref()
|
.as_deref()
|
||||||
.or(report_kind.as_deref())
|
.or(report_kind.as_deref())
|
||||||
.unwrap_or_default()
|
.unwrap_or_default()
|
||||||
.to_string(),
|
.to_string(),
|
||||||
report_context,
|
error_trace_report_context.or(report_context),
|
||||||
status_code,
|
status_code,
|
||||||
|
headers.clone(),
|
||||||
|
provider_body_json,
|
||||||
|
provider_body_base64,
|
||||||
client_headers,
|
client_headers,
|
||||||
body_json,
|
client_body_json,
|
||||||
body_base64,
|
|
||||||
None,
|
None,
|
||||||
);
|
);
|
||||||
record_sync_terminal_usage(state, &plan, payload.report_context.as_ref(), &payload);
|
record_sync_terminal_usage(state, &plan, payload.report_context.as_ref(), &payload);
|
||||||
let terminal_unix_secs = current_request_candidate_unix_ms();
|
let terminal_unix_secs = current_request_candidate_unix_ms();
|
||||||
let error_flow_report_context = with_error_flow_report_context(
|
|
||||||
payload.report_context.as_ref(),
|
|
||||||
build_local_error_flow_metadata(
|
|
||||||
status_code,
|
|
||||||
error_response_text.as_deref(),
|
|
||||||
failover_analysis,
|
|
||||||
),
|
|
||||||
);
|
|
||||||
record_local_request_candidate_status(
|
record_local_request_candidate_status(
|
||||||
state,
|
state,
|
||||||
&plan,
|
&plan,
|
||||||
error_flow_report_context
|
payload.report_context.as_ref(),
|
||||||
.as_ref()
|
|
||||||
.or(payload.report_context.as_ref()),
|
|
||||||
SchedulerRequestCandidateStatusUpdate {
|
SchedulerRequestCandidateStatusUpdate {
|
||||||
status: RequestCandidateStatus::Failed,
|
status: RequestCandidateStatus::Failed,
|
||||||
status_code: Some(status_code),
|
status_code: Some(status_code),
|
||||||
error_type: Some("execution_runtime_stream_error".to_string()),
|
error_type: Some("execution_runtime_stream_non_success_status".to_string()),
|
||||||
error_message: Some(format!(
|
error_message: Some(format!(
|
||||||
"execution runtime stream returned error status {status_code}"
|
"execution runtime stream returned non-success status {status_code}"
|
||||||
)),
|
)),
|
||||||
latency_ms: None,
|
latency_ms: None,
|
||||||
started_at_unix_ms: Some(candidate_started_unix_secs),
|
started_at_unix_ms: Some(candidate_started_unix_secs),
|
||||||
@@ -1182,13 +1313,12 @@ async fn execute_stream_from_frame_stream(
|
|||||||
)?));
|
)?));
|
||||||
}
|
}
|
||||||
return Ok(Some(attach_control_metadata_headers(
|
return Ok(Some(attach_control_metadata_headers(
|
||||||
build_execution_runtime_error_response(
|
build_client_response_from_parts(
|
||||||
|
client_status_code,
|
||||||
|
&client_response_headers,
|
||||||
|
Body::from(client_error_body),
|
||||||
trace_id,
|
trace_id,
|
||||||
decision,
|
Some(decision),
|
||||||
plan_kind,
|
|
||||||
status_code,
|
|
||||||
payload.headers,
|
|
||||||
error_body,
|
|
||||||
)?,
|
)?,
|
||||||
Some(request_id),
|
Some(request_id),
|
||||||
candidate_id,
|
candidate_id,
|
||||||
@@ -2343,7 +2473,9 @@ mod tests {
|
|||||||
ExecutionPlan, ExecutionStreamTerminalSummary, ExecutionTimeouts, RequestBody,
|
ExecutionPlan, ExecutionStreamTerminalSummary, ExecutionTimeouts, RequestBody,
|
||||||
StandardizedUsage,
|
StandardizedUsage,
|
||||||
};
|
};
|
||||||
|
use aether_data::repository::candidates::InMemoryRequestCandidateRepository;
|
||||||
use aether_data::repository::usage::InMemoryUsageReadRepository;
|
use aether_data::repository::usage::InMemoryUsageReadRepository;
|
||||||
|
use aether_data_contracts::repository::candidates::RequestCandidateReadRepository;
|
||||||
use aether_data_contracts::repository::usage::UsageReadRepository;
|
use aether_data_contracts::repository::usage::UsageReadRepository;
|
||||||
use aether_usage_runtime::UsageRuntimeConfig;
|
use aether_usage_runtime::UsageRuntimeConfig;
|
||||||
use async_stream::stream;
|
use async_stream::stream;
|
||||||
@@ -2532,9 +2664,15 @@ mod tests {
|
|||||||
});
|
});
|
||||||
|
|
||||||
let usage_repository = Arc::new(InMemoryUsageReadRepository::default());
|
let usage_repository = Arc::new(InMemoryUsageReadRepository::default());
|
||||||
|
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("app state should build")
|
.expect("app state should build")
|
||||||
.with_usage_data_repository_for_tests(Arc::clone(&usage_repository))
|
.with_data_state_for_tests(
|
||||||
|
crate::data::GatewayDataState::with_request_candidate_and_usage_repository_for_tests(
|
||||||
|
Arc::clone(&request_candidate_repository),
|
||||||
|
Arc::clone(&usage_repository),
|
||||||
|
),
|
||||||
|
)
|
||||||
.with_usage_runtime_for_tests(UsageRuntimeConfig {
|
.with_usage_runtime_for_tests(UsageRuntimeConfig {
|
||||||
enabled: true,
|
enabled: true,
|
||||||
..UsageRuntimeConfig::default()
|
..UsageRuntimeConfig::default()
|
||||||
@@ -2736,6 +2874,213 @@ mod tests {
|
|||||||
server.abort();
|
server.abort();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn execute_execution_runtime_stream_rewrites_redirect_to_structured_failure() {
|
||||||
|
let listener = crate::test_support::bind_loopback_listener()
|
||||||
|
.await
|
||||||
|
.expect("listener should bind");
|
||||||
|
let addr = listener.local_addr().expect("local addr should resolve");
|
||||||
|
let server = tokio::spawn(async move {
|
||||||
|
let app = Router::new().route(
|
||||||
|
"/v1/execute/stream",
|
||||||
|
any(|_request: Request| async move {
|
||||||
|
let frames = concat!(
|
||||||
|
"{\"type\":\"headers\",\"payload\":{\"kind\":\"headers\",\"status_code\":302,\"headers\":{\"location\":\"/\",\"content-type\":\"text/html\",\"content-length\":\"0\"}}}\n",
|
||||||
|
"{\"type\":\"eof\",\"payload\":{\"kind\":\"eof\"}}\n"
|
||||||
|
);
|
||||||
|
let mut response = axum::http::Response::new(Body::from(frames));
|
||||||
|
response.headers_mut().insert(
|
||||||
|
header::CONTENT_TYPE,
|
||||||
|
HeaderValue::from_static("application/x-ndjson"),
|
||||||
|
);
|
||||||
|
response
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
axum::serve(listener, app)
|
||||||
|
.await
|
||||||
|
.expect("server should start");
|
||||||
|
});
|
||||||
|
|
||||||
|
let usage_repository = Arc::new(InMemoryUsageReadRepository::default());
|
||||||
|
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
|
||||||
|
let state = AppState::new()
|
||||||
|
.expect("app state should build")
|
||||||
|
.with_data_state_for_tests(
|
||||||
|
crate::data::GatewayDataState::with_request_candidate_and_usage_repository_for_tests(
|
||||||
|
Arc::clone(&request_candidate_repository),
|
||||||
|
Arc::clone(&usage_repository),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.with_usage_runtime_for_tests(UsageRuntimeConfig {
|
||||||
|
enabled: true,
|
||||||
|
..UsageRuntimeConfig::default()
|
||||||
|
})
|
||||||
|
.with_execution_runtime_override_base_url(format!("http://{addr}"));
|
||||||
|
let plan = ExecutionPlan {
|
||||||
|
request_id: "req-remote-runtime-stream-redirect".into(),
|
||||||
|
candidate_id: Some("cand-remote-runtime-stream-redirect".into()),
|
||||||
|
provider_name: Some("ChatGPTWeb".into()),
|
||||||
|
provider_id: "prov-redirect".into(),
|
||||||
|
endpoint_id: "ep-redirect".into(),
|
||||||
|
key_id: "key-redirect".into(),
|
||||||
|
method: "POST".into(),
|
||||||
|
url: "https://chatgpt.com/backend-api/codex/responses".into(),
|
||||||
|
headers: BTreeMap::from([
|
||||||
|
("content-type".into(), "application/json".into()),
|
||||||
|
("accept".into(), "text/event-stream".into()),
|
||||||
|
]),
|
||||||
|
content_type: Some("application/json".into()),
|
||||||
|
content_encoding: None,
|
||||||
|
body: RequestBody::from_json(json!({
|
||||||
|
"model": "gpt-5.4",
|
||||||
|
"input": "hello",
|
||||||
|
"stream": true
|
||||||
|
})),
|
||||||
|
stream: true,
|
||||||
|
client_api_format: "gemini:generate_content".into(),
|
||||||
|
provider_api_format: "openai:responses".into(),
|
||||||
|
model_name: Some("gemini-3.1-flash-image-preview".into()),
|
||||||
|
proxy: None,
|
||||||
|
transport_profile: None,
|
||||||
|
timeouts: Some(ExecutionTimeouts {
|
||||||
|
connect_ms: Some(5_000),
|
||||||
|
total_ms: Some(5_000),
|
||||||
|
..ExecutionTimeouts::default()
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
let decision = GatewayControlDecision::synthetic(
|
||||||
|
"/v1beta/models/gemini-3.1-flash-image-preview:streamGenerateContent",
|
||||||
|
Some("ai_public".to_string()),
|
||||||
|
Some("gemini".to_string()),
|
||||||
|
Some("generate_content".to_string()),
|
||||||
|
Some("gemini:generate_content".to_string()),
|
||||||
|
)
|
||||||
|
.with_execution_runtime_candidate(true);
|
||||||
|
|
||||||
|
let response = execute_execution_runtime_stream(
|
||||||
|
&state,
|
||||||
|
plan,
|
||||||
|
"trace-remote-runtime-stream-redirect",
|
||||||
|
&decision,
|
||||||
|
"gemini_chat_stream",
|
||||||
|
None,
|
||||||
|
Some(json!({
|
||||||
|
"request_id": "req-remote-runtime-stream-redirect",
|
||||||
|
"candidate_id": "cand-remote-runtime-stream-redirect",
|
||||||
|
"candidate_index": 0,
|
||||||
|
"retry_index": 0,
|
||||||
|
"provider_api_format": "openai:responses",
|
||||||
|
"client_api_format": "gemini:generate_content",
|
||||||
|
"needs_conversion": true
|
||||||
|
})),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("execution should succeed")
|
||||||
|
.expect("execution should return a client response");
|
||||||
|
|
||||||
|
assert_eq!(response.status(), axum::http::StatusCode::BAD_GATEWAY);
|
||||||
|
assert_eq!(
|
||||||
|
response
|
||||||
|
.headers()
|
||||||
|
.get(header::CONTENT_TYPE)
|
||||||
|
.and_then(|value| value.to_str().ok()),
|
||||||
|
Some("application/json")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
response
|
||||||
|
.headers()
|
||||||
|
.get("x-aether-upstream-status")
|
||||||
|
.and_then(|value| value.to_str().ok()),
|
||||||
|
Some("302")
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
response.headers().get(header::LOCATION).is_none(),
|
||||||
|
"redirect location should not be forwarded to AI clients"
|
||||||
|
);
|
||||||
|
let body = to_bytes(response.into_body(), usize::MAX)
|
||||||
|
.await
|
||||||
|
.expect("response body should read");
|
||||||
|
let body_json: Value =
|
||||||
|
serde_json::from_slice(&body).expect("response body should decode as json");
|
||||||
|
assert_eq!(
|
||||||
|
body_json["error"]["type"],
|
||||||
|
json!("execution_runtime_non_success_status")
|
||||||
|
);
|
||||||
|
assert_eq!(body_json["error"]["upstream_status"], json!(302));
|
||||||
|
assert_eq!(body_json["error"]["location"], json!("/"));
|
||||||
|
assert!(body_json["error"]["message"]
|
||||||
|
.as_str()
|
||||||
|
.is_some_and(|value| value.contains("non-success status 302")));
|
||||||
|
|
||||||
|
let usage = tokio::time::timeout(Duration::from_secs(2), async {
|
||||||
|
loop {
|
||||||
|
if let Some(usage) = usage_repository
|
||||||
|
.find_by_request_id("req-remote-runtime-stream-redirect")
|
||||||
|
.await
|
||||||
|
.expect("usage should read")
|
||||||
|
.filter(|usage| usage.status == "failed")
|
||||||
|
{
|
||||||
|
break usage;
|
||||||
|
}
|
||||||
|
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("usage should be written");
|
||||||
|
|
||||||
|
assert_eq!(usage.status_code, Some(302));
|
||||||
|
assert_eq!(usage.error_category.as_deref(), Some("redirect"));
|
||||||
|
assert!(usage
|
||||||
|
.error_message
|
||||||
|
.as_deref()
|
||||||
|
.is_some_and(|value| value.contains("non-success status 302")));
|
||||||
|
assert_eq!(
|
||||||
|
usage
|
||||||
|
.client_response_headers
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|headers| headers.get("x-aether-upstream-status")),
|
||||||
|
Some(&json!("302"))
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
usage
|
||||||
|
.response_headers
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|headers| headers.get("location")),
|
||||||
|
Some(&json!("/"))
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
usage.response_body.is_none(),
|
||||||
|
"upstream redirect did not include a body"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
usage
|
||||||
|
.client_response_body
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|body| body.pointer("/error/upstream_status")),
|
||||||
|
Some(&json!(302))
|
||||||
|
);
|
||||||
|
let candidates = request_candidate_repository
|
||||||
|
.list_by_request_id("req-remote-runtime-stream-redirect")
|
||||||
|
.await
|
||||||
|
.expect("candidate trace should read");
|
||||||
|
let candidate_extra = candidates
|
||||||
|
.first()
|
||||||
|
.and_then(|candidate| candidate.extra_data.as_ref())
|
||||||
|
.expect("failed candidate extra_data should exist");
|
||||||
|
assert_eq!(
|
||||||
|
candidate_extra["upstream_response"]["status_code"],
|
||||||
|
json!(302)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
candidate_extra["upstream_response"]["headers"]["location"],
|
||||||
|
json!("/")
|
||||||
|
);
|
||||||
|
assert!(candidate_extra["upstream_response"].get("body").is_none());
|
||||||
|
assert!(candidate_extra.get("client_response").is_none());
|
||||||
|
|
||||||
|
server.abort();
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn execute_execution_runtime_stream_bridges_openai_image_sync_json_from_remote_runtime_to_image_sse(
|
async fn execute_execution_runtime_stream_bridges_openai_image_sync_json_from_remote_runtime_to_image_sse(
|
||||||
) {
|
) {
|
||||||
|
|||||||
@@ -20,9 +20,9 @@ use crate::execution_runtime::submission::{
|
|||||||
use crate::log_ids::short_request_id;
|
use crate::log_ids::short_request_id;
|
||||||
use crate::orchestration::{
|
use crate::orchestration::{
|
||||||
apply_local_execution_effect, resolve_local_failover_analysis_for_attempt,
|
apply_local_execution_effect, resolve_local_failover_analysis_for_attempt,
|
||||||
LocalAdaptiveRateLimitEffect, LocalAttemptFailureEffect, LocalExecutionEffect,
|
with_upstream_response_report_context, LocalAdaptiveRateLimitEffect, LocalAttemptFailureEffect,
|
||||||
LocalExecutionEffectContext, LocalHealthFailureEffect, LocalOAuthInvalidationEffect,
|
LocalExecutionEffect, LocalExecutionEffectContext, LocalHealthFailureEffect,
|
||||||
LocalPoolErrorEffect,
|
LocalOAuthInvalidationEffect, LocalPoolErrorEffect,
|
||||||
};
|
};
|
||||||
use crate::request_candidate_runtime::record_report_request_candidate_status;
|
use crate::request_candidate_runtime::record_report_request_candidate_status;
|
||||||
use crate::usage::submit_sync_report;
|
use crate::usage::submit_sync_report;
|
||||||
@@ -135,6 +135,15 @@ fn build_stream_failure_sync_payload(
|
|||||||
failure: StreamFailureReport,
|
failure: StreamFailureReport,
|
||||||
) -> GatewaySyncReportRequest {
|
) -> GatewaySyncReportRequest {
|
||||||
let status_code = failure.status_code;
|
let status_code = failure.status_code;
|
||||||
|
let report_context = with_upstream_response_report_context(
|
||||||
|
report_context.as_ref(),
|
||||||
|
status_code,
|
||||||
|
Some(&headers),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.or(report_context);
|
||||||
headers.remove("content-encoding");
|
headers.remove("content-encoding");
|
||||||
headers.remove("content-length");
|
headers.remove("content-length");
|
||||||
headers.insert("content-type".to_string(), "application/json".to_string());
|
headers.insert("content-type".to_string(), "application/json".to_string());
|
||||||
|
|||||||
@@ -39,9 +39,10 @@ use crate::execution_runtime::{
|
|||||||
use crate::log_ids::short_request_id;
|
use crate::log_ids::short_request_id;
|
||||||
use crate::orchestration::{
|
use crate::orchestration::{
|
||||||
apply_local_execution_effect, build_local_error_flow_metadata, with_error_flow_report_context,
|
apply_local_execution_effect, build_local_error_flow_metadata, with_error_flow_report_context,
|
||||||
LocalAdaptiveRateLimitEffect, LocalAdaptiveSuccessEffect, LocalAttemptFailureEffect,
|
with_upstream_response_report_context, LocalAdaptiveRateLimitEffect,
|
||||||
LocalExecutionEffect, LocalExecutionEffectContext, LocalHealthFailureEffect,
|
LocalAdaptiveSuccessEffect, LocalAttemptFailureEffect, LocalExecutionEffect,
|
||||||
LocalHealthSuccessEffect, LocalOAuthInvalidationEffect, LocalPoolErrorEffect,
|
LocalExecutionEffectContext, LocalHealthFailureEffect, LocalHealthSuccessEffect,
|
||||||
|
LocalOAuthInvalidationEffect, LocalPoolErrorEffect,
|
||||||
};
|
};
|
||||||
use crate::request_candidate_runtime::{
|
use crate::request_candidate_runtime::{
|
||||||
ensure_execution_request_candidate_slot, record_local_request_candidate_status,
|
ensure_execution_request_candidate_slot, record_local_request_candidate_status,
|
||||||
@@ -81,6 +82,27 @@ fn record_sync_terminal_usage(
|
|||||||
.record_sync_terminal(state.data.as_ref(), context_seed, payload_seed);
|
.record_sync_terminal(state.data.as_ref(), context_seed, payload_seed);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn with_sync_error_trace_context(
|
||||||
|
report_context: Option<&serde_json::Value>,
|
||||||
|
status_code: u16,
|
||||||
|
headers: &BTreeMap<String, String>,
|
||||||
|
response_text: Option<&str>,
|
||||||
|
local_failover_analysis: crate::orchestration::LocalFailoverAnalysis,
|
||||||
|
) -> Option<serde_json::Value> {
|
||||||
|
let upstream_context = with_upstream_response_report_context(
|
||||||
|
report_context,
|
||||||
|
status_code,
|
||||||
|
Some(headers),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
with_error_flow_report_context(
|
||||||
|
upstream_context.as_ref().or(report_context),
|
||||||
|
build_local_error_flow_metadata(status_code, response_text, local_failover_analysis),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
fn build_sync_report_payload(
|
fn build_sync_report_payload(
|
||||||
trace_id: &str,
|
trace_id: &str,
|
||||||
report_kind: String,
|
report_kind: String,
|
||||||
@@ -572,18 +594,17 @@ pub(crate) async fn execute_execution_runtime_sync(
|
|||||||
LocalFailoverDecision::RetryNextCandidate
|
LocalFailoverDecision::RetryNextCandidate
|
||||||
) {
|
) {
|
||||||
let terminal_unix_secs = current_request_candidate_unix_ms();
|
let terminal_unix_secs = current_request_candidate_unix_ms();
|
||||||
let error_flow_report_context = with_error_flow_report_context(
|
let error_trace_report_context = with_sync_error_trace_context(
|
||||||
report_context.as_ref(),
|
report_context.as_ref(),
|
||||||
build_local_error_flow_metadata(
|
result.status_code,
|
||||||
result.status_code,
|
&headers,
|
||||||
local_failover_response_text.as_deref(),
|
local_failover_response_text.as_deref(),
|
||||||
local_failover_analysis,
|
local_failover_analysis,
|
||||||
),
|
|
||||||
);
|
);
|
||||||
record_local_request_candidate_status(
|
record_local_request_candidate_status(
|
||||||
state,
|
state,
|
||||||
&plan,
|
&plan,
|
||||||
error_flow_report_context
|
error_trace_report_context
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.or(report_context.as_ref()),
|
.or(report_context.as_ref()),
|
||||||
SchedulerRequestCandidateStatusUpdate {
|
SchedulerRequestCandidateStatusUpdate {
|
||||||
@@ -649,18 +670,17 @@ pub(crate) async fn execute_execution_runtime_sync(
|
|||||||
mapped_error_finalize_kind.is_some(),
|
mapped_error_finalize_kind.is_some(),
|
||||||
) {
|
) {
|
||||||
let terminal_unix_secs = current_request_candidate_unix_ms();
|
let terminal_unix_secs = current_request_candidate_unix_ms();
|
||||||
let error_flow_report_context = with_error_flow_report_context(
|
let error_trace_report_context = with_sync_error_trace_context(
|
||||||
report_context.as_ref(),
|
report_context.as_ref(),
|
||||||
build_local_error_flow_metadata(
|
result.status_code,
|
||||||
result.status_code,
|
&headers,
|
||||||
local_failover_response_text.as_deref(),
|
local_failover_response_text.as_deref(),
|
||||||
local_failover_analysis,
|
local_failover_analysis,
|
||||||
),
|
|
||||||
);
|
);
|
||||||
record_local_request_candidate_status(
|
record_local_request_candidate_status(
|
||||||
state,
|
state,
|
||||||
&plan,
|
&plan,
|
||||||
error_flow_report_context
|
error_trace_report_context
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.or(report_context.as_ref()),
|
.or(report_context.as_ref()),
|
||||||
SchedulerRequestCandidateStatusUpdate {
|
SchedulerRequestCandidateStatusUpdate {
|
||||||
@@ -680,13 +700,12 @@ pub(crate) async fn execute_execution_runtime_sync(
|
|||||||
let terminal_unix_secs = current_request_candidate_unix_ms();
|
let terminal_unix_secs = current_request_candidate_unix_ms();
|
||||||
let error_flow_report_context = (result.status_code >= 400)
|
let error_flow_report_context = (result.status_code >= 400)
|
||||||
.then(|| {
|
.then(|| {
|
||||||
with_error_flow_report_context(
|
with_sync_error_trace_context(
|
||||||
report_context.as_ref(),
|
report_context.as_ref(),
|
||||||
build_local_error_flow_metadata(
|
result.status_code,
|
||||||
result.status_code,
|
&headers,
|
||||||
local_failover_response_text.as_deref(),
|
local_failover_response_text.as_deref(),
|
||||||
local_failover_analysis,
|
local_failover_analysis,
|
||||||
),
|
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
.flatten();
|
.flatten();
|
||||||
|
|||||||
@@ -356,6 +356,164 @@ async fn admin_monitoring_trace_request_enriches_proxy_timing_from_usage_audit()
|
|||||||
payload["candidates"][0]["extra_data"]["proxy"]["timing"]["response_wait_ms"],
|
payload["candidates"][0]["extra_data"]["proxy"]["timing"]["response_wait_ms"],
|
||||||
json!(475)
|
json!(475)
|
||||||
);
|
);
|
||||||
|
assert!(payload["candidates"][0]["extra_data"]
|
||||||
|
.get("upstream_response")
|
||||||
|
.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn admin_monitoring_trace_request_exposes_request_path_from_usage_audit() {
|
||||||
|
let mut candidate = sample_candidate(
|
||||||
|
"cand-used",
|
||||||
|
"request-1",
|
||||||
|
0,
|
||||||
|
RequestCandidateStatus::Failed,
|
||||||
|
Some(101),
|
||||||
|
Some(33),
|
||||||
|
Some(403),
|
||||||
|
);
|
||||||
|
candidate.extra_data = Some(json!({
|
||||||
|
"client_api_format": "gemini:generate_content"
|
||||||
|
}));
|
||||||
|
|
||||||
|
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![candidate]));
|
||||||
|
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||||
|
vec![sample_provider()],
|
||||||
|
vec![sample_endpoint()],
|
||||||
|
vec![sample_key()],
|
||||||
|
));
|
||||||
|
let mut usage = sample_usage(
|
||||||
|
"request-1",
|
||||||
|
"provider-1",
|
||||||
|
"OpenAI",
|
||||||
|
40,
|
||||||
|
0.02,
|
||||||
|
"failed",
|
||||||
|
Some(403),
|
||||||
|
100,
|
||||||
|
);
|
||||||
|
usage.candidate_id = Some("cand-used".to_string());
|
||||||
|
usage.request_metadata = Some(json!({
|
||||||
|
"request_path": "/v1beta/models/gemini-2.5-pro:generateContent",
|
||||||
|
"request_query_string": "alt=sse"
|
||||||
|
}));
|
||||||
|
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(vec![usage]));
|
||||||
|
let data_state =
|
||||||
|
crate::data::GatewayDataState::with_request_candidate_and_usage_repository_for_tests(
|
||||||
|
request_candidates,
|
||||||
|
usage_repository,
|
||||||
|
)
|
||||||
|
.with_provider_catalog_reader(provider_catalog);
|
||||||
|
let state = AppState::new()
|
||||||
|
.expect("state should build")
|
||||||
|
.with_data_state_for_tests(data_state);
|
||||||
|
let context = request_context(http::Method::GET, "/api/admin/monitoring/trace/request-1");
|
||||||
|
|
||||||
|
let response = local_monitoring_response(&state, &context)
|
||||||
|
.await
|
||||||
|
.expect("handler should not error")
|
||||||
|
.expect("route should be handled locally");
|
||||||
|
|
||||||
|
assert_eq!(response.status(), http::StatusCode::OK);
|
||||||
|
let body = to_bytes(response.into_body(), usize::MAX)
|
||||||
|
.await
|
||||||
|
.expect("body should read");
|
||||||
|
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
payload["request_path"],
|
||||||
|
json!("/v1beta/models/gemini-2.5-pro:generateContent")
|
||||||
|
);
|
||||||
|
assert_eq!(payload["request_query_string"], json!("alt=sse"));
|
||||||
|
assert_eq!(
|
||||||
|
payload["request_path_and_query"],
|
||||||
|
json!("/v1beta/models/gemini-2.5-pro:generateContent?alt=sse")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
payload["candidates"][0]["extra_data"]["request_path_and_query"],
|
||||||
|
json!("/v1beta/models/gemini-2.5-pro:generateContent?alt=sse")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn admin_monitoring_trace_request_exposes_failed_candidate_upstream_response_boundary() {
|
||||||
|
let candidate = sample_candidate(
|
||||||
|
"cand-used",
|
||||||
|
"request-1",
|
||||||
|
0,
|
||||||
|
RequestCandidateStatus::Failed,
|
||||||
|
Some(101),
|
||||||
|
Some(33),
|
||||||
|
Some(302),
|
||||||
|
);
|
||||||
|
|
||||||
|
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![candidate]));
|
||||||
|
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||||
|
vec![sample_provider()],
|
||||||
|
vec![sample_endpoint()],
|
||||||
|
vec![sample_key()],
|
||||||
|
));
|
||||||
|
let mut usage = sample_usage(
|
||||||
|
"request-1",
|
||||||
|
"provider-1",
|
||||||
|
"OpenAI",
|
||||||
|
40,
|
||||||
|
0.02,
|
||||||
|
"failed",
|
||||||
|
Some(302),
|
||||||
|
100,
|
||||||
|
);
|
||||||
|
usage.candidate_id = Some("cand-used".to_string());
|
||||||
|
usage.response_headers = Some(json!({
|
||||||
|
"location": "/",
|
||||||
|
"content-type": "text/html"
|
||||||
|
}));
|
||||||
|
usage.client_response_headers = Some(json!({
|
||||||
|
"content-type": "application/json",
|
||||||
|
"x-aether-upstream-status": "302"
|
||||||
|
}));
|
||||||
|
usage.client_response_body = Some(json!({
|
||||||
|
"error": {
|
||||||
|
"type": "execution_runtime_non_success_status",
|
||||||
|
"message": "execution runtime stream returned non-success status 302",
|
||||||
|
"upstream_status": 302,
|
||||||
|
"location": "/"
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
usage.request_metadata = Some(json!({
|
||||||
|
"client_response_status_code": 502
|
||||||
|
}));
|
||||||
|
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(vec![usage]));
|
||||||
|
let data_state =
|
||||||
|
crate::data::GatewayDataState::with_request_candidate_and_usage_repository_for_tests(
|
||||||
|
request_candidates,
|
||||||
|
usage_repository,
|
||||||
|
)
|
||||||
|
.with_provider_catalog_reader(provider_catalog);
|
||||||
|
let state = AppState::new()
|
||||||
|
.expect("state should build")
|
||||||
|
.with_data_state_for_tests(data_state);
|
||||||
|
let context = request_context(http::Method::GET, "/api/admin/monitoring/trace/request-1");
|
||||||
|
|
||||||
|
let response = local_monitoring_response(&state, &context)
|
||||||
|
.await
|
||||||
|
.expect("handler should not error")
|
||||||
|
.expect("route should be handled locally");
|
||||||
|
|
||||||
|
assert_eq!(response.status(), http::StatusCode::OK);
|
||||||
|
let body = to_bytes(response.into_body(), usize::MAX)
|
||||||
|
.await
|
||||||
|
.expect("body should read");
|
||||||
|
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||||
|
let extra = &payload["candidates"][0]["extra_data"];
|
||||||
|
assert_eq!(extra["upstream_response"]["status_code"], json!(302));
|
||||||
|
assert_eq!(
|
||||||
|
extra["upstream_response"]["headers"]["location"],
|
||||||
|
json!("/")
|
||||||
|
);
|
||||||
|
assert!(extra["upstream_response"]["body"].is_null());
|
||||||
|
assert!(extra.get("client_response").is_none());
|
||||||
|
assert!(extra.get("provider_response").is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
use aether_contracts::ExecutionPlan;
|
use aether_contracts::ExecutionPlan;
|
||||||
use serde_json::{json, Value};
|
use serde_json::{json, Map, Value};
|
||||||
|
|
||||||
use crate::AppState;
|
use crate::AppState;
|
||||||
|
|
||||||
@@ -120,3 +120,66 @@ pub(crate) fn with_error_flow_report_context(
|
|||||||
object.insert("error_flow".to_string(), error_flow);
|
object.insert("error_flow".to_string(), error_flow);
|
||||||
Some(Value::Object(object))
|
Some(Value::Object(object))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn with_upstream_response_report_context(
|
||||||
|
report_context: Option<&Value>,
|
||||||
|
status_code: u16,
|
||||||
|
headers: Option<&std::collections::BTreeMap<String, String>>,
|
||||||
|
body: Option<&Value>,
|
||||||
|
body_ref: Option<&str>,
|
||||||
|
body_state: Option<&str>,
|
||||||
|
) -> Option<Value> {
|
||||||
|
let mut object = report_context?.as_object()?.clone();
|
||||||
|
let mut upstream_response = serde_json::Map::new();
|
||||||
|
upstream_response.insert("status_code".to_string(), json!(status_code));
|
||||||
|
if let Some(headers) = headers {
|
||||||
|
upstream_response.insert("headers".to_string(), trace_headers_to_json(headers));
|
||||||
|
}
|
||||||
|
if let Some(body) = body {
|
||||||
|
upstream_response.insert("body".to_string(), body.clone());
|
||||||
|
}
|
||||||
|
if let Some(body_ref) = body_ref {
|
||||||
|
upstream_response.insert("body_ref".to_string(), json!(body_ref));
|
||||||
|
}
|
||||||
|
if let Some(body_state) = body_state {
|
||||||
|
upstream_response.insert("body_state".to_string(), json!(body_state));
|
||||||
|
}
|
||||||
|
object.insert(
|
||||||
|
"upstream_response".to_string(),
|
||||||
|
Value::Object(upstream_response),
|
||||||
|
);
|
||||||
|
Some(Value::Object(object))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn trace_headers_to_json(headers: &std::collections::BTreeMap<String, String>) -> Value {
|
||||||
|
Value::Object(Map::from_iter(headers.iter().map(|(key, value)| {
|
||||||
|
(
|
||||||
|
key.clone(),
|
||||||
|
Value::String(mask_trace_header_value(key, value)),
|
||||||
|
)
|
||||||
|
})))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn mask_trace_header_value(name: &str, value: &str) -> String {
|
||||||
|
if !trace_header_is_sensitive(name) {
|
||||||
|
return value.to_string();
|
||||||
|
}
|
||||||
|
if value.len() <= 8 {
|
||||||
|
return "****".to_string();
|
||||||
|
}
|
||||||
|
format!("{}****{}", &value[..4], &value[value.len() - 4..])
|
||||||
|
}
|
||||||
|
|
||||||
|
fn trace_header_is_sensitive(name: &str) -> bool {
|
||||||
|
[
|
||||||
|
"authorization",
|
||||||
|
"x-api-key",
|
||||||
|
"api-key",
|
||||||
|
"x-goog-api-key",
|
||||||
|
"cookie",
|
||||||
|
"set-cookie",
|
||||||
|
"proxy-authorization",
|
||||||
|
]
|
||||||
|
.iter()
|
||||||
|
.any(|candidate| name.trim().eq_ignore_ascii_case(candidate))
|
||||||
|
}
|
||||||
|
|||||||
@@ -928,6 +928,428 @@ async fn gateway_executes_openai_chat_stream_via_local_openai_responses_cross_fo
|
|||||||
upstream_handle.abort();
|
upstream_handle.abort();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn gateway_executes_openai_chat_stream_via_local_cross_format_gemini_candidate_with_stream_path_rewrite(
|
||||||
|
) {
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
struct SeenExecutionRuntimeStreamRequest {
|
||||||
|
trace_id: String,
|
||||||
|
url: String,
|
||||||
|
provider_model: String,
|
||||||
|
auth_header_value: String,
|
||||||
|
accept: String,
|
||||||
|
endpoint_tag: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hash_api_key(value: &str) -> String {
|
||||||
|
let mut hasher = Sha256::new();
|
||||||
|
hasher.update(value.as_bytes());
|
||||||
|
format!("{:x}", hasher.finalize())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sample_auth_snapshot(api_key_id: &str, user_id: &str) -> StoredAuthApiKeySnapshot {
|
||||||
|
StoredAuthApiKeySnapshot::new(
|
||||||
|
user_id.to_string(),
|
||||||
|
"alice".to_string(),
|
||||||
|
Some("alice@example.com".to_string()),
|
||||||
|
"user".to_string(),
|
||||||
|
"local".to_string(),
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
Some(serde_json::json!(["openai", "gemini"])),
|
||||||
|
Some(serde_json::json!(["openai:chat"])),
|
||||||
|
Some(serde_json::json!(["gpt-5"])),
|
||||||
|
api_key_id.to_string(),
|
||||||
|
Some("default".to_string()),
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
Some(60),
|
||||||
|
Some(5),
|
||||||
|
Some(4_102_444_800),
|
||||||
|
Some(serde_json::json!(["openai", "gemini"])),
|
||||||
|
Some(serde_json::json!(["openai:chat"])),
|
||||||
|
Some(serde_json::json!(["gpt-5"])),
|
||||||
|
)
|
||||||
|
.expect("auth snapshot should build")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sample_candidate_row() -> StoredMinimalCandidateSelectionRow {
|
||||||
|
StoredMinimalCandidateSelectionRow {
|
||||||
|
provider_id: "provider-openai-chat-gemini-stream-local-1".to_string(),
|
||||||
|
provider_name: "gemini".to_string(),
|
||||||
|
provider_type: "custom".to_string(),
|
||||||
|
provider_priority: 10,
|
||||||
|
provider_is_active: true,
|
||||||
|
endpoint_id: "endpoint-openai-chat-gemini-stream-local-1".to_string(),
|
||||||
|
endpoint_api_format: "gemini:generate_content".to_string(),
|
||||||
|
endpoint_api_family: Some("gemini".to_string()),
|
||||||
|
endpoint_kind: Some("chat".to_string()),
|
||||||
|
endpoint_is_active: true,
|
||||||
|
key_id: "key-openai-chat-gemini-stream-local-1".to_string(),
|
||||||
|
key_name: "prod".to_string(),
|
||||||
|
key_auth_type: "api_key".to_string(),
|
||||||
|
key_is_active: true,
|
||||||
|
key_api_formats: Some(vec!["gemini:generate_content".to_string()]),
|
||||||
|
key_allowed_models: None,
|
||||||
|
key_capabilities: None,
|
||||||
|
key_internal_priority: 5,
|
||||||
|
key_global_priority_by_format: Some(serde_json::json!({"gemini:generate_content": 1})),
|
||||||
|
model_id: "model-openai-chat-gemini-stream-local-1".to_string(),
|
||||||
|
global_model_id: "global-model-openai-chat-gemini-stream-local-1".to_string(),
|
||||||
|
global_model_name: "gpt-5".to_string(),
|
||||||
|
global_model_mappings: None,
|
||||||
|
global_model_supports_streaming: Some(true),
|
||||||
|
model_provider_model_name: "gemini-2.5-pro-upstream".to_string(),
|
||||||
|
model_provider_model_mappings: Some(vec![StoredProviderModelMapping {
|
||||||
|
name: "gemini-2.5-pro-upstream".to_string(),
|
||||||
|
priority: 1,
|
||||||
|
api_formats: Some(vec!["gemini:generate_content".to_string()]),
|
||||||
|
endpoint_ids: None,
|
||||||
|
}]),
|
||||||
|
model_supports_streaming: Some(true),
|
||||||
|
model_is_active: true,
|
||||||
|
model_is_available: true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sample_provider_catalog_provider() -> StoredProviderCatalogProvider {
|
||||||
|
StoredProviderCatalogProvider::new(
|
||||||
|
"provider-openai-chat-gemini-stream-local-1".to_string(),
|
||||||
|
"gemini".to_string(),
|
||||||
|
Some("https://example.com".to_string()),
|
||||||
|
"custom".to_string(),
|
||||||
|
)
|
||||||
|
.expect("provider should build")
|
||||||
|
.with_transport_fields(
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
true,
|
||||||
|
None,
|
||||||
|
Some(2),
|
||||||
|
None,
|
||||||
|
Some(20.0),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sample_provider_catalog_endpoint() -> StoredProviderCatalogEndpoint {
|
||||||
|
StoredProviderCatalogEndpoint::new(
|
||||||
|
"endpoint-openai-chat-gemini-stream-local-1".to_string(),
|
||||||
|
"provider-openai-chat-gemini-stream-local-1".to_string(),
|
||||||
|
"gemini:generate_content".to_string(),
|
||||||
|
Some("gemini".to_string()),
|
||||||
|
Some("chat".to_string()),
|
||||||
|
true,
|
||||||
|
)
|
||||||
|
.expect("endpoint should build")
|
||||||
|
.with_transport_fields(
|
||||||
|
"https://generativelanguage.googleapis.com".to_string(),
|
||||||
|
Some(serde_json::json!([
|
||||||
|
{"action":"set","key":"x-endpoint-tag","value":"openai-chat-gemini-cross-format-stream"}
|
||||||
|
])),
|
||||||
|
None,
|
||||||
|
Some(2),
|
||||||
|
Some("/custom/v1beta/models/gemini-2.5-pro-upstream:generateContent".to_string()),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.expect("endpoint transport should build")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sample_provider_catalog_key() -> StoredProviderCatalogKey {
|
||||||
|
StoredProviderCatalogKey::new(
|
||||||
|
"key-openai-chat-gemini-stream-local-1".to_string(),
|
||||||
|
"provider-openai-chat-gemini-stream-local-1".to_string(),
|
||||||
|
"prod".to_string(),
|
||||||
|
"api_key".to_string(),
|
||||||
|
None,
|
||||||
|
true,
|
||||||
|
)
|
||||||
|
.expect("key should build")
|
||||||
|
.with_transport_fields(
|
||||||
|
Some(serde_json::json!(["gemini:generate_content"])),
|
||||||
|
encrypt_python_fernet_plaintext(
|
||||||
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
|
"sk-upstream-openai-chat-gemini-stream",
|
||||||
|
)
|
||||||
|
.expect("api key should encrypt"),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
Some(serde_json::json!({"gemini:generate_content": 1})),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.expect("key transport should build")
|
||||||
|
}
|
||||||
|
|
||||||
|
let seen_execution_runtime = Arc::new(Mutex::new(None::<SeenExecutionRuntimeStreamRequest>));
|
||||||
|
let seen_execution_runtime_clone = Arc::clone(&seen_execution_runtime);
|
||||||
|
let seen_report = Arc::new(Mutex::new(false));
|
||||||
|
let seen_report_clone = Arc::clone(&seen_report);
|
||||||
|
let decision_hits = Arc::new(Mutex::new(0usize));
|
||||||
|
let decision_hits_clone = Arc::clone(&decision_hits);
|
||||||
|
let plan_hits = Arc::new(Mutex::new(0usize));
|
||||||
|
let plan_hits_clone = Arc::clone(&plan_hits);
|
||||||
|
let public_hits = Arc::new(Mutex::new(0usize));
|
||||||
|
let public_hits_clone = Arc::clone(&public_hits);
|
||||||
|
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
|
||||||
|
|
||||||
|
let upstream = Router::new()
|
||||||
|
.route(
|
||||||
|
"/api/internal/gateway/resolve",
|
||||||
|
any(|_request: Request| async move {
|
||||||
|
Json(json!({
|
||||||
|
"action": "proxy_public",
|
||||||
|
"route_class": "ai_public",
|
||||||
|
"route_family": "openai",
|
||||||
|
"route_kind": "chat",
|
||||||
|
"auth_endpoint_signature": "openai:chat",
|
||||||
|
"execution_runtime_candidate": true,
|
||||||
|
"auth_context": {
|
||||||
|
"user_id": "user-openai-chat-gemini-stream-local-1",
|
||||||
|
"api_key_id": "api-key-openai-chat-gemini-stream-local-1",
|
||||||
|
"access_allowed": true
|
||||||
|
},
|
||||||
|
"public_path": "/v1/chat/completions"
|
||||||
|
}))
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/api/internal/gateway/decision-stream",
|
||||||
|
any(move |_request: Request| {
|
||||||
|
let decision_hits_inner = Arc::clone(&decision_hits_clone);
|
||||||
|
async move {
|
||||||
|
*decision_hits_inner.lock().expect("mutex should lock") += 1;
|
||||||
|
Json(json!({"action": "proxy_public"}))
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/api/internal/gateway/plan-stream",
|
||||||
|
any(move |_request: Request| {
|
||||||
|
let plan_hits_inner = Arc::clone(&plan_hits_clone);
|
||||||
|
async move {
|
||||||
|
*plan_hits_inner.lock().expect("mutex should lock") += 1;
|
||||||
|
Json(json!({"action": "proxy_public"}))
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/api/internal/gateway/report-stream",
|
||||||
|
any(move |request: Request| {
|
||||||
|
let seen_report_inner = Arc::clone(&seen_report_clone);
|
||||||
|
async move {
|
||||||
|
let (_parts, body) = request.into_parts();
|
||||||
|
let _raw_body = to_bytes(body, usize::MAX).await.expect("body should read");
|
||||||
|
*seen_report_inner.lock().expect("mutex should lock") = true;
|
||||||
|
Json(json!({"ok": true}))
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/v1/chat/completions",
|
||||||
|
any(move |_request: Request| {
|
||||||
|
let public_hits_inner = Arc::clone(&public_hits_clone);
|
||||||
|
async move {
|
||||||
|
*public_hits_inner.lock().expect("mutex should lock") += 1;
|
||||||
|
(StatusCode::IM_A_TEAPOT, Body::from("public-route-hit"))
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
let execution_runtime = Router::new().route(
|
||||||
|
"/v1/execute/stream",
|
||||||
|
any(move |request: Request| {
|
||||||
|
let seen_execution_runtime_inner = Arc::clone(&seen_execution_runtime_clone);
|
||||||
|
async move {
|
||||||
|
let (parts, body) = request.into_parts();
|
||||||
|
let raw_body = to_bytes(body, usize::MAX).await.expect("body should read");
|
||||||
|
let payload: serde_json::Value =
|
||||||
|
serde_json::from_slice(&raw_body).expect("execution runtime payload should parse");
|
||||||
|
*seen_execution_runtime_inner
|
||||||
|
.lock()
|
||||||
|
.expect("mutex should lock") = Some(SeenExecutionRuntimeStreamRequest {
|
||||||
|
trace_id: parts
|
||||||
|
.headers
|
||||||
|
.get(TRACE_ID_HEADER)
|
||||||
|
.and_then(|value| value.to_str().ok())
|
||||||
|
.unwrap_or_default()
|
||||||
|
.to_string(),
|
||||||
|
url: payload
|
||||||
|
.get("url")
|
||||||
|
.and_then(|value| value.as_str())
|
||||||
|
.unwrap_or_default()
|
||||||
|
.to_string(),
|
||||||
|
provider_model: payload
|
||||||
|
.get("body")
|
||||||
|
.and_then(|value| value.get("json_body"))
|
||||||
|
.and_then(|value| value.get("model"))
|
||||||
|
.and_then(|value| value.as_str())
|
||||||
|
.unwrap_or_default()
|
||||||
|
.to_string(),
|
||||||
|
auth_header_value: payload
|
||||||
|
.get("headers")
|
||||||
|
.and_then(|value| value.get("x-goog-api-key"))
|
||||||
|
.and_then(|value| value.as_str())
|
||||||
|
.unwrap_or_default()
|
||||||
|
.to_string(),
|
||||||
|
accept: payload
|
||||||
|
.get("headers")
|
||||||
|
.and_then(|value| value.get("accept"))
|
||||||
|
.and_then(|value| value.as_str())
|
||||||
|
.unwrap_or_default()
|
||||||
|
.to_string(),
|
||||||
|
endpoint_tag: payload
|
||||||
|
.get("headers")
|
||||||
|
.and_then(|value| value.get("x-endpoint-tag"))
|
||||||
|
.and_then(|value| value.as_str())
|
||||||
|
.unwrap_or_default()
|
||||||
|
.to_string(),
|
||||||
|
});
|
||||||
|
let frames = concat!(
|
||||||
|
"{\"type\":\"headers\",\"payload\":{\"kind\":\"headers\",\"status_code\":200,\"headers\":{\"content-type\":\"text/event-stream\"}}}\n",
|
||||||
|
"{\"type\":\"data\",\"payload\":{\"kind\":\"data\",\"text\":\"data: {\\\"responseId\\\":\\\"resp_openai_chat_gemini_stream_123\\\",\\\"candidates\\\":[{\\\"content\\\":{\\\"parts\\\":[{\\\"text\\\":\\\"Hello \\\"}],\\\"role\\\":\\\"model\\\"},\\\"index\\\":0}],\\\"modelVersion\\\":\\\"gemini-2.5-pro-upstream\\\"}\\n\\n\"}}\n",
|
||||||
|
"{\"type\":\"data\",\"payload\":{\"kind\":\"data\",\"text\":\"data: {\\\"responseId\\\":\\\"resp_openai_chat_gemini_stream_123\\\",\\\"candidates\\\":[{\\\"content\\\":{\\\"parts\\\":[{\\\"text\\\":\\\"Hello Gemini stream\\\"}],\\\"role\\\":\\\"model\\\"},\\\"finishReason\\\":\\\"STOP\\\",\\\"index\\\":0}],\\\"modelVersion\\\":\\\"gemini-2.5-pro-upstream\\\",\\\"usageMetadata\\\":{\\\"promptTokenCount\\\":1,\\\"candidatesTokenCount\\\":2,\\\"totalTokenCount\\\":3}}\\n\\n\"}}\n",
|
||||||
|
"{\"type\":\"telemetry\",\"payload\":{\"kind\":\"telemetry\",\"telemetry\":{\"elapsed_ms\":31,\"ttfb_ms\":11,\"upstream_bytes\":37}}}\n",
|
||||||
|
"{\"type\":\"eof\",\"payload\":{\"kind\":\"eof\"}}\n"
|
||||||
|
);
|
||||||
|
let mut response = Response::builder()
|
||||||
|
.status(StatusCode::OK)
|
||||||
|
.body(Body::from(frames))
|
||||||
|
.expect("response should build");
|
||||||
|
response.headers_mut().insert(
|
||||||
|
http::header::CONTENT_TYPE,
|
||||||
|
HeaderValue::from_static("application/x-ndjson"),
|
||||||
|
);
|
||||||
|
response
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
let auth_repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
|
||||||
|
Some(hash_api_key("sk-client-openai-chat-gemini-stream")),
|
||||||
|
sample_auth_snapshot(
|
||||||
|
"api-key-openai-chat-gemini-stream-local-1",
|
||||||
|
"user-openai-chat-gemini-stream-local-1",
|
||||||
|
),
|
||||||
|
)]));
|
||||||
|
let candidate_selection_repository =
|
||||||
|
Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(vec![
|
||||||
|
sample_candidate_row(),
|
||||||
|
]));
|
||||||
|
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||||
|
vec![sample_provider_catalog_provider()],
|
||||||
|
vec![sample_provider_catalog_endpoint()],
|
||||||
|
vec![sample_provider_catalog_key()],
|
||||||
|
));
|
||||||
|
|
||||||
|
let (upstream_url, upstream_handle) = start_server(upstream).await;
|
||||||
|
let (execution_runtime_url, execution_runtime_handle) = start_server(execution_runtime).await;
|
||||||
|
let gateway_state =
|
||||||
|
build_state_with_execution_runtime_override(execution_runtime_url.clone())
|
||||||
|
.with_data_state_for_tests(
|
||||||
|
crate::data::GatewayDataState::with_auth_candidate_selection_provider_catalog_and_request_candidate_repository_for_tests(
|
||||||
|
auth_repository,
|
||||||
|
candidate_selection_repository,
|
||||||
|
provider_catalog_repository,
|
||||||
|
Arc::clone(&request_candidate_repository),
|
||||||
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
|
)
|
||||||
|
.with_system_config_values_for_tests(vec![(
|
||||||
|
"provider_priority_mode".to_string(),
|
||||||
|
json!("global_key"),
|
||||||
|
)]),
|
||||||
|
);
|
||||||
|
let gateway = build_router_with_state(gateway_state);
|
||||||
|
let (gateway_url, gateway_handle) = start_server(gateway).await;
|
||||||
|
|
||||||
|
let response = reqwest::Client::new()
|
||||||
|
.post(format!("{gateway_url}/v1/chat/completions"))
|
||||||
|
.header(http::header::CONTENT_TYPE, "application/json")
|
||||||
|
.header(
|
||||||
|
http::header::AUTHORIZATION,
|
||||||
|
"Bearer sk-client-openai-chat-gemini-stream",
|
||||||
|
)
|
||||||
|
.header(TRACE_ID_HEADER, "trace-openai-chat-gemini-stream-123")
|
||||||
|
.body(
|
||||||
|
"{\"model\":\"gpt-5\",\"messages\":[{\"role\":\"system\",\"content\":\"You are terse.\"},{\"role\":\"user\",\"content\":\"Say hello\"}],\"stream\":true}",
|
||||||
|
)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.expect("request should succeed");
|
||||||
|
|
||||||
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
|
assert_eq!(
|
||||||
|
response
|
||||||
|
.headers()
|
||||||
|
.get(EXECUTION_PATH_HEADER)
|
||||||
|
.and_then(|value| value.to_str().ok()),
|
||||||
|
Some(EXECUTION_PATH_EXECUTION_RUNTIME_STREAM)
|
||||||
|
);
|
||||||
|
let response_text = response.text().await.expect("body should read");
|
||||||
|
assert!(response_text.contains("\"object\":\"chat.completion.chunk\""));
|
||||||
|
assert!(response_text.contains("data: [DONE]"));
|
||||||
|
|
||||||
|
let seen_execution_runtime_request = seen_execution_runtime
|
||||||
|
.lock()
|
||||||
|
.expect("mutex should lock")
|
||||||
|
.clone()
|
||||||
|
.expect("execution runtime stream should be captured");
|
||||||
|
assert_eq!(
|
||||||
|
seen_execution_runtime_request.trace_id,
|
||||||
|
"trace-openai-chat-gemini-stream-123"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
seen_execution_runtime_request.url,
|
||||||
|
"https://generativelanguage.googleapis.com/custom/v1beta/models/gemini-2.5-pro-upstream:streamGenerateContent?alt=sse"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
seen_execution_runtime_request.provider_model,
|
||||||
|
"gemini-2.5-pro-upstream"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
seen_execution_runtime_request.auth_header_value,
|
||||||
|
"sk-upstream-openai-chat-gemini-stream"
|
||||||
|
);
|
||||||
|
assert_eq!(seen_execution_runtime_request.accept, "text/event-stream");
|
||||||
|
assert_eq!(
|
||||||
|
seen_execution_runtime_request.endpoint_tag,
|
||||||
|
"openai-chat-gemini-cross-format-stream"
|
||||||
|
);
|
||||||
|
|
||||||
|
let stored_candidates = request_candidate_repository
|
||||||
|
.list_by_request_id("trace-openai-chat-gemini-stream-123")
|
||||||
|
.await
|
||||||
|
.expect("request candidate trace should read");
|
||||||
|
assert_eq!(stored_candidates.len(), 1);
|
||||||
|
assert_eq!(stored_candidates[0].status, RequestCandidateStatus::Success);
|
||||||
|
let extra_data = stored_candidates[0]
|
||||||
|
.extra_data
|
||||||
|
.as_ref()
|
||||||
|
.expect("request candidate extra_data should exist");
|
||||||
|
assert_eq!(extra_data["client_api_format"], "openai:chat");
|
||||||
|
assert_eq!(extra_data["provider_api_format"], "gemini:generate_content");
|
||||||
|
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||||
|
assert!(
|
||||||
|
!*seen_report.lock().expect("mutex should lock"),
|
||||||
|
"report-stream should stay local when request candidate persistence is available"
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(*decision_hits.lock().expect("mutex should lock"), 0);
|
||||||
|
assert_eq!(*plan_hits.lock().expect("mutex should lock"), 0);
|
||||||
|
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
|
||||||
|
|
||||||
|
gateway_handle.abort();
|
||||||
|
execution_runtime_handle.abort();
|
||||||
|
upstream_handle.abort();
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn gateway_executes_openai_chat_stream_with_custom_path_via_local_decision_gate_with_local_stream_decision(
|
async fn gateway_executes_openai_chat_stream_with_custom_path_via_local_decision_gate_with_local_stream_decision(
|
||||||
) {
|
) {
|
||||||
|
|||||||
@@ -301,6 +301,9 @@ pub fn build_admin_monitoring_trace_request_payload_response_with_key_accounts(
|
|||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
Json(json!({
|
Json(json!({
|
||||||
"request_id": trace.request_id,
|
"request_id": trace.request_id,
|
||||||
|
"request_path": admin_monitoring_trace_request_path(usage),
|
||||||
|
"request_query_string": admin_monitoring_trace_request_query_string(usage),
|
||||||
|
"request_path_and_query": admin_monitoring_trace_request_path_and_query(usage),
|
||||||
"total_candidates": trace.total_candidates,
|
"total_candidates": trace.total_candidates,
|
||||||
"final_status": trace.final_status,
|
"final_status": trace.final_status,
|
||||||
"total_latency_ms": trace.total_latency_ms,
|
"total_latency_ms": trace.total_latency_ms,
|
||||||
@@ -470,6 +473,33 @@ fn build_admin_monitoring_trace_candidate_extra_data(
|
|||||||
.entry("first_byte_time_ms".to_string())
|
.entry("first_byte_time_ms".to_string())
|
||||||
.or_insert_with(|| json!(first_byte_time_ms));
|
.or_insert_with(|| json!(first_byte_time_ms));
|
||||||
}
|
}
|
||||||
|
if let Some(request_path) = admin_monitoring_usage_request_path(usage) {
|
||||||
|
extra_object
|
||||||
|
.entry("request_path".to_string())
|
||||||
|
.or_insert_with(|| json!(request_path));
|
||||||
|
}
|
||||||
|
if let Some(request_query_string) = admin_monitoring_usage_request_query_string(usage) {
|
||||||
|
extra_object
|
||||||
|
.entry("request_query_string".to_string())
|
||||||
|
.or_insert_with(|| json!(request_query_string));
|
||||||
|
}
|
||||||
|
if let Some(request_path_and_query) = admin_monitoring_usage_request_path_and_query(usage) {
|
||||||
|
extra_object
|
||||||
|
.entry("request_path_and_query".to_string())
|
||||||
|
.or_insert_with(|| json!(request_path_and_query));
|
||||||
|
}
|
||||||
|
if admin_monitoring_usage_is_error_node(usage) {
|
||||||
|
if let Some(response) = admin_monitoring_trace_response_data(
|
||||||
|
"upstream_response",
|
||||||
|
usage.status_code,
|
||||||
|
usage.response_headers.as_ref(),
|
||||||
|
usage.response_body.as_ref(),
|
||||||
|
usage.response_body_ref.as_deref(),
|
||||||
|
usage.response_body_state,
|
||||||
|
) {
|
||||||
|
extra_object.insert("upstream_response".to_string(), response);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if let Some(proxy_value) = extra_object.get_mut("proxy") {
|
if let Some(proxy_value) = extra_object.get_mut("proxy") {
|
||||||
if let Some(proxy_object) = proxy_value.as_object_mut() {
|
if let Some(proxy_object) = proxy_value.as_object_mut() {
|
||||||
@@ -497,6 +527,96 @@ fn build_admin_monitoring_trace_candidate_extra_data(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn admin_monitoring_trace_response_data(
|
||||||
|
source: &str,
|
||||||
|
status_code: Option<u16>,
|
||||||
|
headers: Option<&Value>,
|
||||||
|
body: Option<&Value>,
|
||||||
|
body_ref: Option<&str>,
|
||||||
|
body_state: Option<aether_data_contracts::repository::usage::UsageBodyCaptureState>,
|
||||||
|
) -> Option<Value> {
|
||||||
|
if status_code.is_none()
|
||||||
|
&& headers.is_none()
|
||||||
|
&& body.is_none()
|
||||||
|
&& body_ref.is_none()
|
||||||
|
&& body_state.is_none()
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
Some(json!({
|
||||||
|
"source": source,
|
||||||
|
"status_code": status_code,
|
||||||
|
"headers": headers.cloned().unwrap_or(Value::Null),
|
||||||
|
"body": body.cloned().unwrap_or(Value::Null),
|
||||||
|
"body_ref": body_ref,
|
||||||
|
"body_state": body_state.map(|state| state.as_str()),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn admin_monitoring_usage_is_error_node(usage: &StoredRequestUsageAudit) -> bool {
|
||||||
|
!usage.status.eq_ignore_ascii_case("completed")
|
||||||
|
|| usage
|
||||||
|
.status_code
|
||||||
|
.is_some_and(|status| !(200..300).contains(&status))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn admin_monitoring_trace_request_path(usage: Option<&StoredRequestUsageAudit>) -> Option<String> {
|
||||||
|
usage.and_then(admin_monitoring_usage_request_path)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn admin_monitoring_trace_request_query_string(
|
||||||
|
usage: Option<&StoredRequestUsageAudit>,
|
||||||
|
) -> Option<String> {
|
||||||
|
usage.and_then(admin_monitoring_usage_request_query_string)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn admin_monitoring_trace_request_path_and_query(
|
||||||
|
usage: Option<&StoredRequestUsageAudit>,
|
||||||
|
) -> Option<String> {
|
||||||
|
usage.and_then(admin_monitoring_usage_request_path_and_query)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn admin_monitoring_usage_request_path(usage: &StoredRequestUsageAudit) -> Option<String> {
|
||||||
|
admin_monitoring_usage_metadata_string(usage, "request_path")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn admin_monitoring_usage_request_query_string(usage: &StoredRequestUsageAudit) -> Option<String> {
|
||||||
|
admin_monitoring_usage_metadata_string(usage, "request_query_string")
|
||||||
|
.map(|value| value.trim_start_matches('?').to_string())
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn admin_monitoring_usage_request_path_and_query(
|
||||||
|
usage: &StoredRequestUsageAudit,
|
||||||
|
) -> Option<String> {
|
||||||
|
admin_monitoring_usage_metadata_string(usage, "request_path_and_query").or_else(|| {
|
||||||
|
let path = admin_monitoring_usage_metadata_string(usage, "request_path")?;
|
||||||
|
let query = admin_monitoring_usage_metadata_string(usage, "request_query_string")
|
||||||
|
.map(|value| value.trim_start_matches('?').to_string())
|
||||||
|
.filter(|value| !value.is_empty());
|
||||||
|
Some(match query {
|
||||||
|
Some(query) if !path.contains('?') => format!("{path}?{query}"),
|
||||||
|
_ => path,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn admin_monitoring_usage_metadata_string(
|
||||||
|
usage: &StoredRequestUsageAudit,
|
||||||
|
key: &str,
|
||||||
|
) -> Option<String> {
|
||||||
|
usage
|
||||||
|
.request_metadata
|
||||||
|
.as_ref()
|
||||||
|
.and_then(Value::as_object)
|
||||||
|
.and_then(|metadata| metadata.get(key))
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.map(ToOwned::to_owned)
|
||||||
|
}
|
||||||
|
|
||||||
fn json_string_field(object: &serde_json::Map<String, Value>, key: &str) -> Option<String> {
|
fn json_string_field(object: &serde_json::Map<String, Value>, key: &str) -> Option<String> {
|
||||||
object
|
object
|
||||||
.get(key)
|
.get(key)
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
use crate::observability::stats::{aggregate_usage_stats, parse_bounded_u32, round_to};
|
use crate::observability::stats::{aggregate_usage_stats, parse_bounded_u32, round_to};
|
||||||
|
use aether_ai_formats::api::request_path_implies_stream_request;
|
||||||
use aether_billing::{
|
use aether_billing::{
|
||||||
normalize_input_tokens_for_billing, normalize_total_input_context_for_cache_hit_rate,
|
normalize_input_tokens_for_billing, normalize_total_input_context_for_cache_hit_rate,
|
||||||
};
|
};
|
||||||
@@ -227,7 +228,9 @@ pub fn admin_usage_matches_api_format(
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn admin_usage_is_failed(item: &StoredRequestUsageAudit) -> bool {
|
pub fn admin_usage_is_failed(item: &StoredRequestUsageAudit) -> bool {
|
||||||
let has_failure_signal = item.status_code.is_some_and(|value| value >= 400)
|
let has_failure_signal = item
|
||||||
|
.status_code
|
||||||
|
.is_some_and(|value| !(200..300).contains(&value))
|
||||||
|| item
|
|| item
|
||||||
.error_message
|
.error_message
|
||||||
.as_deref()
|
.as_deref()
|
||||||
@@ -256,7 +259,9 @@ pub fn admin_usage_matches_status(item: &StoredRequestUsageAudit, status: Option
|
|||||||
"stream" => item.is_stream,
|
"stream" => item.is_stream,
|
||||||
"standard" => !item.is_stream,
|
"standard" => !item.is_stream,
|
||||||
"error" => {
|
"error" => {
|
||||||
item.status_code.is_some_and(|value| value >= 400) || item.error_message.is_some()
|
item.status_code
|
||||||
|
.is_some_and(|value| !(200..300).contains(&value))
|
||||||
|
|| item.error_message.is_some()
|
||||||
}
|
}
|
||||||
"pending" | "streaming" | "completed" | "cancelled" => item.status == status,
|
"pending" | "streaming" | "completed" | "cancelled" => item.status == status,
|
||||||
"failed" => admin_usage_is_failed(item),
|
"failed" => admin_usage_is_failed(item),
|
||||||
@@ -955,12 +960,26 @@ fn admin_usage_infer_upstream_stream_from_captured_bodies(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn admin_usage_request_path_implies_client_stream(item: &StoredRequestUsageAudit) -> bool {
|
||||||
|
let Some(metadata) = item.request_metadata.as_ref().and_then(Value::as_object) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
["request_path", "request_path_and_query"]
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|field| metadata.get(field).and_then(Value::as_str))
|
||||||
|
.any(request_path_implies_stream_request)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn admin_usage_client_is_stream(item: &StoredRequestUsageAudit) -> bool {
|
pub fn admin_usage_client_is_stream(item: &StoredRequestUsageAudit) -> bool {
|
||||||
item.request_metadata
|
admin_usage_request_path_implies_client_stream(item)
|
||||||
.as_ref()
|
.then_some(true)
|
||||||
.and_then(Value::as_object)
|
.or_else(|| {
|
||||||
.and_then(|metadata| metadata.get("client_requested_stream"))
|
item.request_metadata
|
||||||
.and_then(Value::as_bool)
|
.as_ref()
|
||||||
|
.and_then(Value::as_object)
|
||||||
|
.and_then(|metadata| metadata.get("client_requested_stream"))
|
||||||
|
.and_then(Value::as_bool)
|
||||||
|
})
|
||||||
.or_else(|| admin_usage_request_body_stream_flag(item))
|
.or_else(|| admin_usage_request_body_stream_flag(item))
|
||||||
.or_else(|| admin_usage_headers_stream_flag(item.client_response_headers.as_ref()))
|
.or_else(|| admin_usage_headers_stream_flag(item.client_response_headers.as_ref()))
|
||||||
.or_else(|| admin_usage_request_body_implies_default_non_stream(item).then_some(false))
|
.or_else(|| admin_usage_request_body_implies_default_non_stream(item).then_some(false))
|
||||||
@@ -1665,7 +1684,9 @@ pub fn admin_usage_is_success(item: &StoredRequestUsageAudit) -> bool {
|
|||||||
matches!(
|
matches!(
|
||||||
item.status.as_str(),
|
item.status.as_str(),
|
||||||
"completed" | "success" | "ok" | "billed" | "settled"
|
"completed" | "success" | "ok" | "billed" | "settled"
|
||||||
) && item.status_code.is_none_or(|code| code < 400)
|
) && item
|
||||||
|
.status_code
|
||||||
|
.is_none_or(|code| (200..300).contains(&code))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn admin_usage_matches_optional_id(value: Option<&str>, expected: Option<&str>) -> bool {
|
pub fn admin_usage_matches_optional_id(value: Option<&str>, expected: Option<&str>) -> bool {
|
||||||
@@ -2258,10 +2279,10 @@ mod tests {
|
|||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
admin_usage_active_request_json, admin_usage_client_is_stream, admin_usage_has_body_value,
|
admin_usage_active_request_json, admin_usage_client_is_stream, admin_usage_has_body_value,
|
||||||
admin_usage_has_fallback, admin_usage_is_failed, admin_usage_matches_search,
|
admin_usage_has_fallback, admin_usage_is_failed, admin_usage_is_success,
|
||||||
admin_usage_matches_status, admin_usage_matches_username, admin_usage_record_json,
|
admin_usage_matches_search, admin_usage_matches_status, admin_usage_matches_username,
|
||||||
admin_usage_resolve_request_capture_body, admin_usage_upstream_is_stream,
|
admin_usage_record_json, admin_usage_resolve_request_capture_body,
|
||||||
build_admin_usage_detail_payload,
|
admin_usage_upstream_is_stream, build_admin_usage_detail_payload,
|
||||||
};
|
};
|
||||||
use aether_data_contracts::repository::usage::{StoredRequestUsageAudit, UsageBodyField};
|
use aether_data_contracts::repository::usage::{StoredRequestUsageAudit, UsageBodyField};
|
||||||
|
|
||||||
@@ -2349,6 +2370,33 @@ mod tests {
|
|||||||
assert_eq!(record["client_is_stream"], false);
|
assert_eq!(record["client_is_stream"], false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn client_requested_stream_uses_stream_generate_content_path_over_stale_metadata_flag() {
|
||||||
|
let item = StoredRequestUsageAudit {
|
||||||
|
is_stream: true,
|
||||||
|
request_metadata: Some(json!({
|
||||||
|
"client_requested_stream": false,
|
||||||
|
"request_path": "/v1beta/models/gemini-3.1-flash-image-preview:streamGenerateContent",
|
||||||
|
"request_path_and_query": "/v1beta/models/gemini-3.1-flash-image-preview:streamGenerateContent?alt=sse"
|
||||||
|
})),
|
||||||
|
..sample_usage("completed", Some(200), None)
|
||||||
|
};
|
||||||
|
|
||||||
|
assert!(admin_usage_client_is_stream(&item));
|
||||||
|
|
||||||
|
let record = admin_usage_record_json(
|
||||||
|
&item,
|
||||||
|
&BTreeMap::new(),
|
||||||
|
&BTreeMap::new(),
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
assert_eq!(record["upstream_is_stream"], true);
|
||||||
|
assert_eq!(record["client_requested_stream"], true);
|
||||||
|
assert_eq!(record["client_is_stream"], true);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn client_requested_stream_falls_back_to_request_body_stream_flag() {
|
fn client_requested_stream_falls_back_to_request_body_stream_flag() {
|
||||||
let item = StoredRequestUsageAudit {
|
let item = StoredRequestUsageAudit {
|
||||||
@@ -2528,6 +2576,30 @@ mod tests {
|
|||||||
assert!(admin_usage_matches_status(&item, Some("failed")));
|
assert!(admin_usage_matches_status(&item, Some("failed")));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn redirect_status_is_not_admin_usage_success() {
|
||||||
|
let item = sample_usage("completed", Some(302), None);
|
||||||
|
|
||||||
|
assert!(!admin_usage_is_success(&item));
|
||||||
|
assert!(!admin_usage_is_failed(&item));
|
||||||
|
assert!(admin_usage_matches_status(&item, Some("error")));
|
||||||
|
assert!(admin_usage_matches_status(&item, Some("completed")));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn failed_redirect_status_counts_as_admin_usage_failed() {
|
||||||
|
let item = sample_usage(
|
||||||
|
"failed",
|
||||||
|
Some(302),
|
||||||
|
Some("execution runtime stream returned non-success status 302"),
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(admin_usage_is_failed(&item));
|
||||||
|
assert!(admin_usage_matches_status(&item, Some("failed")));
|
||||||
|
assert!(admin_usage_matches_status(&item, Some("error")));
|
||||||
|
assert!(!admin_usage_is_success(&item));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn active_status_with_failure_signal_counts_as_failed() {
|
fn active_status_with_failure_signal_counts_as_failed() {
|
||||||
let item = sample_usage("pending", Some(503), Some("upstream failed"));
|
let item = sample_usage("pending", Some(503), Some("upstream failed"));
|
||||||
|
|||||||
@@ -15,4 +15,5 @@ serde.workspace = true
|
|||||||
serde_json.workspace = true
|
serde_json.workspace = true
|
||||||
sha1 = "0.10"
|
sha1 = "0.10"
|
||||||
sha2.workspace = true
|
sha2.workspace = true
|
||||||
|
url.workspace = true
|
||||||
uuid.workspace = true
|
uuid.workspace = true
|
||||||
|
|||||||
@@ -96,7 +96,9 @@ pub use crate::formats::shared::response::{
|
|||||||
};
|
};
|
||||||
pub use crate::formats::shared::routing::{
|
pub use crate::formats::shared::routing::{
|
||||||
is_matching_stream_http_request, is_matching_stream_request,
|
is_matching_stream_http_request, is_matching_stream_request,
|
||||||
resolve_execution_runtime_stream_plan_kind, resolve_execution_runtime_sync_plan_kind,
|
request_path_implies_stream_request, resolve_execution_runtime_stream_plan_kind,
|
||||||
|
resolve_execution_runtime_sync_plan_kind, sanitize_request_path,
|
||||||
|
sanitize_request_path_and_query, sanitize_request_query_string,
|
||||||
supports_stream_execution_decision_kind, supports_sync_execution_decision_kind,
|
supports_stream_execution_decision_kind, supports_sync_execution_decision_kind,
|
||||||
};
|
};
|
||||||
pub use crate::formats::shared::sse::{encode_done_sse, encode_json_sse, map_claude_stop_reason};
|
pub use crate::formats::shared::sse::{encode_done_sse, encode_json_sse, map_claude_stop_reason};
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
use http::Method;
|
use http::Method;
|
||||||
|
use url::form_urlencoded;
|
||||||
|
|
||||||
use crate::contracts::{
|
use crate::contracts::{
|
||||||
CLAUDE_CHAT_STREAM_PLAN_KIND, CLAUDE_CHAT_SYNC_PLAN_KIND, CLAUDE_CLI_STREAM_PLAN_KIND,
|
CLAUDE_CHAT_STREAM_PLAN_KIND, CLAUDE_CHAT_SYNC_PLAN_KIND, CLAUDE_CLI_STREAM_PLAN_KIND,
|
||||||
@@ -304,6 +305,67 @@ fn resolve_gemini_generate_content_plan_kind(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn request_path_implies_stream_request(path: &str) -> bool {
|
||||||
|
let trimmed = path.trim();
|
||||||
|
let path = trimmed
|
||||||
|
.split_once('?')
|
||||||
|
.map(|(path, _)| path)
|
||||||
|
.unwrap_or(trimmed);
|
||||||
|
path.ends_with(":streamGenerateContent")
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn sanitize_request_path(path: &str) -> Option<String> {
|
||||||
|
let path = path
|
||||||
|
.trim()
|
||||||
|
.split_once('?')
|
||||||
|
.map(|(path, _)| path)
|
||||||
|
.unwrap_or_else(|| path.trim())
|
||||||
|
.trim();
|
||||||
|
(!path.is_empty()).then(|| path.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn sanitize_request_query_string(query: &str) -> Option<String> {
|
||||||
|
let query = query.trim().trim_start_matches('?').trim();
|
||||||
|
if query.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut serializer = form_urlencoded::Serializer::new(String::new());
|
||||||
|
for (key, value) in form_urlencoded::parse(query.as_bytes()) {
|
||||||
|
if request_query_key_is_safe_to_trace(key.as_ref()) {
|
||||||
|
serializer.append_pair(key.as_ref(), value.as_ref());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let sanitized = serializer.finish();
|
||||||
|
(!sanitized.is_empty()).then_some(sanitized)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn sanitize_request_path_and_query(path: &str, query: Option<&str>) -> Option<String> {
|
||||||
|
let trimmed = path.trim();
|
||||||
|
let (path, embedded_query) = trimmed
|
||||||
|
.split_once('?')
|
||||||
|
.map(|(path, query)| (path.trim(), Some(query)))
|
||||||
|
.unwrap_or((trimmed, None));
|
||||||
|
if path.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let sanitized_query = query
|
||||||
|
.and_then(sanitize_request_query_string)
|
||||||
|
.or_else(|| embedded_query.and_then(sanitize_request_query_string));
|
||||||
|
Some(match sanitized_query {
|
||||||
|
Some(query) => format!("{path}?{query}"),
|
||||||
|
None => path.to_string(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn request_query_key_is_safe_to_trace(key: &str) -> bool {
|
||||||
|
matches!(
|
||||||
|
key.to_ascii_lowercase().as_str(),
|
||||||
|
"alt" | "view" | "pagesize" | "page_size" | "limit" | "offset"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn is_matching_stream_request(
|
pub fn is_matching_stream_request(
|
||||||
plan_kind: &str,
|
plan_kind: &str,
|
||||||
path: &str,
|
path: &str,
|
||||||
@@ -320,7 +382,7 @@ pub fn is_matching_stream_request(
|
|||||||
.and_then(|value| value.as_bool())
|
.and_then(|value| value.as_bool())
|
||||||
.unwrap_or(false),
|
.unwrap_or(false),
|
||||||
GEMINI_CHAT_STREAM_PLAN_KIND | GEMINI_CLI_STREAM_PLAN_KIND => {
|
GEMINI_CHAT_STREAM_PLAN_KIND | GEMINI_CLI_STREAM_PLAN_KIND => {
|
||||||
path.ends_with(":streamGenerateContent")
|
request_path_implies_stream_request(path)
|
||||||
}
|
}
|
||||||
_ => true,
|
_ => true,
|
||||||
}
|
}
|
||||||
@@ -388,7 +450,9 @@ mod tests {
|
|||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
is_matching_stream_http_request, is_matching_stream_request,
|
is_matching_stream_http_request, is_matching_stream_request,
|
||||||
resolve_execution_runtime_stream_plan_kind, resolve_execution_runtime_sync_plan_kind,
|
request_path_implies_stream_request, resolve_execution_runtime_stream_plan_kind,
|
||||||
|
resolve_execution_runtime_sync_plan_kind, sanitize_request_path,
|
||||||
|
sanitize_request_path_and_query, sanitize_request_query_string,
|
||||||
supports_stream_execution_decision_kind, supports_sync_execution_decision_kind,
|
supports_stream_execution_decision_kind, supports_sync_execution_decision_kind,
|
||||||
};
|
};
|
||||||
use crate::contracts::{
|
use crate::contracts::{
|
||||||
@@ -609,6 +673,41 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn stream_path_detection_handles_gemini_method_paths_with_query() {
|
||||||
|
assert!(request_path_implies_stream_request(
|
||||||
|
"/v1beta/models/gemini-2.5-pro:streamGenerateContent?alt=sse"
|
||||||
|
));
|
||||||
|
assert!(request_path_implies_stream_request(
|
||||||
|
" /v1internal:streamGenerateContent?alt=sse "
|
||||||
|
));
|
||||||
|
assert!(!request_path_implies_stream_request(
|
||||||
|
"/v1beta/models/gemini-2.5-pro:generateContent?alt=sse"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn request_path_metadata_sanitizer_drops_sensitive_query_parameters() {
|
||||||
|
assert_eq!(
|
||||||
|
sanitize_request_path("/v1beta/models/gemini-2.5-pro:generateContent?key=secret")
|
||||||
|
.as_deref(),
|
||||||
|
Some("/v1beta/models/gemini-2.5-pro:generateContent")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
sanitize_request_query_string("?key=secret&alt=sse&pageSize=10&token=hidden")
|
||||||
|
.as_deref(),
|
||||||
|
Some("alt=sse&pageSize=10")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
sanitize_request_path_and_query(
|
||||||
|
"/v1beta/models/gemini-2.5-pro:streamGenerateContent?key=secret&alt=sse",
|
||||||
|
None
|
||||||
|
)
|
||||||
|
.as_deref(),
|
||||||
|
Some("/v1beta/models/gemini-2.5-pro:streamGenerateContent?alt=sse")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn stream_matching_requires_openai_stream_flag() {
|
fn stream_matching_requires_openai_stream_flag() {
|
||||||
assert!(!is_matching_stream_request(
|
assert!(!is_matching_stream_request(
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ use crate::claude_code::build_claude_code_messages_url;
|
|||||||
use crate::snapshot::GatewayProviderTransportSnapshot;
|
use crate::snapshot::GatewayProviderTransportSnapshot;
|
||||||
use crate::url::{
|
use crate::url::{
|
||||||
build_claude_messages_url, build_gemini_content_url, build_openai_chat_url,
|
build_claude_messages_url, build_gemini_content_url, build_openai_chat_url,
|
||||||
build_openai_responses_url, build_passthrough_path_url,
|
build_openai_responses_url, build_passthrough_path_url, normalize_gemini_content_action_path,
|
||||||
};
|
};
|
||||||
use crate::vertex::{
|
use crate::vertex::{
|
||||||
build_vertex_api_key_gemini_content_url, resolve_local_vertex_api_key_query_auth,
|
build_vertex_api_key_gemini_content_url, resolve_local_vertex_api_key_query_auth,
|
||||||
@@ -36,6 +36,8 @@ pub fn build_transport_request_url(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let provider_api_format = params.provider_api_format.trim().to_ascii_lowercase();
|
let provider_api_format = params.provider_api_format.trim().to_ascii_lowercase();
|
||||||
|
let normalized_provider_api_format =
|
||||||
|
aether_ai_formats::normalize_api_format_alias(&provider_api_format);
|
||||||
let custom_path = transport
|
let custom_path = transport
|
||||||
.endpoint
|
.endpoint
|
||||||
.custom_path
|
.custom_path
|
||||||
@@ -45,14 +47,19 @@ pub fn build_transport_request_url(
|
|||||||
.map(|path| expand_custom_path_template(path, build_path_params(params)));
|
.map(|path| expand_custom_path_template(path, build_path_params(params)));
|
||||||
|
|
||||||
if let Some(path) = custom_path.as_deref() {
|
if let Some(path) = custom_path.as_deref() {
|
||||||
let blocked_keys = if provider_api_format.starts_with("gemini:") {
|
let blocked_keys = if normalized_provider_api_format.starts_with("gemini:") {
|
||||||
&["key"][..]
|
&["key"][..]
|
||||||
} else {
|
} else {
|
||||||
&[][..]
|
&[][..]
|
||||||
};
|
};
|
||||||
|
let normalized_path = if normalized_provider_api_format == "gemini:generate_content" {
|
||||||
|
normalize_gemini_content_action_path(path, params.upstream_is_stream)
|
||||||
|
} else {
|
||||||
|
path.to_string()
|
||||||
|
};
|
||||||
let url = build_passthrough_path_url(
|
let url = build_passthrough_path_url(
|
||||||
&transport.endpoint.base_url,
|
&transport.endpoint.base_url,
|
||||||
path,
|
normalized_path.as_str(),
|
||||||
params.request_query,
|
params.request_query,
|
||||||
blocked_keys,
|
blocked_keys,
|
||||||
)?;
|
)?;
|
||||||
@@ -63,7 +70,7 @@ pub fn build_transport_request_url(
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
let url = match aether_ai_formats::normalize_api_format_alias(&provider_api_format).as_str() {
|
let url = match normalized_provider_api_format.as_str() {
|
||||||
"openai:chat" => Some(build_openai_chat_url(
|
"openai:chat" => Some(build_openai_chat_url(
|
||||||
&transport.endpoint.base_url,
|
&transport.endpoint.base_url,
|
||||||
params.request_query,
|
params.request_query,
|
||||||
@@ -570,6 +577,81 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rewrites_hardcoded_gemini_custom_path_action_to_match_stream_mode() {
|
||||||
|
let stream_transport = sample_transport(
|
||||||
|
"custom",
|
||||||
|
"gemini:generate_content",
|
||||||
|
"https://generativelanguage.googleapis.com",
|
||||||
|
Some("/v1beta/models/{model}:generateContent"),
|
||||||
|
);
|
||||||
|
|
||||||
|
let stream_url = build_transport_request_url(
|
||||||
|
&stream_transport,
|
||||||
|
TransportRequestUrlParams {
|
||||||
|
provider_api_format: "gemini:generate_content",
|
||||||
|
mapped_model: Some("gemini-2.5-pro"),
|
||||||
|
upstream_is_stream: true,
|
||||||
|
request_query: Some("key=client-key&foo=bar"),
|
||||||
|
kiro_api_region: None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.expect("stream custom path url");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
stream_url,
|
||||||
|
"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:streamGenerateContent?foo=bar&alt=sse"
|
||||||
|
);
|
||||||
|
|
||||||
|
let sync_transport = sample_transport(
|
||||||
|
"custom",
|
||||||
|
"gemini:generate_content",
|
||||||
|
"https://generativelanguage.googleapis.com",
|
||||||
|
Some("/v1beta/models/{model}:streamGenerateContent"),
|
||||||
|
);
|
||||||
|
|
||||||
|
let sync_url = build_transport_request_url(
|
||||||
|
&sync_transport,
|
||||||
|
TransportRequestUrlParams {
|
||||||
|
provider_api_format: "gemini:generate_content",
|
||||||
|
mapped_model: Some("gemini-2.5-pro"),
|
||||||
|
upstream_is_stream: false,
|
||||||
|
request_query: Some("foo=bar"),
|
||||||
|
kiro_api_region: None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.expect("sync custom path url");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
sync_url,
|
||||||
|
"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:generateContent?foo=bar"
|
||||||
|
);
|
||||||
|
|
||||||
|
let v1_transport = sample_transport(
|
||||||
|
"custom",
|
||||||
|
"gemini:generate_content",
|
||||||
|
"https://generativelanguage.googleapis.com",
|
||||||
|
Some("/v1/models/{model}:generateContent"),
|
||||||
|
);
|
||||||
|
|
||||||
|
let v1_stream_url = build_transport_request_url(
|
||||||
|
&v1_transport,
|
||||||
|
TransportRequestUrlParams {
|
||||||
|
provider_api_format: "gemini:generate_content",
|
||||||
|
mapped_model: Some("gemini-2.5-pro"),
|
||||||
|
upstream_is_stream: true,
|
||||||
|
request_query: None,
|
||||||
|
kiro_api_region: None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.expect("v1 stream custom path url");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
v1_stream_url,
|
||||||
|
"https://generativelanguage.googleapis.com/v1/models/gemini-2.5-pro:streamGenerateContent?alt=sse"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn keeps_original_custom_path_when_template_params_are_missing() {
|
fn keeps_original_custom_path_when_template_params_are_missing() {
|
||||||
let transport = sample_transport(
|
let transport = sample_transport(
|
||||||
|
|||||||
@@ -69,9 +69,10 @@ pub fn build_gemini_content_url(
|
|||||||
} else {
|
} else {
|
||||||
"generateContent"
|
"generateContent"
|
||||||
};
|
};
|
||||||
let mut url = if trimmed_base_url.ends_with("/v1beta") {
|
let mut url = if trimmed_base_url.ends_with("/v1") || trimmed_base_url.ends_with("/v1beta") {
|
||||||
format!("{trimmed_base_url}/models/{trimmed_model}:{operation}")
|
format!("{trimmed_base_url}/models/{trimmed_model}:{operation}")
|
||||||
} else if trimmed_base_url.contains("/v1beta/models/") {
|
} else if gemini_content_base_url_contains_model_path(trimmed_base_url) {
|
||||||
|
let trimmed_base_url = strip_gemini_content_action(trimmed_base_url);
|
||||||
format!("{trimmed_base_url}:{operation}")
|
format!("{trimmed_base_url}:{operation}")
|
||||||
} else {
|
} else {
|
||||||
format!("{trimmed_base_url}/v1beta/models/{trimmed_model}:{operation}")
|
format!("{trimmed_base_url}/v1beta/models/{trimmed_model}:{operation}")
|
||||||
@@ -80,6 +81,37 @@ pub fn build_gemini_content_url(
|
|||||||
Some(url)
|
Some(url)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn normalize_gemini_content_action_path(path: &str, stream: bool) -> String {
|
||||||
|
let trimmed = path.trim();
|
||||||
|
let (path, query) = split_path_query(trimmed);
|
||||||
|
let action = if stream {
|
||||||
|
"streamGenerateContent"
|
||||||
|
} else {
|
||||||
|
"generateContent"
|
||||||
|
};
|
||||||
|
let normalized = strip_gemini_content_action(path);
|
||||||
|
let normalized = if normalized.len() == path.len() {
|
||||||
|
path.to_string()
|
||||||
|
} else {
|
||||||
|
format!("{normalized}:{action}")
|
||||||
|
};
|
||||||
|
match query {
|
||||||
|
Some(query) => format!("{normalized}?{query}"),
|
||||||
|
None => normalized,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn strip_gemini_content_action(value: &str) -> &str {
|
||||||
|
value
|
||||||
|
.strip_suffix(":streamGenerateContent")
|
||||||
|
.or_else(|| value.strip_suffix(":generateContent"))
|
||||||
|
.unwrap_or(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn gemini_content_base_url_contains_model_path(value: &str) -> bool {
|
||||||
|
value.contains("/v1/models/") || value.contains("/v1beta/models/")
|
||||||
|
}
|
||||||
|
|
||||||
pub fn build_gemini_video_predict_long_running_url(
|
pub fn build_gemini_video_predict_long_running_url(
|
||||||
upstream_base_url: &str,
|
upstream_base_url: &str,
|
||||||
model: &str,
|
model: &str,
|
||||||
@@ -92,9 +124,9 @@ pub fn build_gemini_video_predict_long_running_url(
|
|||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut url = if trimmed_base_url.ends_with("/v1beta") {
|
let mut url = if trimmed_base_url.ends_with("/v1") || trimmed_base_url.ends_with("/v1beta") {
|
||||||
format!("{trimmed_base_url}/models/{trimmed_model}:predictLongRunning")
|
format!("{trimmed_base_url}/models/{trimmed_model}:predictLongRunning")
|
||||||
} else if trimmed_base_url.contains("/v1beta/models/") {
|
} else if gemini_content_base_url_contains_model_path(trimmed_base_url) {
|
||||||
format!("{trimmed_base_url}:predictLongRunning")
|
format!("{trimmed_base_url}:predictLongRunning")
|
||||||
} else {
|
} else {
|
||||||
format!("{trimmed_base_url}/v1beta/models/{trimmed_model}:predictLongRunning")
|
format!("{trimmed_base_url}/v1beta/models/{trimmed_model}:predictLongRunning")
|
||||||
@@ -248,6 +280,7 @@ mod tests {
|
|||||||
build_gemini_content_url, build_gemini_files_passthrough_url,
|
build_gemini_content_url, build_gemini_files_passthrough_url,
|
||||||
build_gemini_video_predict_long_running_url, build_openai_chat_url,
|
build_gemini_video_predict_long_running_url, build_openai_chat_url,
|
||||||
build_openai_responses_url, build_passthrough_path_url,
|
build_openai_responses_url, build_passthrough_path_url,
|
||||||
|
normalize_gemini_content_action_path,
|
||||||
};
|
};
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -289,6 +322,64 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn gemini_content_urls_rewrite_existing_base_action_for_stream_mode() {
|
||||||
|
assert_eq!(
|
||||||
|
build_gemini_content_url(
|
||||||
|
"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:generateContent",
|
||||||
|
"ignored-model",
|
||||||
|
true,
|
||||||
|
Some("foo=bar")
|
||||||
|
)
|
||||||
|
.as_deref(),
|
||||||
|
Some(
|
||||||
|
"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:streamGenerateContent?foo=bar"
|
||||||
|
)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
build_gemini_content_url(
|
||||||
|
"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:streamGenerateContent",
|
||||||
|
"ignored-model",
|
||||||
|
false,
|
||||||
|
Some("foo=bar")
|
||||||
|
)
|
||||||
|
.as_deref(),
|
||||||
|
Some(
|
||||||
|
"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:generateContent?foo=bar"
|
||||||
|
)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
build_gemini_content_url(
|
||||||
|
"https://generativelanguage.googleapis.com/v1/models/gemini-2.5-pro:generateContent",
|
||||||
|
"ignored-model",
|
||||||
|
true,
|
||||||
|
Some("foo=bar")
|
||||||
|
)
|
||||||
|
.as_deref(),
|
||||||
|
Some(
|
||||||
|
"https://generativelanguage.googleapis.com/v1/models/gemini-2.5-pro:streamGenerateContent?foo=bar"
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn normalizes_gemini_content_action_in_custom_paths() {
|
||||||
|
assert_eq!(
|
||||||
|
normalize_gemini_content_action_path(
|
||||||
|
"/v1beta/models/gemini-2.5-pro:generateContent?alt=sse",
|
||||||
|
true
|
||||||
|
),
|
||||||
|
"/v1beta/models/gemini-2.5-pro:streamGenerateContent?alt=sse"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
normalize_gemini_content_action_path(
|
||||||
|
"/v1beta/models/gemini-2.5-pro:streamGenerateContent",
|
||||||
|
false
|
||||||
|
),
|
||||||
|
"/v1beta/models/gemini-2.5-pro:generateContent"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn merges_base_path_and_request_query_for_passthrough_paths() {
|
fn merges_base_path_and_request_query_for_passthrough_paths() {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
|
|||||||
@@ -17,11 +17,15 @@ pub struct SchedulerRequestCandidateReportContext {
|
|||||||
pub key_id: Option<String>,
|
pub key_id: Option<String>,
|
||||||
pub client_api_format: Option<String>,
|
pub client_api_format: Option<String>,
|
||||||
pub provider_api_format: Option<String>,
|
pub provider_api_format: Option<String>,
|
||||||
|
pub request_path: Option<String>,
|
||||||
|
pub request_query_string: Option<String>,
|
||||||
|
pub request_path_and_query: Option<String>,
|
||||||
pub upstream_url: Option<String>,
|
pub upstream_url: Option<String>,
|
||||||
pub mapped_model: Option<String>,
|
pub mapped_model: Option<String>,
|
||||||
pub key_name: Option<String>,
|
pub key_name: Option<String>,
|
||||||
pub header_rules: Option<Value>,
|
pub header_rules: Option<Value>,
|
||||||
pub body_rules: Option<Value>,
|
pub body_rules: Option<Value>,
|
||||||
|
pub upstream_response: Option<Value>,
|
||||||
pub proxy: Option<Value>,
|
pub proxy: Option<Value>,
|
||||||
pub error_flow: Option<Value>,
|
pub error_flow: Option<Value>,
|
||||||
pub candidate_group_id: Option<String>,
|
pub candidate_group_id: Option<String>,
|
||||||
@@ -60,11 +64,15 @@ pub struct SchedulerExecutionRequestCandidateSeed {
|
|||||||
struct ReportCandidateExtraDataInput {
|
struct ReportCandidateExtraDataInput {
|
||||||
client_api_format: Option<String>,
|
client_api_format: Option<String>,
|
||||||
provider_api_format: Option<String>,
|
provider_api_format: Option<String>,
|
||||||
|
request_path: Option<String>,
|
||||||
|
request_query_string: Option<String>,
|
||||||
|
request_path_and_query: Option<String>,
|
||||||
upstream_url: Option<String>,
|
upstream_url: Option<String>,
|
||||||
mapped_model: Option<String>,
|
mapped_model: Option<String>,
|
||||||
key_name: Option<String>,
|
key_name: Option<String>,
|
||||||
header_rules: Option<Value>,
|
header_rules: Option<Value>,
|
||||||
body_rules: Option<Value>,
|
body_rules: Option<Value>,
|
||||||
|
upstream_response: Option<Value>,
|
||||||
proxy: Option<Value>,
|
proxy: Option<Value>,
|
||||||
error_flow: Option<Value>,
|
error_flow: Option<Value>,
|
||||||
candidate_group_id: Option<String>,
|
candidate_group_id: Option<String>,
|
||||||
@@ -138,6 +146,9 @@ pub fn parse_request_candidate_report_context(
|
|||||||
key_id: string_field(report_context, "key_id"),
|
key_id: string_field(report_context, "key_id"),
|
||||||
client_api_format: string_field(report_context, "client_api_format"),
|
client_api_format: string_field(report_context, "client_api_format"),
|
||||||
provider_api_format: string_field(report_context, "provider_api_format"),
|
provider_api_format: string_field(report_context, "provider_api_format"),
|
||||||
|
request_path: string_field(report_context, "request_path"),
|
||||||
|
request_query_string: string_field(report_context, "request_query_string"),
|
||||||
|
request_path_and_query: string_field(report_context, "request_path_and_query"),
|
||||||
upstream_url: string_field(report_context, "upstream_url"),
|
upstream_url: string_field(report_context, "upstream_url"),
|
||||||
mapped_model: string_field(report_context, "mapped_model"),
|
mapped_model: string_field(report_context, "mapped_model"),
|
||||||
key_name: string_field(report_context, "key_name"),
|
key_name: string_field(report_context, "key_name"),
|
||||||
@@ -149,6 +160,10 @@ pub fn parse_request_candidate_report_context(
|
|||||||
.get("body_rules")
|
.get("body_rules")
|
||||||
.cloned()
|
.cloned()
|
||||||
.filter(|value| !value.is_null()),
|
.filter(|value| !value.is_null()),
|
||||||
|
upstream_response: report_context
|
||||||
|
.get("upstream_response")
|
||||||
|
.cloned()
|
||||||
|
.filter(|value| !value.is_null()),
|
||||||
proxy: report_context
|
proxy: report_context
|
||||||
.get("proxy")
|
.get("proxy")
|
||||||
.cloned()
|
.cloned()
|
||||||
@@ -187,11 +202,15 @@ pub fn resolve_report_request_candidate_slot(
|
|||||||
key_id,
|
key_id,
|
||||||
client_api_format,
|
client_api_format,
|
||||||
provider_api_format,
|
provider_api_format,
|
||||||
|
request_path,
|
||||||
|
request_query_string,
|
||||||
|
request_path_and_query,
|
||||||
upstream_url,
|
upstream_url,
|
||||||
mapped_model,
|
mapped_model,
|
||||||
key_name,
|
key_name,
|
||||||
header_rules,
|
header_rules,
|
||||||
body_rules,
|
body_rules,
|
||||||
|
upstream_response,
|
||||||
proxy,
|
proxy,
|
||||||
error_flow,
|
error_flow,
|
||||||
candidate_group_id,
|
candidate_group_id,
|
||||||
@@ -207,11 +226,15 @@ pub fn resolve_report_request_candidate_slot(
|
|||||||
let synthesized_extra_data = build_report_candidate_extra_data(ReportCandidateExtraDataInput {
|
let synthesized_extra_data = build_report_candidate_extra_data(ReportCandidateExtraDataInput {
|
||||||
client_api_format,
|
client_api_format,
|
||||||
provider_api_format,
|
provider_api_format,
|
||||||
|
request_path,
|
||||||
|
request_query_string,
|
||||||
|
request_path_and_query,
|
||||||
upstream_url,
|
upstream_url,
|
||||||
mapped_model,
|
mapped_model,
|
||||||
key_name,
|
key_name,
|
||||||
header_rules,
|
header_rules,
|
||||||
body_rules,
|
body_rules,
|
||||||
|
upstream_response,
|
||||||
proxy,
|
proxy,
|
||||||
error_flow,
|
error_flow,
|
||||||
candidate_group_id,
|
candidate_group_id,
|
||||||
@@ -326,11 +349,15 @@ pub fn build_execution_request_candidate_seed(
|
|||||||
build_report_candidate_extra_data(ReportCandidateExtraDataInput {
|
build_report_candidate_extra_data(ReportCandidateExtraDataInput {
|
||||||
client_api_format: metadata.client_api_format,
|
client_api_format: metadata.client_api_format,
|
||||||
provider_api_format: metadata.provider_api_format,
|
provider_api_format: metadata.provider_api_format,
|
||||||
|
request_path: metadata.request_path,
|
||||||
|
request_query_string: metadata.request_query_string,
|
||||||
|
request_path_and_query: metadata.request_path_and_query,
|
||||||
upstream_url: metadata.upstream_url,
|
upstream_url: metadata.upstream_url,
|
||||||
mapped_model: metadata.mapped_model,
|
mapped_model: metadata.mapped_model,
|
||||||
key_name: metadata.key_name,
|
key_name: metadata.key_name,
|
||||||
header_rules: metadata.header_rules,
|
header_rules: metadata.header_rules,
|
||||||
body_rules: metadata.body_rules,
|
body_rules: metadata.body_rules,
|
||||||
|
upstream_response: metadata.upstream_response,
|
||||||
proxy: metadata.proxy,
|
proxy: metadata.proxy,
|
||||||
error_flow: metadata.error_flow,
|
error_flow: metadata.error_flow,
|
||||||
candidate_group_id: metadata.candidate_group_id,
|
candidate_group_id: metadata.candidate_group_id,
|
||||||
@@ -431,11 +458,15 @@ pub fn build_local_request_candidate_status_record(
|
|||||||
let extra_data = build_report_candidate_extra_data(ReportCandidateExtraDataInput {
|
let extra_data = build_report_candidate_extra_data(ReportCandidateExtraDataInput {
|
||||||
client_api_format: metadata.client_api_format.clone(),
|
client_api_format: metadata.client_api_format.clone(),
|
||||||
provider_api_format: metadata.provider_api_format.clone(),
|
provider_api_format: metadata.provider_api_format.clone(),
|
||||||
|
request_path: metadata.request_path.clone(),
|
||||||
|
request_query_string: metadata.request_query_string.clone(),
|
||||||
|
request_path_and_query: metadata.request_path_and_query.clone(),
|
||||||
upstream_url: metadata.upstream_url.clone(),
|
upstream_url: metadata.upstream_url.clone(),
|
||||||
mapped_model: metadata.mapped_model.clone(),
|
mapped_model: metadata.mapped_model.clone(),
|
||||||
key_name: metadata.key_name.clone(),
|
key_name: metadata.key_name.clone(),
|
||||||
header_rules: metadata.header_rules.clone(),
|
header_rules: metadata.header_rules.clone(),
|
||||||
body_rules: metadata.body_rules.clone(),
|
body_rules: metadata.body_rules.clone(),
|
||||||
|
upstream_response: metadata.upstream_response.clone(),
|
||||||
proxy: metadata.proxy.clone(),
|
proxy: metadata.proxy.clone(),
|
||||||
error_flow: metadata.error_flow.clone(),
|
error_flow: metadata.error_flow.clone(),
|
||||||
candidate_group_id: metadata.candidate_group_id.clone(),
|
candidate_group_id: metadata.candidate_group_id.clone(),
|
||||||
@@ -667,11 +698,15 @@ fn build_report_candidate_extra_data(input: ReportCandidateExtraDataInput) -> Op
|
|||||||
let ReportCandidateExtraDataInput {
|
let ReportCandidateExtraDataInput {
|
||||||
client_api_format,
|
client_api_format,
|
||||||
provider_api_format,
|
provider_api_format,
|
||||||
|
request_path,
|
||||||
|
request_query_string,
|
||||||
|
request_path_and_query,
|
||||||
upstream_url,
|
upstream_url,
|
||||||
mapped_model,
|
mapped_model,
|
||||||
key_name,
|
key_name,
|
||||||
header_rules,
|
header_rules,
|
||||||
body_rules,
|
body_rules,
|
||||||
|
upstream_response,
|
||||||
proxy,
|
proxy,
|
||||||
error_flow,
|
error_flow,
|
||||||
candidate_group_id,
|
candidate_group_id,
|
||||||
@@ -698,6 +733,21 @@ fn build_report_candidate_extra_data(input: ReportCandidateExtraDataInput) -> Op
|
|||||||
Value::String(provider_api_format),
|
Value::String(provider_api_format),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
if let Some(request_path) = request_path {
|
||||||
|
extra_data.insert("request_path".to_string(), Value::String(request_path));
|
||||||
|
}
|
||||||
|
if let Some(request_query_string) = request_query_string {
|
||||||
|
extra_data.insert(
|
||||||
|
"request_query_string".to_string(),
|
||||||
|
Value::String(request_query_string),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if let Some(request_path_and_query) = request_path_and_query {
|
||||||
|
extra_data.insert(
|
||||||
|
"request_path_and_query".to_string(),
|
||||||
|
Value::String(request_path_and_query),
|
||||||
|
);
|
||||||
|
}
|
||||||
if let Some(upstream_url) = upstream_url {
|
if let Some(upstream_url) = upstream_url {
|
||||||
extra_data.insert("upstream_url".to_string(), Value::String(upstream_url));
|
extra_data.insert("upstream_url".to_string(), Value::String(upstream_url));
|
||||||
}
|
}
|
||||||
@@ -713,6 +763,9 @@ fn build_report_candidate_extra_data(input: ReportCandidateExtraDataInput) -> Op
|
|||||||
if let Some(body_rules) = body_rules {
|
if let Some(body_rules) = body_rules {
|
||||||
extra_data.insert("body_rules".to_string(), body_rules);
|
extra_data.insert("body_rules".to_string(), body_rules);
|
||||||
}
|
}
|
||||||
|
if let Some(upstream_response) = upstream_response {
|
||||||
|
extra_data.insert("upstream_response".to_string(), upstream_response);
|
||||||
|
}
|
||||||
if let Some(proxy) = proxy {
|
if let Some(proxy) = proxy {
|
||||||
extra_data.insert("proxy".to_string(), proxy);
|
extra_data.insert("proxy".to_string(), proxy);
|
||||||
}
|
}
|
||||||
@@ -903,6 +956,11 @@ mod tests {
|
|||||||
"body_rules": [
|
"body_rules": [
|
||||||
{"op": "remove", "path": "/store"}
|
{"op": "remove", "path": "/store"}
|
||||||
],
|
],
|
||||||
|
"upstream_response": {
|
||||||
|
"status_code": 503,
|
||||||
|
"headers": {"retry-after": "2"},
|
||||||
|
"body": {"error": {"message": "overloaded"}}
|
||||||
|
},
|
||||||
"proxy": {
|
"proxy": {
|
||||||
"node_id": "proxy-node-1",
|
"node_id": "proxy-node-1",
|
||||||
"node_name": "edge-1",
|
"node_name": "edge-1",
|
||||||
@@ -962,6 +1020,13 @@ mod tests {
|
|||||||
.map(Vec::len),
|
.map(Vec::len),
|
||||||
Some(1)
|
Some(1)
|
||||||
);
|
);
|
||||||
|
assert_eq!(
|
||||||
|
slot.extra_data
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|value| value.get("upstream_response"))
|
||||||
|
.and_then(|value| value.get("status_code")),
|
||||||
|
Some(&json!(503))
|
||||||
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
slot.extra_data
|
slot.extra_data
|
||||||
.as_ref()
|
.as_ref()
|
||||||
@@ -1063,6 +1128,8 @@ mod tests {
|
|||||||
"api_key_id": "api-key-1",
|
"api_key_id": "api-key-1",
|
||||||
"client_api_format": "openai:chat",
|
"client_api_format": "openai:chat",
|
||||||
"provider_api_format": "openai:responses",
|
"provider_api_format": "openai:responses",
|
||||||
|
"request_path": "/v1/responses",
|
||||||
|
"request_query_string": "debug=true",
|
||||||
"upstream_url": "https://example.com/v1/responses",
|
"upstream_url": "https://example.com/v1/responses",
|
||||||
"mapped_model": "gpt-5-upstream",
|
"mapped_model": "gpt-5-upstream",
|
||||||
"key_name": "primary",
|
"key_name": "primary",
|
||||||
@@ -1098,6 +1165,20 @@ mod tests {
|
|||||||
.and_then(|value| value.get("provider_api_format")),
|
.and_then(|value| value.get("provider_api_format")),
|
||||||
Some(&json!("openai:responses"))
|
Some(&json!("openai:responses"))
|
||||||
);
|
);
|
||||||
|
assert_eq!(
|
||||||
|
record
|
||||||
|
.extra_data
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|value| value.get("request_path")),
|
||||||
|
Some(&json!("/v1/responses"))
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
record
|
||||||
|
.extra_data
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|value| value.get("request_query_string")),
|
||||||
|
Some(&json!("debug=true"))
|
||||||
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
record
|
record
|
||||||
.extra_data
|
.extra_data
|
||||||
|
|||||||
@@ -1,3 +1,6 @@
|
|||||||
|
use aether_ai_formats::api::{
|
||||||
|
sanitize_request_path, sanitize_request_path_and_query, sanitize_request_query_string,
|
||||||
|
};
|
||||||
use aether_contracts::ExecutionPlan;
|
use aether_contracts::ExecutionPlan;
|
||||||
use serde_json::{json, Map, Value};
|
use serde_json::{json, Map, Value};
|
||||||
|
|
||||||
@@ -72,9 +75,13 @@ fn copy_allowed_metadata_fields(source: &Map<String, Value>, target: &mut Map<St
|
|||||||
copy_bool(source, target, "client_requested_stream");
|
copy_bool(source, target, "client_requested_stream");
|
||||||
copy_bool(source, target, "upstream_is_stream");
|
copy_bool(source, target, "upstream_is_stream");
|
||||||
copy_bool(source, target, "api_key_is_standalone");
|
copy_bool(source, target, "api_key_is_standalone");
|
||||||
|
copy_non_empty_string(source, target, "request_path");
|
||||||
|
copy_non_empty_string(source, target, "request_query_string");
|
||||||
|
copy_non_empty_string(source, target, "request_path_and_query");
|
||||||
copy_number(source, target, "provider_request_body_base64_bytes");
|
copy_number(source, target, "provider_request_body_base64_bytes");
|
||||||
copy_number(source, target, "provider_response_body_base64_bytes");
|
copy_number(source, target, "provider_response_body_base64_bytes");
|
||||||
copy_number(source, target, "client_response_body_base64_bytes");
|
copy_number(source, target, "client_response_body_base64_bytes");
|
||||||
|
copy_number(source, target, "client_response_status_code");
|
||||||
copy_non_null_value(source, target, "billing_snapshot");
|
copy_non_null_value(source, target, "billing_snapshot");
|
||||||
copy_non_empty_string(source, target, "billing_snapshot_schema_version");
|
copy_non_empty_string(source, target, "billing_snapshot_schema_version");
|
||||||
copy_non_empty_string(source, target, "billing_snapshot_status");
|
copy_non_empty_string(source, target, "billing_snapshot_status");
|
||||||
@@ -96,6 +103,7 @@ fn copy_allowed_metadata_fields(source: &Map<String, Value>, target: &mut Map<St
|
|||||||
copy_number(source, target, "cache_read_price_per_1m");
|
copy_number(source, target, "cache_read_price_per_1m");
|
||||||
copy_number(source, target, "price_per_request");
|
copy_number(source, target, "price_per_request");
|
||||||
copy_non_null_value(source, target, "proxy");
|
copy_non_null_value(source, target, "proxy");
|
||||||
|
sanitize_request_path_metadata_fields(target);
|
||||||
}
|
}
|
||||||
|
|
||||||
fn move_allowed_metadata_fields(mut source: Map<String, Value>, target: &mut Map<String, Value>) {
|
fn move_allowed_metadata_fields(mut source: Map<String, Value>, target: &mut Map<String, Value>) {
|
||||||
@@ -105,9 +113,13 @@ fn move_allowed_metadata_fields(mut source: Map<String, Value>, target: &mut Map
|
|||||||
remove_bool(&mut source, target, "client_requested_stream");
|
remove_bool(&mut source, target, "client_requested_stream");
|
||||||
remove_bool(&mut source, target, "upstream_is_stream");
|
remove_bool(&mut source, target, "upstream_is_stream");
|
||||||
remove_bool(&mut source, target, "api_key_is_standalone");
|
remove_bool(&mut source, target, "api_key_is_standalone");
|
||||||
|
remove_non_empty_string(&mut source, target, "request_path");
|
||||||
|
remove_non_empty_string(&mut source, target, "request_query_string");
|
||||||
|
remove_non_empty_string(&mut source, target, "request_path_and_query");
|
||||||
remove_number(&mut source, target, "provider_request_body_base64_bytes");
|
remove_number(&mut source, target, "provider_request_body_base64_bytes");
|
||||||
remove_number(&mut source, target, "provider_response_body_base64_bytes");
|
remove_number(&mut source, target, "provider_response_body_base64_bytes");
|
||||||
remove_number(&mut source, target, "client_response_body_base64_bytes");
|
remove_number(&mut source, target, "client_response_body_base64_bytes");
|
||||||
|
remove_number(&mut source, target, "client_response_status_code");
|
||||||
remove_non_null_value(&mut source, target, "billing_snapshot");
|
remove_non_null_value(&mut source, target, "billing_snapshot");
|
||||||
remove_non_empty_string(&mut source, target, "billing_snapshot_schema_version");
|
remove_non_empty_string(&mut source, target, "billing_snapshot_schema_version");
|
||||||
remove_non_empty_string(&mut source, target, "billing_snapshot_status");
|
remove_non_empty_string(&mut source, target, "billing_snapshot_status");
|
||||||
@@ -129,6 +141,38 @@ fn move_allowed_metadata_fields(mut source: Map<String, Value>, target: &mut Map
|
|||||||
remove_number(&mut source, target, "cache_read_price_per_1m");
|
remove_number(&mut source, target, "cache_read_price_per_1m");
|
||||||
remove_number(&mut source, target, "price_per_request");
|
remove_number(&mut source, target, "price_per_request");
|
||||||
remove_non_null_value(&mut source, target, "proxy");
|
remove_non_null_value(&mut source, target, "proxy");
|
||||||
|
sanitize_request_path_metadata_fields(target);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sanitize_request_path_metadata_fields(target: &mut Map<String, Value>) {
|
||||||
|
let path = target
|
||||||
|
.get("request_path")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.and_then(sanitize_request_path);
|
||||||
|
let query = target
|
||||||
|
.get("request_query_string")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.and_then(sanitize_request_query_string);
|
||||||
|
let path_and_query = target
|
||||||
|
.get("request_path_and_query")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.and_then(|value| sanitize_request_path_and_query(value, None))
|
||||||
|
.or_else(|| {
|
||||||
|
path.as_deref()
|
||||||
|
.and_then(|path| sanitize_request_path_and_query(path, query.as_deref()))
|
||||||
|
});
|
||||||
|
|
||||||
|
apply_optional_string_field(target, "request_path", path.as_deref());
|
||||||
|
apply_optional_string_field(target, "request_query_string", query.as_deref());
|
||||||
|
apply_optional_string_field(target, "request_path_and_query", path_and_query.as_deref());
|
||||||
|
}
|
||||||
|
|
||||||
|
fn apply_optional_string_field(target: &mut Map<String, Value>, key: &str, value: Option<&str>) {
|
||||||
|
if let Some(value) = value {
|
||||||
|
target.insert(key.to_string(), Value::String(value.to_string()));
|
||||||
|
} else {
|
||||||
|
target.remove(key);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn copy_non_empty_string(source: &Map<String, Value>, target: &mut Map<String, Value>, key: &str) {
|
fn copy_non_empty_string(source: &Map<String, Value>, target: &mut Map<String, Value>, key: &str) {
|
||||||
@@ -451,6 +495,25 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn sanitizes_request_path_query_metadata() {
|
||||||
|
let metadata = sanitize_usage_request_metadata(Some(json!({
|
||||||
|
"request_path": "/v1beta/models/gemini-2.5-pro:streamGenerateContent?key=secret",
|
||||||
|
"request_query_string": "key=secret&alt=sse&pageSize=10&token=hidden",
|
||||||
|
"request_path_and_query": "/v1beta/models/gemini-2.5-pro:streamGenerateContent?key=secret&alt=sse&pageSize=10&token=hidden",
|
||||||
|
})))
|
||||||
|
.expect("metadata should remain");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
metadata,
|
||||||
|
json!({
|
||||||
|
"request_path": "/v1beta/models/gemini-2.5-pro:streamGenerateContent",
|
||||||
|
"request_query_string": "alt=sse&pageSize=10",
|
||||||
|
"request_path_and_query": "/v1beta/models/gemini-2.5-pro:streamGenerateContent?alt=sse&pageSize=10",
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn sanitizes_large_allowed_metadata_values_to_bounded_representations() {
|
fn sanitizes_large_allowed_metadata_values_to_bounded_representations() {
|
||||||
let metadata = sanitize_usage_request_metadata(Some(json!({
|
let metadata = sanitize_usage_request_metadata(Some(json!({
|
||||||
|
|||||||
@@ -529,7 +529,8 @@ fn build_terminal_usage_event_from_seed_impl(
|
|||||||
};
|
};
|
||||||
let routing = merge_routing_seed_with_metadata_owned(routing, request_metadata.as_ref());
|
let routing = merge_routing_seed_with_metadata_owned(routing, request_metadata.as_ref());
|
||||||
let body_refs = merge_body_refs_seed_with_metadata_owned(body_refs, request_metadata.as_ref());
|
let body_refs = merge_body_refs_seed_with_metadata_owned(body_refs, request_metadata.as_ref());
|
||||||
let error_message = resolve_error_message(status_code, provider_response.as_ref(), None);
|
let error_message = resolve_error_message(status_code, provider_response.as_ref(), None)
|
||||||
|
.or_else(|| resolve_error_message(status_code, client_response.as_ref(), None));
|
||||||
let api_family = infer_api_family(&client_contract).map(ToOwned::to_owned);
|
let api_family = infer_api_family(&client_contract).map(ToOwned::to_owned);
|
||||||
let endpoint_kind = infer_endpoint_kind(&client_contract).map(ToOwned::to_owned);
|
let endpoint_kind = infer_endpoint_kind(&client_contract).map(ToOwned::to_owned);
|
||||||
let provider_api_family = infer_api_family(&provider_contract).map(ToOwned::to_owned);
|
let provider_api_family = infer_api_family(&provider_contract).map(ToOwned::to_owned);
|
||||||
@@ -734,7 +735,8 @@ pub fn build_sync_terminal_usage_payload_seed(
|
|||||||
let context = payload.report_context.as_ref().and_then(Value::as_object);
|
let context = payload.report_context.as_ref().and_then(Value::as_object);
|
||||||
let provider_response_headers = context_usage_value(context, "provider_response_headers")
|
let provider_response_headers = context_usage_value(context, "provider_response_headers")
|
||||||
.or_else(|| headers_to_json(&payload.headers));
|
.or_else(|| headers_to_json(&payload.headers));
|
||||||
let client_response_headers = headers_to_json(&payload.headers);
|
let client_response_headers = context_usage_value(context, "client_response_headers")
|
||||||
|
.or_else(|| headers_to_json(&payload.headers));
|
||||||
SyncTerminalUsagePayloadSeed {
|
SyncTerminalUsagePayloadSeed {
|
||||||
report_kind: payload.report_kind.clone(),
|
report_kind: payload.report_kind.clone(),
|
||||||
status_code: payload.status_code,
|
status_code: payload.status_code,
|
||||||
@@ -941,7 +943,7 @@ fn infer_sync_terminal_state(
|
|||||||
) -> UsageTerminalState {
|
) -> UsageTerminalState {
|
||||||
if status_code == 499 || report_kind.contains("cancel") {
|
if status_code == 499 || report_kind.contains("cancel") {
|
||||||
UsageTerminalState::Cancelled
|
UsageTerminalState::Cancelled
|
||||||
} else if status_code >= 400
|
} else if !(200..300).contains(&status_code)
|
||||||
|| provider_response
|
|| provider_response
|
||||||
.and_then(|value| value.get("error"))
|
.and_then(|value| value.get("error"))
|
||||||
.is_some_and(|value| !value.is_null())
|
.is_some_and(|value| !value.is_null())
|
||||||
@@ -959,7 +961,7 @@ fn infer_stream_terminal_state(
|
|||||||
) -> UsageTerminalState {
|
) -> UsageTerminalState {
|
||||||
if cancelled || status_code == 499 || report_kind.contains("cancel") {
|
if cancelled || status_code == 499 || report_kind.contains("cancel") {
|
||||||
UsageTerminalState::Cancelled
|
UsageTerminalState::Cancelled
|
||||||
} else if status_code >= 400 {
|
} else if !(200..300).contains(&status_code) {
|
||||||
UsageTerminalState::Failed
|
UsageTerminalState::Failed
|
||||||
} else {
|
} else {
|
||||||
UsageTerminalState::Completed
|
UsageTerminalState::Completed
|
||||||
@@ -1982,6 +1984,8 @@ fn resolve_error_category(status_code: u16, event_type: UsageEventType) -> Optio
|
|||||||
UsageEventType::Cancelled => Some("cancelled".to_string()),
|
UsageEventType::Cancelled => Some("cancelled".to_string()),
|
||||||
UsageEventType::Failed if status_code >= 500 => Some("server_error".to_string()),
|
UsageEventType::Failed if status_code >= 500 => Some("server_error".to_string()),
|
||||||
UsageEventType::Failed if status_code >= 400 => Some("client_error".to_string()),
|
UsageEventType::Failed if status_code >= 400 => Some("client_error".to_string()),
|
||||||
|
UsageEventType::Failed if status_code >= 300 => Some("redirect".to_string()),
|
||||||
|
UsageEventType::Failed => Some("non_success_status".to_string()),
|
||||||
_ => None,
|
_ => None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2000,7 +2004,7 @@ fn resolve_error_message(
|
|||||||
if explicit_error_message.is_some() {
|
if explicit_error_message.is_some() {
|
||||||
return explicit_error_message;
|
return explicit_error_message;
|
||||||
}
|
}
|
||||||
if status_code < 400 {
|
if (200..300).contains(&status_code) {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2798,6 +2802,81 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn stream_terminal_usage_marks_redirect_status_as_failed() {
|
||||||
|
let plan = ExecutionPlan {
|
||||||
|
request_id: "req-stream-redirect-usage".to_string(),
|
||||||
|
candidate_id: Some("cand-stream-redirect-usage".to_string()),
|
||||||
|
provider_name: Some("ChatGPTWeb".to_string()),
|
||||||
|
provider_id: "provider-redirect".to_string(),
|
||||||
|
endpoint_id: "endpoint-redirect".to_string(),
|
||||||
|
key_id: "key-redirect".to_string(),
|
||||||
|
method: "POST".to_string(),
|
||||||
|
url: "https://example.com/v1beta/models/gemini:streamGenerateContent".to_string(),
|
||||||
|
headers: BTreeMap::new(),
|
||||||
|
content_type: None,
|
||||||
|
content_encoding: None,
|
||||||
|
body: RequestBody {
|
||||||
|
json_body: None,
|
||||||
|
body_bytes_b64: None,
|
||||||
|
body_ref: None,
|
||||||
|
},
|
||||||
|
stream: true,
|
||||||
|
client_api_format: "gemini:generate_content".to_string(),
|
||||||
|
provider_api_format: "gemini:generate_content".to_string(),
|
||||||
|
model_name: Some("gemini".to_string()),
|
||||||
|
proxy: None,
|
||||||
|
transport_profile: None,
|
||||||
|
timeouts: None,
|
||||||
|
};
|
||||||
|
let client_body = json!({
|
||||||
|
"error": {
|
||||||
|
"type": "execution_runtime_non_success_status",
|
||||||
|
"message": "execution runtime stream returned non-success status 302",
|
||||||
|
"code": 302,
|
||||||
|
"upstream_status": 302,
|
||||||
|
"location": "/"
|
||||||
|
}
|
||||||
|
});
|
||||||
|
let payload = GatewayStreamReportRequest {
|
||||||
|
trace_id: "trace-stream-redirect-usage".to_string(),
|
||||||
|
report_kind: "gemini_chat_stream_success".to_string(),
|
||||||
|
report_context: Some(json!({
|
||||||
|
"client_api_format": "gemini:generate_content",
|
||||||
|
"provider_api_format": "gemini:generate_content"
|
||||||
|
})),
|
||||||
|
status_code: 302,
|
||||||
|
headers: BTreeMap::from([
|
||||||
|
("content-type".to_string(), "application/json".to_string()),
|
||||||
|
("x-aether-upstream-status".to_string(), "302".to_string()),
|
||||||
|
]),
|
||||||
|
provider_body_base64: Some(
|
||||||
|
base64::engine::general_purpose::STANDARD
|
||||||
|
.encode(br#"{"error":{"message":"raw redirect body"}}"#),
|
||||||
|
),
|
||||||
|
provider_body_state: Some(UsageBodyCaptureState::Inline),
|
||||||
|
client_body_base64: Some(
|
||||||
|
base64::engine::general_purpose::STANDARD
|
||||||
|
.encode(serde_json::to_vec(&client_body).expect("body should encode")),
|
||||||
|
),
|
||||||
|
client_body_state: Some(UsageBodyCaptureState::Inline),
|
||||||
|
terminal_summary: None,
|
||||||
|
telemetry: None,
|
||||||
|
};
|
||||||
|
|
||||||
|
let event =
|
||||||
|
build_stream_terminal_usage_event(&plan, payload.report_context.as_ref(), &payload)
|
||||||
|
.expect("usage event should build");
|
||||||
|
|
||||||
|
assert_eq!(event.event_type, UsageEventType::Failed);
|
||||||
|
assert_eq!(event.data.status_code, Some(302));
|
||||||
|
assert_eq!(event.data.error_category.as_deref(), Some("redirect"));
|
||||||
|
assert_eq!(
|
||||||
|
event.data.error_message.as_deref(),
|
||||||
|
Some("raw redirect body")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn builds_stream_terminal_usage_from_terminal_summary_usage_without_decoding_bodies() {
|
fn builds_stream_terminal_usage_from_terminal_summary_usage_without_decoding_bodies() {
|
||||||
let plan = ExecutionPlan {
|
let plan = ExecutionPlan {
|
||||||
|
|||||||
@@ -9,6 +9,15 @@ export interface CandidateRankingMetadata {
|
|||||||
demoted_by?: string
|
demoted_by?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface CandidateResponseBoundary {
|
||||||
|
source?: string
|
||||||
|
status_code?: number | null
|
||||||
|
headers?: Record<string, unknown> | null
|
||||||
|
body?: unknown
|
||||||
|
body_ref?: string | null
|
||||||
|
body_state?: string | null
|
||||||
|
}
|
||||||
|
|
||||||
export interface CandidateRecord {
|
export interface CandidateRecord {
|
||||||
id: string
|
id: string
|
||||||
request_id: string
|
request_id: string
|
||||||
@@ -46,7 +55,9 @@ export interface CandidateRecord {
|
|||||||
latency_ms?: number
|
latency_ms?: number
|
||||||
concurrent_requests?: number
|
concurrent_requests?: number
|
||||||
ranking?: CandidateRankingMetadata | null
|
ranking?: CandidateRankingMetadata | null
|
||||||
extra_data?: Record<string, unknown>
|
extra_data?: Record<string, unknown> & {
|
||||||
|
upstream_response?: CandidateResponseBoundary
|
||||||
|
}
|
||||||
created_at: string
|
created_at: string
|
||||||
started_at?: string
|
started_at?: string
|
||||||
finished_at?: string
|
finished_at?: string
|
||||||
@@ -54,6 +65,9 @@ export interface CandidateRecord {
|
|||||||
|
|
||||||
export interface RequestTrace {
|
export interface RequestTrace {
|
||||||
request_id: string
|
request_id: string
|
||||||
|
request_path?: string
|
||||||
|
request_query_string?: string
|
||||||
|
request_path_and_query?: string
|
||||||
total_candidates: number
|
total_candidates: number
|
||||||
final_status: 'success' | 'failed' | 'streaming' | 'pending' | 'cancelled'
|
final_status: 'success' | 'failed' | 'streaming' | 'pending' | 'cancelled'
|
||||||
total_latency_ms: number
|
total_latency_ms: number
|
||||||
|
|||||||
@@ -25,12 +25,42 @@ const applyDarkMode = (value: boolean) => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const getSystemPreference = (): boolean => {
|
const getSystemPreference = (): boolean => {
|
||||||
if (typeof window === 'undefined') {
|
if (typeof window === 'undefined' || typeof window.matchMedia !== 'function') {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
return window.matchMedia('(prefers-color-scheme: dark)').matches
|
return window.matchMedia('(prefers-color-scheme: dark)').matches
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const getThemeStorage = (): Storage | null => {
|
||||||
|
if (typeof window === 'undefined') {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
const storage = window.localStorage
|
||||||
|
if (!storage || typeof storage.getItem !== 'function' || typeof storage.setItem !== 'function') {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
return storage
|
||||||
|
}
|
||||||
|
|
||||||
|
const readStoredTheme = (): ThemeMode | null => {
|
||||||
|
try {
|
||||||
|
const value = getThemeStorage()?.getItem(THEME_STORAGE_KEY)
|
||||||
|
return value === 'dark' || value === 'light' || value === 'system' ? value : null
|
||||||
|
} catch {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const writeStoredTheme = (value: ThemeMode) => {
|
||||||
|
try {
|
||||||
|
getThemeStorage()?.setItem(THEME_STORAGE_KEY, value)
|
||||||
|
} catch {
|
||||||
|
// Ignore storage failures in restricted or test-like environments.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const updateDarkMode = () => {
|
const updateDarkMode = () => {
|
||||||
if (themeMode.value === 'system') {
|
if (themeMode.value === 'system') {
|
||||||
isDark.value = getSystemPreference()
|
isDark.value = getSystemPreference()
|
||||||
@@ -59,9 +89,7 @@ const ensureWatcher = () => {
|
|||||||
(value) => {
|
(value) => {
|
||||||
updateDarkMode()
|
updateDarkMode()
|
||||||
|
|
||||||
if (typeof window !== 'undefined') {
|
writeStoredTheme(value)
|
||||||
localStorage.setItem(THEME_STORAGE_KEY, value)
|
|
||||||
}
|
|
||||||
},
|
},
|
||||||
{ flush: 'post' }
|
{ flush: 'post' }
|
||||||
)
|
)
|
||||||
@@ -77,9 +105,9 @@ const initialize = () => {
|
|||||||
ensureWatcher()
|
ensureWatcher()
|
||||||
|
|
||||||
if (typeof window !== 'undefined') {
|
if (typeof window !== 'undefined') {
|
||||||
const storedTheme = localStorage.getItem(THEME_STORAGE_KEY) as ThemeMode | null
|
const storedTheme = readStoredTheme()
|
||||||
|
|
||||||
if (storedTheme === 'dark' || storedTheme === 'light' || storedTheme === 'system') {
|
if (storedTheme) {
|
||||||
themeMode.value = storedTheme
|
themeMode.value = storedTheme
|
||||||
} else {
|
} else {
|
||||||
// 兼容旧版本存储格式,旧版本直接存储 'dark' 或 'light'
|
// 兼容旧版本存储格式,旧版本直接存储 'dark' 或 'light'
|
||||||
@@ -87,8 +115,10 @@ const initialize = () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 监听系统主题变化
|
// 监听系统主题变化
|
||||||
mediaQuery = window.matchMedia('(prefers-color-scheme: dark)')
|
if (typeof window.matchMedia === 'function') {
|
||||||
mediaQuery.addEventListener('change', handleSystemChange)
|
mediaQuery = window.matchMedia('(prefers-color-scheme: dark)')
|
||||||
|
mediaQuery.addEventListener('change', handleSystemChange)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
updateDarkMode()
|
updateDarkMode()
|
||||||
|
|||||||
@@ -673,6 +673,7 @@ import type { CandidateRecord, RequestTrace } from '@/api/requestTrace'
|
|||||||
import HorizontalRequestTimeline from '@/features/usage/components/HorizontalRequestTimeline.vue'
|
import HorizontalRequestTimeline from '@/features/usage/components/HorizontalRequestTimeline.vue'
|
||||||
import JsonContent from '@/features/usage/components/RequestDetailDrawer/JsonContent.vue'
|
import JsonContent from '@/features/usage/components/RequestDetailDrawer/JsonContent.vue'
|
||||||
import { useClipboard } from '@/composables/useClipboard'
|
import { useClipboard } from '@/composables/useClipboard'
|
||||||
|
import { useDarkMode } from '@/composables/useDarkMode'
|
||||||
|
|
||||||
type TestEndpointOption = {
|
type TestEndpointOption = {
|
||||||
id: string
|
id: string
|
||||||
@@ -716,7 +717,7 @@ const traceCandidates = computed(() => props.trace?.candidates ?? [])
|
|||||||
const showSetup = computed(() => props.open && !props.testing && !props.result)
|
const showSetup = computed(() => props.open && !props.testing && !props.result)
|
||||||
const showResult = computed(() => !!props.result)
|
const showResult = computed(() => !!props.result)
|
||||||
const showTraceTimeline = computed(() => Boolean(props.requestId) && traceCandidates.value.length > 0)
|
const showTraceTimeline = computed(() => Boolean(props.requestId) && traceCandidates.value.length > 0)
|
||||||
const isDark = computed(() => typeof document !== 'undefined' && document.documentElement.classList.contains('dark'))
|
const { isDark } = useDarkMode()
|
||||||
const { copyToClipboard } = useClipboard()
|
const { copyToClipboard } = useClipboard()
|
||||||
|
|
||||||
const dialogTitle = computed(() => {
|
const dialogTitle = computed(() => {
|
||||||
|
|||||||
@@ -431,38 +431,42 @@
|
|||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<!-- 真实请求错误:节点级调试原因,和对客户端返回的摘要分开 -->
|
<!-- 错误信息:真实上游响应合并在此处展示 -->
|
||||||
<div
|
<div
|
||||||
v-if="currentAttempt.status === 'failed' && currentAttemptRequestError"
|
v-if="currentAttempt.status === 'failed' && currentAttemptRequestError"
|
||||||
class="error-block"
|
class="error-block"
|
||||||
>
|
>
|
||||||
<div class="error-type">
|
<div class="error-heading">
|
||||||
真实请求错误
|
<span class="error-type">错误信息</span>
|
||||||
|
<span
|
||||||
|
v-if="currentAttemptRequestError.statusCode != null"
|
||||||
|
class="error-status-badge"
|
||||||
|
:class="currentAttemptRequestError.statusCode >= 400 ? 'is-error' : currentAttemptRequestError.statusCode >= 300 ? 'is-warning' : 'is-success'"
|
||||||
|
>
|
||||||
|
HTTP {{ currentAttemptRequestError.statusCode }}
|
||||||
|
</span>
|
||||||
</div>
|
</div>
|
||||||
<div class="error-msg">
|
<div
|
||||||
|
v-if="currentAttemptRequestError.message"
|
||||||
|
class="error-msg"
|
||||||
|
>
|
||||||
{{ currentAttemptRequestError.message }}
|
{{ currentAttemptRequestError.message }}
|
||||||
</div>
|
</div>
|
||||||
<div
|
<div
|
||||||
v-if="currentAttemptRequestError.meta.length > 0"
|
v-if="currentAttemptRequestError.upstreamResponse"
|
||||||
class="error-flow-meta"
|
class="error-json"
|
||||||
>
|
>
|
||||||
<span
|
<JsonContentPanel
|
||||||
v-for="item in currentAttemptRequestError.meta"
|
:data="currentAttemptRequestError.upstreamResponse"
|
||||||
:key="item"
|
:is-dark="isDark"
|
||||||
class="error-flow-chip"
|
empty-message="无上游响应信息"
|
||||||
>{{ item }}</span>
|
/>
|
||||||
</div>
|
|
||||||
<div
|
|
||||||
v-if="currentAttemptRequestError.safetyHint"
|
|
||||||
class="error-flow-safety"
|
|
||||||
>
|
|
||||||
{{ currentAttemptRequestError.safetyHint }}
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<!-- 额外数据 -->
|
<!-- 额外数据 -->
|
||||||
<details
|
<details
|
||||||
v-if="currentAttempt.extra_data && Object.keys(currentAttempt.extra_data).length > 0"
|
v-if="currentAttemptExtraDataDisplay"
|
||||||
class="extra-block"
|
class="extra-block"
|
||||||
>
|
>
|
||||||
<summary class="extra-toggle">
|
<summary class="extra-toggle">
|
||||||
@@ -470,7 +474,7 @@
|
|||||||
</summary>
|
</summary>
|
||||||
<JsonContentPanel
|
<JsonContentPanel
|
||||||
class="extra-json-panel"
|
class="extra-json-panel"
|
||||||
:data="currentAttempt.extra_data"
|
:data="currentAttemptExtraDataDisplay"
|
||||||
:is-dark="isDark"
|
:is-dark="isDark"
|
||||||
empty-message="无额外信息"
|
empty-message="无额外信息"
|
||||||
/>
|
/>
|
||||||
@@ -508,6 +512,7 @@ import { requestTraceApi, type RequestTrace, type CandidateRecord } from '@/api/
|
|||||||
import { log } from '@/utils/logger'
|
import { log } from '@/utils/logger'
|
||||||
import { parseApiError } from '@/utils/errorParser'
|
import { parseApiError } from '@/utils/errorParser'
|
||||||
import { formatApiFormat } from '@/api/endpoints/types/api-format'
|
import { formatApiFormat } from '@/api/endpoints/types/api-format'
|
||||||
|
import { useDarkMode } from '@/composables/useDarkMode'
|
||||||
import { resolveTimelineFinalStatus } from '../utils/status'
|
import { resolveTimelineFinalStatus } from '../utils/status'
|
||||||
import {
|
import {
|
||||||
buildPoolGroupVisibleAttempts,
|
buildPoolGroupVisibleAttempts,
|
||||||
@@ -559,17 +564,6 @@ interface UsageData {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
interface AttemptErrorFlow {
|
|
||||||
source?: string
|
|
||||||
statusCode?: number
|
|
||||||
classification?: string
|
|
||||||
decision?: string
|
|
||||||
retryable?: boolean
|
|
||||||
safeToExpose?: boolean
|
|
||||||
propagation?: string
|
|
||||||
message?: string
|
|
||||||
}
|
|
||||||
|
|
||||||
const props = defineProps<{
|
const props = defineProps<{
|
||||||
requestId?: string | null
|
requestId?: string | null
|
||||||
/** 外部传入的状态码,用于覆盖 trace.final_status 的判断 */
|
/** 外部传入的状态码,用于覆盖 trace.final_status 的判断 */
|
||||||
@@ -641,7 +635,7 @@ const getFinalStatusBadgeVariant = (status: string): BadgeVariant => {
|
|||||||
const loading = ref(false)
|
const loading = ref(false)
|
||||||
const error = ref<string | null>(null)
|
const error = ref<string | null>(null)
|
||||||
const internalTrace = ref<RequestTrace | null>(null)
|
const internalTrace = ref<RequestTrace | null>(null)
|
||||||
const isDark = computed(() => document.documentElement.classList.contains('dark'))
|
const { isDark } = useDarkMode()
|
||||||
const trace = computed(() => props.traceData ?? internalTrace.value)
|
const trace = computed(() => props.traceData ?? internalTrace.value)
|
||||||
const selectedGroupIndex = ref(0)
|
const selectedGroupIndex = ref(0)
|
||||||
const selectedAttemptIndex = ref(0)
|
const selectedAttemptIndex = ref(0)
|
||||||
@@ -1073,65 +1067,42 @@ const readNumberField = (obj: Record<string, unknown>, key: string): number | un
|
|||||||
return undefined
|
return undefined
|
||||||
}
|
}
|
||||||
|
|
||||||
const readBooleanField = (obj: Record<string, unknown>, key: string): boolean | undefined => {
|
const hasRenderableValue = (value: unknown): boolean => {
|
||||||
const value = obj[key]
|
if (value == null) return false
|
||||||
return typeof value === 'boolean' ? value : undefined
|
if (typeof value === 'string') return value.trim().length > 0
|
||||||
|
if (typeof value === 'object') return Object.keys(value as Record<string, unknown>).length > 0
|
||||||
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
const normalizeAttemptErrorFlow = (value: unknown): AttemptErrorFlow | null => {
|
const normalizeUpstreamResponseDisplay = (value: unknown): Record<string, unknown> | null => {
|
||||||
const raw = extractObject(value)
|
const raw = extractObject(value)
|
||||||
if (!raw) return null
|
if (!raw) return null
|
||||||
|
const statusCode = readNumberField(raw, 'status_code') ?? readNumberField(raw, 'statusCode')
|
||||||
|
const headers = raw.headers
|
||||||
|
const body = raw.body
|
||||||
|
const bodyRef = readStringField(raw, 'body_ref') ?? readStringField(raw, 'bodyRef')
|
||||||
|
const bodyState = readStringField(raw, 'body_state') ?? readStringField(raw, 'bodyState')
|
||||||
|
|
||||||
const flow: AttemptErrorFlow = {
|
if (
|
||||||
source: readStringField(raw, 'source'),
|
statusCode == null &&
|
||||||
statusCode: readNumberField(raw, 'status_code') ?? readNumberField(raw, 'statusCode'),
|
!hasRenderableValue(headers) &&
|
||||||
classification: readStringField(raw, 'classification'),
|
!hasRenderableValue(body) &&
|
||||||
decision: readStringField(raw, 'decision'),
|
!bodyRef &&
|
||||||
retryable: readBooleanField(raw, 'retryable'),
|
!bodyState
|
||||||
safeToExpose: readBooleanField(raw, 'safe_to_expose') ?? readBooleanField(raw, 'safeToExpose'),
|
) {
|
||||||
propagation: readStringField(raw, 'propagation'),
|
return null
|
||||||
message: readStringField(raw, 'message'),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return Object.values(flow).some(value => value !== undefined) ? flow : null
|
const data: Record<string, unknown> = {}
|
||||||
|
if (statusCode != null) data.status_code = statusCode
|
||||||
|
if (hasRenderableValue(headers)) data.headers = headers
|
||||||
|
if (hasRenderableValue(body)) data.body = body
|
||||||
|
if (bodyRef) data.body_ref = bodyRef
|
||||||
|
if (bodyState) data.body_state = bodyState
|
||||||
|
|
||||||
|
return data
|
||||||
}
|
}
|
||||||
|
|
||||||
const labelFromMap = (value: string | undefined, labels: Record<string, string>): string | undefined => {
|
|
||||||
if (!value) return undefined
|
|
||||||
return labels[value] || value
|
|
||||||
}
|
|
||||||
|
|
||||||
const formatErrorFlowSource = (value?: string): string | undefined => labelFromMap(value, {
|
|
||||||
upstream_response: '上游响应',
|
|
||||||
request_validation: '请求校验',
|
|
||||||
gateway: '网关处理',
|
|
||||||
transport: '传输层',
|
|
||||||
scheduler: '调度层',
|
|
||||||
})
|
|
||||||
|
|
||||||
const formatErrorFlowDecision = (value?: string): string | undefined => labelFromMap(value, {
|
|
||||||
retry_next_candidate: '重试下一个候选',
|
|
||||||
stop_local_failover: '停止本地转移',
|
|
||||||
use_default: '默认处理',
|
|
||||||
return_to_client: '返回客户端',
|
|
||||||
})
|
|
||||||
|
|
||||||
const formatErrorFlowPropagation = (value?: string): string | undefined => labelFromMap(value, {
|
|
||||||
suppressed: '已抑制',
|
|
||||||
converted: '已转换',
|
|
||||||
passthrough: '直接透传',
|
|
||||||
local: '本地生成',
|
|
||||||
captured: '仅采集',
|
|
||||||
})
|
|
||||||
|
|
||||||
const formatErrorFlowClassification = (value?: string): string | undefined => labelFromMap(value, {
|
|
||||||
retryable: '可重试',
|
|
||||||
terminal: '终止',
|
|
||||||
provider_auth: '上游认证',
|
|
||||||
provider_quota: '上游额度',
|
|
||||||
invalid_request: '请求无效',
|
|
||||||
})
|
|
||||||
|
|
||||||
const extractStringList = (value: unknown): string[] => {
|
const extractStringList = (value: unknown): string[] => {
|
||||||
if (Array.isArray(value)) {
|
if (Array.isArray(value)) {
|
||||||
return value
|
return value
|
||||||
@@ -1224,6 +1195,9 @@ const currentAttemptRequestPathDisplay = computed(() => {
|
|||||||
const fromAttempt = resolveRequestPathFromObject(attempt?.extra_data)
|
const fromAttempt = resolveRequestPathFromObject(attempt?.extra_data)
|
||||||
if (fromAttempt) return fromAttempt
|
if (fromAttempt) return fromAttempt
|
||||||
|
|
||||||
|
const fromTrace = resolveRequestPathFromObject(trace.value)
|
||||||
|
if (fromTrace) return fromTrace
|
||||||
|
|
||||||
const fromRequestMetadata = resolveRequestPathFromObject(props.requestMetadata)
|
const fromRequestMetadata = resolveRequestPathFromObject(props.requestMetadata)
|
||||||
if (fromRequestMetadata) return fromRequestMetadata
|
if (fromRequestMetadata) return fromRequestMetadata
|
||||||
|
|
||||||
@@ -1290,45 +1264,64 @@ const currentAttemptFailureDiagnostic = computed<{
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
const formatAttemptErrorMessage = (message: string, statusCode?: number): string => {
|
||||||
|
const normalized = message.trim()
|
||||||
|
if (!normalized) return ''
|
||||||
|
if (/execution runtime (stream )?returned non-success status \d+/i.test(normalized)) {
|
||||||
|
return statusCode != null ? `上游返回非成功状态 ${statusCode}` : '上游返回非成功状态'
|
||||||
|
}
|
||||||
|
return normalized
|
||||||
|
}
|
||||||
|
|
||||||
const currentAttemptRequestError = computed<{
|
const currentAttemptRequestError = computed<{
|
||||||
message: string
|
message: string
|
||||||
meta: string[]
|
statusCode?: number
|
||||||
safetyHint: string
|
upstreamResponse: Record<string, unknown> | null
|
||||||
} | null>(() => {
|
} | null>(() => {
|
||||||
const attempt = currentAttempt.value
|
const attempt = currentAttempt.value
|
||||||
if (!attempt || attempt.status !== 'failed') return null
|
if (!attempt || attempt.status !== 'failed') return null
|
||||||
|
|
||||||
const extra = extractObject(attempt.extra_data)
|
const extra = extractObject(attempt.extra_data)
|
||||||
const flow = normalizeAttemptErrorFlow(extra?.error_flow)
|
const upstreamResponse = extractObject(extra?.upstream_response)
|
||||||
|
const errorFlow = extractObject(extra?.error_flow)
|
||||||
|
const statusCode = readNumberField(upstreamResponse ?? {}, 'status_code')
|
||||||
|
?? readNumberField(upstreamResponse ?? {}, 'statusCode')
|
||||||
|
?? readNumberField(errorFlow ?? {}, 'status_code')
|
||||||
|
?? readNumberField(errorFlow ?? {}, 'statusCode')
|
||||||
|
?? attempt.status_code
|
||||||
|
const flowMessage = errorFlow
|
||||||
|
? readStringField(errorFlow, 'message')
|
||||||
|
: ''
|
||||||
const fallbackMessage = typeof attempt.error_message === 'string' && attempt.error_message.trim()
|
const fallbackMessage = typeof attempt.error_message === 'string' && attempt.error_message.trim()
|
||||||
? attempt.error_message.trim()
|
? attempt.error_message.trim()
|
||||||
: ''
|
: ''
|
||||||
const fallbackType = typeof attempt.error_type === 'string' && attempt.error_type.trim()
|
const fallbackType = typeof attempt.error_type === 'string' && attempt.error_type.trim()
|
||||||
? attempt.error_type.trim()
|
? attempt.error_type.trim()
|
||||||
: ''
|
: ''
|
||||||
const message = flow?.message || fallbackMessage
|
const message = formatAttemptErrorMessage(flowMessage || fallbackMessage, statusCode) || fallbackType
|
||||||
if (!message && !fallbackType && !flow) return null
|
const upstreamResponseDisplay = normalizeUpstreamResponseDisplay(extra?.upstream_response)
|
||||||
|
if (!message && statusCode == null && !upstreamResponseDisplay) return null
|
||||||
const meta = [
|
|
||||||
flow?.statusCode != null ? `HTTP ${flow.statusCode}` : (attempt.status_code ? `HTTP ${attempt.status_code}` : ''),
|
|
||||||
formatErrorFlowSource(flow?.source),
|
|
||||||
formatErrorFlowClassification(flow?.classification) || fallbackType,
|
|
||||||
formatErrorFlowDecision(flow?.decision),
|
|
||||||
formatErrorFlowPropagation(flow?.propagation),
|
|
||||||
flow?.retryable != null ? (flow.retryable ? '会继续重试' : '不再重试') : '',
|
|
||||||
].filter((item): item is string => Boolean(item))
|
|
||||||
|
|
||||||
const safetyHint = flow?.safeToExpose === false
|
|
||||||
? '该错误被标记为敏感上游错误:仅在链路节点展示,不应完整返回给客户端。'
|
|
||||||
: ''
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
message: message || fallbackType || '未知错误',
|
message: upstreamResponseDisplay ? '' : (message || '未知错误'),
|
||||||
meta,
|
statusCode,
|
||||||
safetyHint,
|
upstreamResponse: upstreamResponseDisplay,
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
const currentAttemptExtraDataDisplay = computed<Record<string, unknown> | null>(() => {
|
||||||
|
const extra = extractObject(currentAttempt.value?.extra_data)
|
||||||
|
if (!extra) return null
|
||||||
|
|
||||||
|
const display = { ...extra }
|
||||||
|
delete display.upstream_response
|
||||||
|
delete display.error_flow
|
||||||
|
delete display.client_response
|
||||||
|
delete display.provider_response
|
||||||
|
|
||||||
|
return Object.keys(display).length > 0 ? display : null
|
||||||
|
})
|
||||||
|
|
||||||
// 计算当前尝试启用的能力标签(请求需要的能力)
|
// 计算当前尝试启用的能力标签(请求需要的能力)
|
||||||
const activeCapabilities = computed(() => {
|
const activeCapabilities = computed(() => {
|
||||||
if (!currentAttempt.value?.required_capabilities) return []
|
if (!currentAttempt.value?.required_capabilities) return []
|
||||||
@@ -2495,50 +2488,66 @@ function getDisplayStatus(attempt: CandidateRecord | null | undefined): string {
|
|||||||
border-radius: 8px;
|
border-radius: 8px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.error-heading {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: space-between;
|
||||||
|
gap: 0.75rem;
|
||||||
|
margin-bottom: 0.25rem;
|
||||||
|
}
|
||||||
|
|
||||||
.error-type {
|
.error-type {
|
||||||
font-size: 0.75rem;
|
font-size: 0.75rem;
|
||||||
font-weight: 600;
|
font-weight: 600;
|
||||||
color: #ef4444;
|
color: #ef4444;
|
||||||
margin-bottom: 0.25rem;
|
|
||||||
text-transform: uppercase;
|
text-transform: uppercase;
|
||||||
letter-spacing: 0.025em;
|
letter-spacing: 0.025em;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.error-status-badge {
|
||||||
|
flex-shrink: 0;
|
||||||
|
padding: 0.125rem 0.45rem;
|
||||||
|
border-radius: 999px;
|
||||||
|
font-size: 0.72rem;
|
||||||
|
font-family: ui-monospace, monospace;
|
||||||
|
background: hsl(var(--muted));
|
||||||
|
color: hsl(var(--muted-foreground));
|
||||||
|
}
|
||||||
|
|
||||||
|
.error-status-badge.is-success {
|
||||||
|
color: #166534;
|
||||||
|
background: #22c55e18;
|
||||||
|
}
|
||||||
|
|
||||||
|
.error-status-badge.is-warning {
|
||||||
|
color: #92400e;
|
||||||
|
background: #f59e0b1f;
|
||||||
|
}
|
||||||
|
|
||||||
|
.error-status-badge.is-error {
|
||||||
|
color: #991b1b;
|
||||||
|
background: #ef44441f;
|
||||||
|
}
|
||||||
|
|
||||||
.error-msg {
|
.error-msg {
|
||||||
font-size: 0.85rem;
|
font-size: 0.85rem;
|
||||||
color: #dc2626;
|
color: #dc2626;
|
||||||
word-break: break-word;
|
word-break: break-word;
|
||||||
}
|
}
|
||||||
|
|
||||||
.error-flow-meta {
|
.error-json {
|
||||||
display: flex;
|
margin-top: 0.75rem;
|
||||||
flex-wrap: wrap;
|
|
||||||
gap: 0.375rem;
|
|
||||||
margin-top: 0.625rem;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
.error-flow-chip {
|
.dark .error-status-badge.is-success {
|
||||||
padding: 0.125rem 0.45rem;
|
color: #bbf7d0;
|
||||||
border-radius: 999px;
|
|
||||||
background: #ef444414;
|
|
||||||
border: 1px solid #ef44442e;
|
|
||||||
color: #991b1b;
|
|
||||||
font-size: 0.72rem;
|
|
||||||
line-height: 1.35;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
.error-flow-safety {
|
.dark .error-status-badge.is-warning {
|
||||||
margin-top: 0.625rem;
|
color: #fde68a;
|
||||||
color: #991b1b;
|
|
||||||
font-size: 0.78rem;
|
|
||||||
line-height: 1.5;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
.dark .error-flow-chip {
|
.dark .error-status-badge.is-error {
|
||||||
color: #fecaca;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dark .error-flow-safety {
|
|
||||||
color: #fecaca;
|
color: #fecaca;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -459,55 +459,6 @@
|
|||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<!-- 错误域卡片:保持上游响应与客户端响应两个边界可对照 -->
|
|
||||||
<div
|
|
||||||
v-if="hasVisibleErrorCards"
|
|
||||||
class="space-y-3"
|
|
||||||
>
|
|
||||||
<div
|
|
||||||
class="grid gap-3"
|
|
||||||
:class="visibleErrorCardCount > 1 ? 'lg:grid-cols-2' : 'grid-cols-1'"
|
|
||||||
>
|
|
||||||
<Card
|
|
||||||
v-if="displayClientErrorMessage"
|
|
||||||
class="border-amber-200 dark:border-amber-800"
|
|
||||||
>
|
|
||||||
<div class="p-4">
|
|
||||||
<h4 class="text-sm font-semibold text-amber-700 dark:text-amber-300 mb-2">
|
|
||||||
返回客户端错误
|
|
||||||
</h4>
|
|
||||||
<div class="bg-amber-50 dark:bg-amber-900/20 rounded-lg p-3 space-y-1">
|
|
||||||
<p class="text-sm text-amber-900 dark:text-amber-200">
|
|
||||||
{{ displayClientErrorMessage }}
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</Card>
|
|
||||||
|
|
||||||
<Card
|
|
||||||
v-if="normalizedUpstreamError"
|
|
||||||
class="border-orange-200 dark:border-orange-800"
|
|
||||||
>
|
|
||||||
<div class="p-4">
|
|
||||||
<h4 class="text-sm font-semibold text-orange-700 dark:text-orange-300 mb-2">
|
|
||||||
上游响应错误
|
|
||||||
</h4>
|
|
||||||
<div class="bg-orange-50 dark:bg-orange-900/20 rounded-lg p-3 space-y-1">
|
|
||||||
<p class="text-sm text-orange-900 dark:text-orange-200">
|
|
||||||
{{ normalizedUpstreamError.message }}
|
|
||||||
</p>
|
|
||||||
<p
|
|
||||||
v-if="formatErrorDomainMeta(normalizedUpstreamError)"
|
|
||||||
class="text-xs text-orange-800/70 dark:text-orange-200/70 font-mono"
|
|
||||||
>
|
|
||||||
{{ formatErrorDomainMeta(normalizedUpstreamError) }}
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</Card>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<!-- Tabs 区域 -->
|
<!-- Tabs 区域 -->
|
||||||
<Card>
|
<Card>
|
||||||
<div class="p-3 sm:p-4">
|
<div class="p-3 sm:p-4">
|
||||||
@@ -743,6 +694,7 @@ import { ref, watch, computed, onMounted, onBeforeUnmount } from 'vue'
|
|||||||
import Button from '@/components/ui/button.vue'
|
import Button from '@/components/ui/button.vue'
|
||||||
import { useEscapeKey } from '@/composables/useEscapeKey'
|
import { useEscapeKey } from '@/composables/useEscapeKey'
|
||||||
import { useClipboard } from '@/composables/useClipboard'
|
import { useClipboard } from '@/composables/useClipboard'
|
||||||
|
import { useDarkMode } from '@/composables/useDarkMode'
|
||||||
import Card from '@/components/ui/card.vue'
|
import Card from '@/components/ui/card.vue'
|
||||||
import Badge from '@/components/ui/badge.vue'
|
import Badge from '@/components/ui/badge.vue'
|
||||||
import Separator from '@/components/ui/separator.vue'
|
import Separator from '@/components/ui/separator.vue'
|
||||||
@@ -750,7 +702,7 @@ import Skeleton from '@/components/ui/skeleton.vue'
|
|||||||
import Tabs from '@/components/ui/tabs.vue'
|
import Tabs from '@/components/ui/tabs.vue'
|
||||||
import TabsContent from '@/components/ui/tabs-content.vue'
|
import TabsContent from '@/components/ui/tabs-content.vue'
|
||||||
import { Check, Columns2, RefreshCw, X, Monitor, Server, MessageSquareText, Code2, Terminal, Play } from 'lucide-vue-next'
|
import { Check, Columns2, RefreshCw, X, Monitor, Server, MessageSquareText, Code2, Terminal, Play } from 'lucide-vue-next'
|
||||||
import { dashboardApi, type RequestDetail, type RequestErrorDomain } from '@/api/dashboard'
|
import { dashboardApi, type RequestDetail } from '@/api/dashboard'
|
||||||
import { formatApiFormat } from '@/api/endpoints/types/api-format'
|
import { formatApiFormat } from '@/api/endpoints/types/api-format'
|
||||||
import { formatShortRequestId } from '@/utils/format'
|
import { formatShortRequestId } from '@/utils/format'
|
||||||
import { log } from '@/utils/logger'
|
import { log } from '@/utils/logger'
|
||||||
@@ -832,42 +784,11 @@ type PricingTierLike = {
|
|||||||
|
|
||||||
type JsonRecord = Record<string, unknown>
|
type JsonRecord = Record<string, unknown>
|
||||||
|
|
||||||
type NormalizedErrorDomain = {
|
|
||||||
source?: string | null
|
|
||||||
status_code?: number | null
|
|
||||||
type?: string | null
|
|
||||||
message: string
|
|
||||||
code?: string | number | null
|
|
||||||
category?: string | null
|
|
||||||
}
|
|
||||||
|
|
||||||
function asRecord(value: unknown): JsonRecord | null {
|
function asRecord(value: unknown): JsonRecord | null {
|
||||||
if (!value || typeof value !== 'object' || Array.isArray(value)) return null
|
if (!value || typeof value !== 'object' || Array.isArray(value)) return null
|
||||||
return value as JsonRecord
|
return value as JsonRecord
|
||||||
}
|
}
|
||||||
|
|
||||||
function normalizeErrorDomain(domain: RequestErrorDomain | null | undefined): NormalizedErrorDomain | null {
|
|
||||||
if (!domain || typeof domain !== 'object') return null
|
|
||||||
const message = typeof domain.message === 'string' ? domain.message.trim() : ''
|
|
||||||
if (!message) return null
|
|
||||||
return {
|
|
||||||
source: domain.source ?? null,
|
|
||||||
status_code: domain.status_code ?? null,
|
|
||||||
type: domain.type ?? null,
|
|
||||||
message,
|
|
||||||
code: domain.code ?? null,
|
|
||||||
category: domain.category ?? null,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function formatErrorDomainMeta(domain: NormalizedErrorDomain): string {
|
|
||||||
const parts: string[] = []
|
|
||||||
if (domain.status_code != null) parts.push(`HTTP ${domain.status_code}`)
|
|
||||||
if (domain.type) parts.push(domain.type)
|
|
||||||
if (domain.source) parts.push(`source=${domain.source}`)
|
|
||||||
return parts.join(' · ')
|
|
||||||
}
|
|
||||||
|
|
||||||
function handleTraceState(state: { loaded: boolean, hasTrace: boolean }) {
|
function handleTraceState(state: { loaded: boolean, hasTrace: boolean }) {
|
||||||
timelineLoaded.value = state.loaded
|
timelineLoaded.value = state.loaded
|
||||||
timelineHasTrace.value = state.hasTrace
|
timelineHasTrace.value = state.hasTrace
|
||||||
@@ -987,10 +908,7 @@ watch(activeTab, (newTab) => {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
// 检测暗色模式
|
const { isDark } = useDarkMode()
|
||||||
const isDark = computed(() => {
|
|
||||||
return document.documentElement.classList.contains('dark')
|
|
||||||
})
|
|
||||||
|
|
||||||
const traceRequestMetadata = computed<Record<string, unknown> | null>(() => {
|
const traceRequestMetadata = computed<Record<string, unknown> | null>(() => {
|
||||||
const meta = detail.value?.metadata
|
const meta = detail.value?.metadata
|
||||||
@@ -1021,27 +939,6 @@ const metadataPanelData = computed<Record<string, unknown> | null>(() => {
|
|||||||
return Object.keys(merged).length > 0 ? merged : null
|
return Object.keys(merged).length > 0 ? merged : null
|
||||||
})
|
})
|
||||||
|
|
||||||
const normalizedClientError = computed(() =>
|
|
||||||
normalizeErrorDomain(detail.value?.errors?.client_error ?? detail.value?.client_error),
|
|
||||||
)
|
|
||||||
|
|
||||||
const normalizedUpstreamError = computed(() =>
|
|
||||||
normalizeErrorDomain(detail.value?.errors?.upstream_error ?? detail.value?.upstream_error),
|
|
||||||
)
|
|
||||||
|
|
||||||
const displayClientErrorMessage = computed(() =>
|
|
||||||
normalizedClientError.value?.message ?? '',
|
|
||||||
)
|
|
||||||
|
|
||||||
const hasVisibleErrorCards = computed(() =>
|
|
||||||
Boolean(displayClientErrorMessage.value || normalizedUpstreamError.value),
|
|
||||||
)
|
|
||||||
|
|
||||||
const visibleErrorCardCount = computed(() =>
|
|
||||||
(displayClientErrorMessage.value ? 1 : 0)
|
|
||||||
+ (normalizedUpstreamError.value ? 1 : 0),
|
|
||||||
)
|
|
||||||
|
|
||||||
const settlementInfo = computed<JsonRecord | null>(() =>
|
const settlementInfo = computed<JsonRecord | null>(() =>
|
||||||
asRecord(detail.value?.settlement ?? null),
|
asRecord(detail.value?.settlement ?? null),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -45,8 +45,14 @@ vi.mock('../JsonContentPanel.vue', async () => {
|
|||||||
return {
|
return {
|
||||||
default: defineComponent({
|
default: defineComponent({
|
||||||
name: 'JsonContentPanelStub',
|
name: 'JsonContentPanelStub',
|
||||||
setup() {
|
props: {
|
||||||
return () => h('div')
|
data: {
|
||||||
|
type: null,
|
||||||
|
default: null,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
setup(props) {
|
||||||
|
return () => h('pre', JSON.stringify(props.data))
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
@@ -321,4 +327,81 @@ describe('HorizontalRequestTimeline', () => {
|
|||||||
const requestPathCode = root.querySelector<HTMLElement>('.request-path-code')
|
const requestPathCode = root.querySelector<HTMLElement>('.request-path-code')
|
||||||
expect(requestPathCode?.textContent).toContain('/v1beta/models/gemini-2.5-pro:generateContent?alt=sse')
|
expect(requestPathCode?.textContent).toContain('/v1beta/models/gemini-2.5-pro:generateContent?alt=sse')
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('shows request path from trace payload', async () => {
|
||||||
|
const trace: RequestTrace = {
|
||||||
|
...buildTrace([
|
||||||
|
buildCandidate({
|
||||||
|
id: 'cand-trace-path',
|
||||||
|
provider_id: 'provider-path',
|
||||||
|
provider_name: 'Provider Path',
|
||||||
|
key_id: 'key-path',
|
||||||
|
key_name: 'Path Key',
|
||||||
|
candidate_index: 0,
|
||||||
|
status: 'failed',
|
||||||
|
}),
|
||||||
|
]),
|
||||||
|
request_path: '/v1/images/generations',
|
||||||
|
}
|
||||||
|
|
||||||
|
const root = mountTimeline(trace)
|
||||||
|
await nextTick()
|
||||||
|
|
||||||
|
expect(root.textContent).toContain('请求路径')
|
||||||
|
const requestPathCode = root.querySelector<HTMLElement>('.request-path-code')
|
||||||
|
expect(requestPathCode?.textContent).toContain('/v1/images/generations')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('shows upstream response JSON inside the error block on trace nodes', async () => {
|
||||||
|
const trace = buildTrace([
|
||||||
|
buildCandidate({
|
||||||
|
id: 'cand-upstream-response',
|
||||||
|
provider_id: 'provider-upstream',
|
||||||
|
provider_name: 'Provider Upstream',
|
||||||
|
key_id: 'key-upstream',
|
||||||
|
key_name: 'Upstream Key',
|
||||||
|
candidate_index: 0,
|
||||||
|
status: 'failed',
|
||||||
|
error_message: 'execution runtime stream returned non-success status 302',
|
||||||
|
extra_data: {
|
||||||
|
upstream_response: {
|
||||||
|
status_code: 302,
|
||||||
|
headers: { location: '/' },
|
||||||
|
},
|
||||||
|
error_flow: {
|
||||||
|
source: 'upstream_response',
|
||||||
|
status_code: 302,
|
||||||
|
classification: 'use_default',
|
||||||
|
decision: 'use_default',
|
||||||
|
propagation: 'none',
|
||||||
|
retryable: false,
|
||||||
|
safe_to_expose: false,
|
||||||
|
message: 'execution runtime stream returned non-success status 302',
|
||||||
|
},
|
||||||
|
client_response: {
|
||||||
|
status_code: 502,
|
||||||
|
headers: { 'content-type': 'application/json' },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
])
|
||||||
|
|
||||||
|
const root = mountTimeline(trace)
|
||||||
|
await nextTick()
|
||||||
|
|
||||||
|
expect(root.textContent).toContain('错误信息')
|
||||||
|
expect(root.textContent).toContain('HTTP 302')
|
||||||
|
expect(root.textContent).not.toContain('上游返回非成功状态 302')
|
||||||
|
expect(root.querySelector('.error-block .error-json')?.textContent).toContain('"status_code":302')
|
||||||
|
expect(root.querySelector('.error-block .error-json')?.textContent).toContain('"headers"')
|
||||||
|
expect(root.textContent).not.toContain('上游真实响应')
|
||||||
|
expect(root.textContent).not.toContain('execution runtime stream returned non-success status 302')
|
||||||
|
expect(root.textContent).not.toContain('真实请求错误')
|
||||||
|
expect(root.textContent).not.toContain('返回客户端响应')
|
||||||
|
expect(root.textContent).not.toContain('上游响应')
|
||||||
|
expect(root.textContent).not.toContain('默认处理')
|
||||||
|
expect(root.textContent).not.toContain('none')
|
||||||
|
expect(root.textContent).not.toContain('不再重试')
|
||||||
|
expect(root.textContent).not.toContain('该错误被标记为敏感上游错误')
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|||||||
Reference in New Issue
Block a user