mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-10 13:10:21 +08:00
elky
fc92c4f431
Shard and singleflight hot-path caches, batch and prioritize candidate and usage lifecycle persistence, and extend database and pressure-test instrumentation for 20k concurrent streams.
1330 lines
46 KiB
Rust
1330 lines
46 KiB
Rust
use std::collections::{BTreeMap, BTreeSet};
|
||
use std::time::{Duration, Instant};
|
||
|
||
use aether_contracts::ExecutionPlan;
|
||
use aether_data_contracts::repository::candidates::{
|
||
RequestCandidateStatus, StoredRequestCandidate,
|
||
};
|
||
use aether_data_contracts::repository::provider_catalog::{
|
||
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
|
||
};
|
||
use aether_usage_runtime::{
|
||
build_usage_event_data_seed, UsageEvent, UsageEventData, UsageEventType,
|
||
};
|
||
use axum::body::Body;
|
||
use axum::body::Bytes;
|
||
use axum::http::{self, HeaderMap, Response};
|
||
use base64::Engine as _;
|
||
use serde_json::{json, Map, Value};
|
||
use tracing::warn;
|
||
|
||
use crate::constants::{
|
||
EXECUTION_PATH_LOCAL_EXECUTION_RUNTIME_MISS, LOCAL_EXECUTION_RUNTIME_MISS_REASON_HEADER,
|
||
};
|
||
use crate::control::GatewayControlDecision;
|
||
use crate::state::LocalExecutionRuntimeMissDiagnostic;
|
||
use crate::AppState;
|
||
|
||
#[derive(Debug)]
|
||
pub(crate) enum LocalExecutionRequestOutcome {
|
||
Responded(Response<Body>),
|
||
Exhausted(LocalExecutionExhaustion),
|
||
NoPath,
|
||
}
|
||
|
||
#[derive(Debug, Clone)]
|
||
pub(crate) struct LocalExecutionExhaustion {
|
||
request_id: String,
|
||
data: UsageEventData,
|
||
candidate_id: Option<String>,
|
||
candidate_index: Option<u32>,
|
||
upstream_status_code: Option<u16>,
|
||
upstream_error_type: Option<String>,
|
||
upstream_error_message: Option<String>,
|
||
}
|
||
|
||
#[derive(Debug, Clone, Default)]
|
||
pub(crate) struct LocalExecutionRuntimeMissContext {
|
||
pub(crate) auth_user_id: Option<String>,
|
||
pub(crate) auth_api_key_id: Option<String>,
|
||
pub(crate) auth_username: Option<String>,
|
||
pub(crate) auth_api_key_name: Option<String>,
|
||
candidate_contexts: Vec<RuntimeMissCandidateContext>,
|
||
}
|
||
|
||
#[derive(Debug, Clone)]
|
||
struct RuntimeMissCandidateContext {
|
||
candidate: StoredRequestCandidate,
|
||
provider_name: Option<String>,
|
||
key_name: Option<String>,
|
||
client_api_format: Option<String>,
|
||
provider_api_format: Option<String>,
|
||
global_model_name: Option<String>,
|
||
selected_provider_model_name: Option<String>,
|
||
endpoint_url: Option<String>,
|
||
}
|
||
|
||
impl LocalExecutionRequestOutcome {
|
||
pub(crate) fn responded(response: Response<Body>) -> Self {
|
||
Self::Responded(response)
|
||
}
|
||
}
|
||
|
||
impl LocalExecutionRuntimeMissContext {
|
||
pub(crate) fn persisted_candidate_count(&self) -> usize {
|
||
self.candidate_contexts.len()
|
||
}
|
||
|
||
pub(crate) fn all_candidates_skipped_for_reason(&self, reason: &str) -> bool {
|
||
let reason = reason.trim();
|
||
if reason.is_empty() || self.candidate_contexts.is_empty() {
|
||
return false;
|
||
}
|
||
|
||
self.candidate_contexts.iter().all(|candidate| {
|
||
candidate.candidate.status == RequestCandidateStatus::Skipped
|
||
&& candidate
|
||
.candidate
|
||
.skip_reason
|
||
.as_deref()
|
||
.map(str::trim)
|
||
.is_some_and(|value| value == reason)
|
||
})
|
||
}
|
||
|
||
pub(crate) fn candidate_summary(&self) -> Option<String> {
|
||
const MAX_ITEMS: usize = 5;
|
||
|
||
if self.candidate_contexts.is_empty() {
|
||
return None;
|
||
}
|
||
|
||
let mut summaries = self
|
||
.candidate_contexts
|
||
.iter()
|
||
.take(MAX_ITEMS)
|
||
.map(format_runtime_miss_candidate_summary)
|
||
.collect::<Vec<_>>();
|
||
let remaining = self.candidate_contexts.len().saturating_sub(MAX_ITEMS);
|
||
if remaining > 0 {
|
||
summaries.push(format!("+{remaining} more"));
|
||
}
|
||
Some(summaries.join(" | "))
|
||
}
|
||
|
||
pub(crate) fn all_provider_request_body_build_failures_detail(&self) -> Option<String> {
|
||
if self.candidate_contexts.is_empty()
|
||
|| !self.candidate_contexts.iter().all(|candidate| {
|
||
candidate.candidate.status == RequestCandidateStatus::Skipped
|
||
&& candidate
|
||
.candidate
|
||
.skip_reason
|
||
.as_deref()
|
||
.map(str::trim)
|
||
.is_some_and(|value| value == "provider_request_body_build_failed")
|
||
})
|
||
{
|
||
return None;
|
||
}
|
||
|
||
let diagnostic = self
|
||
.candidate_contexts
|
||
.iter()
|
||
.find_map(runtime_miss_candidate_failure_diagnostic)?;
|
||
let mut detail = format!("上游请求体转换失败：{}", diagnostic.message);
|
||
if diagnostic.path != "$" {
|
||
detail.push_str(&format!("；字段路径：{}", diagnostic.path));
|
||
}
|
||
detail.push_str("（原因代码: provider_request_body_build_failed）");
|
||
Some(detail)
|
||
}
|
||
}
|
||
|
||
struct RuntimeMissFailureDiagnostic {
|
||
path: String,
|
||
message: String,
|
||
}
|
||
|
||
pub(crate) async fn build_local_execution_exhaustion(
|
||
state: &AppState,
|
||
plan: &ExecutionPlan,
|
||
report_context: Option<&Value>,
|
||
) -> LocalExecutionExhaustion {
|
||
let mut exhaustion = build_fast_local_execution_exhaustion(plan, report_context);
|
||
let mut data = build_usage_event_data_seed(plan, report_context);
|
||
let last_failed_candidate = match state
|
||
.read_request_candidates_by_request_id(plan.request_id.as_str())
|
||
.await
|
||
{
|
||
Ok(candidates) => select_last_failed_request_candidate(&candidates).cloned(),
|
||
Err(err) => {
|
||
warn!(
|
||
request_id = %plan.request_id,
|
||
error = ?err,
|
||
"gateway failed to load request candidates for exhausted local execution"
|
||
);
|
||
None
|
||
}
|
||
};
|
||
|
||
if let Some(candidate) = last_failed_candidate.as_ref() {
|
||
data.user_id = data.user_id.or_else(|| candidate.user_id.clone());
|
||
data.api_key_id = data.api_key_id.or_else(|| candidate.api_key_id.clone());
|
||
data.username = data.username.or_else(|| candidate.username.clone());
|
||
data.api_key_name = data.api_key_name.or_else(|| candidate.api_key_name.clone());
|
||
data.provider_id = data.provider_id.or_else(|| candidate.provider_id.clone());
|
||
data.provider_endpoint_id = data
|
||
.provider_endpoint_id
|
||
.or_else(|| candidate.endpoint_id.clone());
|
||
data.provider_api_key_id = data
|
||
.provider_api_key_id
|
||
.or_else(|| candidate.key_id.clone());
|
||
}
|
||
|
||
exhaustion.data = data;
|
||
exhaustion.candidate_id = last_failed_candidate
|
||
.as_ref()
|
||
.map(|candidate| candidate.id.clone());
|
||
exhaustion.candidate_index = last_failed_candidate
|
||
.as_ref()
|
||
.map(|candidate| candidate.candidate_index);
|
||
exhaustion.upstream_status_code = last_failed_candidate
|
||
.as_ref()
|
||
.and_then(|candidate| candidate.status_code);
|
||
exhaustion.upstream_error_type = last_failed_candidate
|
||
.as_ref()
|
||
.and_then(|candidate| candidate.error_type.clone())
|
||
.map(|value| value.trim().to_string())
|
||
.filter(|value| !value.is_empty());
|
||
exhaustion.upstream_error_message = last_failed_candidate
|
||
.as_ref()
|
||
.and_then(|candidate| candidate.error_message.clone())
|
||
.map(|value| value.trim().to_string())
|
||
.filter(|value| !value.is_empty());
|
||
exhaustion
|
||
}
|
||
|
||
pub(crate) fn build_fast_local_execution_exhaustion(
|
||
plan: &ExecutionPlan,
|
||
report_context: Option<&Value>,
|
||
) -> LocalExecutionExhaustion {
|
||
let data = build_usage_event_data_seed(plan, report_context);
|
||
LocalExecutionExhaustion {
|
||
request_id: plan.request_id.clone(),
|
||
candidate_id: plan.candidate_id.clone(),
|
||
candidate_index: report_context
|
||
.and_then(Value::as_object)
|
||
.and_then(|value| value.get("candidate_index"))
|
||
.and_then(Value::as_u64)
|
||
.and_then(|value| u32::try_from(value).ok()),
|
||
data,
|
||
upstream_status_code: None,
|
||
upstream_error_type: None,
|
||
upstream_error_message: None,
|
||
}
|
||
}
|
||
|
||
pub(crate) async fn build_local_execution_runtime_miss_context(
|
||
state: &AppState,
|
||
request_id: &str,
|
||
decision: Option<&GatewayControlDecision>,
|
||
) -> LocalExecutionRuntimeMissContext {
|
||
let auth_context = decision.and_then(|value| value.auth_context.as_ref());
|
||
|
||
LocalExecutionRuntimeMissContext {
|
||
auth_user_id: auth_context.map(|value| value.user_id.clone()),
|
||
auth_api_key_id: auth_context.map(|value| value.api_key_id.clone()),
|
||
auth_username: auth_context.and_then(|value| value.username.clone()),
|
||
auth_api_key_name: auth_context.and_then(|value| value.api_key_name.clone()),
|
||
candidate_contexts: load_runtime_miss_candidate_contexts_with_retry(
|
||
state, request_id, decision,
|
||
)
|
||
.await,
|
||
}
|
||
}
|
||
|
||
pub(crate) fn build_fast_local_execution_runtime_miss_context(
|
||
decision: Option<&GatewayControlDecision>,
|
||
) -> LocalExecutionRuntimeMissContext {
|
||
let auth_context = decision.and_then(|value| value.auth_context.as_ref());
|
||
|
||
LocalExecutionRuntimeMissContext {
|
||
auth_user_id: auth_context.map(|value| value.user_id.clone()),
|
||
auth_api_key_id: auth_context.map(|value| value.api_key_id.clone()),
|
||
auth_username: auth_context.and_then(|value| value.username.clone()),
|
||
auth_api_key_name: auth_context.and_then(|value| value.api_key_name.clone()),
|
||
candidate_contexts: Vec::new(),
|
||
}
|
||
}
|
||
|
||
pub(crate) async fn record_failed_usage_for_exhausted_request(
|
||
state: &AppState,
|
||
exhaustion: LocalExecutionExhaustion,
|
||
started_at: &Instant,
|
||
local_execution_runtime_miss_detail: &str,
|
||
execution_path: &str,
|
||
diagnostic: Option<&LocalExecutionRuntimeMissDiagnostic>,
|
||
) {
|
||
if !state.usage_runtime.is_enabled() {
|
||
return;
|
||
}
|
||
|
||
let LocalExecutionExhaustion {
|
||
request_id,
|
||
mut data,
|
||
candidate_id,
|
||
candidate_index,
|
||
upstream_status_code,
|
||
upstream_error_type,
|
||
upstream_error_message,
|
||
} = exhaustion;
|
||
|
||
let status_code = http::StatusCode::SERVICE_UNAVAILABLE.as_u16();
|
||
let candidate_status_code = upstream_status_code.unwrap_or(status_code);
|
||
data.status_code = Some(status_code);
|
||
data.error_message = upstream_error_message
|
||
.clone()
|
||
.or_else(|| Some(local_execution_runtime_miss_detail.to_string()));
|
||
data.error_category = error_category_for_failed_status(status_code);
|
||
data.response_time_ms = Some(started_at.elapsed().as_millis() as u64);
|
||
data.response_headers = Some(json_header_map());
|
||
data.response_body = Some(json!({
|
||
"error": {
|
||
"type": upstream_error_type
|
||
.as_deref()
|
||
.filter(|value| !value.trim().is_empty())
|
||
.unwrap_or("upstream_error"),
|
||
"message": upstream_error_message
|
||
.as_deref()
|
||
.filter(|value| !value.trim().is_empty())
|
||
.unwrap_or(local_execution_runtime_miss_detail),
|
||
"code": candidate_status_code,
|
||
}
|
||
}));
|
||
|
||
let mut client_headers = Map::from_iter([(
|
||
"content-type".to_string(),
|
||
Value::String("application/json".to_string()),
|
||
)]);
|
||
if let Some(reason) = diagnostic
|
||
.map(|value| value.reason.trim())
|
||
.filter(|value| !value.is_empty())
|
||
{
|
||
client_headers.insert(
|
||
LOCAL_EXECUTION_RUNTIME_MISS_REASON_HEADER.to_string(),
|
||
Value::String(reason.to_string()),
|
||
);
|
||
}
|
||
data.client_response_headers = Some(Value::Object(client_headers));
|
||
data.client_response_body = Some(json!({
|
||
"error": {
|
||
"type": "http_error",
|
||
"message": beautify_local_execution_client_error_message(local_execution_runtime_miss_detail),
|
||
}
|
||
}));
|
||
|
||
let mut request_metadata = match data.request_metadata.take() {
|
||
Some(Value::Object(object)) => object,
|
||
Some(other) => Map::from_iter([("seed".to_string(), other)]),
|
||
None => Map::new(),
|
||
};
|
||
request_metadata.insert("trace_id".to_string(), Value::String(request_id.clone()));
|
||
apply_runtime_miss_usage_routing(
|
||
&mut data,
|
||
&mut request_metadata,
|
||
execution_path,
|
||
candidate_id.as_deref(),
|
||
candidate_index,
|
||
None,
|
||
diagnostic,
|
||
None,
|
||
None,
|
||
);
|
||
data.request_metadata = Some(Value::Object(request_metadata));
|
||
|
||
state
|
||
.usage_runtime
|
||
.record_terminal_event_direct(
|
||
state.usage_lifecycle_data_state().as_ref(),
|
||
UsageEvent::new(UsageEventType::Failed, request_id, data),
|
||
)
|
||
.await;
|
||
}
|
||
|
||
pub(crate) async fn record_failed_usage_for_runtime_miss_request(
|
||
state: &AppState,
|
||
request_id: &str,
|
||
started_at: &Instant,
|
||
local_execution_runtime_miss_detail: &str,
|
||
execution_path: &str,
|
||
decision: Option<&GatewayControlDecision>,
|
||
diagnostic: Option<&LocalExecutionRuntimeMissDiagnostic>,
|
||
context: &LocalExecutionRuntimeMissContext,
|
||
request_headers: &HeaderMap,
|
||
request_body: Option<&Bytes>,
|
||
) {
|
||
if !state.usage_runtime.is_enabled() {
|
||
return;
|
||
}
|
||
|
||
let selected_candidate =
|
||
select_last_runtime_miss_executed_candidate(&context.candidate_contexts);
|
||
let api_format = selected_candidate
|
||
.and_then(|value| value.client_api_format.clone())
|
||
.or_else(|| {
|
||
trimmed_non_empty(decision.and_then(|value| value.auth_endpoint_signature.as_deref()))
|
||
});
|
||
let provider_api_format = selected_candidate
|
||
.and_then(|value| value.provider_api_format.clone())
|
||
.or_else(|| api_format.clone());
|
||
let provider_name = selected_candidate
|
||
.and_then(|value| value.provider_name.clone())
|
||
.or_else(|| selected_candidate.and_then(|value| value.candidate.provider_id.clone()))
|
||
.unwrap_or_else(|| "unknown".to_string());
|
||
let model = trimmed_non_empty(diagnostic.and_then(|value| value.requested_model.as_deref()))
|
||
.or_else(|| selected_candidate.and_then(|value| value.global_model_name.clone()))
|
||
.or_else(|| selected_candidate.and_then(|value| value.selected_provider_model_name.clone()))
|
||
.unwrap_or_else(|| "unknown".to_string());
|
||
let target_model = selected_candidate
|
||
.and_then(|value| value.selected_provider_model_name.clone())
|
||
.filter(|value| !value.eq_ignore_ascii_case(model.as_str()));
|
||
|
||
let status_code = http::StatusCode::SERVICE_UNAVAILABLE.as_u16();
|
||
let client_message =
|
||
beautify_local_execution_client_error_message(local_execution_runtime_miss_detail);
|
||
let client_body = json!({
|
||
"error": {
|
||
"type": "http_error",
|
||
"message": client_message,
|
||
}
|
||
});
|
||
let mut client_headers = Map::from_iter([(
|
||
"content-type".to_string(),
|
||
Value::String("application/json".to_string()),
|
||
)]);
|
||
if let Some(reason) = diagnostic
|
||
.map(|value| value.reason.trim())
|
||
.filter(|value| !value.is_empty())
|
||
{
|
||
client_headers.insert(
|
||
LOCAL_EXECUTION_RUNTIME_MISS_REASON_HEADER.to_string(),
|
||
Value::String(reason.to_string()),
|
||
);
|
||
}
|
||
|
||
let mut request_metadata = Map::new();
|
||
request_metadata.insert(
|
||
"trace_id".to_string(),
|
||
Value::String(request_id.to_string()),
|
||
);
|
||
let mut data = UsageEventData {
|
||
user_id: context.auth_user_id.clone(),
|
||
api_key_id: context.auth_api_key_id.clone(),
|
||
username: context.auth_username.clone(),
|
||
api_key_name: context.auth_api_key_name.clone(),
|
||
provider_name,
|
||
model,
|
||
target_model,
|
||
provider_id: selected_candidate.and_then(|value| value.candidate.provider_id.clone()),
|
||
provider_endpoint_id: selected_candidate
|
||
.and_then(|value| value.candidate.endpoint_id.clone()),
|
||
provider_api_key_id: selected_candidate.and_then(|value| value.candidate.key_id.clone()),
|
||
request_type: Some(infer_request_type(api_format.as_deref())),
|
||
api_format: api_format.clone(),
|
||
api_family: api_format
|
||
.as_deref()
|
||
.and_then(infer_api_family)
|
||
.map(ToOwned::to_owned),
|
||
endpoint_kind: api_format
|
||
.as_deref()
|
||
.and_then(infer_endpoint_kind)
|
||
.map(ToOwned::to_owned),
|
||
endpoint_api_format: provider_api_format.clone(),
|
||
provider_api_family: provider_api_format
|
||
.as_deref()
|
||
.and_then(infer_api_family)
|
||
.map(ToOwned::to_owned),
|
||
provider_endpoint_kind: provider_api_format
|
||
.as_deref()
|
||
.and_then(infer_endpoint_kind)
|
||
.map(ToOwned::to_owned),
|
||
has_format_conversion: selected_candidate.and_then(|value| {
|
||
value
|
||
.client_api_format
|
||
.as_deref()
|
||
.zip(value.provider_api_format.as_deref())
|
||
.map(|(left, right)| !left.eq_ignore_ascii_case(right))
|
||
}),
|
||
status_code: Some(status_code),
|
||
error_message: Some(local_execution_runtime_miss_detail.to_string()),
|
||
error_category: error_category_for_failed_status(status_code),
|
||
response_time_ms: Some(started_at.elapsed().as_millis() as u64),
|
||
request_headers: Some(runtime_miss_original_headers_json(request_headers)),
|
||
request_body: runtime_miss_original_request_body_json(request_headers, request_body),
|
||
response_headers: Some(json_header_map()),
|
||
response_body: Some(client_body.clone()),
|
||
client_response_headers: Some(Value::Object(client_headers)),
|
||
client_response_body: Some(client_body),
|
||
..UsageEventData::default()
|
||
};
|
||
apply_runtime_miss_usage_routing(
|
||
&mut data,
|
||
&mut request_metadata,
|
||
execution_path,
|
||
selected_candidate.map(|value| value.candidate.id.as_str()),
|
||
selected_candidate.map(|value| value.candidate.candidate_index),
|
||
selected_candidate.and_then(|value| value.key_name.as_deref()),
|
||
diagnostic,
|
||
decision.and_then(|value| value.route_family.as_deref()),
|
||
decision.and_then(|value| value.route_kind.as_deref()),
|
||
);
|
||
data.request_metadata =
|
||
(!request_metadata.is_empty()).then_some(Value::Object(request_metadata));
|
||
|
||
state
|
||
.usage_runtime
|
||
.record_terminal_event_direct(
|
||
state.usage_lifecycle_data_state().as_ref(),
|
||
UsageEvent::new(UsageEventType::Failed, request_id, data),
|
||
)
|
||
.await;
|
||
}
|
||
|
||
pub(crate) fn beautify_local_execution_client_error_message(message: &str) -> String {
|
||
let without_reason_code = strip_parenthesized_reason_code(message);
|
||
let mut simplified = collapse_whitespace(without_reason_code.as_str());
|
||
if let Some(unavailable_message) =
|
||
simplify_all_candidates_skipped_client_error_message(simplified.as_str())
|
||
{
|
||
return unavailable_message;
|
||
}
|
||
for marker in [
|
||
"。请检查",
|
||
"。请确认",
|
||
". 请检查",
|
||
". 请确认",
|
||
"! 请检查",
|
||
"! 请确认",
|
||
"? 请检查",
|
||
"? 请确认",
|
||
"。Reason",
|
||
". Reason",
|
||
"。Code",
|
||
". Code",
|
||
] {
|
||
if let Some(index) = simplified.find(marker) {
|
||
simplified.truncate(index);
|
||
break;
|
||
}
|
||
}
|
||
trim_trailing_message_punctuation(simplified.as_str()).to_string()
|
||
}
|
||
|
||
fn simplify_all_candidates_skipped_client_error_message(message: &str) -> Option<String> {
|
||
if !message.contains("候选提供商")
|
||
|| !(message.contains("全部不可用") || message.contains("都不满足本次"))
|
||
{
|
||
return None;
|
||
}
|
||
|
||
let request_mode = extract_local_execution_request_mode(message)?;
|
||
if let Some(model) = extract_candidate_supported_model(message) {
|
||
return Some(format!(
|
||
"没有可用提供商支持模型 {model} 的{request_mode}请求"
|
||
));
|
||
}
|
||
|
||
Some(format!("没有可用提供商支持本次{request_mode}请求"))
|
||
}
|
||
|
||
fn extract_local_execution_request_mode(message: &str) -> Option<&str> {
|
||
let rest = message.get(message.find("本次")? + "本次".len()..)?;
|
||
let mode = rest.get(..rest.find("请求")?)?.trim();
|
||
(!mode.is_empty()).then_some(mode)
|
||
}
|
||
|
||
fn extract_candidate_supported_model(message: &str) -> Option<&str> {
|
||
let rest = message.get(message.find("支持模型 ")? + "支持模型 ".len()..)?;
|
||
let model = rest.get(..rest.find(" 的")?)?.trim();
|
||
(!model.is_empty()).then_some(model)
|
||
}
|
||
|
||
fn strip_parenthesized_reason_code(message: &str) -> String {
|
||
let Some(reason_index) = message.find("原因代码") else {
|
||
return message.to_string();
|
||
};
|
||
let Some((start, open)) = message[..reason_index]
|
||
.char_indices()
|
||
.rev()
|
||
.find(|(_, ch)| *ch == '（' || *ch == '(')
|
||
else {
|
||
return message.to_string();
|
||
};
|
||
let close = if open == '(' { ')' } else { '）' };
|
||
let Some(close_offset) = message[start..].find(close) else {
|
||
return message[..start].to_string();
|
||
};
|
||
let end = start + close_offset + close.len_utf8();
|
||
format!("{}{}", &message[..start], &message[end..])
|
||
}
|
||
|
||
fn collapse_whitespace(message: &str) -> String {
|
||
message.split_whitespace().collect::<Vec<_>>().join(" ")
|
||
}
|
||
|
||
fn trim_trailing_message_punctuation(message: &str) -> &str {
|
||
message
|
||
.trim_end_matches(|ch: char| {
|
||
ch.is_whitespace()
|
||
|| matches!(
|
||
ch,
|
||
'。' | '.' | '!' | '?' | '；' | ';' | '，' | ',' | '：' | ':'
|
||
)
|
||
})
|
||
.trim()
|
||
}
|
||
|
||
fn select_last_failed_request_candidate(
|
||
candidates: &[StoredRequestCandidate],
|
||
) -> Option<&StoredRequestCandidate> {
|
||
candidates
|
||
.iter()
|
||
.filter(|candidate| {
|
||
matches!(
|
||
candidate.status,
|
||
RequestCandidateStatus::Failed | RequestCandidateStatus::Cancelled
|
||
)
|
||
})
|
||
.max_by_key(|candidate| {
|
||
(
|
||
candidate.retry_index,
|
||
candidate.candidate_index,
|
||
candidate
|
||
.finished_at_unix_ms
|
||
.or(candidate.started_at_unix_ms)
|
||
.unwrap_or(candidate.created_at_unix_ms),
|
||
)
|
||
})
|
||
}
|
||
|
||
fn select_last_runtime_miss_executed_candidate(
|
||
candidates: &[RuntimeMissCandidateContext],
|
||
) -> Option<&RuntimeMissCandidateContext> {
|
||
candidates
|
||
.iter()
|
||
.filter(|candidate| request_candidate_represents_provider_execution(&candidate.candidate))
|
||
.max_by_key(|candidate| {
|
||
(
|
||
candidate.candidate.retry_index,
|
||
candidate.candidate.candidate_index,
|
||
candidate
|
||
.candidate
|
||
.finished_at_unix_ms
|
||
.or(candidate.candidate.started_at_unix_ms)
|
||
.unwrap_or(candidate.candidate.created_at_unix_ms),
|
||
)
|
||
})
|
||
}
|
||
|
||
fn request_candidate_represents_provider_execution(candidate: &StoredRequestCandidate) -> bool {
|
||
matches!(
|
||
candidate.status,
|
||
RequestCandidateStatus::Pending
|
||
| RequestCandidateStatus::Streaming
|
||
| RequestCandidateStatus::Success
|
||
| RequestCandidateStatus::Failed
|
||
| RequestCandidateStatus::Cancelled
|
||
)
|
||
}
|
||
|
||
fn error_category_for_failed_status(status_code: u16) -> Option<String> {
|
||
if status_code >= 500 {
|
||
Some("server_error".to_string())
|
||
} else if status_code >= 400 {
|
||
Some("client_error".to_string())
|
||
} else {
|
||
None
|
||
}
|
||
}
|
||
|
||
fn json_header_map() -> Value {
|
||
Value::Object(Map::from_iter([(
|
||
"content-type".to_string(),
|
||
Value::String("application/json".to_string()),
|
||
)]))
|
||
}
|
||
|
||
fn runtime_miss_original_headers_json(headers: &HeaderMap) -> Value {
|
||
let mut headers = crate::headers::collect_control_headers(headers);
|
||
for (name, value) in headers.iter_mut() {
|
||
if runtime_miss_sensitive_header(name) {
|
||
*value = runtime_miss_mask_header_value(value);
|
||
}
|
||
}
|
||
serde_json::to_value(headers).unwrap_or_else(|_| json!({}))
|
||
}
|
||
|
||
fn runtime_miss_original_request_body_json(
|
||
headers: &HeaderMap,
|
||
body: Option<&Bytes>,
|
||
) -> Option<Value> {
|
||
let body = body?;
|
||
if crate::headers::is_json_request(headers) {
|
||
if body.is_empty() {
|
||
return Some(json!({}));
|
||
}
|
||
return serde_json::from_slice::<Value>(body.as_ref()).ok();
|
||
}
|
||
|
||
(!body.is_empty()).then(|| {
|
||
json!({
|
||
"body_bytes_b64": base64::engine::general_purpose::STANDARD.encode(body.as_ref())
|
||
})
|
||
})
|
||
}
|
||
|
||
fn runtime_miss_sensitive_header(name: &str) -> bool {
|
||
const SENSITIVE_HEADERS: &[&str] = &[
|
||
"authorization",
|
||
"x-api-key",
|
||
"api-key",
|
||
"x-goog-api-key",
|
||
"cookie",
|
||
"proxy-authorization",
|
||
];
|
||
|
||
SENSITIVE_HEADERS
|
||
.iter()
|
||
.any(|candidate| name.eq_ignore_ascii_case(candidate))
|
||
}
|
||
|
||
fn runtime_miss_mask_header_value(value: &str) -> String {
|
||
let value = value.trim();
|
||
let char_count = value.chars().count();
|
||
if char_count <= 8 {
|
||
return "****".to_string();
|
||
}
|
||
|
||
let prefix: String = value.chars().take(4).collect();
|
||
let suffix: String = value
|
||
.chars()
|
||
.rev()
|
||
.take(4)
|
||
.collect::<Vec<_>>()
|
||
.into_iter()
|
||
.rev()
|
||
.collect();
|
||
format!("{prefix}****{suffix}")
|
||
}
|
||
|
||
async fn load_runtime_miss_candidate_contexts(
|
||
state: &AppState,
|
||
request_id: &str,
|
||
decision: Option<&GatewayControlDecision>,
|
||
) -> Vec<RuntimeMissCandidateContext> {
|
||
let mut candidates = match state
|
||
.read_request_candidates_by_request_id(request_id)
|
||
.await
|
||
{
|
||
Ok(candidates) => candidates,
|
||
Err(err) => {
|
||
warn!(
|
||
request_id = %request_id,
|
||
error = ?err,
|
||
"gateway failed to load request candidates for local execution runtime miss"
|
||
);
|
||
return Vec::new();
|
||
}
|
||
};
|
||
if candidates.is_empty() {
|
||
return Vec::new();
|
||
}
|
||
|
||
candidates.sort_by_key(|candidate| {
|
||
(
|
||
candidate.candidate_index,
|
||
candidate.retry_index,
|
||
candidate.created_at_unix_ms,
|
||
)
|
||
});
|
||
|
||
let (providers_by_id, endpoints_by_id, keys_by_id) = if state.has_provider_catalog_data_reader()
|
||
{
|
||
let provider_ids = collect_present_ids(
|
||
candidates
|
||
.iter()
|
||
.filter_map(|value| value.provider_id.as_deref()),
|
||
);
|
||
let endpoint_ids = collect_present_ids(
|
||
candidates
|
||
.iter()
|
||
.filter_map(|value| value.endpoint_id.as_deref()),
|
||
);
|
||
let key_ids = collect_present_ids(
|
||
candidates
|
||
.iter()
|
||
.filter_map(|value| value.key_id.as_deref()),
|
||
);
|
||
let (providers_result, endpoints_result, keys_result) = tokio::join!(
|
||
state.read_provider_catalog_providers_by_ids(&provider_ids),
|
||
state.read_provider_catalog_endpoints_by_ids(&endpoint_ids),
|
||
state.read_provider_catalog_keys_by_ids(&key_ids),
|
||
);
|
||
(
|
||
match providers_result {
|
||
Ok(values) => values
|
||
.into_iter()
|
||
.map(|value| (value.id.clone(), value))
|
||
.collect::<BTreeMap<_, _>>(),
|
||
Err(err) => {
|
||
warn!(
|
||
request_id = %request_id,
|
||
error = ?err,
|
||
"gateway failed to load provider catalog providers for local execution runtime miss"
|
||
);
|
||
BTreeMap::new()
|
||
}
|
||
},
|
||
match endpoints_result {
|
||
Ok(values) => values
|
||
.into_iter()
|
||
.map(|value| (value.id.clone(), value))
|
||
.collect::<BTreeMap<_, _>>(),
|
||
Err(err) => {
|
||
warn!(
|
||
request_id = %request_id,
|
||
error = ?err,
|
||
"gateway failed to load provider catalog endpoints for local execution runtime miss"
|
||
);
|
||
BTreeMap::new()
|
||
}
|
||
},
|
||
match keys_result {
|
||
Ok(values) => values
|
||
.into_iter()
|
||
.map(|value| (value.id.clone(), value))
|
||
.collect::<BTreeMap<_, _>>(),
|
||
Err(err) => {
|
||
warn!(
|
||
request_id = %request_id,
|
||
error = ?err,
|
||
"gateway failed to load provider catalog keys for local execution runtime miss"
|
||
);
|
||
BTreeMap::new()
|
||
}
|
||
},
|
||
)
|
||
} else {
|
||
(BTreeMap::new(), BTreeMap::new(), BTreeMap::new())
|
||
};
|
||
|
||
candidates
|
||
.into_iter()
|
||
.map(|candidate| {
|
||
let provider = candidate
|
||
.provider_id
|
||
.as_deref()
|
||
.and_then(|value| providers_by_id.get(value));
|
||
let endpoint = candidate
|
||
.endpoint_id
|
||
.as_deref()
|
||
.and_then(|value| endpoints_by_id.get(value));
|
||
let key = candidate
|
||
.key_id
|
||
.as_deref()
|
||
.and_then(|value| keys_by_id.get(value));
|
||
RuntimeMissCandidateContext {
|
||
provider_name: candidate_extra_data_string(&candidate, "provider_name")
|
||
.or_else(|| provider.map(|value| value.name.clone())),
|
||
key_name: candidate_extra_data_string(&candidate, "key_name")
|
||
.or_else(|| key.map(|value| value.name.clone())),
|
||
client_api_format: candidate_extra_data_string(&candidate, "client_api_format")
|
||
.or_else(|| candidate_extra_data_string(&candidate, "client_contract")),
|
||
provider_api_format: candidate_extra_data_string(&candidate, "provider_api_format")
|
||
.or_else(|| candidate_extra_data_string(&candidate, "provider_contract"))
|
||
.or_else(|| endpoint.map(|value| value.api_format.clone())),
|
||
global_model_name: candidate_extra_data_string(&candidate, "global_model_name"),
|
||
selected_provider_model_name: candidate_extra_data_string(
|
||
&candidate,
|
||
"selected_provider_model_name",
|
||
),
|
||
endpoint_url: endpoint.and_then(|value| {
|
||
build_runtime_miss_candidate_endpoint_url(&candidate, value, decision)
|
||
}),
|
||
candidate,
|
||
}
|
||
})
|
||
.collect()
|
||
}
|
||
|
||
async fn load_runtime_miss_candidate_contexts_with_retry(
|
||
state: &AppState,
|
||
request_id: &str,
|
||
decision: Option<&GatewayControlDecision>,
|
||
) -> Vec<RuntimeMissCandidateContext> {
|
||
let mut contexts = load_runtime_miss_candidate_contexts(state, request_id, decision).await;
|
||
if !contexts.is_empty() {
|
||
return contexts;
|
||
}
|
||
|
||
for _ in 0..4 {
|
||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||
contexts = load_runtime_miss_candidate_contexts(state, request_id, decision).await;
|
||
if !contexts.is_empty() {
|
||
break;
|
||
}
|
||
}
|
||
|
||
contexts
|
||
}
|
||
|
||
fn collect_present_ids<'a>(ids: impl Iterator<Item = &'a str>) -> Vec<String> {
|
||
ids.filter_map(|value| {
|
||
let trimmed = value.trim();
|
||
(!trimmed.is_empty()).then_some(trimmed.to_string())
|
||
})
|
||
.collect::<BTreeSet<_>>()
|
||
.into_iter()
|
||
.collect()
|
||
}
|
||
|
||
fn candidate_extra_data_string(candidate: &StoredRequestCandidate, key: &str) -> Option<String> {
|
||
candidate
|
||
.extra_data
|
||
.as_ref()
|
||
.and_then(Value::as_object)
|
||
.and_then(|value| value.get(key))
|
||
.and_then(Value::as_str)
|
||
.map(str::trim)
|
||
.filter(|value| !value.is_empty())
|
||
.map(ToOwned::to_owned)
|
||
}
|
||
|
||
fn runtime_miss_candidate_failure_diagnostic(
|
||
candidate: &RuntimeMissCandidateContext,
|
||
) -> Option<RuntimeMissFailureDiagnostic> {
|
||
let extra_data = candidate.candidate.extra_data.as_ref()?.as_object()?;
|
||
let diagnostic = extra_data
|
||
.get("failure_diagnostic")
|
||
.and_then(Value::as_object)
|
||
.filter(|diagnostic| diagnostic.get("safe_to_show") != Some(&Value::Bool(false)))
|
||
.or_else(|| {
|
||
extra_data
|
||
.get("request_conversion_error")
|
||
.and_then(Value::as_object)
|
||
})
|
||
.or_else(|| {
|
||
extra_data
|
||
.get("request_body_build_error")
|
||
.and_then(Value::as_object)
|
||
})?;
|
||
let message = diagnostic
|
||
.get("message")
|
||
.and_then(Value::as_str)
|
||
.map(str::trim)
|
||
.filter(|value| !value.is_empty())?;
|
||
let path = diagnostic
|
||
.get("path")
|
||
.and_then(Value::as_str)
|
||
.map(str::trim)
|
||
.filter(|value| !value.is_empty())
|
||
.unwrap_or("$");
|
||
Some(RuntimeMissFailureDiagnostic {
|
||
path: path.to_string(),
|
||
message: message.to_string(),
|
||
})
|
||
}
|
||
|
||
fn build_runtime_miss_candidate_endpoint_url(
|
||
candidate: &StoredRequestCandidate,
|
||
endpoint: &StoredProviderCatalogEndpoint,
|
||
decision: Option<&GatewayControlDecision>,
|
||
) -> Option<String> {
|
||
if let Some(upstream_url) = candidate_extra_data_string(candidate, "upstream_url") {
|
||
return Some(upstream_url);
|
||
}
|
||
|
||
let path = endpoint
|
||
.custom_path
|
||
.as_deref()
|
||
.map(str::trim)
|
||
.filter(|value| !value.is_empty())
|
||
.or_else(|| {
|
||
decision
|
||
.map(|value| value.public_path.as_str())
|
||
.map(str::trim)
|
||
.filter(|value| !value.is_empty())
|
||
});
|
||
let query = decision
|
||
.and_then(|value| value.public_query_string.as_deref())
|
||
.map(str::trim)
|
||
.filter(|value| !value.is_empty());
|
||
|
||
path.and_then(|value| {
|
||
crate::provider_transport::url::build_passthrough_path_url(
|
||
&endpoint.base_url,
|
||
value,
|
||
query,
|
||
&[],
|
||
)
|
||
})
|
||
.or_else(|| trimmed_non_empty(Some(endpoint.base_url.as_str())))
|
||
}
|
||
|
||
fn format_runtime_miss_candidate_summary(candidate: &RuntimeMissCandidateContext) -> String {
|
||
let mut parts = Vec::new();
|
||
parts.push(format!("idx={}", candidate.candidate.candidate_index));
|
||
parts.push(format!("retry={}", candidate.candidate.retry_index));
|
||
parts.push(format!(
|
||
"status={}",
|
||
request_candidate_status_label(candidate.candidate.status)
|
||
));
|
||
if let Some(provider_label) = format_name_with_id(
|
||
candidate.provider_name.as_deref(),
|
||
candidate.candidate.provider_id.as_deref(),
|
||
) {
|
||
parts.push(format!("provider={provider_label}"));
|
||
}
|
||
if let Some(endpoint_id) = candidate
|
||
.candidate
|
||
.endpoint_id
|
||
.as_deref()
|
||
.map(str::trim)
|
||
.filter(|value| !value.is_empty())
|
||
{
|
||
parts.push(format!("endpoint={endpoint_id}"));
|
||
}
|
||
if let Some(endpoint_url) = candidate
|
||
.endpoint_url
|
||
.as_deref()
|
||
.map(str::trim)
|
||
.filter(|value| !value.is_empty())
|
||
{
|
||
parts.push(format!("url={endpoint_url}"));
|
||
}
|
||
if let Some(key_label) = format_name_with_id(
|
||
candidate.key_name.as_deref(),
|
||
candidate.candidate.key_id.as_deref(),
|
||
) {
|
||
parts.push(format!("key={key_label}"));
|
||
}
|
||
if let Some(skip_reason) = candidate
|
||
.candidate
|
||
.skip_reason
|
||
.as_deref()
|
||
.map(str::trim)
|
||
.filter(|value| !value.is_empty())
|
||
{
|
||
parts.push(format!("skip={skip_reason}"));
|
||
}
|
||
if let Some(status_code) = candidate.candidate.status_code {
|
||
parts.push(format!("code={status_code}"));
|
||
}
|
||
if let Some(error_type) = candidate
|
||
.candidate
|
||
.error_type
|
||
.as_deref()
|
||
.map(str::trim)
|
||
.filter(|value| !value.is_empty())
|
||
{
|
||
parts.push(format!("error_type={error_type}"));
|
||
}
|
||
parts.join(" ")
|
||
}
|
||
|
||
fn format_name_with_id(name: Option<&str>, id: Option<&str>) -> Option<String> {
|
||
let name = name.map(str::trim).filter(|value| !value.is_empty());
|
||
let id = id.map(str::trim).filter(|value| !value.is_empty());
|
||
|
||
match (name, id) {
|
||
(Some(name), Some(id)) => Some(format!("{name}({id})")),
|
||
(Some(name), None) => Some(name.to_string()),
|
||
(None, Some(id)) => Some(id.to_string()),
|
||
(None, None) => None,
|
||
}
|
||
}
|
||
|
||
fn request_candidate_status_label(status: RequestCandidateStatus) -> &'static str {
|
||
match status {
|
||
RequestCandidateStatus::Available => "available",
|
||
RequestCandidateStatus::Unused => "unused",
|
||
RequestCandidateStatus::Pending => "pending",
|
||
RequestCandidateStatus::Streaming => "streaming",
|
||
RequestCandidateStatus::Success => "success",
|
||
RequestCandidateStatus::Failed => "failed",
|
||
RequestCandidateStatus::Cancelled => "cancelled",
|
||
RequestCandidateStatus::Skipped => "skipped",
|
||
}
|
||
}
|
||
|
||
fn infer_request_type(api_format: Option<&str>) -> String {
|
||
match infer_endpoint_kind(api_format.unwrap_or_default()) {
|
||
Some("video") => "video".to_string(),
|
||
Some("image") => "image".to_string(),
|
||
_ => "chat".to_string(),
|
||
}
|
||
}
|
||
|
||
fn infer_api_family(api_format: &str) -> Option<&str> {
|
||
api_format.split_once(':').map(|(family, _)| family)
|
||
}
|
||
|
||
fn infer_endpoint_kind(api_format: &str) -> Option<&str> {
|
||
api_format.split_once(':').map(|(_, kind)| kind)
|
||
}
|
||
|
||
fn apply_runtime_miss_usage_routing(
|
||
data: &mut UsageEventData,
|
||
request_metadata: &mut Map<String, Value>,
|
||
execution_path: &str,
|
||
candidate_id: Option<&str>,
|
||
candidate_index: Option<u32>,
|
||
key_name: Option<&str>,
|
||
diagnostic: Option<&LocalExecutionRuntimeMissDiagnostic>,
|
||
route_family_fallback: Option<&str>,
|
||
route_kind_fallback: Option<&str>,
|
||
) {
|
||
data.candidate_id = data
|
||
.candidate_id
|
||
.clone()
|
||
.or_else(|| trimmed_non_empty(candidate_id));
|
||
data.candidate_index = data
|
||
.candidate_index
|
||
.or_else(|| candidate_index.map(u64::from));
|
||
data.key_name = data
|
||
.key_name
|
||
.clone()
|
||
.or_else(|| trimmed_non_empty(key_name));
|
||
data.execution_path = data
|
||
.execution_path
|
||
.clone()
|
||
.or_else(|| trimmed_non_empty(Some(execution_path)));
|
||
data.local_execution_runtime_miss_reason = data
|
||
.local_execution_runtime_miss_reason
|
||
.clone()
|
||
.or_else(|| trimmed_non_empty(diagnostic.map(|value| value.reason.as_str())));
|
||
data.route_family = data.route_family.clone().or_else(|| {
|
||
trimmed_non_empty(
|
||
diagnostic
|
||
.and_then(|value| value.route_family.as_deref())
|
||
.or(route_family_fallback),
|
||
)
|
||
});
|
||
data.route_kind = data.route_kind.clone().or_else(|| {
|
||
trimmed_non_empty(
|
||
diagnostic
|
||
.and_then(|value| value.route_kind.as_deref())
|
||
.or(route_kind_fallback),
|
||
)
|
||
});
|
||
data.planner_kind = data
|
||
.planner_kind
|
||
.clone()
|
||
.or_else(|| trimmed_non_empty(diagnostic.and_then(|value| value.plan_kind.as_deref())));
|
||
let _ = request_metadata;
|
||
}
|
||
|
||
fn trimmed_non_empty(value: Option<&str>) -> Option<String> {
|
||
value
|
||
.map(str::trim)
|
||
.filter(|value| !value.is_empty())
|
||
.map(ToOwned::to_owned)
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::{
|
||
apply_runtime_miss_usage_routing, beautify_local_execution_client_error_message,
|
||
request_candidate_represents_provider_execution,
|
||
select_last_runtime_miss_executed_candidate, LocalExecutionRuntimeMissContext,
|
||
RuntimeMissCandidateContext,
|
||
};
|
||
use crate::constants::EXECUTION_PATH_LOCAL_EXECUTION_RUNTIME_MISS;
|
||
use crate::state::LocalExecutionRuntimeMissDiagnostic;
|
||
use aether_data_contracts::repository::candidates::{
|
||
RequestCandidateStatus, StoredRequestCandidate,
|
||
};
|
||
use aether_usage_runtime::UsageEventData;
|
||
use serde_json::{json, Map, Value};
|
||
|
||
#[test]
|
||
fn local_execution_client_error_message_is_client_friendly() {
|
||
assert_eq!(
|
||
beautify_local_execution_client_error_message(
|
||
"没有可用提供商支持模型 gpt-5.4 的同步请求。请检查模型映射、端点启用状态和 API Key 权限（原因代码: candidate_list_empty）",
|
||
),
|
||
"没有可用提供商支持模型 gpt-5.4 的同步请求"
|
||
);
|
||
assert_eq!(
|
||
beautify_local_execution_client_error_message(
|
||
"请求缺少 model 字段，无法选择上游提供商（openai/chat，原因代码: missing_requested_model）",
|
||
),
|
||
"请求缺少 model 字段，无法选择上游提供商"
|
||
);
|
||
assert_eq!(
|
||
beautify_local_execution_client_error_message(
|
||
"找到 1 个支持模型 gpt-5.4 的候选提供商，但本次流式请求全部不可用：provider_quota_blocked 2 次（原因代码: all_candidates_skipped）",
|
||
),
|
||
"没有可用提供商支持模型 gpt-5.4 的流式请求"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn runtime_miss_routing_moves_to_typed_usage_fields_and_keeps_metadata_lightweight() {
|
||
let mut data = UsageEventData::default();
|
||
let mut request_metadata =
|
||
Map::from_iter([("trace_id".to_string(), Value::String("trace-1".to_string()))]);
|
||
|
||
apply_runtime_miss_usage_routing(
|
||
&mut data,
|
||
&mut request_metadata,
|
||
EXECUTION_PATH_LOCAL_EXECUTION_RUNTIME_MISS,
|
||
Some("cand-1"),
|
||
Some(2),
|
||
Some("primary"),
|
||
Some(&LocalExecutionRuntimeMissDiagnostic {
|
||
reason: "all_candidates_skipped".to_string(),
|
||
route_family: Some("claude".to_string()),
|
||
route_kind: Some("cli".to_string()),
|
||
plan_kind: Some("claude_cli_sync".to_string()),
|
||
..LocalExecutionRuntimeMissDiagnostic::default()
|
||
}),
|
||
None,
|
||
None,
|
||
);
|
||
|
||
assert_eq!(data.candidate_id.as_deref(), Some("cand-1"));
|
||
assert_eq!(data.candidate_index, Some(2));
|
||
assert_eq!(data.key_name.as_deref(), Some("primary"));
|
||
assert_eq!(data.planner_kind.as_deref(), Some("claude_cli_sync"));
|
||
assert_eq!(data.route_family.as_deref(), Some("claude"));
|
||
assert_eq!(data.route_kind.as_deref(), Some("cli"));
|
||
assert_eq!(
|
||
data.execution_path.as_deref(),
|
||
Some(EXECUTION_PATH_LOCAL_EXECUTION_RUNTIME_MISS)
|
||
);
|
||
assert_eq!(
|
||
data.local_execution_runtime_miss_reason.as_deref(),
|
||
Some("all_candidates_skipped")
|
||
);
|
||
assert_eq!(
|
||
Value::Object(request_metadata),
|
||
json!({
|
||
"trace_id": "trace-1"
|
||
})
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn runtime_miss_executed_candidate_selection_ignores_skipped_only_histories() {
|
||
let skipped_candidate = StoredRequestCandidate::new(
|
||
"cand-skipped".to_string(),
|
||
"req-1".to_string(),
|
||
Some("user-1".to_string()),
|
||
Some("api-key-1".to_string()),
|
||
Some("alice".to_string()),
|
||
Some("default".to_string()),
|
||
0,
|
||
0,
|
||
Some("provider-1".to_string()),
|
||
Some("endpoint-1".to_string()),
|
||
Some("provider-key-1".to_string()),
|
||
RequestCandidateStatus::Skipped,
|
||
Some("api_key_concurrency_limit_reached".to_string()),
|
||
false,
|
||
None,
|
||
None,
|
||
None,
|
||
None,
|
||
None,
|
||
None,
|
||
None,
|
||
100,
|
||
None,
|
||
None,
|
||
)
|
||
.expect("candidate should build");
|
||
|
||
assert!(!request_candidate_represents_provider_execution(
|
||
&skipped_candidate
|
||
));
|
||
|
||
let contexts = vec![RuntimeMissCandidateContext {
|
||
candidate: skipped_candidate,
|
||
provider_name: Some("openai".to_string()),
|
||
key_name: Some("prod".to_string()),
|
||
client_api_format: Some("openai:responses".to_string()),
|
||
provider_api_format: Some("openai:responses".to_string()),
|
||
global_model_name: Some("gpt-5".to_string()),
|
||
selected_provider_model_name: Some("gpt-5-upstream".to_string()),
|
||
endpoint_url: Some("https://api.openai.example/v1/responses".to_string()),
|
||
}];
|
||
|
||
assert!(select_last_runtime_miss_executed_candidate(&contexts).is_none());
|
||
}
|
||
|
||
#[test]
|
||
fn runtime_miss_context_surfaces_request_conversion_field_diagnostic() {
|
||
let skipped_candidate = StoredRequestCandidate::new(
|
||
"cand-skipped".to_string(),
|
||
"req-1".to_string(),
|
||
Some("user-1".to_string()),
|
||
Some("api-key-1".to_string()),
|
||
Some("alice".to_string()),
|
||
Some("default".to_string()),
|
||
0,
|
||
0,
|
||
Some("provider-1".to_string()),
|
||
Some("endpoint-1".to_string()),
|
||
Some("provider-key-1".to_string()),
|
||
RequestCandidateStatus::Skipped,
|
||
Some("provider_request_body_build_failed".to_string()),
|
||
false,
|
||
None,
|
||
None,
|
||
None,
|
||
None,
|
||
None,
|
||
Some(json!({
|
||
"failure_diagnostic": {
|
||
"kind": "request_conversion",
|
||
"path": "$.n",
|
||
"message": "openai:chat 字段 n 不能无损转换到 openai:responses：OpenAI Responses request has no canonical equivalent for this Chat field",
|
||
"safe_to_show": true
|
||
},
|
||
"request_conversion_error": {
|
||
"path": "$.n",
|
||
"message": "compat"
|
||
}
|
||
})),
|
||
None,
|
||
100,
|
||
None,
|
||
None,
|
||
)
|
||
.expect("candidate should build");
|
||
|
||
let context = LocalExecutionRuntimeMissContext {
|
||
candidate_contexts: vec![RuntimeMissCandidateContext {
|
||
candidate: skipped_candidate,
|
||
provider_name: Some("openai".to_string()),
|
||
key_name: Some("prod".to_string()),
|
||
client_api_format: Some("openai:chat".to_string()),
|
||
provider_api_format: Some("openai:responses".to_string()),
|
||
global_model_name: Some("gpt-5".to_string()),
|
||
selected_provider_model_name: Some("gpt-5-upstream".to_string()),
|
||
endpoint_url: Some("https://api.openai.example/v1/responses".to_string()),
|
||
}],
|
||
..LocalExecutionRuntimeMissContext::default()
|
||
};
|
||
|
||
let detail = context
|
||
.all_provider_request_body_build_failures_detail()
|
||
.expect("detail should include conversion diagnostic");
|
||
|
||
assert!(detail.contains("字段 n"));
|
||
assert!(detail.contains("字段路径：$.n"));
|
||
assert!(detail.contains("provider_request_body_build_failed"));
|
||
}
|
||
}
|