feat(security): harden gateway boundaries and usage policies

Consolidate subscription usage policy enforcement, privacy-safe persistence, and gateway security hardening into one reviewable change.

Includes bounded HTTP and execution envelopes, header and protocol guards, DNS and relay validation, authentication and secret projection hardening, secure backup/install paths, and regression coverage.
This commit is contained in:
elky
2026-09-04 03:45:52 +08:00
parent ddcbeb3ae9
commit 579f2c7cc1
1019 changed files with 190437 additions and 26080 deletions
File diff suppressed because it is too large Load Diff
+5 -4
View File
@@ -17,10 +17,11 @@ pub(crate) use candidate_loop::{
};
pub(crate) use orchestration::*;
pub(crate) use outcome::{
beautify_local_execution_client_error_message, build_fast_local_execution_exhaustion,
build_fast_local_execution_runtime_miss_context, build_local_execution_exhaustion,
build_local_execution_runtime_miss_context, is_deferred_upstream_response,
mark_deferred_upstream_response, record_failed_usage_for_exhausted_request,
attach_deferred_usage_context, beautify_local_execution_client_error_message,
build_fast_local_execution_exhaustion, build_fast_local_execution_runtime_miss_context,
build_local_execution_exhaustion, build_local_execution_runtime_miss_context,
is_deferred_upstream_response, mark_deferred_upstream_response,
record_failed_usage_for_deferred_response, record_failed_usage_for_exhausted_request,
record_failed_usage_for_runtime_miss_request, LocalExecutionExhaustion,
LocalExecutionRequestOutcome, LocalExecutionRuntimeMissContext,
};
@@ -899,10 +899,10 @@ async fn standard_text_sync_heartbeat_final_bytes(
STANDARD_TEXT_SYNC_HEARTBEAT_EXHAUSTED_STATUS,
"standard text sync exhausted all local candidates",
),
Err(err) => standard_text_sync_heartbeat_error_body(
Err(_err) => standard_text_sync_heartbeat_error_body(
client_api_format,
STANDARD_TEXT_SYNC_HEARTBEAT_INTERNAL_ERROR_STATUS,
&format!("{err:?}"),
"internal gateway error while executing request",
),
}
}
@@ -922,11 +922,11 @@ async fn standard_text_sync_heartbeat_response_body_bytes(
bytes.as_ref(),
) {
Ok(body) => body,
Err(err) => {
Err(_err) => {
return standard_text_sync_heartbeat_error_body(
client_api_format,
STANDARD_TEXT_SYNC_HEARTBEAT_INTERNAL_ERROR_STATUS,
&format!("{err:?}"),
"internal gateway error while restoring response",
);
}
};
@@ -946,10 +946,10 @@ async fn standard_text_sync_heartbeat_response_body_bytes(
"empty standard text sync response",
)
}
Err(err) => standard_text_sync_heartbeat_error_body(
Err(_err) => standard_text_sync_heartbeat_error_body(
client_api_format,
STANDARD_TEXT_SYNC_HEARTBEAT_INTERNAL_ERROR_STATUS,
&err.to_string(),
"internal gateway error while reading response",
),
}
}
@@ -1200,9 +1200,9 @@ async fn openai_image_sync_heartbeat_final_bytes(
OPENAI_IMAGE_SYNC_HEARTBEAT_EXHAUSTED_STATUS,
"OpenAI image sync exhausted all local candidates",
),
Err(err) => openai_image_sync_heartbeat_error_body(
Err(_err) => openai_image_sync_heartbeat_error_body(
OPENAI_IMAGE_SYNC_HEARTBEAT_INTERNAL_ERROR_STATUS,
&format!("{err:?}"),
"internal gateway error while executing image request",
),
}
}
@@ -1223,9 +1223,9 @@ async fn openai_image_sync_heartbeat_response_body_bytes(response: Response<Body
OPENAI_IMAGE_SYNC_HEARTBEAT_INTERNAL_ERROR_STATUS,
"empty sync image response",
),
Err(err) => openai_image_sync_heartbeat_error_body(
Err(_err) => openai_image_sync_heartbeat_error_body(
OPENAI_IMAGE_SYNC_HEARTBEAT_INTERNAL_ERROR_STATUS,
&err.to_string(),
"internal gateway error while reading image response",
),
}
}
@@ -2267,6 +2267,70 @@ mod tests {
let _ = release_tx.send(());
}
#[tokio::test]
async fn standard_text_sync_heartbeat_background_holds_request_admission_after_disconnect() {
let state = AppState::new().expect("state should build");
let gate = aether_runtime::ConcurrencyGate::new("heartbeat_request", 1);
let admission = aether_runtime::AdmissionPermit::from(
gate.try_acquire().expect("request admission permit"),
);
let (mut parts, _) = http::Request::builder()
.method(http::Method::POST)
.uri("/v1/responses")
.body(())
.expect("request should build")
.into_parts();
parts.extensions.insert(
crate::executor::candidate_loop::BackgroundAdmissionPermit::new(admission.clone()),
);
drop(admission);
let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
let (release_tx, release_rx) = tokio::sync::oneshot::channel::<()>();
let response = build_standard_text_sync_heartbeat_shell_response(
state,
parts,
"trace-standard-text-heartbeat-admission".to_string(),
test_standard_text_heartbeat_decision(),
TEST_STANDARD_TEXT_SYNC_PLAN_KIND.to_string(),
move |_state, parts, _trace_id, _decision, _plan_kind, _started_at| async move {
assert!(
parts
.extensions
.get::<crate::executor::candidate_loop::BackgroundAdmissionPermit>()
.is_some(),
"background request parts should carry admission"
);
let _ = started_tx.send(());
let _ = release_rx.await;
Ok(LocalExecutionRequestOutcome::responded(
Response::builder()
.status(StatusCode::OK)
.body(Body::from(r#"{"id":"resp_done","output":[]}"#))
.expect("response should build"),
))
},
)
.expect("heartbeat shell should build");
started_rx.await.expect("background execution should start");
drop(response);
assert_eq!(gate.snapshot().in_flight, 1);
assert!(
gate.try_acquire().is_err(),
"disconnect must not release background admission"
);
let _ = release_tx.send(());
tokio::time::timeout(Duration::from_secs(1), async {
while gate.snapshot().in_flight != 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("background completion should release admission");
}
#[tokio::test]
async fn standard_text_sync_heartbeat_propagates_request_diagnostics_to_terminal_usage() {
let (state, usage_repository) = heartbeat_usage_test_state(json!({
+79 -118
View File
@@ -3,14 +3,13 @@ use std::time::{Duration, Instant};
use aether_contracts::ExecutionPlan;
use aether_data_contracts::repository::candidates::{
sanitize_request_candidate_error_type, sanitize_request_candidate_skip_reason,
RequestCandidateStatus, StoredRequestCandidate,
};
use aether_data_contracts::repository::provider_catalog::{
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
};
use aether_data_contracts::repository::usage::{
ROUTING_CANDIDATE_SKIP_REASON_METADATA_KEY, ROUTING_FAILURE_DIAGNOSTIC_METADATA_KEY,
};
use aether_data_contracts::repository::usage::ROUTING_CANDIDATE_SKIP_REASON_METADATA_KEY;
use aether_usage_runtime::{
build_usage_event_data_seed, UsageEvent, UsageEventData, UsageEventType,
};
@@ -39,6 +38,12 @@ pub(crate) enum LocalExecutionRequestOutcome {
#[derive(Debug, Clone, Copy)]
pub(crate) struct DeferredUpstreamResponse;
#[derive(Debug, Clone)]
pub(crate) struct DeferredUsageContext {
plan: ExecutionPlan,
report_context: Option<Value>,
}
#[derive(Debug, Clone)]
pub(crate) struct LocalExecutionExhaustion {
request_id: String,
@@ -47,7 +52,6 @@ pub(crate) struct LocalExecutionExhaustion {
candidate_index: Option<u32>,
upstream_status_code: Option<u16>,
upstream_error_type: Option<String>,
upstream_error_message: Option<String>,
}
#[derive(Debug, Clone, Default)]
@@ -82,6 +86,53 @@ pub(crate) fn mark_deferred_upstream_response(mut response: Response<Body>) -> R
response
}
pub(crate) fn attach_deferred_usage_context(
response: &mut Response<Body>,
plan: &ExecutionPlan,
report_context: Option<&Value>,
) {
response.extensions_mut().insert(DeferredUsageContext {
plan: plan.clone(),
report_context: report_context.cloned(),
});
}
pub(crate) fn record_failed_usage_for_deferred_response<'a>(
state: &'a AppState,
response: &Response<Body>,
) -> impl std::future::Future<Output = ()> + Send + 'a {
let context = response.extensions().get::<DeferredUsageContext>().cloned();
let status_code = response.status().as_u16();
async move {
if !state.usage_runtime.is_enabled() {
return;
}
let Some(context) = context else {
return;
};
let mut data = build_usage_event_data_seed(&context.plan, context.report_context.as_ref());
data.status_code = Some(status_code);
data.error_message =
Some("all local candidates failed; returning preserved upstream error".to_string());
data.error_category = error_category_for_failed_status(status_code)
.or_else(|| Some("upstream_error".to_string()));
data.response_headers = Some(json_header_map());
data.client_response_headers = Some(json_header_map());
state
.usage_runtime
.record_terminal_event_direct(
state.usage_lifecycle_data_state().as_ref(),
UsageEvent::new(
UsageEventType::Failed,
context.plan.request_id.clone(),
data,
),
)
.await;
}
}
pub(crate) fn is_deferred_upstream_response(response: &Response<Body>) -> bool {
response
.extensions()
@@ -162,24 +213,10 @@ impl LocalExecutionRuntimeMissContext {
return None;
}
let diagnostic = self
.candidate_contexts
.iter()
.find_map(runtime_miss_candidate_failure_diagnostic)?;
let mut detail = format!("上游请求体转换失败:{}", diagnostic.message);
if diagnostic.path != "$" {
detail.push_str(&format!(";字段路径:{}", diagnostic.path));
}
detail.push_str("(原因代码: provider_request_body_build_failed)");
Some(detail)
Some("上游请求体转换失败(原因代码: provider_request_body_build_failed)".to_string())
}
}
struct RuntimeMissFailureDiagnostic {
path: String,
message: String,
}
pub(crate) async fn build_local_execution_exhaustion(
state: &AppState,
plan: &ExecutionPlan,
@@ -227,16 +264,11 @@ pub(crate) async fn build_local_execution_exhaustion(
exhaustion.upstream_status_code = last_failed_candidate
.as_ref()
.and_then(|candidate| candidate.status_code);
exhaustion.upstream_error_type = last_failed_candidate
.as_ref()
.and_then(|candidate| candidate.error_type.clone())
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty());
exhaustion.upstream_error_message = last_failed_candidate
.as_ref()
.and_then(|candidate| candidate.error_message.clone())
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty());
exhaustion.upstream_error_type = sanitize_request_candidate_error_type(
last_failed_candidate
.as_ref()
.and_then(|candidate| candidate.error_type.clone()),
);
exhaustion
}
@@ -256,7 +288,6 @@ pub(crate) fn build_fast_local_execution_exhaustion(
data,
upstream_status_code: None,
upstream_error_type: None,
upstream_error_message: None,
}
}
@@ -312,15 +343,12 @@ pub(crate) async fn record_failed_usage_for_exhausted_request(
candidate_index,
upstream_status_code,
upstream_error_type,
upstream_error_message,
} = exhaustion;
let status_code = http::StatusCode::SERVICE_UNAVAILABLE.as_u16();
let candidate_status_code = upstream_status_code.unwrap_or(status_code);
data.status_code = Some(status_code);
data.error_message = upstream_error_message
.clone()
.or_else(|| Some(local_execution_runtime_miss_detail.to_string()));
data.error_message = Some(local_execution_runtime_miss_detail.to_string());
data.error_category = error_category_for_failed_status(status_code);
data.response_time_ms = Some(started_at.elapsed().as_millis() as u64);
data.response_headers = Some(json_header_map());
@@ -330,10 +358,7 @@ pub(crate) async fn record_failed_usage_for_exhausted_request(
.as_deref()
.filter(|value| !value.trim().is_empty())
.unwrap_or("upstream_error"),
"message": upstream_error_message
.as_deref()
.filter(|value| !value.trim().is_empty())
.unwrap_or(local_execution_runtime_miss_detail),
"message": local_execution_runtime_miss_detail,
"code": candidate_status_code,
}
}));
@@ -993,82 +1018,13 @@ fn insert_runtime_miss_candidate_usage_metadata(
metadata: &mut Map<String, Value>,
candidate: &StoredRequestCandidate,
) {
if let Some(skip_reason) = candidate
.skip_reason
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
if let Some(skip_reason) = sanitize_request_candidate_skip_reason(candidate.skip_reason.clone())
{
metadata.insert(
ROUTING_CANDIDATE_SKIP_REASON_METADATA_KEY.to_string(),
Value::String(skip_reason.to_string()),
Value::String(skip_reason),
);
}
let diagnostic = candidate
.extra_data
.as_ref()
.and_then(Value::as_object)
.and_then(|extra_data| {
extra_data
.get("failure_diagnostic")
.filter(|value| {
value.as_object().is_some_and(|diagnostic| {
diagnostic.get("safe_to_show") != Some(&Value::Bool(false))
})
})
.or_else(|| {
extra_data
.get("request_conversion_error")
.filter(|v| v.is_object())
})
.or_else(|| {
extra_data
.get("request_body_build_error")
.filter(|v| v.is_object())
})
});
if let Some(diagnostic) = diagnostic {
metadata.insert(
ROUTING_FAILURE_DIAGNOSTIC_METADATA_KEY.to_string(),
diagnostic.clone(),
);
}
}
fn runtime_miss_candidate_failure_diagnostic(
candidate: &RuntimeMissCandidateContext,
) -> Option<RuntimeMissFailureDiagnostic> {
let extra_data = candidate.candidate.extra_data.as_ref()?.as_object()?;
let diagnostic = extra_data
.get("failure_diagnostic")
.and_then(Value::as_object)
.filter(|diagnostic| diagnostic.get("safe_to_show") != Some(&Value::Bool(false)))
.or_else(|| {
extra_data
.get("request_conversion_error")
.and_then(Value::as_object)
})
.or_else(|| {
extra_data
.get("request_body_build_error")
.and_then(Value::as_object)
})?;
let message = diagnostic
.get("message")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())?;
let path = diagnostic
.get("path")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.unwrap_or("$");
Some(RuntimeMissFailureDiagnostic {
path: path.to_string(),
message: message.to_string(),
})
}
fn build_runtime_miss_candidate_endpoint_url(
@@ -1136,7 +1092,11 @@ fn format_runtime_miss_candidate_summary(candidate: &RuntimeMissCandidateContext
.map(str::trim)
.filter(|value| !value.is_empty())
{
parts.push(format!("url={endpoint_url}"));
// Candidate URLs can contain provider API keys or other query
// credentials. Runtime-miss summaries are emitted to logs and may
// cross an operator/client boundary, so retain only the safe origin.
let origin = crate::handlers::shared::security_log_url_origin(endpoint_url);
parts.push(format!("url={origin}"));
}
if let Some(key_label) = format_name_with_id(
candidate.key_name.as_deref(),
@@ -1408,10 +1368,10 @@ mod tests {
request_metadata["routing_candidate_skip_reason"],
"provider_request_body_build_failed"
);
assert_eq!(
request_metadata["routing_failure_diagnostic"]["path"],
"$.reasoning.summary"
);
assert!(request_metadata.get("routing_failure_diagnostic").is_none());
assert!(!Value::Object(request_metadata.clone())
.to_string()
.contains("invalid reasoning summary"));
assert!(!request_candidate_represents_provider_execution(
&skipped_candidate
@@ -1437,7 +1397,7 @@ mod tests {
}
#[test]
fn runtime_miss_context_surfaces_request_conversion_field_diagnostic() {
fn runtime_miss_context_uses_fixed_request_body_build_failure_detail() {
let skipped_candidate = StoredRequestCandidate::new(
"cand-skipped".to_string(),
"req-1".to_string(),
@@ -1493,10 +1453,11 @@ mod tests {
let detail = context
.all_provider_request_body_build_failures_detail()
.expect("detail should include conversion diagnostic");
.expect("detail should identify the fixed failure category");
assert!(detail.contains("字段 n"));
assert!(detail.contains("字段路径:$.n"));
assert!(!detail.contains("字段 n"));
assert!(!detail.contains("字段路径"));
assert!(!detail.contains("OpenAI Responses"));
assert!(detail.contains("provider_request_body_build_failed"));
}
}
+56 -11
View File
@@ -18,6 +18,7 @@ use crate::ai_serving::api::{
use crate::api::response::build_client_response_from_parts;
use crate::control::GatewayControlDecision;
use crate::stage_metrics::observe_gateway_stage_ms;
use crate::state::VideoTaskRouteAccess;
use crate::{AppState, GatewayError, GatewayFallbackReason};
use super::{
@@ -101,7 +102,7 @@ pub(crate) async fn maybe_execute_via_stream_decision_path(
if skip_direct_plan {
return Ok(LocalExecutionRequestOutcome::NoPath);
}
let transfer_tracker = ProviderTransferTracker::default();
let transfer_tracker = ProviderTransferTracker::for_request(parts);
if plan_kind == OPENAI_CHAT_STREAM_PLAN_KIND
&& supports_stream_execution_decision_kind(plan_kind)
@@ -490,29 +491,73 @@ async fn maybe_execute_local_video_task_content_stream(
return Ok(LocalExecutionRequestOutcome::NoPath);
}
let _ = state
.hydrate_video_task_for_route(decision.route_family.as_deref(), parts.uri.path())
.await?;
let Some(user_id) = decision
.auth_context
.as_ref()
.filter(|auth_context| auth_context.access_allowed)
.map(|auth_context| auth_context.user_id.trim())
.filter(|value| !value.is_empty())
else {
return Ok(LocalExecutionRequestOutcome::Responded(
build_json_response(
trace_id,
decision,
404,
&crate::video_tasks::not_found_body(),
)?,
));
};
if state
.hydrate_video_task_for_route_for_user(
decision.route_family.as_deref(),
parts.uri.path(),
user_id,
)
.await?
!= VideoTaskRouteAccess::Allowed
{
return Ok(LocalExecutionRequestOutcome::Responded(
build_json_response(
trace_id,
decision,
404,
&crate::video_tasks::not_found_body(),
)?,
));
}
if let Some(task_id) =
crate::video_tasks::extract_openai_task_id_from_content_path(parts.uri.path())
{
let refresh_path = format!("/v1/videos/{task_id}");
if let Some(refresh_plan) = state.video_tasks.prepare_read_refresh_sync_plan(
if let Some(refresh_plan) = state.video_tasks.prepare_read_refresh_sync_plan_for_user(
Some("openai"),
&refresh_path,
user_id,
trace_id,
) {
state.execute_video_task_refresh_plan(&refresh_plan).await?;
}
}
let Some(action) = state.video_tasks.prepare_openai_content_stream_action(
parts.uri.path(),
parts.uri.query(),
trace_id,
) else {
return Ok(LocalExecutionRequestOutcome::NoPath);
let Some(action) = state
.video_tasks
.prepare_openai_content_stream_action_for_user(
parts.uri.path(),
parts.uri.query(),
trace_id,
user_id,
)
else {
return Ok(LocalExecutionRequestOutcome::Responded(
build_json_response(
trace_id,
decision,
404,
&crate::video_tasks::not_found_body(),
)?,
));
};
match action {
+82 -18
View File
@@ -16,6 +16,7 @@ use crate::ai_serving::api::{
use crate::api::response::build_client_response_from_parts;
use crate::control::resolve_execution_runtime_auth_context;
use crate::control::GatewayControlDecision;
use crate::state::VideoTaskRouteAccess;
use crate::{AppState, GatewayError, GatewayFallbackReason};
use super::{
@@ -90,7 +91,7 @@ pub(crate) async fn maybe_execute_via_sync_decision_path(
plan_kind,
bypass_cache_key,
scheduler_supported: supports_sync_execution_decision_kind(plan_kind),
transfer_tracker: ProviderTransferTracker::default(),
transfer_tracker: ProviderTransferTracker::for_request(parts),
};
Ok(from_ai_serving_outcome(
@@ -321,13 +322,41 @@ async fn maybe_build_local_video_task_read_response(
return Ok(LocalExecutionRequestOutcome::NoPath);
}
let _ = state
.hydrate_video_task_for_route(decision.route_family.as_deref(), parts.uri.path())
.await?;
let refresh_plan = state.video_tasks.prepare_read_refresh_sync_plan(
if crate::video_tasks::resolve_video_task_read_lookup_key(
decision.route_family.as_deref(),
parts.uri.path(),
)
.is_none()
{
return Ok(LocalExecutionRequestOutcome::NoPath);
}
let Some(user_id) = decision
.auth_context
.as_ref()
.filter(|auth_context| auth_context.access_allowed)
.map(|auth_context| auth_context.user_id.trim())
.filter(|value| !value.is_empty())
else {
return build_video_task_not_found_outcome(trace_id, decision);
};
if state
.hydrate_video_task_for_route_for_user(
decision.route_family.as_deref(),
parts.uri.path(),
user_id,
)
.await?
== VideoTaskRouteAccess::Denied
{
return build_video_task_not_found_outcome(trace_id, decision);
}
let refresh_plan = state.video_tasks.prepare_read_refresh_sync_plan_for_user(
decision.route_family.as_deref(),
parts.uri.path(),
user_id,
trace_id,
);
@@ -335,22 +364,25 @@ async fn maybe_build_local_video_task_read_response(
state.execute_video_task_refresh_plan(&refresh_plan).await?;
}
let read_response = state
.video_tasks
.read_response(decision.route_family.as_deref(), parts.uri.path());
let read_response = state.video_tasks.read_response_for_user(
decision.route_family.as_deref(),
parts.uri.path(),
user_id,
);
let read_response = match read_response {
Some(read_response) => Some(read_response),
None => {
state
.read_data_backed_video_task_response(
.read_data_backed_video_task_response_for_user(
decision.route_family.as_deref(),
parts.uri.path(),
user_id,
)
.await?
}
};
let Some(read_response) = read_response else {
return Ok(LocalExecutionRequestOutcome::NoPath);
return build_video_task_not_found_outcome(trace_id, decision);
};
let body_bytes = serde_json::to_vec(&read_response.body_json)
@@ -389,10 +421,6 @@ async fn maybe_execute_local_video_task_follow_up_sync(
return Ok(LocalExecutionRequestOutcome::NoPath);
}
let _ = state
.hydrate_video_task_for_route(decision.route_family.as_deref(), parts.uri.path())
.await?;
let auth_context = resolve_execution_runtime_auth_context(
state,
decision,
@@ -401,14 +429,30 @@ async fn maybe_execute_local_video_task_follow_up_sync(
trace_id,
)
.await?;
let Some(follow_up) = state.video_tasks.prepare_follow_up_sync_plan(
let Some(auth_context) = auth_context.filter(|auth_context| {
auth_context.access_allowed && !auth_context.user_id.trim().is_empty()
}) else {
return build_video_task_not_found_outcome(trace_id, decision);
};
if state
.hydrate_video_task_for_route_for_user(
decision.route_family.as_deref(),
parts.uri.path(),
&auth_context.user_id,
)
.await?
!= VideoTaskRouteAccess::Allowed
{
return build_video_task_not_found_outcome(trace_id, decision);
}
let Some(follow_up) = state.video_tasks.prepare_follow_up_sync_plan_for_user(
plan_kind,
parts.uri.path(),
Some(body_json),
auth_context.as_ref(),
Some(&auth_context),
trace_id,
) else {
return Ok(LocalExecutionRequestOutcome::NoPath);
return build_video_task_not_found_outcome(trace_id, decision);
};
execute_sync_plan_and_reports_with_transfer_tracker(
@@ -426,3 +470,23 @@ async fn maybe_execute_local_video_task_follow_up_sync(
)
.await
}
fn build_video_task_not_found_outcome(
trace_id: &str,
decision: &GatewayControlDecision,
) -> Result<LocalExecutionRequestOutcome, GatewayError> {
let body_bytes = serde_json::to_vec(&crate::video_tasks::not_found_body())
.map_err(|err| GatewayError::Internal(err.to_string()))?;
let mut headers = BTreeMap::new();
headers.insert("content-type".to_string(), "application/json".to_string());
headers.insert("content-length".to_string(), body_bytes.len().to_string());
Ok(LocalExecutionRequestOutcome::Responded(
build_client_response_from_parts(
404,
&headers,
Body::from(body_bytes),
trace_id,
Some(decision),
)?,
))
}