mirror of
https://github.com/fawney19/Aether.git
synced 2026-10-04 08:27:46 +08:00
feat(security): harden gateway request and runtime controls
This commit is contained in:
@@ -652,7 +652,9 @@ pub(super) async fn extract_admin_provider_oauth_batch_error_detail(
|
||||
response: Response<Body>,
|
||||
) -> String {
|
||||
let status = response.status();
|
||||
let raw_body = to_bytes(response.into_body(), usize::MAX).await.ok();
|
||||
let raw_body = to_bytes(response.into_body(), crate::MAX_ERROR_BODY_BYTES)
|
||||
.await
|
||||
.ok();
|
||||
if let Some(raw_body) = raw_body {
|
||||
if let Ok(value) = serde_json::from_slice::<serde_json::Value>(&raw_body) {
|
||||
if let Some(detail) = value.get("detail").and_then(serde_json::Value::as_str) {
|
||||
|
||||
@@ -1704,9 +1704,12 @@ async fn provider_query_finalize_kiro_result(
|
||||
let Some(outcome) = maybe_build_sync_finalize_outcome(trace_id, &decision, &payload)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let bytes = to_bytes(outcome.response.into_body(), usize::MAX)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
let bytes = to_bytes(
|
||||
outcome.response.into_body(),
|
||||
crate::headers::max_internal_buffered_body_bytes(),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
serde_json::from_slice::<Value>(&bytes)
|
||||
.map(Some)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))
|
||||
@@ -1926,9 +1929,12 @@ async fn provider_query_finalize_windsurf_result(
|
||||
let Some(outcome) = maybe_build_sync_finalize_outcome(trace_id, &decision, &payload)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let bytes = to_bytes(outcome.response.into_body(), usize::MAX)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
let bytes = to_bytes(
|
||||
outcome.response.into_body(),
|
||||
crate::headers::max_internal_buffered_body_bytes(),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
serde_json::from_slice::<Value>(&bytes)
|
||||
.map(Some)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))
|
||||
@@ -2044,9 +2050,12 @@ async fn provider_query_finalize_openai_image_result(
|
||||
let Some(outcome) = maybe_build_sync_finalize_outcome(trace_id, &decision, &payload)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let bytes = to_bytes(outcome.response.into_body(), usize::MAX)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
let bytes = to_bytes(
|
||||
outcome.response.into_body(),
|
||||
crate::headers::max_internal_buffered_body_bytes(),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
serde_json::from_slice::<Value>(&bytes)
|
||||
.map(Some)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))
|
||||
@@ -2412,9 +2421,12 @@ async fn provider_query_finalize_antigravity_result(
|
||||
let Some(outcome) = maybe_build_sync_finalize_outcome(trace_id, &decision, &payload)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let bytes = to_bytes(outcome.response.into_body(), usize::MAX)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
let bytes = to_bytes(
|
||||
outcome.response.into_body(),
|
||||
crate::headers::max_internal_buffered_body_bytes(),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
serde_json::from_slice::<Value>(&bytes)
|
||||
.map(Some)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))
|
||||
@@ -3794,9 +3806,12 @@ pub(crate) async fn build_admin_provider_query_test_model_local_response(
|
||||
if !response.status().is_success() {
|
||||
return Ok(response);
|
||||
}
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
let body = to_bytes(
|
||||
response.into_body(),
|
||||
crate::headers::max_internal_buffered_body_bytes(),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
let parsed: Value =
|
||||
serde_json::from_slice(&body).map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
|
||||
|
||||
@@ -0,0 +1,310 @@
|
||||
use crate::api::response::build_local_http_error_response;
|
||||
use crate::control::GatewayPublicRequestContext;
|
||||
use crate::headers::RequestBodyNormalizationError;
|
||||
use crate::{AppState, GatewayError};
|
||||
use axum::body::{to_bytes, Body, Bytes};
|
||||
use axum::http::{self, Response};
|
||||
use std::error::Error as StdError;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::sync::Semaphore;
|
||||
use tracing::{info, warn};
|
||||
|
||||
const REQUEST_BODY_READ_TIMEOUT_DETAIL: &str =
|
||||
"Request body read timed out before the gateway could route the request";
|
||||
const REQUEST_BODY_READ_FAILED_DETAIL: &str = "Failed to read request body";
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) struct RequestBodyBufferPolicy {
|
||||
max_bytes: u64,
|
||||
read_timeout: Duration,
|
||||
queue_timeout: Duration,
|
||||
budget_bytes: usize,
|
||||
budget: Arc<Semaphore>,
|
||||
}
|
||||
|
||||
impl RequestBodyBufferPolicy {
|
||||
pub(super) fn from_state(state: &AppState) -> Self {
|
||||
Self {
|
||||
max_bytes: crate::headers::max_request_body_bytes(),
|
||||
read_timeout: state.frontdoor_runtime_guards.request_body_read_timeout,
|
||||
queue_timeout: state.frontdoor_runtime_guards.internal_gate_queue_budget,
|
||||
budget_bytes: state
|
||||
.frontdoor_runtime_guards
|
||||
.request_body_buffer_budget_bytes,
|
||||
budget: Arc::clone(&state.request_body_buffer_budget),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn for_tests(max_bytes: u64, read_timeout: Duration) -> Self {
|
||||
let budget_bytes = usize::try_from(max_bytes)
|
||||
.unwrap_or(usize::MAX)
|
||||
.max(crate::state::REQUEST_BODY_BUFFER_PERMIT_BYTES);
|
||||
Self {
|
||||
max_bytes,
|
||||
read_timeout,
|
||||
queue_timeout: read_timeout,
|
||||
budget_bytes,
|
||||
budget: Arc::new(Semaphore::new(
|
||||
budget_bytes.saturating_add(crate::state::REQUEST_BODY_BUFFER_PERMIT_BYTES - 1)
|
||||
/ crate::state::REQUEST_BODY_BUFFER_PERMIT_BYTES,
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn for_tests_with_budget(
|
||||
max_bytes: u64,
|
||||
read_timeout: Duration,
|
||||
queue_timeout: Duration,
|
||||
budget_bytes: usize,
|
||||
budget: Arc<Semaphore>,
|
||||
) -> Self {
|
||||
Self {
|
||||
max_bytes,
|
||||
read_timeout,
|
||||
queue_timeout,
|
||||
budget_bytes,
|
||||
budget,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(super) enum RequestBodyBufferError {
|
||||
Normalization(RequestBodyNormalizationError),
|
||||
TooLarge {
|
||||
limit_bytes: u64,
|
||||
},
|
||||
Overloaded {
|
||||
requested_bytes: usize,
|
||||
budget_bytes: usize,
|
||||
timeout_ms: u64,
|
||||
},
|
||||
Timeout {
|
||||
timeout_ms: u64,
|
||||
},
|
||||
ReadFailed {
|
||||
message: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl RequestBodyBufferError {
|
||||
pub(super) fn http_status(&self) -> http::StatusCode {
|
||||
match self {
|
||||
Self::Normalization(error) => error.http_status(),
|
||||
Self::TooLarge { .. } => http::StatusCode::PAYLOAD_TOO_LARGE,
|
||||
Self::Overloaded { .. } => http::StatusCode::SERVICE_UNAVAILABLE,
|
||||
Self::Timeout { .. } => http::StatusCode::REQUEST_TIMEOUT,
|
||||
Self::ReadFailed { .. } => http::StatusCode::BAD_REQUEST,
|
||||
}
|
||||
}
|
||||
|
||||
fn client_message(&self) -> String {
|
||||
match self {
|
||||
Self::Normalization(error) => error.client_message(),
|
||||
Self::TooLarge { limit_bytes } => format!("Request body exceeds {limit_bytes} bytes"),
|
||||
Self::Overloaded { .. } => {
|
||||
"Request body buffering capacity is temporarily exhausted".to_string()
|
||||
}
|
||||
Self::Timeout { .. } => REQUEST_BODY_READ_TIMEOUT_DETAIL.to_string(),
|
||||
Self::ReadFailed { .. } => REQUEST_BODY_READ_FAILED_DETAIL.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn reason(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Normalization(error) => match error {
|
||||
RequestBodyNormalizationError::UnsupportedContentEncoding(_) => {
|
||||
"unsupported_content_encoding"
|
||||
}
|
||||
RequestBodyNormalizationError::DecodeFailed { .. } => "decode_failed",
|
||||
RequestBodyNormalizationError::DecompressedBodyTooLarge { .. } => {
|
||||
"decompressed_body_too_large"
|
||||
}
|
||||
RequestBodyNormalizationError::RequestBodyTooLarge { .. } => {
|
||||
"request_body_too_large"
|
||||
}
|
||||
},
|
||||
Self::TooLarge { .. } => "request_body_too_large",
|
||||
Self::Overloaded { .. } => "request_body_buffer_overloaded",
|
||||
Self::Timeout { .. } => "request_body_read_timeout",
|
||||
Self::ReadFailed { .. } => "request_body_read_failed",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn request_body_buffer_reservation_bytes(headers: &http::HeaderMap, max_bytes: u64) -> usize {
|
||||
let max_bytes = usize::try_from(max_bytes).unwrap_or(usize::MAX);
|
||||
let encoded = headers
|
||||
.get(http::header::CONTENT_ENCODING)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::trim)
|
||||
.is_some_and(|value| !value.is_empty() && !value.eq_ignore_ascii_case("identity"));
|
||||
if encoded {
|
||||
return max_bytes;
|
||||
}
|
||||
headers
|
||||
.get(http::header::CONTENT_LENGTH)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.trim().parse::<usize>().ok())
|
||||
.map(|value| value.min(max_bytes))
|
||||
.unwrap_or(max_bytes)
|
||||
}
|
||||
|
||||
fn request_body_buffer_reservation_permits(reservation_bytes: usize) -> u32 {
|
||||
let permits = reservation_bytes
|
||||
.max(1)
|
||||
.saturating_add(crate::state::REQUEST_BODY_BUFFER_PERMIT_BYTES - 1)
|
||||
/ crate::state::REQUEST_BODY_BUFFER_PERMIT_BYTES;
|
||||
u32::try_from(permits).unwrap_or(u32::MAX).max(1)
|
||||
}
|
||||
|
||||
pub(super) async fn buffer_and_normalize_request_body(
|
||||
request_body: &mut Option<Body>,
|
||||
headers: &mut http::HeaderMap,
|
||||
body_owner_expectation: &'static str,
|
||||
trace_id: &str,
|
||||
method: &http::Method,
|
||||
path_and_query: &str,
|
||||
phase: &'static str,
|
||||
policy: RequestBodyBufferPolicy,
|
||||
) -> Result<Bytes, RequestBodyBufferError> {
|
||||
if let Err(err) =
|
||||
crate::headers::check_request_content_length_with_limit(headers, policy.max_bytes)
|
||||
{
|
||||
return Err(RequestBodyBufferError::Normalization(err));
|
||||
}
|
||||
|
||||
let reservation_bytes = request_body_buffer_reservation_bytes(headers, policy.max_bytes);
|
||||
let reservation_permits = request_body_buffer_reservation_permits(reservation_bytes);
|
||||
let queue_timeout_ms = policy.queue_timeout.as_millis() as u64;
|
||||
let _budget_permit = match tokio::time::timeout(
|
||||
policy.queue_timeout,
|
||||
Arc::clone(&policy.budget).acquire_many_owned(reservation_permits),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(permit)) => permit,
|
||||
Ok(Err(_)) | Err(_) => {
|
||||
return Err(RequestBodyBufferError::Overloaded {
|
||||
requested_bytes: reservation_bytes,
|
||||
budget_bytes: policy.budget_bytes,
|
||||
timeout_ms: queue_timeout_ms,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
let read_started_at = Instant::now();
|
||||
let timeout_ms = policy.read_timeout.as_millis() as u64;
|
||||
info!(
|
||||
event_name = "frontdoor_request_body_buffer_started",
|
||||
log_type = "event",
|
||||
trace_id,
|
||||
method = %method,
|
||||
path = %path_and_query,
|
||||
phase,
|
||||
max_body_bytes = policy.max_bytes,
|
||||
reserved_body_bytes = reservation_bytes,
|
||||
body_buffer_budget_bytes = policy.budget_bytes,
|
||||
timeout_ms,
|
||||
"gateway started buffering request body"
|
||||
);
|
||||
|
||||
let body_limit = usize::try_from(policy.max_bytes).unwrap_or(usize::MAX);
|
||||
let body = match tokio::time::timeout(
|
||||
policy.read_timeout,
|
||||
to_bytes(
|
||||
request_body.take().expect(body_owner_expectation),
|
||||
body_limit,
|
||||
),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(body)) => body,
|
||||
Ok(Err(err)) if request_body_collection_exceeded_limit(&err) => {
|
||||
return Err(RequestBodyBufferError::TooLarge {
|
||||
limit_bytes: policy.max_bytes,
|
||||
});
|
||||
}
|
||||
Ok(Err(err)) => {
|
||||
return Err(RequestBodyBufferError::ReadFailed {
|
||||
message: err.to_string(),
|
||||
});
|
||||
}
|
||||
Err(_) => {
|
||||
return Err(RequestBodyBufferError::Timeout { timeout_ms });
|
||||
}
|
||||
};
|
||||
|
||||
let normalized = crate::headers::normalize_request_body_headers_and_bytes_with_limit(
|
||||
headers,
|
||||
body,
|
||||
policy.max_bytes,
|
||||
)
|
||||
.map_err(RequestBodyBufferError::Normalization)?;
|
||||
info!(
|
||||
event_name = "frontdoor_request_body_buffer_completed",
|
||||
log_type = "event",
|
||||
trace_id,
|
||||
method = %method,
|
||||
path = %path_and_query,
|
||||
phase,
|
||||
body_bytes = normalized.len(),
|
||||
elapsed_ms = read_started_at.elapsed().as_millis() as u64,
|
||||
"gateway completed buffering request body"
|
||||
);
|
||||
Ok(normalized)
|
||||
}
|
||||
|
||||
fn request_body_collection_exceeded_limit(error: &(dyn StdError + 'static)) -> bool {
|
||||
let mut current = Some(error);
|
||||
while let Some(error) = current {
|
||||
if error.to_string().contains("length limit exceeded") {
|
||||
return true;
|
||||
}
|
||||
current = error.source();
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
pub(super) fn build_request_body_buffer_error_response(
|
||||
trace_id: &str,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
error: &RequestBodyBufferError,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
warn!(
|
||||
event_name = "frontdoor_request_body_buffer_failed",
|
||||
log_type = "ops",
|
||||
trace_id,
|
||||
method = %request_context.request_method,
|
||||
path = %request_context.request_path_and_query(),
|
||||
status_code = error.http_status().as_u16(),
|
||||
reason = error.reason(),
|
||||
detail = %error.client_message(),
|
||||
read_error = match error {
|
||||
RequestBodyBufferError::ReadFailed { message } => message.as_str(),
|
||||
_ => "",
|
||||
},
|
||||
buffer_requested_bytes = match error {
|
||||
RequestBodyBufferError::Overloaded { requested_bytes, .. } => *requested_bytes,
|
||||
_ => 0,
|
||||
},
|
||||
buffer_budget_bytes = match error {
|
||||
RequestBodyBufferError::Overloaded { budget_bytes, .. } => *budget_bytes,
|
||||
_ => 0,
|
||||
},
|
||||
buffer_queue_timeout_ms = match error {
|
||||
RequestBodyBufferError::Overloaded { timeout_ms, .. } => *timeout_ms,
|
||||
_ => 0,
|
||||
},
|
||||
"gateway rejected request body before local execution planning"
|
||||
);
|
||||
build_local_http_error_response(
|
||||
trace_id,
|
||||
request_context.control_decision.as_ref(),
|
||||
error.http_status(),
|
||||
error.client_message().as_str(),
|
||||
)
|
||||
}
|
||||
@@ -1,5 +1,10 @@
|
||||
mod body_buffer;
|
||||
mod local;
|
||||
|
||||
use self::body_buffer::{
|
||||
buffer_and_normalize_request_body, build_request_body_buffer_error_response,
|
||||
RequestBodyBufferError, RequestBodyBufferPolicy,
|
||||
};
|
||||
use self::local::{
|
||||
maybe_build_local_admin_proxy_response, maybe_build_local_internal_proxy_response,
|
||||
};
|
||||
@@ -52,7 +57,7 @@ use crate::handlers::shared::{
|
||||
};
|
||||
use crate::headers::{
|
||||
effective_client_ip, extract_or_generate_trace_id, request_origin_from_headers_and_remote_addr,
|
||||
should_skip_request_header, RequestBodyNormalizationError,
|
||||
should_skip_request_header,
|
||||
};
|
||||
use crate::router::RequestAdmissionError;
|
||||
use crate::scheduler::candidate::{
|
||||
@@ -70,11 +75,7 @@ use axum::extract::{ConnectInfo, Request, State};
|
||||
use axum::http::{self, header::HeaderName, header::HeaderValue, Response};
|
||||
use futures_util::StreamExt;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
error::Error as StdError,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use std::{collections::BTreeMap, time::Instant};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
const OPENAI_CHAT_LOCAL_EXECUTION_RUNTIME_MISS_DETAIL: &str =
|
||||
@@ -98,201 +99,11 @@ const LOCAL_EXECUTION_LOOP_DETECTED_DETAIL: &str =
|
||||
"Gateway detected an execution runtime request loop back into the local frontdoor";
|
||||
const AUTH_API_KEY_CONCURRENCY_LIMIT_REACHED_DETAIL: &str =
|
||||
"当前调用方 API Key 并发请求数已达上限,请稍后重试";
|
||||
const REQUEST_BODY_READ_TIMEOUT_DETAIL: &str =
|
||||
"Request body read timed out before the gateway could route the request";
|
||||
const REQUEST_BODY_READ_FAILED_DETAIL: &str = "Failed to read request body";
|
||||
const LOCAL_EXECUTION_PLANNING_TIMEOUT_DETAIL: &str =
|
||||
"当前 AI 请求在本地执行规划阶段超时,请稍后重试";
|
||||
const EXECUTION_PATH_TUNNEL_AFFINITY_FORWARD: &str = "tunnel_affinity_forward";
|
||||
const MANAGEMENT_TOKEN_PREFIX: &str = "ae-";
|
||||
const LEGACY_MANAGEMENT_TOKEN_PREFIX: &str = "ae_";
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct RequestBodyBufferPolicy {
|
||||
max_bytes: u64,
|
||||
read_timeout: Duration,
|
||||
}
|
||||
|
||||
impl RequestBodyBufferPolicy {
|
||||
fn from_state(state: &AppState) -> Self {
|
||||
Self {
|
||||
max_bytes: crate::headers::max_request_body_bytes(),
|
||||
read_timeout: state.frontdoor_runtime_guards.request_body_read_timeout,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn for_tests(max_bytes: u64, read_timeout: Duration) -> Self {
|
||||
Self {
|
||||
max_bytes,
|
||||
read_timeout,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
enum RequestBodyBufferError {
|
||||
Normalization(RequestBodyNormalizationError),
|
||||
TooLarge { limit_bytes: u64 },
|
||||
Timeout { timeout_ms: u64 },
|
||||
ReadFailed { message: String },
|
||||
}
|
||||
|
||||
impl RequestBodyBufferError {
|
||||
fn http_status(&self) -> http::StatusCode {
|
||||
match self {
|
||||
Self::Normalization(error) => error.http_status(),
|
||||
Self::TooLarge { .. } => http::StatusCode::PAYLOAD_TOO_LARGE,
|
||||
Self::Timeout { .. } => http::StatusCode::REQUEST_TIMEOUT,
|
||||
Self::ReadFailed { .. } => http::StatusCode::BAD_REQUEST,
|
||||
}
|
||||
}
|
||||
|
||||
fn client_message(&self) -> String {
|
||||
match self {
|
||||
Self::Normalization(error) => error.client_message(),
|
||||
Self::TooLarge { limit_bytes } => format!("Request body exceeds {limit_bytes} bytes"),
|
||||
Self::Timeout { .. } => REQUEST_BODY_READ_TIMEOUT_DETAIL.to_string(),
|
||||
Self::ReadFailed { .. } => REQUEST_BODY_READ_FAILED_DETAIL.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn reason(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Normalization(error) => match error {
|
||||
RequestBodyNormalizationError::UnsupportedContentEncoding(_) => {
|
||||
"unsupported_content_encoding"
|
||||
}
|
||||
RequestBodyNormalizationError::DecodeFailed { .. } => "decode_failed",
|
||||
RequestBodyNormalizationError::DecompressedBodyTooLarge { .. } => {
|
||||
"decompressed_body_too_large"
|
||||
}
|
||||
RequestBodyNormalizationError::RequestBodyTooLarge { .. } => {
|
||||
"request_body_too_large"
|
||||
}
|
||||
},
|
||||
Self::TooLarge { .. } => "request_body_too_large",
|
||||
Self::Timeout { .. } => "request_body_read_timeout",
|
||||
Self::ReadFailed { .. } => "request_body_read_failed",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn buffer_and_normalize_request_body(
|
||||
request_body: &mut Option<Body>,
|
||||
headers: &mut http::HeaderMap,
|
||||
body_owner_expectation: &'static str,
|
||||
trace_id: &str,
|
||||
method: &http::Method,
|
||||
path_and_query: &str,
|
||||
phase: &'static str,
|
||||
policy: RequestBodyBufferPolicy,
|
||||
) -> Result<Bytes, RequestBodyBufferError> {
|
||||
if let Err(err) =
|
||||
crate::headers::check_request_content_length_with_limit(headers, policy.max_bytes)
|
||||
{
|
||||
return Err(RequestBodyBufferError::Normalization(err));
|
||||
}
|
||||
|
||||
let read_started_at = Instant::now();
|
||||
let timeout_ms = policy.read_timeout.as_millis() as u64;
|
||||
info!(
|
||||
event_name = "frontdoor_request_body_buffer_started",
|
||||
log_type = "event",
|
||||
trace_id,
|
||||
method = %method,
|
||||
path = %path_and_query,
|
||||
phase,
|
||||
max_body_bytes = policy.max_bytes,
|
||||
timeout_ms,
|
||||
"gateway started buffering request body"
|
||||
);
|
||||
|
||||
let body_limit = usize::try_from(policy.max_bytes).unwrap_or(usize::MAX);
|
||||
let body = match tokio::time::timeout(
|
||||
policy.read_timeout,
|
||||
to_bytes(
|
||||
request_body.take().expect(body_owner_expectation),
|
||||
body_limit,
|
||||
),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(body)) => body,
|
||||
Ok(Err(err)) if request_body_collection_exceeded_limit(&err) => {
|
||||
return Err(RequestBodyBufferError::TooLarge {
|
||||
limit_bytes: policy.max_bytes,
|
||||
});
|
||||
}
|
||||
Ok(Err(err)) => {
|
||||
return Err(RequestBodyBufferError::ReadFailed {
|
||||
message: err.to_string(),
|
||||
});
|
||||
}
|
||||
Err(_) => {
|
||||
return Err(RequestBodyBufferError::Timeout { timeout_ms });
|
||||
}
|
||||
};
|
||||
|
||||
let normalized = crate::headers::normalize_request_body_headers_and_bytes_with_limit(
|
||||
headers,
|
||||
body,
|
||||
policy.max_bytes,
|
||||
)
|
||||
.map_err(RequestBodyBufferError::Normalization)?;
|
||||
info!(
|
||||
event_name = "frontdoor_request_body_buffer_completed",
|
||||
log_type = "event",
|
||||
trace_id,
|
||||
method = %method,
|
||||
path = %path_and_query,
|
||||
phase,
|
||||
body_bytes = normalized.len(),
|
||||
elapsed_ms = read_started_at.elapsed().as_millis() as u64,
|
||||
"gateway completed request body buffering"
|
||||
);
|
||||
Ok(normalized)
|
||||
}
|
||||
|
||||
fn request_body_collection_exceeded_limit(error: &(dyn StdError + 'static)) -> bool {
|
||||
let mut current = Some(error);
|
||||
while let Some(error) = current {
|
||||
if error.to_string().contains("length limit exceeded") {
|
||||
return true;
|
||||
}
|
||||
current = error.source();
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn build_request_body_buffer_error_response(
|
||||
trace_id: &str,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
error: &RequestBodyBufferError,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
warn!(
|
||||
event_name = "frontdoor_request_body_buffer_failed",
|
||||
log_type = "ops",
|
||||
trace_id,
|
||||
method = %request_context.request_method,
|
||||
path = %request_context.request_path_and_query(),
|
||||
status_code = error.http_status().as_u16(),
|
||||
reason = error.reason(),
|
||||
detail = %error.client_message(),
|
||||
read_error = match error {
|
||||
RequestBodyBufferError::ReadFailed { message } => message.as_str(),
|
||||
_ => "",
|
||||
},
|
||||
"gateway rejected request body before local execution planning"
|
||||
);
|
||||
build_local_http_error_response(
|
||||
trace_id,
|
||||
request_context.control_decision.as_ref(),
|
||||
error.http_status(),
|
||||
error.client_message().as_str(),
|
||||
)
|
||||
}
|
||||
|
||||
fn finalize_request_body_buffer_rejection(
|
||||
state: &AppState,
|
||||
request_context: &GatewayPublicRequestContext,
|
||||
@@ -303,12 +114,17 @@ fn finalize_request_body_buffer_rejection(
|
||||
error: &RequestBodyBufferError,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let response = build_request_body_buffer_error_response(trace_id, request_context, error)?;
|
||||
let execution_path = if matches!(error, RequestBodyBufferError::Overloaded { .. }) {
|
||||
EXECUTION_PATH_LOCAL_OVERLOADED
|
||||
} else {
|
||||
EXECUTION_PATH_LOCAL_INVALID_REQUEST
|
||||
};
|
||||
Ok(finalize_gateway_response_with_context(
|
||||
state,
|
||||
response,
|
||||
remote_addr,
|
||||
request_context,
|
||||
EXECUTION_PATH_LOCAL_INVALID_REQUEST,
|
||||
execution_path,
|
||||
started_at,
|
||||
request_permit,
|
||||
))
|
||||
@@ -776,9 +592,17 @@ async fn restore_redacted_sync_execution_response(
|
||||
return Ok(Response::from_parts(parts, body));
|
||||
};
|
||||
let mut headers = collect_response_headers(&parts.headers);
|
||||
let body_bytes = to_bytes(body, usize::MAX)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
let response_body_limit = crate::headers::max_redacted_sync_response_body_bytes();
|
||||
let body_bytes = to_bytes(
|
||||
body,
|
||||
usize::try_from(response_body_limit).unwrap_or(usize::MAX),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
GatewayError::Internal(format!(
|
||||
"failed to buffer redacted sync response within {response_body_limit} bytes: {err}"
|
||||
))
|
||||
})?;
|
||||
let restored =
|
||||
crate::privacy::restore_sync_response_body(&mut headers, body_bytes.as_ref(), &session)?;
|
||||
replace_response_headers(&mut parts.headers, &headers)?;
|
||||
@@ -1041,49 +865,6 @@ async fn proxy_request_inner(
|
||||
let started_at = Instant::now();
|
||||
let client_ip = effective_client_ip(request.headers(), &remote_addr);
|
||||
let trace_id = extract_or_generate_trace_id(request.headers());
|
||||
match state.admin_security_ip_blacklisted(client_ip).await {
|
||||
Ok(true) => {
|
||||
warn!(
|
||||
event_name = "frontdoor_ip_blacklist_rejected",
|
||||
log_type = "event",
|
||||
trace_id = %trace_id,
|
||||
client_ip = %client_ip,
|
||||
path = %request.uri().path(),
|
||||
"gateway rejected blacklisted client IP"
|
||||
);
|
||||
let response = build_local_http_error_response(
|
||||
&trace_id,
|
||||
None,
|
||||
http::StatusCode::FORBIDDEN,
|
||||
"当前 IP 已被禁止访问",
|
||||
)?;
|
||||
return Ok(finalize_gateway_response(
|
||||
&state,
|
||||
response,
|
||||
&trace_id,
|
||||
&remote_addr,
|
||||
request.method(),
|
||||
request
|
||||
.uri()
|
||||
.path_and_query()
|
||||
.map(|value| value.as_str())
|
||||
.unwrap_or("/"),
|
||||
None,
|
||||
EXECUTION_PATH_LOCAL_AUTH_DENIED,
|
||||
&started_at,
|
||||
None,
|
||||
));
|
||||
}
|
||||
Ok(false) => {}
|
||||
Err(err) => warn!(
|
||||
event_name = "frontdoor_ip_blacklist_check_failed",
|
||||
log_type = "ops",
|
||||
trace_id = %trace_id,
|
||||
client_ip = %client_ip,
|
||||
error = ?err,
|
||||
"gateway failed open after IP blacklist check error"
|
||||
),
|
||||
}
|
||||
let accepted_at = request
|
||||
.extensions()
|
||||
.get::<crate::middleware::GatewayRequestAcceptedAt>()
|
||||
@@ -1161,6 +942,49 @@ async fn proxy_request_inner(
|
||||
};
|
||||
let request_admission_ms = started_at.elapsed().as_millis() as u64;
|
||||
observe_gateway_stage_ms("frontdoor_admission", request_admission_ms);
|
||||
match state.admin_security_ip_blacklisted(client_ip).await {
|
||||
Ok(true) => {
|
||||
warn!(
|
||||
event_name = "frontdoor_ip_blacklist_rejected",
|
||||
log_type = "event",
|
||||
trace_id = %trace_id,
|
||||
client_ip = %client_ip,
|
||||
path = %request.uri().path(),
|
||||
"gateway rejected blacklisted client IP"
|
||||
);
|
||||
let response = build_local_http_error_response(
|
||||
&trace_id,
|
||||
None,
|
||||
http::StatusCode::FORBIDDEN,
|
||||
"当前 IP 已被禁止访问",
|
||||
)?;
|
||||
return Ok(finalize_gateway_response(
|
||||
&state,
|
||||
response,
|
||||
&trace_id,
|
||||
&remote_addr,
|
||||
request.method(),
|
||||
request
|
||||
.uri()
|
||||
.path_and_query()
|
||||
.map(|value| value.as_str())
|
||||
.unwrap_or("/"),
|
||||
None,
|
||||
EXECUTION_PATH_LOCAL_AUTH_DENIED,
|
||||
&started_at,
|
||||
request_permit.take(),
|
||||
));
|
||||
}
|
||||
Ok(false) => {}
|
||||
Err(err) => warn!(
|
||||
event_name = "frontdoor_ip_blacklist_check_failed",
|
||||
log_type = "ops",
|
||||
trace_id = %trace_id,
|
||||
client_ip = %client_ip,
|
||||
error = ?err,
|
||||
"gateway failed open after IP blacklist check error"
|
||||
),
|
||||
}
|
||||
let (mut parts, body) = request.into_parts();
|
||||
let redaction_slot = crate::privacy::RedactionSessionSlot::default();
|
||||
parts.extensions.insert(redaction_slot.clone());
|
||||
@@ -2432,6 +2256,7 @@ fn local_execution_runtime_miss_route_detail(
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use super::{
|
||||
@@ -2442,8 +2267,9 @@ mod tests {
|
||||
RequestBodyBufferPolicy,
|
||||
};
|
||||
use axum::body::{to_bytes, Body, Bytes};
|
||||
use axum::http::{header, HeaderMap, Method, Response};
|
||||
use axum::http::{header, HeaderMap, HeaderValue, Method, Response};
|
||||
use serde_json::json;
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
#[test]
|
||||
fn api_key_remote_ip_allows_unrestricted_keys() {
|
||||
@@ -2622,6 +2448,45 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn request_body_buffer_rejects_when_weighted_budget_is_exhausted() {
|
||||
let budget = Arc::new(Semaphore::new(1));
|
||||
let _held = Arc::clone(&budget)
|
||||
.acquire_owned()
|
||||
.await
|
||||
.expect("test budget should be open");
|
||||
let mut body = Some(Body::from(Bytes::from_static(b"{}")));
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(header::CONTENT_LENGTH, HeaderValue::from_static("2"));
|
||||
|
||||
let err = buffer_and_normalize_request_body(
|
||||
&mut body,
|
||||
&mut headers,
|
||||
"test owns body",
|
||||
"trace-body-budget",
|
||||
&Method::POST,
|
||||
"/v1/responses",
|
||||
"test",
|
||||
RequestBodyBufferPolicy::for_tests_with_budget(
|
||||
1024,
|
||||
Duration::from_secs(1),
|
||||
Duration::from_millis(5),
|
||||
crate::state::REQUEST_BODY_BUFFER_PERMIT_BYTES,
|
||||
budget,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect_err("exhausted body budget should reject quickly");
|
||||
|
||||
assert!(matches!(
|
||||
err,
|
||||
RequestBodyBufferError::Overloaded {
|
||||
requested_bytes: 2,
|
||||
..
|
||||
}
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_miss_detail_returns_model_specific_stream_message_when_candidates_are_unavailable() {
|
||||
let decision = GatewayControlDecision::synthetic(
|
||||
|
||||
@@ -464,7 +464,7 @@ async fn complete_oauth_login(
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
let body = login_response.into_body();
|
||||
let body = match to_bytes(body, usize::MAX).await {
|
||||
let body = match to_bytes(body, crate::headers::max_internal_buffered_body_bytes()).await {
|
||||
Ok(value) => value,
|
||||
Err(_) => return redirect_oauth_error(Some(frontend_callback_url), "provider_unavailable"),
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user