mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 17:30:23 +08:00
feat(tunnel): 添加流帧分发超时机制和结构化请求日志
- dispatcher: 为 stream handler 的帧接收添加超时保护, 防止单个 handler 阻塞 WebSocket 读循环; 超时后发送 StreamError 并清理流 - stream_handler: 在所有请求完成和错误路径添加结构化日志, 记录 method/host/path/status/duration 等关键信息 - 将常规隧道连接/断开日志从 info 降级为 debug, 减少日志噪音
This commit is contained in:
@@ -16,7 +16,7 @@ use futures_util::StreamExt;
|
||||
use http_body_util::BodyExt;
|
||||
use hyper::body::Frame as BodyFrame;
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::{debug, warn};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::state::{AppState, ServerContext};
|
||||
use crate::target_filter;
|
||||
@@ -116,6 +116,110 @@ struct UpstreamResponseContext {
|
||||
request_timing: upstream_client::RequestTiming,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct StreamLogContext<'a> {
|
||||
server: &'a ServerContext,
|
||||
stream_id: u32,
|
||||
method: &'a hyper::Method,
|
||||
url: Option<&'a url::Url>,
|
||||
redirect_count: usize,
|
||||
request_body_size: usize,
|
||||
}
|
||||
|
||||
fn parse_request_method(method: &str) -> hyper::Method {
|
||||
method.parse().unwrap_or(hyper::Method::GET)
|
||||
}
|
||||
|
||||
fn request_log_host(url: &url::Url) -> &str {
|
||||
url.host_str().unwrap_or("")
|
||||
}
|
||||
|
||||
fn request_log_port(url: &url::Url) -> u16 {
|
||||
url.port_or_known_default().unwrap_or(0)
|
||||
}
|
||||
|
||||
fn request_log_path(url: &url::Url) -> &str {
|
||||
let path = url.path();
|
||||
if path.is_empty() {
|
||||
"/"
|
||||
} else {
|
||||
path
|
||||
}
|
||||
}
|
||||
|
||||
fn stream_log_context<'a>(
|
||||
server: &'a ServerContext,
|
||||
stream_id: u32,
|
||||
method: &'a hyper::Method,
|
||||
url: Option<&'a url::Url>,
|
||||
redirect_count: usize,
|
||||
request_body_size: usize,
|
||||
) -> StreamLogContext<'a> {
|
||||
StreamLogContext {
|
||||
server,
|
||||
stream_id,
|
||||
method,
|
||||
url,
|
||||
redirect_count,
|
||||
request_body_size,
|
||||
}
|
||||
}
|
||||
|
||||
fn log_stream_success(ctx: StreamLogContext<'_>, status: u16, duration: Duration) {
|
||||
let url = ctx
|
||||
.url
|
||||
.expect("successful requests should always have a URL");
|
||||
info!(
|
||||
server = %ctx.server.server_label,
|
||||
stream_id = ctx.stream_id,
|
||||
method = %ctx.method,
|
||||
scheme = url.scheme(),
|
||||
host = request_log_host(url),
|
||||
port = request_log_port(url),
|
||||
path = request_log_path(url),
|
||||
query_present = url.query().is_some(),
|
||||
status,
|
||||
duration_ms = duration.as_millis() as u64,
|
||||
redirect_count = ctx.redirect_count,
|
||||
request_body_bytes = ctx.request_body_size,
|
||||
"proxy request completed"
|
||||
);
|
||||
}
|
||||
|
||||
fn log_stream_failure(ctx: StreamLogContext<'_>, error: &str, duration: Duration) {
|
||||
match ctx.url {
|
||||
Some(url) => {
|
||||
warn!(
|
||||
server = %ctx.server.server_label,
|
||||
stream_id = ctx.stream_id,
|
||||
method = %ctx.method,
|
||||
scheme = url.scheme(),
|
||||
host = request_log_host(url),
|
||||
port = request_log_port(url),
|
||||
path = request_log_path(url),
|
||||
query_present = url.query().is_some(),
|
||||
error = %error,
|
||||
duration_ms = duration.as_millis() as u64,
|
||||
redirect_count = ctx.redirect_count,
|
||||
request_body_bytes = ctx.request_body_size,
|
||||
"proxy request failed"
|
||||
);
|
||||
}
|
||||
None => {
|
||||
warn!(
|
||||
server = %ctx.server.server_label,
|
||||
stream_id = ctx.stream_id,
|
||||
method = %ctx.method,
|
||||
error = %error,
|
||||
duration_ms = duration.as_millis() as u64,
|
||||
redirect_count = ctx.redirect_count,
|
||||
request_body_bytes = ctx.request_body_size,
|
||||
"proxy request failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl PreparedRequestBody {
|
||||
fn streaming(
|
||||
body_rx: mpsc::Receiver<TunnelFrame>,
|
||||
@@ -466,6 +570,8 @@ async fn execute_upstream_request(
|
||||
async fn relay_upstream_response(
|
||||
server: &ServerContext,
|
||||
stream_id: u32,
|
||||
method: &hyper::Method,
|
||||
request_url: &url::Url,
|
||||
frame_tx: &FrameSender,
|
||||
response: hyper::Response<hyper::body::Incoming>,
|
||||
total_dns_ms: u64,
|
||||
@@ -516,6 +622,18 @@ async fn relay_upstream_response(
|
||||
)
|
||||
.await
|
||||
{
|
||||
log_stream_failure(
|
||||
stream_log_context(
|
||||
server,
|
||||
stream_id,
|
||||
method,
|
||||
Some(request_url),
|
||||
redirect_count,
|
||||
request_body_size.load(Ordering::Relaxed),
|
||||
),
|
||||
"tunnel response headers relay failed",
|
||||
total_elapsed,
|
||||
);
|
||||
return Some(total_elapsed);
|
||||
}
|
||||
|
||||
@@ -531,6 +649,18 @@ async fn relay_upstream_response(
|
||||
)
|
||||
.await
|
||||
{
|
||||
log_stream_failure(
|
||||
stream_log_context(
|
||||
server,
|
||||
stream_id,
|
||||
method,
|
||||
Some(request_url),
|
||||
redirect_count,
|
||||
request_body_size.load(Ordering::Relaxed),
|
||||
),
|
||||
"tunnel response body relay failed",
|
||||
total_elapsed,
|
||||
);
|
||||
return Some(total_elapsed);
|
||||
}
|
||||
} else {
|
||||
@@ -550,6 +680,18 @@ async fn relay_upstream_response(
|
||||
)
|
||||
.await
|
||||
{
|
||||
log_stream_failure(
|
||||
stream_log_context(
|
||||
server,
|
||||
stream_id,
|
||||
method,
|
||||
Some(request_url),
|
||||
redirect_count,
|
||||
request_body_size.load(Ordering::Relaxed),
|
||||
),
|
||||
"tunnel response body relay failed",
|
||||
total_elapsed,
|
||||
);
|
||||
return Some(total_elapsed);
|
||||
}
|
||||
offset = end;
|
||||
@@ -559,13 +701,26 @@ async fn relay_upstream_response(
|
||||
Err(error) => {
|
||||
server.metrics.stream_errors.fetch_add(1, Ordering::Release);
|
||||
warn!(stream_id, error = %error, "upstream body read error");
|
||||
let error_message = format!("upstream body read error: {error}");
|
||||
log_stream_failure(
|
||||
stream_log_context(
|
||||
server,
|
||||
stream_id,
|
||||
method,
|
||||
Some(request_url),
|
||||
redirect_count,
|
||||
request_body_size.load(Ordering::Relaxed),
|
||||
),
|
||||
&error_message,
|
||||
total_elapsed,
|
||||
);
|
||||
send_error(frame_tx, stream_id, &format!("body read error: {error}")).await;
|
||||
return Some(total_elapsed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let _ = send_frame(
|
||||
if !send_frame(
|
||||
frame_tx,
|
||||
TunnelFrame::new(
|
||||
stream_id,
|
||||
@@ -574,7 +729,22 @@ async fn relay_upstream_response(
|
||||
Bytes::new(),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
.await
|
||||
{
|
||||
log_stream_failure(
|
||||
stream_log_context(
|
||||
server,
|
||||
stream_id,
|
||||
method,
|
||||
Some(request_url),
|
||||
redirect_count,
|
||||
request_body_size.load(Ordering::Relaxed),
|
||||
),
|
||||
"tunnel stream end relay failed",
|
||||
total_elapsed,
|
||||
);
|
||||
return Some(total_elapsed);
|
||||
}
|
||||
|
||||
debug!(
|
||||
stream_id,
|
||||
@@ -582,6 +752,18 @@ async fn relay_upstream_response(
|
||||
redirects = redirect_count,
|
||||
"stream completed"
|
||||
);
|
||||
log_stream_success(
|
||||
stream_log_context(
|
||||
server,
|
||||
stream_id,
|
||||
method,
|
||||
Some(request_url),
|
||||
redirect_count,
|
||||
request_body_size.load(Ordering::Relaxed),
|
||||
),
|
||||
status,
|
||||
total_elapsed,
|
||||
);
|
||||
Some(total_elapsed)
|
||||
}
|
||||
|
||||
@@ -606,6 +788,8 @@ pub async fn handle_stream(
|
||||
body_rx: mpsc::Receiver<TunnelFrame>,
|
||||
frame_tx: FrameSender,
|
||||
) {
|
||||
let request_method = parse_request_method(&meta.method);
|
||||
let request_url = url::Url::parse(&meta.url).ok();
|
||||
let permit = match state.try_acquire_stream_permit().await {
|
||||
Ok(permit) => permit,
|
||||
Err(err) => {
|
||||
@@ -615,6 +799,18 @@ pub async fn handle_stream(
|
||||
"proxy admission unavailable"
|
||||
}
|
||||
};
|
||||
log_stream_failure(
|
||||
stream_log_context(
|
||||
&server,
|
||||
stream_id,
|
||||
&request_method,
|
||||
request_url.as_ref(),
|
||||
0,
|
||||
0,
|
||||
),
|
||||
message,
|
||||
Duration::ZERO,
|
||||
);
|
||||
send_error(&frame_tx, stream_id, message).await;
|
||||
return;
|
||||
}
|
||||
@@ -660,9 +856,15 @@ async fn handle_stream_inner(
|
||||
body_rx: mpsc::Receiver<TunnelFrame>,
|
||||
frame_tx: &FrameSender,
|
||||
) -> Option<Duration> {
|
||||
let mut current_method: hyper::Method = parse_request_method(&meta.method);
|
||||
let mut current_url = match url::Url::parse(&meta.url) {
|
||||
Ok(u) => u,
|
||||
Err(e) => {
|
||||
log_stream_failure(
|
||||
stream_log_context(server, stream_id, ¤t_method, None, 0, 0),
|
||||
&format!("invalid URL: {e}"),
|
||||
Duration::ZERO,
|
||||
);
|
||||
send_error(frame_tx, stream_id, &format!("invalid URL: {e}")).await;
|
||||
return None;
|
||||
}
|
||||
@@ -672,12 +874,13 @@ async fn handle_stream_inner(
|
||||
match current_url.scheme() {
|
||||
"http" | "https" => {}
|
||||
other => {
|
||||
send_error(
|
||||
frame_tx,
|
||||
stream_id,
|
||||
&format!("unsupported URL scheme: {other}"),
|
||||
)
|
||||
.await;
|
||||
let error_message = format!("unsupported URL scheme: {other}");
|
||||
log_stream_failure(
|
||||
stream_log_context(server, stream_id, ¤t_method, Some(¤t_url), 0, 0),
|
||||
&error_message,
|
||||
Duration::ZERO,
|
||||
);
|
||||
send_error(frame_tx, stream_id, &error_message).await;
|
||||
return None;
|
||||
}
|
||||
}
|
||||
@@ -685,7 +888,6 @@ async fn handle_stream_inner(
|
||||
let deadline = Instant::now()
|
||||
+ Duration::from_secs(meta.timeout.clamp(MIN_TIMEOUT_SECS, MAX_TIMEOUT_SECS));
|
||||
let follow_redirects = follow_redirects_enabled(&meta);
|
||||
let mut current_method: hyper::Method = meta.method.parse().unwrap_or(hyper::Method::GET);
|
||||
let mut current_headers = sanitize_upstream_headers(&meta.headers);
|
||||
let timeout = Duration::from_secs(meta.timeout.clamp(MIN_TIMEOUT_SECS, MAX_TIMEOUT_SECS));
|
||||
let request_body_size = Arc::new(AtomicUsize::new(0));
|
||||
@@ -700,6 +902,18 @@ async fn handle_stream_inner(
|
||||
{
|
||||
Ok(body) => body,
|
||||
Err(message) => {
|
||||
log_stream_failure(
|
||||
stream_log_context(
|
||||
server,
|
||||
stream_id,
|
||||
¤t_method,
|
||||
Some(¤t_url),
|
||||
0,
|
||||
request_body_size.load(Ordering::Relaxed),
|
||||
),
|
||||
&message,
|
||||
Duration::ZERO,
|
||||
);
|
||||
send_error(frame_tx, stream_id, &message).await;
|
||||
return None;
|
||||
}
|
||||
@@ -715,6 +929,18 @@ async fn handle_stream_inner(
|
||||
|
||||
loop {
|
||||
let Some(remaining) = remaining_timeout(deadline) else {
|
||||
log_stream_failure(
|
||||
stream_log_context(
|
||||
server,
|
||||
stream_id,
|
||||
¤t_method,
|
||||
Some(¤t_url),
|
||||
redirects_followed,
|
||||
request_body_size.load(Ordering::Relaxed),
|
||||
),
|
||||
"upstream timeout",
|
||||
overall_start.elapsed(),
|
||||
);
|
||||
send_error(frame_tx, stream_id, "upstream timeout").await;
|
||||
return None;
|
||||
};
|
||||
@@ -722,12 +948,20 @@ async fn handle_stream_inner(
|
||||
Some(mode) => match prepared_body.build_redirect_request_body(mode) {
|
||||
Some(body) => body,
|
||||
None => {
|
||||
send_error(
|
||||
frame_tx,
|
||||
stream_id,
|
||||
"upstream redirect error: body not replayable",
|
||||
)
|
||||
.await;
|
||||
let error_message = "upstream redirect error: body not replayable";
|
||||
log_stream_failure(
|
||||
stream_log_context(
|
||||
server,
|
||||
stream_id,
|
||||
¤t_method,
|
||||
Some(¤t_url),
|
||||
redirects_followed,
|
||||
request_body_size.load(Ordering::Relaxed),
|
||||
),
|
||||
error_message,
|
||||
overall_start.elapsed(),
|
||||
);
|
||||
send_error(frame_tx, stream_id, error_message).await;
|
||||
return None;
|
||||
}
|
||||
},
|
||||
@@ -748,6 +982,18 @@ async fn handle_stream_inner(
|
||||
{
|
||||
Ok(context) => context,
|
||||
Err(message) => {
|
||||
log_stream_failure(
|
||||
stream_log_context(
|
||||
server,
|
||||
stream_id,
|
||||
¤t_method,
|
||||
Some(¤t_url),
|
||||
redirects_followed,
|
||||
request_body_size.load(Ordering::Relaxed),
|
||||
),
|
||||
&message,
|
||||
overall_start.elapsed(),
|
||||
);
|
||||
send_error(frame_tx, stream_id, &message).await;
|
||||
return None;
|
||||
}
|
||||
@@ -767,6 +1013,8 @@ async fn handle_stream_inner(
|
||||
return relay_upstream_response(
|
||||
server,
|
||||
stream_id,
|
||||
¤t_method,
|
||||
¤t_url,
|
||||
frame_tx,
|
||||
response_ctx.response,
|
||||
total_dns_ms,
|
||||
@@ -791,12 +1039,20 @@ async fn handle_stream_inner(
|
||||
continue;
|
||||
}
|
||||
RedirectDecision::Error(message) => {
|
||||
send_error(
|
||||
frame_tx,
|
||||
stream_id,
|
||||
&format!("upstream redirect error: {message}"),
|
||||
)
|
||||
.await;
|
||||
let error_message = format!("upstream redirect error: {message}");
|
||||
log_stream_failure(
|
||||
stream_log_context(
|
||||
server,
|
||||
stream_id,
|
||||
¤t_method,
|
||||
Some(¤t_url),
|
||||
redirects_followed,
|
||||
request_body_size.load(Ordering::Relaxed),
|
||||
),
|
||||
&error_message,
|
||||
overall_start.elapsed(),
|
||||
);
|
||||
send_error(frame_tx, stream_id, &error_message).await;
|
||||
return None;
|
||||
}
|
||||
}
|
||||
@@ -805,6 +1061,8 @@ async fn handle_stream_inner(
|
||||
return relay_upstream_response(
|
||||
server,
|
||||
stream_id,
|
||||
¤t_method,
|
||||
¤t_url,
|
||||
frame_tx,
|
||||
response_ctx.response,
|
||||
total_dns_ms,
|
||||
|
||||
Reference in New Issue
Block a user