perf(transport): 全链路传输压缩优化

- 上游请求启用 HTTP/2 (HPACK 头部压缩 + 多路复用),添加 h2 依赖
- 上游请求体超过阈值时自动 gzip 压缩,使用紧凑 JSON 序列化
- 添加 brotli 依赖,Accept-Encoding 支持 gzip/deflate/br
- 隧道帧压缩: Rust 端响应帧和 Python 端请求/响应帧均支持 gzip
- Rust 端压缩/解压逻辑统一提取到 protocol.rs
- Rust 端请求头构建改用 .headers() 替换 reqwest 默认值
- 新增 ENABLE_HTTP2、ENABLE_REQUEST_COMPRESSION 等环境变量配置
This commit is contained in:
fawney19
2026-02-28 20:53:11 +08:00
parent 3ff67fec2f
commit a2d1cff3b0
10 changed files with 279 additions and 59 deletions

View File

@@ -7,13 +7,16 @@
from __future__ import annotations
import gzip
import hashlib
import json
import time
from typing import Any
from urllib.parse import quote, urlparse
import httpx
from src.config import config
from src.core.exceptions import ProxyNodeUnavailableError
from src.core.logger import logger
@@ -614,6 +617,30 @@ def resolve_delegate_config(proxy_config: dict[str, Any] | None) -> dict[str, An
# ---------------------------------------------------------------------------
def _maybe_compress_payload(
payload: Any,
headers: dict[str, str],
) -> tuple[bytes, dict[str, str]]:
"""
将 payload 序列化为 JSON bytes按配置决定是否 gzip 压缩。
NOTE: 使用紧凑分隔符 ``(",", ":")`` 序列化(无空格),相比 httpx ``json=``
参数的默认 ``json.dumps``(带空格分隔符)体积更小,所有上游 API 均兼容。
Returns:
(body_bytes, updated_headers)
"""
json_bytes = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
if config.enable_request_compression and len(json_bytes) >= config.request_compression_min_size:
compressed = gzip.compress(json_bytes, compresslevel=6)
if len(compressed) < len(json_bytes):
headers = {**headers, "Content-Encoding": "gzip"}
return compressed, headers
return json_bytes, headers
def build_post_kwargs(
_delegate_cfg: dict[str, Any] | None = None,
*,
@@ -631,10 +658,11 @@ def build_post_kwargs(
``_delegate_cfg`` 和 ``refresh_auth`` 已废弃tunnel 模式下认证由 transport 层处理),
保留仅为兼容现有调用方签名。
"""
content, final_headers = _maybe_compress_payload(payload, headers)
return {
"url": url,
"json": payload,
"headers": headers,
"content": content,
"headers": final_headers,
"timeout": httpx.Timeout(timeout),
}
@@ -655,11 +683,12 @@ def build_stream_kwargs(
``_delegate_cfg`` 已废弃,保留仅为兼容现有调用方签名。
"""
content, final_headers = _maybe_compress_payload(payload, headers)
kwargs: dict[str, Any] = {
"method": "POST",
"url": url,
"json": payload,
"headers": headers,
"content": content,
"headers": final_headers,
}
if timeout is not None:
kwargs["timeout"] = httpx.Timeout(timeout)

View File

@@ -8,6 +8,7 @@ WebSocket 隧道管理器
from __future__ import annotations
import asyncio
import gzip
import json
import time
from typing import TYPE_CHECKING, Any
@@ -21,6 +22,10 @@ from src.core.logger import logger
from .tunnel_protocol import Frame, FrameFlags, MsgType
# 隧道帧压缩的最小 payload 大小(字节)
# 小于此值的帧压缩收益不大,反而增加 CPU 开销
_TUNNEL_COMPRESS_MIN_SIZE = 512
class TunnelConnection:
"""单条 tunnel 连接"""
@@ -309,7 +314,7 @@ class TunnelManager:
stream_state = conn.create_stream(stream_id)
try:
# 发送 REQUEST_HEADERS
# 发送 REQUEST_HEADERS(大元数据帧压缩)
meta = json.dumps(
{
"method": method,
@@ -318,13 +323,19 @@ class TunnelManager:
"timeout": int(timeout),
}
).encode()
await conn.send_frame(Frame(stream_id, MsgType.REQUEST_HEADERS, 0, meta))
# 发送 REQUEST_BODY + END_STREAM
body_data = body or b""
meta_payload, meta_flags = _compress_frame_payload(meta)
await conn.send_frame(
Frame(stream_id, MsgType.REQUEST_BODY, FrameFlags.END_STREAM, body_data)
Frame(stream_id, MsgType.REQUEST_HEADERS, meta_flags, meta_payload)
)
# 发送 REQUEST_BODY + END_STREAM大请求体帧压缩
body_data = body or b""
if body_data:
body_payload, body_flags = _compress_frame_payload(body_data)
else:
body_payload, body_flags = body_data, 0
body_flags |= FrameFlags.END_STREAM
await conn.send_frame(Frame(stream_id, MsgType.REQUEST_BODY, body_flags, body_payload))
except Exception:
conn.remove_stream(stream_id)
raise
@@ -348,14 +359,16 @@ class TunnelManager:
if not stream:
return
try:
meta = json.loads(frame.payload)
payload = _decompress_frame_payload(frame)
meta = json.loads(payload)
stream.set_response_headers(meta["status"], meta.get("headers", []))
except Exception as e:
stream.set_error(f"invalid response headers: {e}")
elif frame.msg_type == MsgType.RESPONSE_BODY:
if stream:
stream.push_body_chunk(frame.payload)
payload = _decompress_frame_payload(frame)
stream.push_body_chunk(payload)
elif frame.msg_type == MsgType.STREAM_END:
if stream:
@@ -426,6 +439,32 @@ class TunnelManager:
logger.debug("heartbeat ACK send failed for node_id={}", conn.node_id)
# ---------------------------------------------------------------------------
# 隧道帧压缩 / 解压
# ---------------------------------------------------------------------------
def _compress_frame_payload(data: bytes) -> tuple[bytes, int]:
"""按配置对帧 payload 进行 gzip 压缩。
Returns:
(payload, flags) — 若压缩则 flags 含 GZIP_COMPRESSED否则 flags=0。
"""
if len(data) >= _TUNNEL_COMPRESS_MIN_SIZE:
compressed = gzip.compress(data, compresslevel=6)
# 仅在压缩确实缩小时使用
if len(compressed) < len(data):
return compressed, FrameFlags.GZIP_COMPRESSED
return data, 0
def _decompress_frame_payload(frame: Frame) -> bytes:
"""如果帧设置了 GZIP_COMPRESSED 标志则解压,否则原样返回。"""
if frame.is_gzip:
return gzip.decompress(frame.payload)
return frame.payload
# 全局单例
_tunnel_manager: TunnelManager | None = None