mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 17:30:23 +08:00
perf(transport): 全链路传输压缩优化
- 上游请求启用 HTTP/2 (HPACK 头部压缩 + 多路复用),添加 h2 依赖 - 上游请求体超过阈值时自动 gzip 压缩,使用紧凑 JSON 序列化 - 添加 brotli 依赖,Accept-Encoding 支持 gzip/deflate/br - 隧道帧压缩: Rust 端响应帧和 Python 端请求/响应帧均支持 gzip - Rust 端压缩/解压逻辑统一提取到 protocol.rs - Rust 端请求头构建改用 .headers() 替换 reqwest 默认值 - 新增 ENABLE_HTTP2、ENABLE_REQUEST_COMPRESSION 等环境变量配置
This commit is contained in:
@@ -7,13 +7,16 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
from typing import Any
|
||||
from urllib.parse import quote, urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from src.config import config
|
||||
from src.core.exceptions import ProxyNodeUnavailableError
|
||||
from src.core.logger import logger
|
||||
|
||||
@@ -614,6 +617,30 @@ def resolve_delegate_config(proxy_config: dict[str, Any] | None) -> dict[str, An
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _maybe_compress_payload(
|
||||
payload: Any,
|
||||
headers: dict[str, str],
|
||||
) -> tuple[bytes, dict[str, str]]:
|
||||
"""
|
||||
将 payload 序列化为 JSON bytes,按配置决定是否 gzip 压缩。
|
||||
|
||||
NOTE: 使用紧凑分隔符 ``(",", ":")`` 序列化(无空格),相比 httpx ``json=``
|
||||
参数的默认 ``json.dumps``(带空格分隔符)体积更小,所有上游 API 均兼容。
|
||||
|
||||
Returns:
|
||||
(body_bytes, updated_headers)
|
||||
"""
|
||||
json_bytes = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
||||
|
||||
if config.enable_request_compression and len(json_bytes) >= config.request_compression_min_size:
|
||||
compressed = gzip.compress(json_bytes, compresslevel=6)
|
||||
if len(compressed) < len(json_bytes):
|
||||
headers = {**headers, "Content-Encoding": "gzip"}
|
||||
return compressed, headers
|
||||
|
||||
return json_bytes, headers
|
||||
|
||||
|
||||
def build_post_kwargs(
|
||||
_delegate_cfg: dict[str, Any] | None = None,
|
||||
*,
|
||||
@@ -631,10 +658,11 @@ def build_post_kwargs(
|
||||
``_delegate_cfg`` 和 ``refresh_auth`` 已废弃(tunnel 模式下认证由 transport 层处理),
|
||||
保留仅为兼容现有调用方签名。
|
||||
"""
|
||||
content, final_headers = _maybe_compress_payload(payload, headers)
|
||||
return {
|
||||
"url": url,
|
||||
"json": payload,
|
||||
"headers": headers,
|
||||
"content": content,
|
||||
"headers": final_headers,
|
||||
"timeout": httpx.Timeout(timeout),
|
||||
}
|
||||
|
||||
@@ -655,11 +683,12 @@ def build_stream_kwargs(
|
||||
|
||||
``_delegate_cfg`` 已废弃,保留仅为兼容现有调用方签名。
|
||||
"""
|
||||
content, final_headers = _maybe_compress_payload(payload, headers)
|
||||
kwargs: dict[str, Any] = {
|
||||
"method": "POST",
|
||||
"url": url,
|
||||
"json": payload,
|
||||
"headers": headers,
|
||||
"content": content,
|
||||
"headers": final_headers,
|
||||
}
|
||||
if timeout is not None:
|
||||
kwargs["timeout"] = httpx.Timeout(timeout)
|
||||
|
||||
@@ -8,6 +8,7 @@ WebSocket 隧道管理器
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import gzip
|
||||
import json
|
||||
import time
|
||||
from typing import TYPE_CHECKING, Any
|
||||
@@ -21,6 +22,10 @@ from src.core.logger import logger
|
||||
|
||||
from .tunnel_protocol import Frame, FrameFlags, MsgType
|
||||
|
||||
# 隧道帧压缩的最小 payload 大小(字节)
|
||||
# 小于此值的帧压缩收益不大,反而增加 CPU 开销
|
||||
_TUNNEL_COMPRESS_MIN_SIZE = 512
|
||||
|
||||
|
||||
class TunnelConnection:
|
||||
"""单条 tunnel 连接"""
|
||||
@@ -309,7 +314,7 @@ class TunnelManager:
|
||||
stream_state = conn.create_stream(stream_id)
|
||||
|
||||
try:
|
||||
# 发送 REQUEST_HEADERS
|
||||
# 发送 REQUEST_HEADERS(大元数据帧压缩)
|
||||
meta = json.dumps(
|
||||
{
|
||||
"method": method,
|
||||
@@ -318,13 +323,19 @@ class TunnelManager:
|
||||
"timeout": int(timeout),
|
||||
}
|
||||
).encode()
|
||||
await conn.send_frame(Frame(stream_id, MsgType.REQUEST_HEADERS, 0, meta))
|
||||
|
||||
# 发送 REQUEST_BODY + END_STREAM
|
||||
body_data = body or b""
|
||||
meta_payload, meta_flags = _compress_frame_payload(meta)
|
||||
await conn.send_frame(
|
||||
Frame(stream_id, MsgType.REQUEST_BODY, FrameFlags.END_STREAM, body_data)
|
||||
Frame(stream_id, MsgType.REQUEST_HEADERS, meta_flags, meta_payload)
|
||||
)
|
||||
|
||||
# 发送 REQUEST_BODY + END_STREAM(大请求体帧压缩)
|
||||
body_data = body or b""
|
||||
if body_data:
|
||||
body_payload, body_flags = _compress_frame_payload(body_data)
|
||||
else:
|
||||
body_payload, body_flags = body_data, 0
|
||||
body_flags |= FrameFlags.END_STREAM
|
||||
await conn.send_frame(Frame(stream_id, MsgType.REQUEST_BODY, body_flags, body_payload))
|
||||
except Exception:
|
||||
conn.remove_stream(stream_id)
|
||||
raise
|
||||
@@ -348,14 +359,16 @@ class TunnelManager:
|
||||
if not stream:
|
||||
return
|
||||
try:
|
||||
meta = json.loads(frame.payload)
|
||||
payload = _decompress_frame_payload(frame)
|
||||
meta = json.loads(payload)
|
||||
stream.set_response_headers(meta["status"], meta.get("headers", []))
|
||||
except Exception as e:
|
||||
stream.set_error(f"invalid response headers: {e}")
|
||||
|
||||
elif frame.msg_type == MsgType.RESPONSE_BODY:
|
||||
if stream:
|
||||
stream.push_body_chunk(frame.payload)
|
||||
payload = _decompress_frame_payload(frame)
|
||||
stream.push_body_chunk(payload)
|
||||
|
||||
elif frame.msg_type == MsgType.STREAM_END:
|
||||
if stream:
|
||||
@@ -426,6 +439,32 @@ class TunnelManager:
|
||||
logger.debug("heartbeat ACK send failed for node_id={}", conn.node_id)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 隧道帧压缩 / 解压
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _compress_frame_payload(data: bytes) -> tuple[bytes, int]:
|
||||
"""按配置对帧 payload 进行 gzip 压缩。
|
||||
|
||||
Returns:
|
||||
(payload, flags) — 若压缩则 flags 含 GZIP_COMPRESSED,否则 flags=0。
|
||||
"""
|
||||
if len(data) >= _TUNNEL_COMPRESS_MIN_SIZE:
|
||||
compressed = gzip.compress(data, compresslevel=6)
|
||||
# 仅在压缩确实缩小时使用
|
||||
if len(compressed) < len(data):
|
||||
return compressed, FrameFlags.GZIP_COMPRESSED
|
||||
return data, 0
|
||||
|
||||
|
||||
def _decompress_frame_payload(frame: Frame) -> bytes:
|
||||
"""如果帧设置了 GZIP_COMPRESSED 标志则解压,否则原样返回。"""
|
||||
if frame.is_gzip:
|
||||
return gzip.decompress(frame.payload)
|
||||
return frame.payload
|
||||
|
||||
|
||||
# 全局单例
|
||||
_tunnel_manager: TunnelManager | None = None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user