i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 01:10:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

perf(transport): 全链路传输压缩优化

Browse Source 
- 上游请求启用 HTTP/2 (HPACK 头部压缩 + 多路复用)，添加 h2 依赖
- 上游请求体超过阈值时自动 gzip 压缩，使用紧凑 JSON 序列化
- 添加 brotli 依赖，Accept-Encoding 支持 gzip/deflate/br
- 隧道帧压缩: Rust 端响应帧和 Python 端请求/响应帧均支持 gzip
- Rust 端压缩/解压逻辑统一提取到 protocol.rs
- Rust 端请求头构建改用 .headers() 替换 reqwest 默认值
- 新增 ENABLE_HTTP2、ENABLE_REQUEST_COMPRESSION 等环境变量配置
This commit is contained in:
fawney19
2026-02-28 20:53:11 +08:00
parent 3ff67fec2f
commit a2d1cff3b0
10 changed files with 279 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/clients/http_client.py
View File

@@ -73,7 +73,7 @@ class HTTPClientPool:
# 双重检查，避免重复创建
if cls._default_client is None:
cls._default_client = httpx.AsyncClient(
http2=False, # 暂时禁用HTTP/2以提高兼容性
http2=config.enable_http2,
verify=get_ssl_context(), # 使用 certifi 证书
timeout=httpx.Timeout(
connect=config.http_connect_timeout,
@@ -106,7 +106,7 @@ class HTTPClientPool:
"""
if cls._default_client is None:
cls._default_client = httpx.AsyncClient(
http2=False, # 暂时禁用HTTP/2以提高兼容性
http2=config.enable_http2,
verify=get_ssl_context(), # 使用 certifi 证书
timeout=httpx.Timeout(
connect=config.http_connect_timeout,
@@ -143,7 +143,7 @@ class HTTPClientPool:
if name not in cls._clients:
# 合并默认配置和自定义配置
default_config = {
"http2": False,
"http2": config.enable_http2,
"verify": get_ssl_context(),
"timeout": httpx.Timeout(
connect=config.http_connect_timeout,
@@ -282,7 +282,7 @@ class HTTPClientPool:
# 创建新客户端（使用默认超时，请求时可覆盖）
client_config: dict[str, Any] = {
"http2": False,
"http2": config.enable_http2,
"verify": get_ssl_context_for_profile(tls_profile),
"follow_redirects": True,
"limits": httpx.Limits(
@@ -377,7 +377,7 @@ class HTTPClientPool:
response = await client.get('https://example.com')
"""
default_config = {
"http2": False,
"http2": config.enable_http2,
"verify": get_ssl_context(),
"timeout": httpx.Timeout(
connect=config.http_connect_timeout,
@@ -416,7 +416,7 @@ class HTTPClientPool:
配置好的 httpx.AsyncClient 实例（调用者需要负责关闭）
"""
client_config: dict[str, Any] = {
"http2": False,
"http2": config.enable_http2,
"verify": get_ssl_context_for_profile(tls_profile),
"follow_redirects": True,
}

16
src/config/settings.py
View File

@@ -177,6 +177,22 @@ class Config:
)
self.http_keepalive_expiry = float(os.getenv("HTTP_KEEPALIVE_EXPIRY", "30.0"))
# 上游传输优化配置
# ENABLE_HTTP2: 是否对上游请求启用 HTTP/2（HPACK 头部压缩 + 多路复用）
# - 三家上游（Claude/OpenAI/Gemini）均已确认支持 HTTP/2
# - 出现兼容性问题时可通过环境变量快速回退到 HTTP/1.1
self.enable_http2 = os.getenv("ENABLE_HTTP2", "true").lower() == "true"
# ENABLE_REQUEST_COMPRESSION: 是否对上游请求体启用 gzip 压缩
# - 仅对超过 REQUEST_COMPRESSION_MIN_SIZE 的请求体生效
# - 上游 Cloudflare/Google Front End 层会透明解压
self.enable_request_compression = (
os.getenv("ENABLE_REQUEST_COMPRESSION", "true").lower() == "true"
)
# REQUEST_COMPRESSION_MIN_SIZE: 触发请求体压缩的最小字节数
# - gzip 有固定头部开销，小请求压缩后可能反而变大
# - 默认 1024 字节（1KB）
self.request_compression_min_size = int(os.getenv("REQUEST_COMPRESSION_MIN_SIZE", "1024"))
# 流式处理配置
# STREAM_PREFETCH_LINES: 预读行数，用于检测嵌套错误
# STREAM_STATS_DELAY: 统计记录延迟（秒），等待流完全关闭

3
src/core/api_format/headers.py
View File

@@ -39,6 +39,7 @@ BROWSER_FINGERPRINT_HEADERS: dict[str, str] = {
"Chrome/140.0.7339.249 Electron/38.7.0 Safari/537.36"
),
"Accept": "application/json",
"Accept-Encoding": "gzip, deflate, br",
"Accept-Language": "zh-CN",
"sec-ch-ua": '"Not=A?Brand";v="24", "Chromium";v="140"',
"sec-ch-ua-mobile": "?0",
@@ -73,7 +74,7 @@ UPSTREAM_DROP_HEADERS: frozenset[str] = frozenset(
"content-length",
"transfer-encoding",
"connection",
# 编码头 - 避免客户端请求 brotli/zstd 但 httpx 不支持
# 编码头 - 丢弃客户端值，由 BROWSER_FINGERPRINT_HEADERS 统一设置
"accept-encoding",
# 反向代理 / 网关注入的头部 - 属于本站基础设施，不应泄露给上游
"x-real-ip",

37
src/services/proxy_node/resolver.py
View File

@@ -7,13 +7,16 @@
from __future__ import annotations
import gzip
import hashlib
import json
import time
from typing import Any
from urllib.parse import quote, urlparse
import httpx
from src.config import config
from src.core.exceptions import ProxyNodeUnavailableError
from src.core.logger import logger
@@ -614,6 +617,30 @@ def resolve_delegate_config(proxy_config: dict[str, Any] | None) -> dict[str, An
# ---------------------------------------------------------------------------
def _maybe_compress_payload(
payload: Any,
headers: dict[str, str],
) -> tuple[bytes, dict[str, str]]:
"""
将 payload 序列化为 JSON bytes，按配置决定是否 gzip 压缩。
NOTE: 使用紧凑分隔符 ``(",", ":")`` 序列化（无空格），相比 httpx ``json=``
参数的默认 ``json.dumps``（带空格分隔符）体积更小，所有上游 API 均兼容。
Returns:
(body_bytes, updated_headers)
"""
json_bytes = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
if config.enable_request_compression and len(json_bytes) >= config.request_compression_min_size:
compressed = gzip.compress(json_bytes, compresslevel=6)
if len(compressed) < len(json_bytes):
headers = {**headers, "Content-Encoding": "gzip"}
return compressed, headers
return json_bytes, headers
def build_post_kwargs(
_delegate_cfg: dict[str, Any] | None = None,
*,
@@ -631,10 +658,11 @@ def build_post_kwargs(
``_delegate_cfg`` 和 ``refresh_auth`` 已废弃（tunnel 模式下认证由 transport 层处理），
保留仅为兼容现有调用方签名。
"""
content, final_headers = _maybe_compress_payload(payload, headers)
return {
"url": url,
"json": payload,
"headers": headers,
"content": content,
"headers": final_headers,
"timeout": httpx.Timeout(timeout),
}
@@ -655,11 +683,12 @@ def build_stream_kwargs(
``_delegate_cfg`` 已废弃，保留仅为兼容现有调用方签名。
"""
content, final_headers = _maybe_compress_payload(payload, headers)
kwargs: dict[str, Any] = {
"method": "POST",
"url": url,
"json": payload,
"headers": headers,
"content": content,
"headers": final_headers,
}
if timeout is not None:
kwargs["timeout"] = httpx.Timeout(timeout)

55
src/services/proxy_node/tunnel_manager.py
View File

@@ -8,6 +8,7 @@ WebSocket 隧道管理器
from __future__ import annotations
import asyncio
import gzip
import json
import time
from typing import TYPE_CHECKING, Any
@@ -21,6 +22,10 @@ from src.core.logger import logger
from .tunnel_protocol import Frame, FrameFlags, MsgType
# 隧道帧压缩的最小 payload 大小（字节）
# 小于此值的帧压缩收益不大，反而增加 CPU 开销
_TUNNEL_COMPRESS_MIN_SIZE = 512
class TunnelConnection:
"""单条 tunnel 连接"""
@@ -309,7 +314,7 @@ class TunnelManager:
stream_state = conn.create_stream(stream_id)
try:
# 发送 REQUEST_HEADERS
# 发送 REQUEST_HEADERS（大元数据帧压缩）
meta = json.dumps(
{
"method": method,
@@ -318,13 +323,19 @@ class TunnelManager:
"timeout": int(timeout),
}
).encode()
await conn.send_frame(Frame(stream_id, MsgType.REQUEST_HEADERS, 0, meta))
# 发送 REQUEST_BODY + END_STREAM
body_data = body or b""
meta_payload, meta_flags = _compress_frame_payload(meta)
await conn.send_frame(
Frame(stream_id, MsgType.REQUEST_BODY, FrameFlags.END_STREAM, body_data)
Frame(stream_id, MsgType.REQUEST_HEADERS, meta_flags, meta_payload)
)
# 发送 REQUEST_BODY + END_STREAM（大请求体帧压缩）
body_data = body or b""
if body_data:
body_payload, body_flags = _compress_frame_payload(body_data)
else:
body_payload, body_flags = body_data, 0
body_flags |= FrameFlags.END_STREAM
await conn.send_frame(Frame(stream_id, MsgType.REQUEST_BODY, body_flags, body_payload))
except Exception:
conn.remove_stream(stream_id)
raise
@@ -348,14 +359,16 @@ class TunnelManager:
if not stream:
return
try:
meta = json.loads(frame.payload)
payload = _decompress_frame_payload(frame)
meta = json.loads(payload)
stream.set_response_headers(meta["status"], meta.get("headers", []))
except Exception as e:
stream.set_error(f"invalid response headers: {e}")
elif frame.msg_type == MsgType.RESPONSE_BODY:
if stream:
stream.push_body_chunk(frame.payload)
payload = _decompress_frame_payload(frame)
stream.push_body_chunk(payload)
elif frame.msg_type == MsgType.STREAM_END:
if stream:
@@ -426,6 +439,32 @@ class TunnelManager:
logger.debug("heartbeat ACK send failed for node_id={}", conn.node_id)
# ---------------------------------------------------------------------------
# 隧道帧压缩 / 解压
# ---------------------------------------------------------------------------
def _compress_frame_payload(data: bytes) -> tuple[bytes, int]:
"""按配置对帧 payload 进行 gzip 压缩。
Returns:
(payload, flags) — 若压缩则 flags 含 GZIP_COMPRESSED，否则 flags=0。
"""
if len(data) >= _TUNNEL_COMPRESS_MIN_SIZE:
compressed = gzip.compress(data, compresslevel=6)
# 仅在压缩确实缩小时使用
if len(compressed) < len(data):
return compressed, FrameFlags.GZIP_COMPRESSED
return data, 0
def _decompress_frame_payload(frame: Frame) -> bytes:
"""如果帧设置了 GZIP_COMPRESSED 标志则解压，否则原样返回。"""
if frame.is_gzip:
return gzip.decompress(frame.payload)
return frame.payload
# 全局单例
_tunnel_manager: TunnelManager | None = None