i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-10-08 18:37:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(proxy): 实现代理节点批量升级回滚、隧道重定向跟随及远程配置管理

Browse Source 
核心功能：
- 新增代理节点批量升级回滚工作流，支持分批升级、健康探针、跳过/重试/取消等操作
- proxy 隧道流处理器支持 HTTP 重定向跟随（最多 10 跳），区分 307/308 可重播与不可重播请求体
- proxy 协议新增 follow_redirects / http1_only 字段，网关侧同步支持
- 新增代理节点远端配置变更接口（名称、允许端口、调度状态、升级目标等）
- 新增代理节点注册/反注册/心跳的 Admin API，及节点过期清理维护任务
- gateway 隧道 owner-relay 支持流式代理大请求体，新增 5 MiB 默认限制
- 新增 ProxyNodeRegistrationMutation / ProxyNodeRemoteConfigMutation 数据类型
- proxy 配置新增重定向重播预算、心跳间隔等参数，TUI 安装向导同步更新
- 前端 ProxyNodes 页面新增批量升级操作面板及滚动进度展示
This commit is contained in:
fawney19
2026-04-12 16:02:38 +08:00
parent 7c5bb7f383
commit 9703840a36
83 changed files with 11832 additions and 1520 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/aether-gateway/src/tunnel/embedded/control_plane.rs
+13 -3
View File
@@ -6,7 +6,7 @@ use std::sync::Arc;
type HeartbeatAckCallback =
dyn Fn(Vec<u8>) -> BoxFuture<'static, Result<Vec<u8>, String>> + Send + Sync;
type NodeStatusCallback =
dyn Fn(String, bool, usize) -> BoxFuture<'static, Result<(), String>> + Send + Sync;
dyn Fn(String, bool, usize, u64) -> BoxFuture<'static, Result<(), String>> + Send + Sync;
enum ControlPlaneMode {
Disabled,
@@ -51,7 +51,7 @@ impl ControlPlaneClient {
where
HeartbeatAck:
Fn(Vec<u8>) -> BoxFuture<'static, Result<Vec<u8>, String>> + Send + Sync + 'static,
PushNodeStatus: Fn(String, bool, usize) -> BoxFuture<'static, Result<(), String>>
PushNodeStatus: Fn(String, bool, usize, u64) -> BoxFuture<'static, Result<(), String>>
+ Send
+ Sync
+ 'static,
@@ -103,6 +103,7 @@ impl ControlPlaneClient {
node_id: &str,
connected: bool,
conn_count: usize,
observed_at_unix_secs: u64,
) -> Result<(), String> {
match self.inner.as_ref() {
ControlPlaneMode::Disabled => Ok(()),
@@ -120,6 +121,7 @@ impl ControlPlaneClient {
"node_id": node_id,
"connected": connected,
"conn_count": conn_count,
"observed_at_unix_secs": observed_at_unix_secs,
}))
.send()
.await
@@ -135,7 +137,15 @@ impl ControlPlaneClient {
}
ControlPlaneMode::Local {
push_node_status, ..
} => push_node_status(node_id.to_string(), connected, conn_count).await,
} => {
push_node_status(
node_id.to_string(),
connected,
conn_count,
observed_at_unix_secs,
)
.await
}
}
}
}
apps/aether-gateway/src/tunnel/embedded/hub.rs
+98 -18
View File
@@ -1,7 +1,7 @@
use std::collections::HashMap;
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::Duration;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use aether_runtime::{BoundedQueueSender, MetricKind, MetricSample, QueueSendError};
use axum::extract::ws::Message;
@@ -324,10 +324,46 @@ pub struct HubRouter {
next_conn_id: AtomicU64,
next_local_stream_id: AtomicU64,
control_plane: ControlPlaneClient,
node_status_tx: mpsc::UnboundedSender<NodeStatusEvent>,
}
#[derive(Debug)]
struct NodeStatusEvent {
node_id: String,
connected: bool,
conn_count: usize,
observed_at_unix_secs: u64,
}
impl HubRouter {
pub fn new(control_plane: ControlPlaneClient) -> Arc<Self> {
let (node_status_tx, mut node_status_rx) = mpsc::unbounded_channel::<NodeStatusEvent>();
let worker_control_plane = control_plane.clone();
if let Ok(handle) = tokio::runtime::Handle::try_current() {
handle.spawn(async move {
while let Some(event) = node_status_rx.recv().await {
if let Err(error) = worker_control_plane
.push_node_status(
&event.node_id,
event.connected,
event.conn_count,
event.observed_at_unix_secs,
)
.await
{
warn!(
node_id = %event.node_id,
connected = event.connected,
conn_count = event.conn_count,
observed_at_unix_secs = event.observed_at_unix_secs,
error = %error,
"failed to push node status to app control plane"
);
}
}
});
}
Arc::new(Self {
proxy_conns: RwLock::new(HashMap::new()),
proxy_conns_by_id: DashMap::new(),
@@ -336,6 +372,7 @@ impl HubRouter {
next_conn_id: AtomicU64::new(1),
next_local_stream_id: AtomicU64::new(1),
control_plane,
node_status_tx,
})
}
@@ -405,21 +442,21 @@ impl HubRouter {
}
fn notify_node_status(&self, node_id: String, connected: bool, conn_count: usize) {
let control_plane = self.control_plane.clone();
tokio::spawn(async move {
if let Err(error) = control_plane
.push_node_status(&node_id, connected, conn_count)
.await
{
warn!(
node_id = %node_id,
connected = connected,
conn_count = conn_count,
error = %error,
"failed to push node status to app control plane"
);
}
});
let event = NodeStatusEvent {
node_id,
connected,
conn_count,
observed_at_unix_secs: current_unix_secs(),
};
if let Err(error) = self.node_status_tx.send(event) {
warn!(
node_id = %error.0.node_id,
connected = error.0.connected,
conn_count = error.0.conn_count,
observed_at_unix_secs = error.0.observed_at_unix_secs,
"node status worker unavailable"
);
}
}
fn get_proxy_conn(&self, node_id: &str) -> Option<Arc<ProxyConn>> {
@@ -750,8 +787,12 @@ impl HubRouter {
let ack_payload = match self.control_plane.heartbeat_ack(&payload).await {
Ok(payload) => payload,
Err(error) => {
warn!(proxy_conn_id = proxy_conn_id, error = %error, "control-plane heartbeat callback failed");
b"{}".to_vec()
warn!(
proxy_conn_id = proxy_conn_id,
error = %error,
"control-plane heartbeat callback failed; keeping heartbeat pending"
);
return;
}
};
if let Some(pc) = self.proxy_conns_by_id.get(&proxy_conn_id) {
@@ -796,6 +837,13 @@ impl HubRouter {
}
}
fn current_unix_secs() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
#[derive(serde::Serialize)]
pub struct HubStats {
pub proxy_connections: usize,
@@ -845,6 +893,8 @@ mod tests {
url: "https://example.com".to_string(),
headers: HashMap::new(),
timeout: 30,
follow_redirects: None,
http1_only: false,
}
}
@@ -924,4 +974,34 @@ mod tests {
assert_eq!(second_header.msg_type, protocol::REQUEST_BODY);
assert_ne!(second_header.flags & protocol::FLAG_END_STREAM, 0);
}
#[tokio::test]
async fn heartbeat_callback_failure_does_not_send_fake_ack() {
let hub = HubRouter::new(ControlPlaneClient::local(
|_payload| Box::pin(async { Err("db unavailable".to_string()) }),
|_node_id, _connected, _conn_count, _observed_at_unix_secs| Box::pin(async { Ok(()) }),
));
let (proxy_tx, mut proxy_rx) = bounded_queue(8);
let (proxy_close_tx, _) = watch::channel(false);
let proxy = Arc::new(ProxyConn::new(
300,
"node-3".to_string(),
"Node 3".to_string(),
proxy_tx,
proxy_close_tx,
16,
));
hub.register_proxy(proxy);
let payload = serde_json::to_vec(&serde_json::json!({
"node_id": "node-3",
"heartbeat_id": 99u64,
}))
.expect("payload should serialize");
let mut frame = protocol::encode_frame(1, protocol::HEARTBEAT_DATA, 0, &payload);
hub.handle_proxy_frame(300, &mut frame).await;
assert!(proxy_rx.try_recv().is_err());
}
}
apps/aether-gateway/src/tunnel/embedded/local_relay.rs
+210 -1
View File
@@ -14,6 +14,7 @@ use futures_util::StreamExt;
use tracing::warn;
use crate::api::response::apply_streaming_response_headers;
use crate::maintenance::record_proxy_upgrade_traffic_success;
use super::hub::{LocalBodyEvent, LocalStream};
use super::protocol;
@@ -222,6 +223,13 @@ pub async fn relay_request(
);
}
};
if let Err(error) = record_proxy_upgrade_traffic_success(state.data.as_ref(), &node_id).await {
warn!(
node_id = %node_id,
error = %error,
"failed to record proxy upgrade traffic confirmation"
);
}
let Some(mut body_rx) = stream.take_body_receiver() else {
state
@@ -340,12 +348,25 @@ fn tunnel_error_response(status: StatusCode, kind: &str, message: &str) -> Respo
#[cfg(test)]
mod tests {
use super::super::{AppState, ConnConfig, ControlPlaneClient};
use super::super::hub::ProxyConn;
use super::super::{protocol, AppState, ConnConfig, ControlPlaneClient};
use super::{relay_request, Body, Request, SocketAddr, StatusCode, TUNNEL_ERROR_HEADER};
use crate::data::GatewayDataState;
use crate::maintenance::start_proxy_upgrade_rollout;
use aether_contracts::tunnel::TUNNEL_RELAY_FORWARDED_BY_HEADER;
use aether_data::repository::proxy_nodes::{
InMemoryProxyNodeRepository, ProxyNodeHeartbeatMutation, ProxyNodeWriteRepository,
StoredProxyNode,
};
use axum::extract::ws::Message;
use axum::extract::{ConnectInfo, Path, State};
use axum::response::IntoResponse;
use bytes::Bytes;
use serde_json::json;
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::watch;
fn test_app_state() -> AppState {
AppState::new(
@@ -359,6 +380,49 @@ mod tests {
)
}
fn sample_connected_proxy_node(node_id: &str) -> StoredProxyNode {
StoredProxyNode::new(
node_id.to_string(),
format!("proxy-{node_id}"),
"127.0.0.1".to_string(),
0,
false,
"online".to_string(),
30,
0,
0,
0,
0,
0,
true,
true,
0,
)
.expect("node should build")
.with_runtime_fields(
Some("test".to_string()),
None,
Some(1_800_000_000),
None,
None,
None,
None,
Some(1_800_000_000),
None,
Some(1_800_000_000),
Some(1_800_000_000),
)
}
fn encode_relay_envelope(meta: &protocol::RequestMeta, body: &[u8]) -> Vec<u8> {
let meta_bytes = serde_json::to_vec(meta).expect("meta should serialize");
let mut payload = Vec::with_capacity(4 + meta_bytes.len() + body.len());
payload.extend_from_slice(&(meta_bytes.len() as u32).to_be_bytes());
payload.extend_from_slice(&meta_bytes);
payload.extend_from_slice(body);
payload
}
#[tokio::test]
async fn relay_rejects_non_loopback_without_forwarded_header() {
let request = Request::builder()
@@ -400,4 +464,149 @@ mod tests {
Some("bad_request")
);
}
#[tokio::test]
async fn relay_records_real_traffic_confirmation_for_upgrade_rollout() {
let mut node = sample_connected_proxy_node("node-123");
node.proxy_metadata = Some(json!({"version": "1.0.0"}));
let repository = Arc::new(InMemoryProxyNodeRepository::seed(vec![node]));
let data = Arc::new(
GatewayDataState::with_proxy_node_repository_for_tests(Arc::clone(&repository))
.with_system_config_values_for_tests(Vec::<(String, serde_json::Value)>::new()),
);
let started = start_proxy_upgrade_rollout(data.as_ref(), "2.0.0".to_string(), 1, 0, None)
.await
.expect("rollout should start");
assert_eq!(started.node_ids, vec!["node-123".to_string()]);
repository
.apply_heartbeat(&ProxyNodeHeartbeatMutation {
node_id: "node-123".to_string(),
heartbeat_interval: None,
active_connections: Some(1),
total_requests_delta: Some(1),
avg_latency_ms: Some(2.0),
failed_requests_delta: Some(0),
dns_failures_delta: Some(0),
stream_errors_delta: Some(0),
proxy_metadata: Some(json!({"version": "2.0.0"})),
proxy_version: Some("2.0.0".to_string()),
})
.await
.expect("heartbeat should succeed");
let observed = start_proxy_upgrade_rollout(data.as_ref(), "2.0.0".to_string(), 1, 0, None)
.await
.expect("rollout should observe version confirmation");
assert!(observed.blocked);
assert_eq!(observed.pending_node_ids, vec!["node-123".to_string()]);
let state = test_app_state().with_data(Arc::clone(&data));
let (proxy_tx, mut proxy_rx) = aether_runtime::bounded_queue(8);
let (proxy_close_tx, _) = watch::channel(false);
state.hub.register_proxy(Arc::new(ProxyConn::new(
500,
"node-123".to_string(),
"Node 123".to_string(),
proxy_tx,
proxy_close_tx,
16,
)));
let meta = protocol::RequestMeta {
method: "GET".to_string(),
url: "https://example.com/health".to_string(),
headers: HashMap::new(),
timeout: 30,
follow_redirects: None,
http1_only: false,
};
let request = Request::builder()
.body(Body::from(encode_relay_envelope(&meta, &[])))
.expect("request should build");
let relay_state = state.clone();
let relay_task = tokio::spawn(async move {
relay_request(
Path("node-123".to_string()),
State(relay_state),
ConnectInfo(SocketAddr::from(([127, 0, 0, 1], 4242))),
request,
)
.await
.into_response()
});
let request_headers = match proxy_rx.recv().await.expect("headers frame should arrive") {
Message::Binary(data) => data,
other => panic!("unexpected message: {other:?}"),
};
let request_header = protocol::FrameHeader::parse(&request_headers)
.expect("request header frame should parse");
assert_eq!(request_header.msg_type, protocol::REQUEST_HEADERS);
let request_body = match proxy_rx.recv().await.expect("body frame should arrive") {
Message::Binary(data) => data,
other => panic!("unexpected message: {other:?}"),
};
let request_body_header =
protocol::FrameHeader::parse(&request_body).expect("request body frame should parse");
assert_eq!(request_body_header.msg_type, protocol::REQUEST_BODY);
let response_meta = protocol::ResponseMeta {
status: 200,
headers: vec![("content-type".to_string(), "text/plain".to_string())],
};
let response_payload =
serde_json::to_vec(&response_meta).expect("response meta should serialize");
let mut response_headers_frame = protocol::encode_frame(
request_header.stream_id,
protocol::RESPONSE_HEADERS,
0,
&response_payload,
);
state
.hub
.handle_proxy_frame(500, &mut response_headers_frame)
.await;
let mut response_body_frame = protocol::encode_frame(
request_header.stream_id,
protocol::RESPONSE_BODY,
0,
Bytes::new().as_ref(),
);
state
.hub
.handle_proxy_frame(500, &mut response_body_frame)
.await;
let mut response_end_frame =
protocol::encode_frame(request_header.stream_id, protocol::STREAM_END, 0, &[]);
state
.hub
.handle_proxy_frame(500, &mut response_end_frame)
.await;
let response = relay_task.await.expect("relay task should complete");
assert_eq!(response.status(), StatusCode::OK);
let body = axum::body::to_bytes(response.into_body(), usize::MAX)
.await
.expect("response body should read");
assert!(body.is_empty());
let rollout_entry = data
.list_system_config_entries()
.await
.expect("system config list should succeed")
.into_iter()
.find(|entry| entry.key == "proxy_node_upgrade_rollout")
.expect("rollout entry should exist");
let tracked_nodes = rollout_entry.value["tracked_nodes"]
.as_array()
.expect("tracked nodes should be an array");
assert_eq!(tracked_nodes.len(), 1);
assert!(tracked_nodes[0]["version_confirmed_at_unix_secs"].is_u64());
assert!(tracked_nodes[0]["traffic_confirmed_at_unix_secs"].is_u64());
}
}
apps/aether-gateway/src/tunnel/embedded/mod.rs
+10 -1
View File
@@ -19,8 +19,10 @@ use axum::routing::{get, post};
use axum::Router;
use tracing::warn;
use crate::data::GatewayDataState;
pub use control_plane::ControlPlaneClient;
pub use hub::{ConnConfig, HubRouter};
pub use hub::{ConnConfig, HubRouter, ProxyConn};
pub use local_relay::relay_request;
#[derive(Clone)]
@@ -28,6 +30,7 @@ pub struct AppState {
pub hub: Arc<HubRouter>,
pub proxy_conn_cfg: ConnConfig,
pub max_streams: usize,
data: Arc<GatewayDataState>,
request_gate: Option<Arc<ConcurrencyGate>>,
distributed_request_gate: Option<Arc<DistributedConcurrencyGate>>,
}
@@ -48,11 +51,17 @@ impl AppState {
hub: HubRouter::new(control_plane),
proxy_conn_cfg,
max_streams,
data: Arc::new(GatewayDataState::disabled()),
request_gate: None,
distributed_request_gate: None,
}
}
pub(crate) fn with_data(mut self, data: Arc<GatewayDataState>) -> Self {
self.data = data;
self
}
pub fn with_request_concurrency_limit(mut self, limit: Option<usize>) -> Self {
self.request_gate = limit
.filter(|limit| *limit > 0)