Compare commits

..
Author SHA1 Message Date
elky 7b8048c6ae chore(tunnel): release v0.3.17 2026-09-07 15:57:39 +08:00
elky ec95f2ca1f fix(tunnel): prevent stream stalls and harden session cleanup
Reliably deliver flow-control credits and terminal states, isolate slow streams and heartbeats, negotiate stream windows, and clean up cancelled streams and session tasks.

Add regression coverage for queue pressure, early cancellation, small-window streaming, drain, and reconnect. Validate 185 agent tests, 88 gateway tunnel tests, and 21 protocol tests.
2026-09-07 15:39:40 +08:00
elky aa7dbe67d3 feat(providers): add persistent card view and shared drag ordering 2026-09-07 14:13:59 +08:00
28 changed files with 3165 additions and 365 deletions
Generated
+1 -1
View File
@@ -661,7 +661,7 @@ dependencies = [
[[package]]
name = "aether-tunnel"
version = "0.3.16"
version = "0.3.17"
dependencies = [
"aether-contracts",
"aether-gateway",
@@ -0,0 +1,186 @@
use std::collections::VecDeque;
use std::sync::Arc;
use bytes::{Bytes, BytesMut};
use parking_lot::Mutex;
use tokio::sync::Notify;
const CHUNK_BYTES: usize = 32 * 1024;
#[derive(Debug)]
pub enum LocalBodyEvent {
Chunk(Bytes),
End,
Error(String),
}
#[derive(Default)]
struct BufferState {
chunks: VecDeque<BytesMut>,
bytes: usize,
terminal: Option<Result<(), String>>,
receiver_taken: bool,
receiver_closed: bool,
}
pub(super) struct ResponseBuffer {
state: Mutex<BufferState>,
notify: Notify,
capacity: usize,
}
impl ResponseBuffer {
pub(super) fn new(capacity: usize) -> Arc<Self> {
Arc::new(Self {
state: Mutex::new(BufferState::default()),
notify: Notify::new(),
capacity,
})
}
pub(super) fn take_receiver(self: &Arc<Self>) -> Option<BodyReceiver> {
let mut state = self.state.lock();
if state.receiver_taken {
return None;
}
state.receiver_taken = true;
Some(BodyReceiver {
buffer: Arc::clone(self),
finished: false,
})
}
pub(super) fn push(&self, mut payload: Bytes) -> bool {
let mut state = self.state.lock();
if state.terminal.is_some()
|| state.receiver_closed
|| payload.len() > self.capacity.saturating_sub(state.bytes)
{
return false;
}
state.bytes += payload.len();
while !payload.is_empty() {
if let Some(tail) = state
.chunks
.back_mut()
.filter(|chunk| chunk.len() < CHUNK_BYTES)
{
let count = payload.len().min(CHUNK_BYTES - tail.len());
tail.extend_from_slice(&payload.split_to(count));
} else {
let count = payload.len().min(CHUNK_BYTES);
let chunk = payload.split_to(count);
state.chunks.push_back(
chunk
.try_into_mut()
.unwrap_or_else(|chunk| BytesMut::from(chunk.as_ref())),
);
}
}
drop(state);
self.notify.notify_waiters();
true
}
pub(super) fn finish(&self, result: Result<(), String>) {
let mut state = self.state.lock();
if state.terminal.is_none() {
state.terminal = Some(result);
}
drop(state);
self.notify.notify_waiters();
}
}
pub(super) struct BodyReceiver {
buffer: Arc<ResponseBuffer>,
finished: bool,
}
impl BodyReceiver {
pub(super) async fn recv(&mut self) -> Option<LocalBodyEvent> {
if self.finished {
return None;
}
loop {
let notified = self.buffer.notify.notified();
tokio::pin!(notified);
notified.as_mut().enable();
{
let mut state = self.buffer.state.lock();
if let Some(chunk) = state.chunks.pop_front() {
state.bytes -= chunk.len();
return Some(LocalBodyEvent::Chunk(chunk.freeze()));
}
if let Some(terminal) = state.terminal.take() {
self.finished = true;
state.receiver_closed = true;
return Some(match terminal {
Ok(()) => LocalBodyEvent::End,
Err(error) => LocalBodyEvent::Error(error),
});
}
}
notified.await;
}
}
}
impl Drop for BodyReceiver {
fn drop(&mut self) {
let mut state = self.buffer.state.lock();
state.receiver_closed = true;
state.chunks.clear();
state.bytes = 0;
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn error_survives_a_full_buffer() {
let buffer = ResponseBuffer::new(CHUNK_BYTES);
let mut receiver = buffer.take_receiver().unwrap();
assert!(buffer.push(Bytes::from(vec![b'x'; CHUNK_BYTES])));
buffer.finish(Err("proxy disconnected".into()));
assert!(matches!(
receiver.recv().await,
Some(LocalBodyEvent::Chunk(_))
));
assert!(
matches!(receiver.recv().await, Some(LocalBodyEvent::Error(error)) if error == "proxy disconnected")
);
assert!(receiver.recv().await.is_none());
}
#[tokio::test]
async fn small_frames_are_coalesced_within_the_byte_budget() {
let buffer = ResponseBuffer::new(4096);
let mut receiver = buffer.take_receiver().unwrap();
for _ in 0..4096 {
assert!(buffer.push(Bytes::from_static(b"x")));
}
assert!(!buffer.push(Bytes::from_static(b"x")));
assert_eq!(buffer.state.lock().chunks.len(), 1);
buffer.finish(Ok(()));
assert!(
matches!(receiver.recv().await, Some(LocalBodyEvent::Chunk(chunk)) if chunk.len() == 4096)
);
assert!(matches!(receiver.recv().await, Some(LocalBodyEvent::End)));
}
#[tokio::test]
async fn terminal_wakes_an_empty_receiver_and_is_not_overwritten() {
let buffer = ResponseBuffer::new(1024);
let mut receiver = buffer.take_receiver().unwrap();
let task = tokio::spawn(async move { receiver.recv().await });
tokio::task::yield_now().await;
buffer.finish(Err("cancelled".into()));
buffer.finish(Ok(()));
assert!(
matches!(task.await.unwrap(), Some(LocalBodyEvent::Error(error)) if error == "cancelled")
);
}
}
@@ -0,0 +1,250 @@
use super::*;
async fn fixture(
window: u32,
capacity: usize,
) -> (
Arc<HubRouter>,
Arc<ProxyConn>,
Arc<LocalStream>,
aether_runtime::BoundedQueueReceiver<Message>,
) {
let hub = HubRouter::new(ControlPlaneClient::disabled());
let (sender, receiver) = bounded_queue(capacity);
let (close_tx, _) = watch::channel(false);
let connection = Arc::new(
ProxyConn::new(
99,
"flow-test".into(),
"flow-test".into(),
sender,
close_tx,
16,
3,
)
.with_settings(protocol::SettingsPayload {
initial_stream_window_bytes: window,
min_window_update_bytes: (window / 4).max(1),
drain_deadline_ms: 1000,
}),
);
hub.register_proxy(Arc::clone(&connection));
let stream = hub.open_local_stream("flow-test", &meta()).await.unwrap();
(hub, connection, stream, receiver)
}
fn meta() -> protocol::RequestMeta {
protocol::RequestMeta {
provider_id: None,
endpoint_id: None,
key_id: None,
method: "GET".into(),
url: "https://example.com".into(),
headers: HashMap::new(),
stream: true,
request_timeout_ms: None,
stream_first_byte_timeout_ms: None,
timeout: 30,
follow_redirects: None,
http1_only: false,
transport_profile: None,
}
}
async fn headers(hub: &Arc<HubRouter>, stream: &LocalStream) {
let payload = serde_json::to_vec(&protocol::ResponseMeta {
status: 200,
headers: vec![],
})
.unwrap();
let mut frame = protocol::encode_frame(
stream.proxy_stream_id,
protocol::RESPONSE_HEADERS,
0,
&payload,
);
hub.handle_proxy_frame(stream.proxy_conn_id, &mut frame)
.await;
}
#[tokio::test]
async fn window_credit_is_retried_after_queue_pressure_and_cancelled_receive() {
let (hub, _, stream, mut outbound) = fixture(128, 1).await;
headers(&hub, &stream).await;
assert!(stream.push_body_chunk(Bytes::from(vec![b'x'; 64])));
let mut receiver = stream.take_body_receiver().unwrap();
assert!(
tokio::time::timeout(Duration::from_millis(10), receiver.recv())
.await
.is_err()
);
assert_eq!(*stream.response_consumed_since_update.lock(), 64);
outbound.recv().await.unwrap();
let event = tokio::time::timeout(Duration::from_secs(1), receiver.recv())
.await
.unwrap();
assert!(matches!(event, Some(LocalBodyEvent::Chunk(chunk)) if chunk.len() == 64));
assert_eq!(*stream.response_consumed_since_update.lock(), 0);
let Message::Binary(data) = outbound.recv().await.unwrap() else {
panic!("expected binary update")
};
let frame = aether_contracts::tunnel::Frame::decode(data).unwrap();
let update: protocol::WindowUpdatePayload = serde_json::from_slice(&frame.payload).unwrap();
assert_eq!(
frame.msg_type,
aether_contracts::tunnel::MsgType::WindowUpdate
);
assert_eq!(update.delta_bytes, 64);
hub.cancel_local_stream(stream.id, "test complete");
}
#[tokio::test]
async fn response_credit_is_not_returned_until_consumed() {
let (hub, _, stream, mut outbound) = fixture(128, 4).await;
outbound.recv().await.unwrap();
let mut body = protocol::encode_frame(
stream.proxy_stream_id,
protocol::RESPONSE_BODY,
0,
&[b'x'; 128],
);
hub.handle_proxy_frame(99, &mut body).await;
assert!(outbound.try_recv().is_err());
let mut receiver = stream.take_body_receiver().unwrap();
assert!(
matches!(receiver.recv().await, Some(LocalBodyEvent::Chunk(chunk)) if chunk.len() == 128)
);
assert!(outbound.try_recv().is_ok());
hub.cancel_local_stream(stream.id, "test complete");
}
#[tokio::test]
async fn cancelled_stream_open_releases_slot_without_resetting_connection() {
let (hub, connection, first_stream, mut outbound) = fixture(128, 1).await;
let opening_hub = Arc::clone(&hub);
let opening =
tokio::spawn(async move { opening_hub.open_local_stream("flow-test", &meta()).await });
tokio::time::timeout(Duration::from_secs(1), async {
while hub.local_streams.len() != 2 {
tokio::task::yield_now().await;
}
})
.await
.unwrap();
opening.abort();
assert!(matches!(opening.await, Err(error) if error.is_cancelled()));
assert_eq!(connection.stream_count.load(Ordering::Relaxed), 1);
assert_eq!(hub.local_streams.len(), 1);
assert_eq!(hub.proxy_to_local.len(), 1);
assert!(connection.is_available());
outbound.recv().await.unwrap();
assert!(outbound.try_recv().is_err());
let next_stream = hub.open_local_stream("flow-test", &meta()).await.unwrap();
outbound.recv().await.unwrap();
hub.cancel_local_stream(first_stream.id, "test complete");
outbound.recv().await.unwrap();
hub.cancel_local_stream(next_stream.id, "test complete");
}
#[tokio::test]
async fn full_response_buffer_preserves_disconnect_error() {
let (hub, connection, stream, mut outbound) = fixture(4 * 1024 * 1024, 512).await;
outbound.recv().await.unwrap();
headers(&hub, &stream).await;
let mut receiver = stream.take_body_receiver().unwrap();
for _ in 0..128 {
let mut frame = protocol::encode_frame(
stream.proxy_stream_id,
protocol::RESPONSE_BODY,
0,
&vec![b'x'; 32 * 1024],
);
hub.handle_proxy_frame(99, &mut frame).await;
}
hub.unregister_proxy(connection.id, &connection.node_id);
let mut bytes = 0;
loop {
match receiver.recv().await {
Some(LocalBodyEvent::Chunk(chunk)) => bytes += chunk.len(),
Some(LocalBodyEvent::Error(error)) => {
assert!(error.contains("disconnected"));
break;
}
event => panic!("disconnect must not become normal EOF: {event:?}"),
}
}
assert_eq!(bytes, 4 * 1024 * 1024);
}
#[tokio::test]
async fn slow_stream_does_not_block_another_stream_on_the_same_connection() {
let (hub, _, slow, mut outbound) = fixture(128, 512).await;
outbound.recv().await.unwrap();
let fast = hub.open_local_stream("flow-test", &meta()).await.unwrap();
assert!(slow.push_body_chunk(Bytes::from(vec![b'x'; 128])));
let mut overflowing = protocol::encode_frame(
slow.proxy_stream_id,
protocol::RESPONSE_BODY,
0,
b"overflow",
);
tokio::time::timeout(Duration::from_secs(1), async {
hub.handle_proxy_frame(99, &mut overflowing).await;
headers(&hub, &fast).await;
assert_eq!(
fast.wait_headers(Duration::from_secs(1))
.await
.unwrap()
.status,
200
);
})
.await
.expect("slow stream must not block connection reader");
assert!(!hub.local_streams.contains_key(&slow.id));
assert!(hub.local_streams.contains_key(&fast.id));
hub.cancel_local_stream(fast.id, "test complete");
}
#[tokio::test]
async fn cancelling_a_stream_wakes_request_window_waiters() {
let (_, _, stream, _) = fixture(128, 512).await;
*stream.request_window.available.lock() = 0;
let waiter = tokio::spawn({
let stream = Arc::clone(&stream);
async move {
stream
.acquire_request_window(1, Duration::from_secs(30))
.await
}
});
tokio::task::yield_now().await;
stream.fail("cancelled");
assert!(tokio::time::timeout(Duration::from_secs(1), waiter)
.await
.unwrap()
.unwrap()
.is_err());
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn concurrent_headers_and_credit_updates_do_not_lose_notifications() {
for index in 0..256 {
let stream = Arc::new(LocalStream::new(index, "test".into(), 1, 1, 1));
let window = Arc::new(StreamFlowWindow::new(0));
let waiter = tokio::spawn({
let stream = Arc::clone(&stream);
let window = Arc::clone(&window);
async move {
stream.wait_headers(Duration::from_secs(1)).await.unwrap();
window.acquire(1, Duration::from_secs(1)).await.unwrap();
}
});
stream.set_response_headers(protocol::ResponseMeta {
status: 200,
headers: vec![],
});
window.add(1);
waiter.await.unwrap();
}
}
+203 -83
View File
@@ -12,10 +12,11 @@ use axum::extract::ws::Message;
use bytes::Bytes;
use dashmap::DashMap;
use parking_lot::{Mutex, RwLock};
use tokio::sync::mpsc;
use tokio::sync::{watch, Notify};
use tracing::{debug, info, warn};
pub use super::body::LocalBodyEvent;
use super::body::{BodyReceiver, ResponseBuffer};
use super::control_plane::ControlPlaneClient;
use super::protocol;
@@ -29,6 +30,10 @@ const DEFAULT_DRAIN_DEADLINE_MS: u64 = 30_000;
const DEFAULT_NODE_STATUS_QUEUE_CAPACITY: usize = 1_024;
const CONNECTION_WARMUP: Duration = Duration::from_secs(1);
#[cfg(test)]
#[path = "flow_control_tests.rs"]
mod flow_control_tests;
static STREAM_INITIAL_WINDOW_BYTES: LazyLock<u32> = LazyLock::new(|| {
std::env::var("AETHER_TUNNEL_STREAM_INITIAL_WINDOW_BYTES")
.ok()
@@ -53,11 +58,16 @@ static NODE_STATUS_QUEUE_CAPACITY: LazyLock<usize> = LazyLock::new(|| {
.unwrap_or(DEFAULT_NODE_STATUS_QUEUE_CAPACITY)
});
static STREAM_MIN_WINDOW_UPDATE_BYTES: LazyLock<u32> = LazyLock::new(|| {
STREAM_INITIAL_WINDOW_BYTES
.saturating_div(4)
.clamp(1, 1024 * 1024)
});
pub(super) fn local_settings() -> protocol::SettingsPayload {
protocol::SettingsPayload {
initial_stream_window_bytes: (*STREAM_INITIAL_WINDOW_BYTES)
.min(aether_contracts::tunnel::MAX_TUNNEL_DECOMPRESSED_PAYLOAD_BYTES as u32),
min_window_update_bytes: STREAM_INITIAL_WINDOW_BYTES
.saturating_div(4)
.clamp(1, 1024 * 1024),
drain_deadline_ms: *DRAIN_DEADLINE_MS,
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SendStatus {
@@ -91,6 +101,7 @@ impl ConnHealthState {
struct StreamFlowWindow {
available: Mutex<u64>,
notify: Notify,
closed: AtomicBool,
}
impl StreamFlowWindow {
@@ -98,6 +109,7 @@ impl StreamFlowWindow {
Self {
available: Mutex::new(u64::from(initial)),
notify: Notify::new(),
closed: AtomicBool::new(false),
}
}
@@ -109,6 +121,12 @@ impl StreamFlowWindow {
let requested = bytes as u64;
let started_at = Instant::now();
loop {
let notified = self.notify.notified();
tokio::pin!(notified);
notified.as_mut().enable();
if self.closed.load(Ordering::Acquire) {
return Err(());
}
{
let mut available = self.available.lock();
if *available >= requested {
@@ -120,10 +138,7 @@ impl StreamFlowWindow {
let Some(remaining) = timeout.checked_sub(started_at.elapsed()) else {
return Err(());
};
if tokio::time::timeout(remaining, self.notify.notified())
.await
.is_err()
{
if tokio::time::timeout(remaining, notified).await.is_err() {
return Err(());
}
}
@@ -138,6 +153,11 @@ impl StreamFlowWindow {
drop(available);
self.notify.notify_waiters();
}
fn close(&self) {
self.closed.store(true, Ordering::Release);
self.notify.notify_waiters();
}
}
#[derive(Debug, Clone, Copy)]
@@ -209,6 +229,10 @@ impl BoundedOutbound {
pub fn snapshot(&self) -> QueueSnapshot {
self.tx.snapshot()
}
pub(super) fn subscribe_close(&self) -> watch::Receiver<bool> {
self.close_tx.subscribe()
}
}
pub struct ProxyConn {
@@ -231,6 +255,7 @@ pub struct ProxyConn {
flow_window_blocked_ms: AtomicU64,
write_latency_last_us: AtomicU64,
write_latency_ewma_us: AtomicU64,
settings: Mutex<protocol::SettingsPayload>,
}
impl ProxyConn {
@@ -244,6 +269,7 @@ impl ProxyConn {
protocol_version: u8,
) -> Self {
Self {
settings: Mutex::new(local_settings()),
id,
node_id,
node_name,
@@ -271,6 +297,11 @@ impl ProxyConn {
self
}
pub(super) fn with_settings(mut self, settings: protocol::SettingsPayload) -> Self {
*self.settings.get_mut() = settings;
self
}
pub fn with_tunnel_generation(mut self, tunnel_generation: String) -> Self {
self.node_generation = tunnel_generation;
self
@@ -565,13 +596,6 @@ pub struct LocalResponseHead {
pub headers: Vec<(String, String)>,
}
#[derive(Debug)]
pub enum LocalBodyEvent {
Chunk(Bytes),
End,
Error(String),
}
#[derive(Debug, Default)]
struct LocalWaitState {
response: Option<LocalResponseHead>,
@@ -585,10 +609,11 @@ pub struct LocalStream {
proxy_stream_id: u32,
request_window: StreamFlowWindow,
response_consumed_since_update: Mutex<u64>,
min_window_update_bytes: u32,
response_connection: Mutex<Option<std::sync::Weak<ProxyConn>>>,
wait_state: Mutex<LocalWaitState>,
headers_notify: Notify,
body_tx: mpsc::Sender<LocalBodyEvent>,
body_rx: Mutex<Option<mpsc::Receiver<LocalBodyEvent>>>,
body: Arc<ResponseBuffer>,
terminal: AtomicBool,
}
@@ -600,7 +625,6 @@ impl LocalStream {
proxy_stream_id: u32,
initial_window_bytes: u32,
) -> Self {
let (body_tx, body_rx) = mpsc::channel(128);
Self {
id,
tunnel_generation,
@@ -608,10 +632,11 @@ impl LocalStream {
proxy_stream_id,
request_window: StreamFlowWindow::new(initial_window_bytes),
response_consumed_since_update: Mutex::new(0),
min_window_update_bytes: (initial_window_bytes / 4).clamp(1, 1024 * 1024),
response_connection: Mutex::new(None),
wait_state: Mutex::new(LocalWaitState::default()),
headers_notify: Notify::new(),
body_tx,
body_rx: Mutex::new(Some(body_rx)),
body: ResponseBuffer::new(initial_window_bytes as usize),
terminal: AtomicBool::new(false),
}
}
@@ -632,26 +657,51 @@ impl LocalStream {
self.request_window.add(delta);
}
fn response_window_update_delta(&self, bytes: usize) -> Option<u32> {
if bytes == 0 {
return None;
async fn flush_response_credit(&self) -> Result<(), String> {
if self.terminal.load(Ordering::Acquire) {
return Ok(());
}
let mut consumed = self.response_consumed_since_update.lock();
*consumed = consumed.saturating_add(bytes as u64);
let threshold = u64::from(*STREAM_MIN_WINDOW_UPDATE_BYTES);
if *consumed < threshold {
return None;
let connection = self
.response_connection
.lock()
.as_ref()
.and_then(std::sync::Weak::upgrade);
let Some(connection) = connection else {
return Ok(());
};
if connection.protocol_version() < 3 {
return Ok(());
}
let delta = (*consumed).min(u64::from(u32::MAX)) as u32;
*consumed = consumed.saturating_sub(u64::from(delta));
Some(delta)
let delta = {
let consumed = self.response_consumed_since_update.lock();
if *consumed < u64::from(self.min_window_update_bytes) {
return Ok(());
}
(*consumed).min(u64::from(u32::MAX)) as u32
};
let frame = protocol::encode_window_update(self.proxy_stream_id, delta);
if connection
.send_wait(Message::Binary(frame.into()), OUTBOUND_BACKPRESSURE_TIMEOUT)
.await
== SendStatus::Queued
{
let mut consumed = self.response_consumed_since_update.lock();
*consumed = consumed.saturating_sub(u64::from(delta));
return Ok(());
}
if self.terminal.load(Ordering::Acquire) {
return Ok(());
}
connection.request_close();
Err("proxy flow-control update failed".to_string())
}
pub async fn wait_headers(&self, timeout: Duration) -> Result<LocalResponseHead, String> {
tokio::time::timeout(timeout, async {
loop {
let notified = self.headers_notify.notified();
tokio::pin!(notified);
notified.as_mut().enable();
let outcome = {
let state = self.wait_state.lock();
if let Some(response) = &state.response {
@@ -662,15 +712,20 @@ impl LocalStream {
if let Some(error) = outcome {
return Err(error);
}
self.headers_notify.notified().await;
notified.await;
}
})
.await
.map_err(|_| "timed out waiting for response headers".to_string())?
}
pub fn take_body_receiver(&self) -> Option<mpsc::Receiver<LocalBodyEvent>> {
self.body_rx.lock().take()
pub fn take_body_receiver(self: &Arc<Self>) -> Option<LocalBodyReceiver> {
self.body.take_receiver().map(|receiver| LocalBodyReceiver {
receiver,
stream: Arc::clone(self),
failed: false,
pending: None,
})
}
fn set_response_headers(&self, meta: protocol::ResponseMeta) {
@@ -690,21 +745,11 @@ impl LocalStream {
}
}
async fn push_body_chunk(&self, payload: Bytes) -> bool {
fn push_body_chunk(&self, payload: Bytes) -> bool {
if self.terminal.load(Ordering::Acquire) {
return false;
}
// Use a timeout to prevent a slow consumer from blocking the shared
// proxy-connection reader (head-of-line blocking across streams).
match tokio::time::timeout(
Duration::from_secs(5),
self.body_tx.send(LocalBodyEvent::Chunk(payload)),
)
.await
{
Ok(Ok(())) => true,
_ => false,
}
self.body.push(payload)
}
fn finish(&self) {
@@ -722,7 +767,8 @@ impl LocalStream {
if notify {
self.headers_notify.notify_waiters();
}
let _ = self.body_tx.try_send(LocalBodyEvent::End);
self.request_window.close();
self.body.finish(Ok(()));
}
fn fail(&self, error: impl Into<String>) {
@@ -742,7 +788,38 @@ impl LocalStream {
if notify {
self.headers_notify.notify_waiters();
}
let _ = self.body_tx.try_send(LocalBodyEvent::Error(error));
self.request_window.close();
self.body.finish(Err(error));
}
}
pub struct LocalBodyReceiver {
receiver: BodyReceiver,
stream: Arc<LocalStream>,
failed: bool,
pending: Option<LocalBodyEvent>,
}
impl LocalBodyReceiver {
pub async fn recv(&mut self) -> Option<LocalBodyEvent> {
if self.failed {
return None;
}
if self.pending.is_none() {
let event = self.receiver.recv().await?;
if let LocalBodyEvent::Chunk(chunk) = &event {
let mut consumed = self.stream.response_consumed_since_update.lock();
*consumed = consumed.saturating_add(chunk.len() as u64);
}
self.pending = Some(event);
}
if matches!(self.pending, Some(LocalBodyEvent::Chunk(_))) {
if let Err(error) = self.stream.flush_response_credit().await {
self.failed = true;
return Some(LocalBodyEvent::Error(error));
}
}
self.pending.take()
}
}
@@ -765,6 +842,21 @@ pub struct HubRouter {
drain_reasons: Mutex<HashMap<String, u64>>,
}
struct PendingStreamGuard<'router> {
hub: &'router HubRouter,
connection: &'router ProxyConn,
stream_id: u64,
committed: bool,
}
impl Drop for PendingStreamGuard<'_> {
fn drop(&mut self) {
if !self.committed && self.hub.cleanup_local_stream(self.stream_id) {
self.connection.release_stream();
}
}
}
struct NodeStatusEvent {
node_id: String,
authenticated_key: Option<String>,
@@ -1166,17 +1258,27 @@ impl HubRouter {
// Frames encoded successfully -- now register the stream.
let local_stream_id = self.next_local_stream_id.fetch_add(1, Ordering::Relaxed);
let local_stream = Arc::new(LocalStream::new(
let settings = proxy_conn.settings.lock().clone();
let mut local_stream = LocalStream::new(
local_stream_id,
proxy_conn.node_generation.clone(),
proxy_conn.id,
proxy_stream_id,
*STREAM_INITIAL_WINDOW_BYTES,
));
settings.initial_stream_window_bytes,
);
local_stream.min_window_update_bytes = settings.min_window_update_bytes;
*local_stream.response_connection.get_mut() = Some(Arc::downgrade(&proxy_conn));
let local_stream = Arc::new(local_stream);
self.local_streams
.insert(local_stream_id, local_stream.clone());
self.proxy_to_local
.insert((proxy_conn.id, proxy_stream_id), local_stream_id);
let mut pending_stream = PendingStreamGuard {
hub: self,
connection: &proxy_conn,
stream_id: local_stream_id,
committed: false,
};
let send_status = proxy_conn
.send_wait(
@@ -1195,10 +1297,11 @@ impl HubRouter {
"open_local_stream dispatched"
);
match send_status {
SendStatus::Queued => Ok(local_stream),
SendStatus::Queued => {
pending_stream.committed = true;
Ok(local_stream)
}
SendStatus::Closed | SendStatus::Congested => {
self.cleanup_local_stream(local_stream_id);
proxy_conn.release_stream();
Err("proxy connection congested".to_string())
}
}
@@ -1243,7 +1346,9 @@ impl HubRouter {
.map(|entry| entry.value().clone())
.ok_or_else(|| "proxy connection unavailable".to_string())?;
let total_chunks = payload.len().div_ceil(MAX_REQUEST_BODY_FRAME_SIZE);
let chunk_size = MAX_REQUEST_BODY_FRAME_SIZE
.min(proxy_conn.settings.lock().initial_stream_window_bytes as usize);
let total_chunks = payload.len().div_ceil(chunk_size);
let result = if total_chunks == 0 {
if end_stream {
self.send_request_body_frame(&proxy_conn, &stream, &[], true)
@@ -1252,7 +1357,7 @@ impl HubRouter {
Ok(())
}
} else {
for (index, chunk) in payload.chunks(MAX_REQUEST_BODY_FRAME_SIZE).enumerate() {
for (index, chunk) in payload.chunks(chunk_size).enumerate() {
let is_last_chunk = index + 1 == total_chunks;
if let Err(error) = self
.send_request_body_frame(
@@ -1352,17 +1457,20 @@ impl HubRouter {
} else {
protocol::encode_stream_error(stream.proxy_stream_id, reason)
};
let _ = pc.send(Message::Binary(frame.into()));
if pc.send(Message::Binary(frame.into())) != SendStatus::Queued {
pc.request_close();
}
}
stream.fail(reason.to_string());
}
fn cleanup_local_stream(&self, local_stream_id: u64) {
fn cleanup_local_stream(&self, local_stream_id: u64) -> bool {
let Some((_, stream)) = self.local_streams.remove(&local_stream_id) else {
return;
return false;
};
self.proxy_to_local
.remove(&(stream.proxy_conn_id, stream.proxy_stream_id));
true
}
pub async fn handle_proxy_frame(self: &Arc<Self>, proxy_conn_id: u64, data: &mut [u8]) {
@@ -1433,9 +1541,7 @@ impl HubRouter {
.get(&proxy_conn_id)
.map(|entry| entry.value().clone());
if let Some(pc) = pc {
let _ = pc
.send_wait(Message::Binary(pong.into()), Duration::from_millis(250))
.await;
let _ = pc.send(Message::Binary(pong.into()));
}
}
protocol::PONG => {}
@@ -1509,11 +1615,36 @@ impl HubRouter {
);
}
protocol::SETTINGS => {
debug!(
msg_type = header.msg_type,
proxy_conn_id = proxy_conn_id,
"received tunnel protocol v3 SETTINGS from proxy"
);
let settings = protocol::decode_payload_with_limit(
data,
&header,
MAX_TUNNEL_CONTROL_PAYLOAD_SIZE,
)
.ok()
.and_then(|payload| {
serde_json::from_slice::<protocol::SettingsPayload>(&payload).ok()
})
.filter(|settings| settings.is_valid());
if let Some(connection) = self.proxy_conns_by_id.get(&proxy_conn_id) {
if header.stream_id != 0 || header.flags != 0 {
connection.request_close();
return;
}
let Some(settings) = settings else {
connection.request_close();
return;
};
let local = local_settings();
let settings = settings
.negotiate(local.initial_stream_window_bytes, local.drain_deadline_ms);
let mut current = connection.settings.lock();
if connection.stream_count.load(Ordering::Acquire) > 0 && *current != settings {
drop(current);
connection.request_close();
return;
}
*current = settings;
}
}
protocol::WINDOW_UPDATE => {
self.handle_window_update(proxy_conn_id, header.stream_id, data, &header);
@@ -1739,19 +1870,8 @@ impl HubRouter {
None => return,
};
let payload_len = payload.len();
if !stream.push_body_chunk(Bytes::from(payload)).await {
if !stream.push_body_chunk(Bytes::from(payload)) {
self.cancel_local_stream(local_id, "local relay response congested");
return;
}
if let Some(pc) = self.proxy_conns_by_id.get(&proxy_conn_id) {
if pc.protocol_version() >= 3 {
if let Some(delta) = stream.response_window_update_delta(payload_len) {
let frame = protocol::encode_window_update(header.stream_id, delta);
let _ = pc.send(Message::Binary(frame.into()));
}
}
}
}
@@ -9,14 +9,13 @@ use axum::body::{Body, Bytes};
use axum::extract::{ConnectInfo, Path, Request, State};
use axum::http::{HeaderMap, HeaderName, HeaderValue, Response, StatusCode};
use axum::response::IntoResponse;
use tokio::sync::mpsc;
use tracing::warn;
use crate::api::response::apply_streaming_response_headers;
use crate::headers::should_skip_response_header;
use crate::maintenance::record_proxy_upgrade_traffic_success_for_generation;
use super::hub::{LocalBodyEvent, LocalStream};
use super::hub::{LocalBodyEvent, LocalBodyReceiver, LocalStream};
use super::protocol;
use super::{AppState, RelayRequestAuthenticated};
@@ -40,7 +39,7 @@ impl Drop for StreamGuard {
pub(crate) struct DirectRelayResponse {
status: u16,
headers: Vec<(String, String)>,
body_rx: mpsc::Receiver<LocalBodyEvent>,
body_rx: LocalBodyReceiver,
request_guard: StreamGuard,
_request_permit: Option<AdmissionPermit>,
}
@@ -55,10 +54,13 @@ impl DirectRelayResponse {
}
pub(crate) async fn next_chunk(&mut self) -> Result<Option<Bytes>, String> {
if self.request_guard.finished {
return Ok(None);
}
let event = self.body_rx.recv().await;
match event {
Some(LocalBodyEvent::Chunk(chunk)) => Ok(Some(chunk)),
Some(LocalBodyEvent::End) | None => {
Some(LocalBodyEvent::End) => {
self.request_guard.finished = true;
Ok(None)
}
@@ -66,6 +68,7 @@ impl DirectRelayResponse {
self.request_guard.finished = true;
Err(error)
}
None => Err("tunnel response ended without a terminal frame".to_string()),
}
}
}
@@ -84,6 +87,11 @@ pub(crate) async fn open_direct_relay_stream(
.open_authorized_local_stream(node_id, &meta)
.await
.map_err(|error| format!("connect: {error}"))?;
let request_guard = StreamGuard {
hub: state.hub.clone(),
stream_id: stream.id,
finished: false,
};
if let Err(error) = state
.hub
.push_local_request_body(stream.id, body, true)
@@ -126,11 +134,7 @@ pub(crate) async fn open_direct_relay_stream(
status: response_head.status,
headers: response_head.headers,
body_rx,
request_guard: StreamGuard {
hub: state.hub.clone(),
stream_id: stream.id,
finished: false,
},
request_guard,
_request_permit: request_permit,
})
}
@@ -259,6 +263,11 @@ pub async fn relay_request(
);
}
};
let request_guard = StreamGuard {
hub: state.hub.clone(),
stream_id: stream.id,
finished: false,
};
let body_stream = match spool.body_stream().await {
Ok(stream) => stream,
Err(error) => {
@@ -306,12 +315,6 @@ pub async fn relay_request(
);
}
let request_guard = StreamGuard {
hub: state.hub.clone(),
stream_id: stream.id,
finished: false,
};
let wait_timeout = relay_header_timeout(&meta);
let response_head = match stream.wait_headers(wait_timeout).await {
Ok(response) => response,
@@ -373,6 +376,9 @@ pub async fn relay_request(
}
}
}
if !guard.finished {
yield Err(io::Error::other("tunnel response ended without a terminal frame"));
}
guard.finished = true;
};
@@ -563,6 +569,100 @@ mod tests {
request
}
#[tokio::test]
async fn cancelled_relays_reset_streams_during_upload_and_header_wait() {
for direct in [true, false] {
for during_upload in [true, false] {
let repository = Arc::new(InMemoryProxyNodeRepository::seed(vec![
sample_connected_proxy_node("node-123"),
]));
let data = Arc::new(
GatewayDataState::with_proxy_node_repository_for_tests(repository)
.with_system_config_values_for_tests(
Vec::<(String, serde_json::Value)>::new(),
)
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY),
);
let state = test_app_state().with_data(data);
let (proxy_tx, mut proxy_rx) = aether_runtime::bounded_queue(8);
let (proxy_close_tx, _) = watch::channel(false);
let connection = Arc::new(
ProxyConn::new(
500,
"node-123".into(),
"Node 123".into(),
proxy_tx,
proxy_close_tx,
16,
3,
)
.with_tunnel_generation(LOCAL_TUNNEL_TEST_GENERATION.to_string())
.with_authenticated_key(LOCAL_TUNNEL_TEST_PSK.to_string())
.with_settings(protocol::SettingsPayload {
initial_stream_window_bytes: 128,
min_window_update_bytes: 32,
drain_deadline_ms: 1000,
}),
);
state.hub.register_proxy(Arc::clone(&connection));
let meta = protocol::RequestMeta {
provider_id: None,
endpoint_id: None,
key_id: None,
method: "POST".into(),
url: "https://example.com/".into(),
headers: HashMap::new(),
stream: true,
request_timeout_ms: None,
stream_first_byte_timeout_ms: None,
timeout: 30,
follow_redirects: None,
http1_only: false,
transport_profile: None,
};
let body = Bytes::from(vec![b'x'; if during_upload { 256 } else { 0 }]);
let relay = tokio::spawn(async move {
if direct {
let _response =
super::open_direct_relay_stream(&state, "node-123", meta, body)
.await
.unwrap();
} else {
let request =
authenticated_request(encode_relay_envelope(&meta, &body)).await;
let _response = relay_request(
Path("node-123".into()),
State(state),
ConnectInfo(SocketAddr::from(([127, 0, 0, 1], 4242))),
request,
)
.await;
}
});
recv_tunnel_test_frame(&mut proxy_rx, "request headers").await;
recv_tunnel_test_frame(&mut proxy_rx, "request body").await;
relay.abort();
assert!(relay.await.unwrap_err().is_cancelled());
let Message::Binary(frame) = recv_tunnel_test_frame(&mut proxy_rx, "reset").await
else {
panic!("expected binary reset frame")
};
let frame = aether_contracts::tunnel::Frame::decode(frame).unwrap();
assert_eq!(
frame.msg_type,
aether_contracts::tunnel::MsgType::ResetStream
);
assert_eq!(
connection
.stream_count
.load(std::sync::atomic::Ordering::Relaxed),
0
);
assert!(connection.is_available());
}
}
}
#[test]
fn relay_header_timeout_ignores_request_timeout_for_stream_requests() {
let meta = protocol::RequestMeta {
@@ -1,3 +1,4 @@
mod body;
mod control_plane;
mod hub;
mod local_relay;
@@ -9,6 +9,7 @@ use aether_runtime::bounded_queue;
use axum::extract::ws::{Message, WebSocket};
use futures_util::{SinkExt, StreamExt};
use tokio::sync::watch;
use tokio::task::JoinSet;
use tracing::{debug, info, warn};
use super::hub::{ConnConfig, HubRouter, ProxyConn, ProxyManagementTokenCredential, SendStatus};
@@ -84,6 +85,35 @@ pub async fn handle_proxy_connection(
let (tx, mut rx) = bounded_queue::<Message>(cfg.outbound_queue_capacity);
let (close_tx, mut close_rx) = watch::channel(false);
let settings = if protocol_version >= 3 {
let Some(settings) = read_proxy_settings(
&mut ws_tx,
&mut ws_rx,
security.as_deref(),
protocol_version,
)
.await
else {
warn!(conn_id, "proxy SETTINGS negotiation failed");
return;
};
let local = super::hub::local_settings();
let negotiated =
settings.negotiate(local.initial_stream_window_bytes, local.drain_deadline_ms);
let message = Message::Binary(protocol::encode_settings(&negotiated).into());
let Ok(message) = encrypt_message(message, security.as_deref()) else {
return;
};
if !matches!(
tokio::time::timeout(PROXY_HELLO_TIMEOUT, ws_tx.send(message)).await,
Ok(Ok(()))
) {
return;
}
negotiated
} else {
super::hub::local_settings()
};
let conn = ProxyConn::new(
conn_id,
node_id.clone(),
@@ -93,7 +123,8 @@ pub async fn handle_proxy_connection(
max_streams,
protocol_version,
)
.with_tunnel_generation(node_generation);
.with_tunnel_generation(node_generation)
.with_settings(settings);
let conn = match (security_key.clone(), management_token_credential) {
(Some(key), None) => Arc::new(conn.with_authenticated_key(key)),
(None, Some(credential)) => Arc::new(conn.with_management_token_credential(credential)),
@@ -416,19 +447,22 @@ async fn run_proxy_reader(
let idle_enabled = !idle_timeout.is_zero();
let mut oversized_count = 0u32;
let mut frames_received: u64 = 0;
let mut close_rx = conn.outbound.subscribe_close();
let mut heartbeats = JoinSet::new();
loop {
let msg = if idle_enabled {
tokio::select! {
msg = ws_rx.next() => msg,
_ = tokio::time::sleep(idle_timeout) => {
warn!(conn_id = conn.id, node_id = %conn.node_id, "proxy idle timeout");
let _ = conn.send(Message::Binary(protocol::encode_goaway().into()));
conn.request_close();
break;
}
if conn.outbound.is_closing() {
break;
}
while heartbeats.try_join_next().is_some() {}
let msg = tokio::select! {
biased;
_ = close_rx.changed() => break,
msg = ws_rx.next() => msg,
_ = tokio::time::sleep(idle_timeout), if idle_enabled => {
warn!(conn_id = conn.id, node_id = %conn.node_id, "proxy idle timeout");
conn.request_close();
break;
}
} else {
ws_rx.next().await
};
match msg {
@@ -463,7 +497,27 @@ async fn run_proxy_reader(
continue;
}
hub.handle_proxy_frame(conn.id, &mut data).await;
let is_heartbeat = protocol::FrameHeader::parse(&data)
.is_some_and(|header| header.msg_type == protocol::HEARTBEAT_DATA);
if is_heartbeat {
if heartbeats.is_empty() {
let heartbeat_hub = Arc::clone(&hub);
let conn_id = conn.id;
heartbeats.spawn(async move {
if tokio::time::timeout(
Duration::from_secs(10),
heartbeat_hub.handle_proxy_frame(conn_id, &mut data),
)
.await
.is_err()
{
warn!(conn_id, "proxy heartbeat processing timed out");
}
});
}
} else {
hub.handle_proxy_frame(conn.id, &mut data).await;
}
}
Some(Ok(Message::Close(_))) | None => {
info!(
@@ -489,6 +543,56 @@ async fn run_proxy_reader(
_ => {}
}
}
heartbeats.shutdown().await;
}
async fn read_proxy_settings(
ws_tx: &mut futures_util::stream::SplitSink<WebSocket, Message>,
ws_rx: &mut futures_util::stream::SplitStream<WebSocket>,
security: Option<&SecureFrameCodec>,
protocol_version: u8,
) -> Option<protocol::SettingsPayload> {
tokio::time::timeout(PROXY_HELLO_TIMEOUT, async {
let mut hello_received = security.is_some();
for _ in 0..MAX_PREAUTH_PINGS {
match ws_rx.next().await? {
Ok(Message::Binary(data)) => {
if data.len() > 256 * 1024 {
return None;
}
let data = decrypt_message(data, security).ok()?;
let frame = Frame::decode(data.into()).ok()?;
if frame.stream_id != 0 || frame.flags != 0 {
return None;
}
match frame.msg_type {
MsgType::Hello if !hello_received => {
let hello =
serde_json::from_slice::<HelloPayload>(&frame.payload).ok()?;
if hello.protocol_version != protocol_version {
return None;
}
hello_received = true;
}
MsgType::Settings if hello_received => {
let settings =
serde_json::from_slice::<protocol::SettingsPayload>(&frame.payload)
.ok()?;
return settings.is_valid().then_some(settings);
}
_ => return None,
}
}
Ok(Message::Ping(payload)) => ws_tx.send(Message::Pong(payload)).await.ok()?,
Ok(Message::Pong(_)) => {}
_ => return None,
}
}
None
})
.await
.ok()
.flatten()
}
fn encrypt_message(
@@ -523,6 +627,158 @@ fn decrypt_message(
#[cfg(test)]
mod tests {
#[cfg(feature = "testkit")]
#[tokio::test]
async fn slow_heartbeat_does_not_block_response_frames() {
use std::sync::atomic::{AtomicUsize, Ordering};
use tokio_tungstenite::tungstenite::{client::IntoClientRequest, Message as ClientMessage};
let called = Arc::new(AtomicUsize::new(0));
let callback_called = Arc::clone(&called);
let control_plane = super::super::control_plane::ControlPlaneClient::local(
move |_, _| {
callback_called.fetch_add(1, Ordering::SeqCst);
Box::pin(std::future::pending())
},
|_, _, _, _| Box::pin(async { Ok(()) }),
);
let data = crate::data::GatewayDataState::with_tunnel_management_auth_for_testkit(
"heartbeat-test",
"heartbeat-generation",
"ae-tunnel-harness-management-token",
aether_crypto::DEVELOPMENT_ENCRYPTION_KEY,
)
.unwrap();
let state = super::super::AppState::new(
control_plane,
ConnConfig {
ping_interval: Duration::from_secs(60),
idle_timeout: Duration::ZERO,
outbound_queue_capacity: 128,
},
16,
)
.with_data(Arc::new(data));
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let router = super::super::build_router_with_state(state.clone());
let server = tokio::spawn(async move {
axum::serve(
listener,
router.into_make_service_with_connect_info::<std::net::SocketAddr>(),
)
.await
.unwrap();
});
let mut request = format!("ws://{address}/api/internal/proxy-tunnel")
.into_client_request()
.unwrap();
let headers = request.headers_mut();
headers.insert("x-node-id", "heartbeat-test".parse().unwrap());
headers.insert(
aether_contracts::tunnel_security::TUNNEL_GENERATION_HEADER,
"heartbeat-generation".parse().unwrap(),
);
headers.insert(
aether_contracts::tunnel::TUNNEL_PROTOCOL_VERSION_HEADER,
"3".parse().unwrap(),
);
headers.insert(
"authorization",
"Bearer ae-tunnel-harness-management-token".parse().unwrap(),
);
let (mut websocket, _) = tokio_tungstenite::connect_async(request).await.unwrap();
let hello = HelloPayload {
protocol_version: 3,
capabilities: vec![],
session_id: None,
replica_id: None,
};
websocket
.send(ClientMessage::Binary(protocol::encode_hello(&hello).into()))
.await
.unwrap();
websocket
.send(ClientMessage::Binary(
protocol::encode_settings(&super::super::hub::local_settings()).into(),
))
.await
.unwrap();
let ClientMessage::Binary(settings) = websocket.next().await.unwrap().unwrap() else {
panic!("expected SETTINGS")
};
assert_eq!(Frame::decode(settings).unwrap().msg_type, MsgType::Settings);
tokio::time::timeout(Duration::from_secs(1), async {
while !state.hub.has_local_proxy("heartbeat-test") {
tokio::task::yield_now().await;
}
})
.await
.unwrap();
let meta: protocol::RequestMeta = serde_json::from_value(serde_json::json!({
"method": "GET", "url": "https://example.com", "headers": {}, "stream": true, "timeout": 10
})).unwrap();
let stream = state
.hub
.open_local_stream("heartbeat-test", &meta)
.await
.unwrap();
let ClientMessage::Binary(request) = websocket.next().await.unwrap().unwrap() else {
panic!("expected request headers")
};
let stream_id = Frame::decode(request).unwrap().stream_id;
let heartbeat = Frame::control(
MsgType::HeartbeatData,
serde_json::to_vec(&serde_json::json!({"node_id": "heartbeat-test"})).unwrap(),
);
websocket
.send(ClientMessage::Binary(heartbeat.encode()))
.await
.unwrap();
tokio::time::timeout(Duration::from_secs(1), async {
while called.load(Ordering::SeqCst) == 0 {
tokio::task::yield_now().await;
}
})
.await
.unwrap();
for _ in 0..8 {
websocket
.send(ClientMessage::Binary(heartbeat.encode()))
.await
.unwrap();
}
let response = Frame::new(
stream_id,
MsgType::ResponseHeaders,
0,
serde_json::to_vec(&serde_json::json!({"status": 200, "headers": []})).unwrap(),
);
websocket
.send(ClientMessage::Binary(response.encode()))
.await
.unwrap();
assert_eq!(
stream
.wait_headers(Duration::from_secs(1))
.await
.unwrap()
.status,
200
);
assert_eq!(called.load(Ordering::SeqCst), 1);
state.hub.request_close_all_proxies();
tokio::time::timeout(Duration::from_secs(1), async {
while state.hub.has_local_proxy("heartbeat-test") {
tokio::task::yield_now().await;
}
})
.await
.unwrap();
server.abort();
let _ = server.await;
}
use super::*;
const KEY: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc=";
+2 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "aether-tunnel"
version = "0.3.16"
version = "0.3.17"
edition = "2021"
description = "Tunnel agent for Aether"
@@ -47,3 +47,4 @@ uuid.workspace = true
[dev-dependencies]
aether-gateway = { workspace = true, features = ["testkit"] }
tokio = { version = "1", features = ["test-util"] }
+9
View File
@@ -4,6 +4,15 @@ Aether Tunnel 代理节点,部署在海外 VPS 上,通过 WebSocket 隧道
Tunnel 模式下代理节点**无需对外监听端口**,仅需出站连接到 Aether 服务器。
## 流式传输与升级注意事项
- 协议 v3 连接在 `HELLO` / `SETTINGS` 协商后才接收业务请求。实际双向流窗口取 gateway 与 agent 配置的较小值,信用更新阈值不超过该窗口的四分之一;单帧也不会超过协商窗口。
- 响应缓冲按字节限额并合并小帧,结束和错误状态独立保存。慢消费者不会阻塞同一隧道其他流的读取;超出窗口或缓冲预算的流会被明确终止,不会静默截断。
- 信用更新在消费数据后可靠入队;启用重定向重放时,进入有界重放缓存也视为请求体消费。持续无法投递关键控制帧时会关闭连接并向在途请求报告错误。
- 客户端取消会终止对应上游请求,断连会回收 session 的 writer、heartbeat 和请求任务。正常 drain 在配置期限内继续处理已有流,期限到达后终止残留任务。
- 建议先升级 gateway,再升级 agent。既有 v3 agent 已发送 `HELLO` / `SETTINGS`,可连接新 gateway;自定义 v3 节点必须完成这两步握手。协议 v1/v2 保留旧握手。与旧 gateway 混用时应保持默认窗口配置,不能依赖旧 gateway 应用新的窗口协商。
- 自动重连恢复后续请求,不会自动续传已经输出的 SSE,也不会无条件重放已经发送的请求。
## 安装
`aether-tunnel` 会根据宿主机自动选择服务管理器:
+7
View File
@@ -764,6 +764,13 @@ impl Config {
if self.tunnel_stream_initial_window_bytes == 0 {
anyhow::bail!("tunnel_stream_initial_window_bytes must be > 0");
}
if u64::from(self.tunnel_stream_initial_window_bytes)
> aether_contracts::tunnel::MAX_TUNNEL_DECOMPRESSED_PAYLOAD_BYTES as u64
{
anyhow::bail!(
"tunnel_stream_initial_window_bytes exceeds the maximum tunnel payload size"
);
}
if self.tunnel_drain_deadline_ms == 0 {
anyhow::bail!("tunnel_drain_deadline_ms must be > 0");
}
+46 -13
View File
@@ -203,19 +203,34 @@ pub async fn connect_and_run(
let (ws_sink, ws_read) = futures_util::StreamExt::split(ws_stream);
// Spawn writer task (with WebSocket ping keepalive)
let (frame_tx, mut writer_handle) = writer::spawn_writer_with_metrics_and_security(
let (frame_tx, writer_handle) = writer::spawn_writer_with_metrics_and_security(
ws_sink,
ping_interval,
Some(Arc::clone(&server.tunnel_metrics)),
security.clone(),
);
let mut writer_handle = super::task::SessionTask::new(writer_handle);
send_protocol_v3_hello(&frame_tx, &security_session, state).await;
let drain_signal = spawn_drain_signal(
let (session_drain_tx, session_drain_rx) = watch::channel(*drain.borrow());
let forward_drain_tx = session_drain_tx.clone();
let mut external_drain = drain;
let forward_drain = super::task::SessionTask::new(tokio::spawn(async move {
loop {
if *external_drain.borrow() {
let _ = forward_drain_tx.send(true);
break;
}
if external_drain.changed().await.is_err() {
break;
}
}
}));
let drain_signal = super::task::SessionTask::new(spawn_drain_signal(
conn_idx,
frame_tx.clone(),
drain.clone(),
session_drain_rx.clone(),
state.config.tunnel_drain_deadline_ms,
);
));
// Spawn heartbeat task (only for primary connection to avoid
// resetting shared atomic metrics via swap(0))
@@ -237,16 +252,19 @@ pub async fn connect_and_run(
// ensures we detect this and trigger a reconnect promptly.
let state_clone = Arc::clone(state);
let server_clone = Arc::clone(server);
let outcome = tokio::select! {
result = dispatcher::run_with_security(
let outcome = {
let dispatch = dispatcher::run_with_security(
state_clone,
server_clone,
ws_read,
frame_tx.clone(),
hb_handle,
drain.clone(),
session_drain_rx,
security.clone(),
) => {
);
tokio::pin!(dispatch);
tokio::select! {
result = &mut dispatch => {
match result {
Ok(()) => Ok(TunnelOutcome::Disconnected),
Err(e) => {
@@ -258,6 +276,8 @@ pub async fn connect_and_run(
}
}
writer_result = &mut writer_handle => {
frame_tx.close();
let _ = tokio::time::timeout(Duration::from_secs(1), &mut dispatch).await;
match writer_result {
Ok(()) => warn!("writer task exited normally, triggering reconnect"),
Err(e) => {
@@ -278,24 +298,37 @@ pub async fn connect_and_run(
}
_ = shutdown.changed() => {
debug!("shutdown during tunnel dispatch");
let _ = session_drain_tx.send(true);
let deadline = Duration::from_millis(state.config.tunnel_drain_deadline_ms).saturating_add(Duration::from_secs(1));
let _ = tokio::time::timeout(deadline, &mut dispatch).await;
Ok(TunnelOutcome::Shutdown)
}
}
};
// Drop our sender; the writer will exit once all stream handler clones
// are also dropped (i.e. after they finish their in-flight work).
drop(frame_tx);
forward_drain.abort();
let _ = forward_drain.await;
if !drain_signal.is_finished() {
drain_signal.abort();
let _ = drain_signal.await;
}
// Wait for the writer task to finish with a generous timeout — the
// dispatcher already waits up to 30s for stream handlers, so 35s here
// covers that plus a small margin.
// Skip if the writer already exited (the select branch that fired).
if !writer_handle.is_finished() {
let _ = tokio::time::timeout(Duration::from_secs(35), writer_handle).await;
let flush_timeout = if *session_drain_tx.borrow() {
Duration::from_millis(state.config.tunnel_drain_deadline_ms)
} else {
Duration::from_secs(1)
};
if tokio::time::timeout(flush_timeout, &mut writer_handle)
.await
.is_err()
{
writer_handle.abort();
let _ = writer_handle.await;
}
}
let connected_for = connected_at.elapsed();
+151 -131
View File
@@ -9,7 +9,7 @@ use std::time::Duration;
use bytes::Bytes;
use futures_util::StreamExt;
use tokio::sync::{mpsc, watch, OwnedSemaphorePermit, Semaphore};
use tokio::task::JoinHandle;
use tokio::task::{AbortHandle, JoinSet};
use tokio_tungstenite::tungstenite::Message;
use tracing::{debug, error, info, warn};
@@ -41,13 +41,25 @@ impl AsRef<[u8]> for BudgetedFramePayload {
enum StreamDispatchStatus {
Delivered,
Closed,
TimedOut,
Congested,
}
#[derive(Clone)]
struct StreamDispatchTarget {
body_tx: mpsc::Sender<Frame>,
response_window: Arc<StreamSendWindow>,
handler: Option<AbortHandle>,
}
struct StreamCompletion {
stream_id: u32,
finished_tx: mpsc::UnboundedSender<u32>,
}
impl Drop for StreamCompletion {
fn drop(&mut self) {
let _ = self.finished_tx.send(self.stream_id);
}
}
/// A request stream is identified by a non-zero id and may only be opened
@@ -109,7 +121,7 @@ where
// reopen the same id and bypass the stream admission limit.
let mut active_handler_ids: HashSet<u32> = HashSet::new();
// Track spawned stream handlers so we can wait for them on shutdown
let mut handler_handles: Vec<JoinHandle<()>> = Vec::new();
let mut handler_handles = JoinSet::new();
let (handler_finished_tx, mut handler_finished_rx) = mpsc::unbounded_channel::<u32>();
let max_streams = state.config.tunnel_max_streams.unwrap_or(128) as usize;
let mut frames_since_cleanup: u32 = 0;
@@ -121,30 +133,48 @@ where
// Track last time we received any data to detect stale connections
let mut last_data_at = tokio::time::Instant::now();
let mut draining = *drain.borrow();
let mut drain_open = true;
let mut drain_deadline = draining.then(|| {
tokio::time::Instant::now() + Duration::from_millis(state.config.tunnel_drain_deadline_ms)
});
let mut initial_window_bytes = state.config.tunnel_stream_initial_window_bytes;
let mut close_rx = frame_tx.subscribe_close();
let read_err = loop {
if *close_rx.borrow() {
break None;
}
if draining && streams.is_empty() && active_handler_ids.is_empty() {
info!("tunnel drained after in-flight streams completed");
break None;
}
let msg_result = tokio::select! {
_ = close_rx.changed() => break None,
msg = ws_stream.next() => {
match msg {
Some(r) => r,
None => break None,
}
}
changed = drain.changed() => {
changed = drain.changed(), if drain_open => {
if changed.is_err() {
drain_open = false;
continue;
}
if *drain.borrow() {
info!("tunnel drain requested, waiting for in-flight streams");
draining = true;
drain_deadline.get_or_insert_with(|| tokio::time::Instant::now() + Duration::from_millis(state.config.tunnel_drain_deadline_ms));
}
continue;
}
_ = async {
match drain_deadline {
Some(deadline) => tokio::time::sleep_until(deadline).await,
None => std::future::pending().await,
}
} => break None,
finished = handler_finished_rx.recv() => {
if let Some(stream_id) = finished {
active_handler_ids.remove(&stream_id);
@@ -238,20 +268,7 @@ where
continue;
}
if draining {
if frame_tx
.try_send(Frame::new(
frame.stream_id,
MsgType::StreamError,
0,
Bytes::from("tunnel draining"),
))
.is_err()
{
warn!(
stream_id = frame.stream_id,
"writer channel full, StreamError dropped during drain"
);
}
try_send_stream_error(&frame_tx, frame.stream_id, "tunnel draining");
continue;
}
@@ -263,6 +280,11 @@ where
Ok(p) => p,
Err(e) => {
warn!(stream_id = frame.stream_id, error = %e, "frame decompress failed");
try_send_stream_error(
&frame_tx,
frame.stream_id,
"invalid request metadata",
);
continue;
}
};
@@ -270,21 +292,11 @@ where
Ok(m) => m,
Err(e) => {
warn!(stream_id = frame.stream_id, error = %e, "invalid request metadata");
// Use try_send to avoid blocking the read loop
if frame_tx
.try_send(Frame::new(
frame.stream_id,
MsgType::StreamError,
0,
Bytes::from(format!("invalid request metadata: {e}")),
))
.is_err()
{
warn!(
stream_id = frame.stream_id,
"writer channel full, StreamError dropped"
);
}
try_send_stream_error(
&frame_tx,
frame.stream_id,
"invalid request metadata",
);
continue;
}
};
@@ -294,33 +306,27 @@ where
stream_id = frame.stream_id,
"max concurrent streams reached"
);
if frame_tx
.try_send(Frame::new(
frame.stream_id,
MsgType::StreamError,
0,
Bytes::from("max concurrent streams reached"),
))
.is_err()
{
warn!(
stream_id = frame.stream_id,
"writer channel full, StreamError dropped"
);
}
try_send_stream_error(
&frame_tx,
frame.stream_id,
"max concurrent streams reached",
);
continue;
}
// Create body channel and spawn handler
let (body_tx, body_rx) = mpsc::channel::<Frame>(64);
let response_window = Arc::new(StreamSendWindow::new(
state.config.tunnel_stream_initial_window_bytes,
));
let body_capacity = (initial_window_bytes as usize)
.div_ceil(32 * 1024)
.saturating_add(1)
.max(64);
let (body_tx, body_rx) = mpsc::channel::<Frame>(body_capacity);
let response_window = Arc::new(StreamSendWindow::new(initial_window_bytes));
streams.insert(
frame.stream_id,
StreamDispatchTarget {
body_tx,
response_window: Arc::clone(&response_window),
handler: None,
},
);
active_handler_ids.insert(frame.stream_id);
@@ -329,9 +335,13 @@ where
let state_clone = Arc::clone(&state);
let server_clone = Arc::clone(&server);
let tx_clone = frame_tx.clone();
let finished_tx = handler_finished_tx.clone();
let sid = frame.stream_id;
let handle = tokio::spawn(async move {
let completion = StreamCompletion {
stream_id: sid,
finished_tx: handler_finished_tx.clone(),
};
let handle = handler_handles.spawn(async move {
let _completion = completion;
stream_handler::handle_stream(
state_clone,
server_clone,
@@ -342,9 +352,8 @@ where
response_window,
)
.await;
let _ = finished_tx.send(sid);
});
handler_handles.push(handle);
streams.get_mut(&sid).expect("new stream exists").handler = Some(handle);
if request_headers_end_stream {
if let Some(target) = streams.get(&sid) {
@@ -365,19 +374,21 @@ where
let is_end = frame.is_end_stream();
let sid = frame.stream_id;
let dispatch = dispatch_stream_frame(&target.body_tx, frame).await;
if dispatch != StreamDispatchStatus::Delivered {
streams.remove(&sid);
if dispatch == StreamDispatchStatus::TimedOut {
server.tunnel_metrics.record_error(
"stream_dispatch_timeout",
&format!("request body dispatch timed out for stream {}", sid),
);
try_send_stream_error(
&frame_tx,
sid,
"tunnel request body dispatch stalled",
);
if dispatch == StreamDispatchStatus::Congested {
if let Some(target) = streams.remove(&sid) {
if let Some(handler) = target.handler {
handler.abort();
}
}
server.tunnel_metrics.record_error(
"stream_dispatch_timeout",
&format!("request body dispatch congested for stream {}", sid),
);
try_send_stream_error(
&frame_tx,
sid,
"tunnel request body dispatch stalled",
);
if is_end && draining && streams.is_empty() && active_handler_ids.is_empty()
{
info!("tunnel drained after request body completion");
@@ -387,10 +398,29 @@ where
}
}
MsgType::StreamEnd | MsgType::StreamError | MsgType::ResetStream => {
MsgType::StreamEnd => {
if let Some(target) = streams.get(&frame.stream_id) {
if dispatch_stream_frame(&target.body_tx, frame.clone()).await
== StreamDispatchStatus::Congested
{
if let Some(handler) = &target.handler {
handler.abort();
}
try_send_stream_error(
&frame_tx,
frame.stream_id,
"tunnel request body dispatch stalled",
);
}
}
}
MsgType::StreamError | MsgType::ResetStream => {
// Client-side cancellation or end
if let Some(target) = streams.remove(&frame.stream_id) {
let _ = dispatch_stream_frame(&target.body_tx, frame).await;
if let Some(handler) = target.handler {
handler.abort();
}
if draining && streams.is_empty() && active_handler_ids.is_empty() {
info!("tunnel drained after stream termination");
break None;
@@ -409,12 +439,22 @@ where
}
MsgType::HeartbeatAck => {
heartbeat.on_ack(frame.payload).await;
heartbeat.on_ack(frame.payload);
}
MsgType::GoAway => {
info!("received GOAWAY");
break None;
draining = true;
let deadline_ms =
serde_json::from_slice::<aether_contracts::tunnel::GoAwayPayload>(
&frame.payload,
)
.map(|payload| payload.drain_deadline_ms)
.unwrap_or(state.config.tunnel_drain_deadline_ms)
.min(state.config.tunnel_drain_deadline_ms);
drain_deadline.get_or_insert_with(|| {
tokio::time::Instant::now() + Duration::from_millis(deadline_ms)
});
}
MsgType::WindowUpdate => {
@@ -433,7 +473,31 @@ where
);
}
MsgType::Hello | MsgType::Settings | MsgType::LoadReport => {
MsgType::Settings => {
if frame.stream_id != 0 || frame.flags != 0 {
break None;
}
let settings = serde_json::from_slice::<aether_contracts::tunnel::SettingsPayload>(
&frame.payload,
)
.ok()
.filter(|settings| settings.is_valid());
let Some(settings) = settings else {
warn!("invalid tunnel SETTINGS");
break None;
};
if !streams.is_empty()
&& settings.initial_stream_window_bytes != initial_window_bytes
{
warn!("tunnel SETTINGS changed with active streams");
break None;
}
initial_window_bytes = settings
.initial_stream_window_bytes
.min(state.config.tunnel_stream_initial_window_bytes);
}
MsgType::Hello | MsgType::LoadReport => {
debug!(
msg_type = ?frame.msg_type,
stream_id = frame.stream_id,
@@ -455,7 +519,7 @@ where
// Trigger every 64 frames OR when the count exceeds max_streams.
frames_since_cleanup += 1;
if frames_since_cleanup >= 64 || handler_handles.len() > max_streams {
handler_handles.retain(|h| !h.is_finished());
while handler_handles.try_join_next().is_some() {}
frames_since_cleanup = 0;
if draining && streams.is_empty() && active_handler_ids.is_empty() {
info!("tunnel drained after cleanup");
@@ -467,9 +531,7 @@ where
// Drop body senders so stream handlers waiting on body_rx will unblock
streams.clear();
// Wait for active stream handlers to finish so their frame_tx clones
// are dropped before the writer closes the sink.
drain_handlers(handler_handles).await;
handler_handles.shutdown().await;
match read_err {
Some(e) => Err(e.into()),
@@ -478,30 +540,13 @@ where
}
async fn dispatch_stream_frame(tx: &mpsc::Sender<Frame>, frame: Frame) -> StreamDispatchStatus {
let stream_id = frame.stream_id;
let dispatched = tokio::time::timeout(stream_frame_dispatch_timeout(), async {
let frame = attach_request_body_queue_budget(frame).await?;
tx.send(frame).await.ok()?;
Some(())
})
.await;
match dispatched {
Ok(Some(())) => StreamDispatchStatus::Delivered,
Ok(None) => {
warn!(
stream_id,
"stream handler channel or request body budget closed while dispatching tunnel frame"
);
StreamDispatchStatus::Closed
}
Err(_) => {
warn!(
stream_id,
timeout_ms = stream_frame_dispatch_timeout().as_millis(),
"stream handler channel blocked while dispatching tunnel frame"
);
StreamDispatchStatus::TimedOut
}
let Some(frame) = attach_request_body_queue_budget(frame).await else {
return StreamDispatchStatus::Congested;
};
match tx.try_send(frame) {
Ok(()) => StreamDispatchStatus::Delivered,
Err(mpsc::error::TrySendError::Closed(_)) => StreamDispatchStatus::Closed,
Err(mpsc::error::TrySendError::Full(_)) => StreamDispatchStatus::Congested,
}
}
@@ -523,7 +568,7 @@ async fn attach_request_body_queue_budget_with(
return Some(frame);
}
let permits = request_body_queue_permits(&frame, budget_bytes)?;
let permit = budget.acquire_many_owned(permits).await.ok()?;
let permit = budget.try_acquire_many_owned(permits).ok()?;
frame.payload = Bytes::from_owner(BudgetedFramePayload {
bytes: frame.payload,
_permit: permit,
@@ -549,20 +594,6 @@ fn request_body_queue_permits(frame: &Frame, budget_bytes: usize) -> Option<u32>
u32::try_from(retained_bytes).ok()
}
/// Bound how long a single stream handler is allowed to block the shared
/// WebSocket read loop while receiving request-body frames.
fn stream_frame_dispatch_timeout() -> Duration {
#[cfg(test)]
{
Duration::from_millis(25)
}
#[cfg(not(test))]
{
Duration::from_millis(500)
}
}
fn try_send_stream_error(frame_tx: &FrameSender, stream_id: u32, message: &'static str) {
if frame_tx
.try_send(Frame::new(
@@ -573,6 +604,7 @@ fn try_send_stream_error(frame_tx: &FrameSender, stream_id: u32, message: &'stat
))
.is_err()
{
frame_tx.close();
warn!(
stream_id,
"writer channel full, StreamError dropped while aborting stalled stream"
@@ -587,21 +619,6 @@ fn prune_closed_stream_senders(streams: &mut HashMap<u32, StreamDispatchTarget>)
before.saturating_sub(streams.len())
}
/// Wait for all active stream handlers to finish (with a timeout).
async fn drain_handlers(handles: Vec<JoinHandle<()>>) {
if handles.is_empty() {
return;
}
let count = handles.len();
debug!(count, "waiting for active stream handlers to finish");
let _ = tokio::time::timeout(Duration::from_secs(30), async {
for h in handles {
let _ = h.await;
}
})
.await;
}
#[cfg(test)]
mod tests {
use super::*;
@@ -633,7 +650,7 @@ mod tests {
assert_eq!(
stalled_send.await.expect("dispatch task should join"),
StreamDispatchStatus::TimedOut
StreamDispatchStatus::Congested
);
let retained = rx
@@ -737,6 +754,7 @@ mod tests {
StreamDispatchTarget {
body_tx: closed_tx,
response_window: Arc::new(StreamSendWindow::new(1024)),
handler: None,
},
),
(
@@ -744,6 +762,7 @@ mod tests {
StreamDispatchTarget {
body_tx: open_tx,
response_window: Arc::new(StreamSendWindow::new(1024)),
handler: None,
},
),
]);
@@ -763,6 +782,7 @@ mod tests {
StreamDispatchTarget {
body_tx: tx,
response_window: Arc::new(StreamSendWindow::new(1024)),
handler: None,
},
)]);
let mut active_handler_ids = HashSet::from([7]);
+19 -7
View File
@@ -31,14 +31,22 @@ enum AckDecision {
}
/// Handle for the dispatcher to forward HeartbeatAck frames.
#[derive(Clone)]
pub struct HeartbeatHandle {
ack_tx: tokio::sync::mpsc::Sender<Bytes>,
task: Option<tokio::task::JoinHandle<()>>,
}
impl HeartbeatHandle {
pub async fn on_ack(&self, payload: Bytes) {
let _ = self.ack_tx.send(payload).await;
pub fn on_ack(&self, payload: Bytes) {
let _ = self.ack_tx.try_send(payload);
}
}
impl Drop for HeartbeatHandle {
fn drop(&mut self) {
if let Some(task) = self.task.take() {
task.abort();
}
}
}
@@ -48,7 +56,7 @@ impl HeartbeatHandle {
pub fn spawn_noop() -> HeartbeatHandle {
let (ack_tx, _) = tokio::sync::mpsc::channel::<Bytes>(1);
// receiver is immediately dropped; on_ack() calls will silently fail
HeartbeatHandle { ack_tx }
HeartbeatHandle { ack_tx, task: None }
}
#[derive(Debug, Clone, Copy, Default)]
@@ -74,7 +82,7 @@ pub fn spawn(
) -> HeartbeatHandle {
let (ack_tx, mut ack_rx) = tokio::sync::mpsc::channel::<Bytes>(4);
tokio::spawn(async move {
let task = tokio::spawn(async move {
// Read initial interval from dynamic config (may be updated by remote config).
let initial_interval = Duration::from_secs(server.dynamic.load().heartbeat_interval);
let mut current_interval = initial_interval;
@@ -151,7 +159,8 @@ pub fn spawn(
current_interval = new_interval;
}
}
Some(ack_payload) = ack_rx.recv() => {
ack_payload = ack_rx.recv() => {
let Some(ack_payload) = ack_payload else { break; };
match handle_ack(&server, &ack_payload) {
AckDecision::Accept {
heartbeat_id: ack_id,
@@ -179,7 +188,10 @@ pub fn spawn(
}
});
HeartbeatHandle { ack_tx }
HeartbeatHandle {
ack_tx,
task: Some(task),
}
}
async fn build_heartbeat_payload(
+156 -27
View File
@@ -3,6 +3,7 @@ pub mod dispatcher;
pub mod heartbeat;
pub mod protocol;
pub mod stream_handler;
mod task;
pub mod writer;
use std::sync::Arc;
@@ -332,9 +333,9 @@ mod tests {
)
.await;
assert_eq!(gateway_state.force_close_all_tunnel_proxies(), 1);
tokio::time::sleep(Duration::from_millis(200)).await;
gateway_handle.abort();
let _ = (&mut gateway_handle).await;
assert_eq!(gateway_state.force_close_all_tunnel_proxies(), 1);
let (_restarted_gateway_state, restarted_gateway_handle) =
start_gateway_on_port_retry(gateway_port)
@@ -349,6 +350,7 @@ mod tests {
)
.await;
assert!(server.tunnel_metrics.snapshot().connect_successes >= 2);
let _ = shutdown_tx.send(true);
tokio::time::timeout(Duration::from_secs(5), tunnel_task)
.await
@@ -380,7 +382,17 @@ mod tests {
gateway_base_url: &str,
node_id: &str,
) -> Option<(StatusCode, String)> {
let payload = relay_probe_envelope();
let response = relay_response(gateway_base_url, node_id, relay_probe_envelope()).await?;
let status = response.status();
let body = response.text().await.unwrap_or_default();
Some((status, body))
}
async fn relay_response(
gateway_base_url: &str,
node_id: &str,
payload: Vec<u8>,
) -> Option<reqwest::Response> {
let timestamp = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("test clock should be after epoch")
@@ -398,7 +410,7 @@ mod tests {
&nonce,
&digest,
);
let response = reqwest::Client::new()
reqwest::Client::new()
.post(format!(
"{gateway_base_url}/api/internal/tunnel/relay/{node_id}"
))
@@ -421,10 +433,7 @@ mod tests {
.body(payload)
.send()
.await
.ok()?;
let status = response.status();
let body = response.text().await.unwrap_or_default();
Some((status, body))
.ok()
}
fn relay_probe_envelope() -> Vec<u8> {
@@ -456,30 +465,150 @@ mod tests {
) -> Result<(GatewayAppState, tokio::task::JoinHandle<()>), std::io::Error> {
// The embedded gateway now fails closed when relay authentication is
// not configured. Keep this integration fixture explicitly authenticated.
let previous_secret = std::env::var_os("AETHER_TUNNEL_RELAY_AUTH_SECRET");
let previous_instance = std::env::var_os("AETHER_GATEWAY_INSTANCE_ID");
std::env::set_var(
"AETHER_TUNNEL_RELAY_AUTH_SECRET",
"tunnel-reconnect-test-secret-at-least-32-bytes",
);
std::env::set_var(
"AETHER_GATEWAY_INSTANCE_ID",
"tunnel-reconnect-test-gateway",
);
let mut state = GatewayAppState::new().expect("gateway test state should build");
aether_gateway::configure_test_tunnel_security(
&mut state,
"node-recovery",
"test-generation-1",
"BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc=",
);
restore_test_env("AETHER_TUNNEL_RELAY_AUTH_SECRET", previous_secret);
restore_test_env("AETHER_GATEWAY_INSTANCE_ID", previous_instance);
static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
let state = {
let _guard = ENV_LOCK.lock().unwrap();
let previous_secret = std::env::var_os("AETHER_TUNNEL_RELAY_AUTH_SECRET");
let previous_instance = std::env::var_os("AETHER_GATEWAY_INSTANCE_ID");
std::env::set_var(
"AETHER_TUNNEL_RELAY_AUTH_SECRET",
"tunnel-reconnect-test-secret-at-least-32-bytes",
);
std::env::set_var(
"AETHER_GATEWAY_INSTANCE_ID",
"tunnel-reconnect-test-gateway",
);
let mut state = GatewayAppState::new().expect("gateway test state should build");
aether_gateway::configure_test_tunnel_security(
&mut state,
"node-recovery",
"test-generation-1",
"BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc=",
);
restore_test_env("AETHER_TUNNEL_RELAY_AUTH_SECRET", previous_secret);
restore_test_env("AETHER_GATEWAY_INSTANCE_ID", previous_instance);
state
};
let router = build_router_with_state(state.clone());
let handle = spawn_router_on_port(port, router).await?;
Ok((state, handle))
}
#[tokio::test]
async fn negotiated_small_window_streams_large_responses_and_cancels_idle_upstream() {
use axum::body::{Body, Bytes};
use axum::routing::get;
use futures_util::StreamExt;
ensure_rustls_provider();
let upstream_port = reserve_local_port().unwrap();
let upstream = Router::new()
.route(
"/large",
get(|| async { Body::from(vec![b'x'; 2 * 1024 * 1024]) }),
)
.route(
"/idle",
get(|| async {
let first = futures_util::stream::once(async {
Ok::<_, std::io::Error>(Bytes::from_static(b"data: started\n\n"))
});
(
[("content-type", "text/event-stream")],
Body::from_stream(first.chain(futures_util::stream::pending())),
)
}),
);
let upstream_task = super::task::SessionTask::new(
spawn_router_on_port(upstream_port, upstream).await.unwrap(),
);
let gateway_port = reserve_local_port().unwrap();
let gateway_url = format!("http://127.0.0.1:{gateway_port}");
let (_, gateway_task) = start_gateway_on_port(gateway_port).await.unwrap();
let gateway_task = super::task::SessionTask::new(gateway_task);
let mut config = sample_config(&gateway_url);
config.tunnel_security = crate::config::TunnelSecurity::NonTlsRequired;
config.tunnel_encryption_key = Some("BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc=".into());
config.tunnel_stream_initial_window_bytes = 512 * 1024;
config.tunnel_drain_deadline_ms = 100;
config.allow_private_targets = true;
config.allowed_ports.push(upstream_port);
let state = sample_state(config);
let server = sample_server(&state, "node-recovery");
let (shutdown_tx, shutdown_rx) = watch::channel(false);
let (_drain_tx, drain_rx) = watch::channel(false);
let tunnel_task = super::task::SessionTask::new(tokio::spawn({
let state = Arc::clone(&state);
let server = Arc::clone(&server);
async move {
run(&state, &server, 0, shutdown_rx, drain_rx).await;
}
}));
wait_until_relay_status(&gateway_url, "node-recovery", StatusCode::GATEWAY_TIMEOUT).await;
let envelope = |path: &str| {
let mut meta: protocol::RequestMeta =
serde_json::from_slice(&relay_probe_envelope()[4..]).unwrap();
meta.url = format!("http://127.0.0.1:{upstream_port}/{path}");
meta.stream = true;
meta.timeout = 10;
meta.stream_first_byte_timeout_ms = Some(10_000);
let encoded = serde_json::to_vec(&meta).unwrap();
let mut result = (encoded.len() as u32).to_be_bytes().to_vec();
result.extend(encoded);
result
};
let response = relay_response(&gateway_url, "node-recovery", envelope("large"))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body = tokio::time::timeout(Duration::from_secs(10), response.bytes())
.await
.unwrap()
.unwrap();
assert_eq!(body.len(), 2 * 1024 * 1024);
assert!(body.iter().all(|byte| *byte == b'x'));
let mut response = relay_response(&gateway_url, "node-recovery", envelope("idle"))
.await
.unwrap();
assert_eq!(
response.chunk().await.unwrap().unwrap(),
"data: started\n\n"
);
drop(response);
tokio::time::timeout(Duration::from_secs(3), async {
while server
.active_connections
.load(std::sync::atomic::Ordering::Acquire)
!= 0
{
tokio::task::yield_now().await;
}
})
.await
.expect("cancelled SSE must release the upstream handler");
let mut response = relay_response(&gateway_url, "node-recovery", envelope("idle"))
.await
.unwrap();
assert!(response.chunk().await.unwrap().is_some());
shutdown_tx.send(true).unwrap();
tokio::time::timeout(Duration::from_secs(3), tunnel_task)
.await
.unwrap()
.unwrap();
assert_eq!(
server
.active_connections
.load(std::sync::atomic::Ordering::Acquire),
0
);
drop(response);
drop(gateway_task);
drop(upstream_task);
}
fn restore_test_env(key: &str, value: Option<std::ffi::OsString>) {
if let Some(value) = value {
std::env::set_var(key, value);
+237 -60
View File
@@ -52,6 +52,7 @@ static REDIRECT_REPLAY_BUFFERED_BYTES: AtomicUsize = AtomicUsize::new(0);
#[derive(Debug)]
pub(crate) struct StreamSendWindow {
initial_window_bytes: u32,
available: Mutex<u64>,
notify: Notify,
}
@@ -59,6 +60,7 @@ pub(crate) struct StreamSendWindow {
impl StreamSendWindow {
pub(crate) fn new(initial_window_bytes: u32) -> Self {
Self {
initial_window_bytes: initial_window_bytes.max(1),
available: Mutex::new(u64::from(initial_window_bytes.max(1))),
notify: Notify::new(),
}
@@ -82,6 +84,9 @@ impl StreamSendWindow {
let requested = bytes as u64;
let started_at = Instant::now();
loop {
let notified = self.notify.notified();
tokio::pin!(notified);
notified.as_mut().enable();
{
let mut available = self.available.lock().expect("stream window lock poisoned");
if *available >= requested {
@@ -93,10 +98,7 @@ impl StreamSendWindow {
let Some(remaining) = timeout.checked_sub(started_at.elapsed()) else {
return Err(());
};
if tokio::time::timeout(remaining, self.notify.notified())
.await
.is_err()
{
if tokio::time::timeout(remaining, notified).await.is_err() {
return Err(());
}
}
@@ -173,31 +175,33 @@ fn safe_stream_error_message(message: &str) -> &'static str {
"upstream request failed"
}
fn try_send_window_update(frame_tx: &FrameSender, stream_id: u32, bytes: usize) {
async fn send_window_update(frame_tx: &FrameSender, stream_id: u32, bytes: usize) -> bool {
if bytes == 0 {
return;
return true;
}
let delta = bytes.min(u32::MAX as usize) as u32;
if frame_tx
.try_send(TunnelFrame::new(
stream_id,
MsgType::WindowUpdate,
0,
Bytes::from(
serde_json::to_vec(&aether_contracts::tunnel::WindowUpdatePayload {
delta_bytes: delta,
})
.expect("window update payload should serialize"),
),
))
.is_err()
{
warn!(
stream_id,
delta_bytes = delta,
"writer channel full, WINDOW_UPDATE dropped"
);
if matches!(
tokio::time::timeout(
FLOW_CONTROL_WAIT_TIMEOUT,
frame_tx.send(TunnelFrame::new(
stream_id,
MsgType::WindowUpdate,
0,
Bytes::from(
serde_json::to_vec(&aether_contracts::tunnel::WindowUpdatePayload {
delta_bytes: delta,
})
.expect("window update payload should serialize"),
),
))
)
.await,
Ok(Ok(()))
) {
return true;
}
frame_tx.close();
false
}
/// Match reqwest's default redirect budget so direct execution and tunnel relay
@@ -242,6 +246,23 @@ enum ReplayableRequestBody {
struct PreparedRequestBody {
first_request_body: Option<upstream_client::UpstreamRequestBody>,
replay_body: ReplayableRequestBody,
spool_task: Option<tokio::task::JoinHandle<()>>,
}
impl Drop for PreparedRequestBody {
fn drop(&mut self) {
if let Some(task) = self.spool_task.take() {
task.abort();
}
}
}
struct ActiveStreamGuard(Arc<ServerContext>);
impl Drop for ActiveStreamGuard {
fn drop(&mut self) {
self.0.active_connections.fetch_sub(1, Ordering::Release);
}
}
#[derive(Debug, Clone, Copy)]
@@ -331,7 +352,10 @@ impl hyper::body::Body for ReplayRequestBody {
#[derive(Debug)]
enum SpoolBodyEvent {
Data(Bytes),
Data {
payload: Bytes,
credit_returned: bool,
},
Error(String),
End,
}
@@ -563,8 +587,9 @@ impl RequestBodyReplayState {
}
}
fn push_chunk(&self, payload: Bytes) {
fn push_chunk(&self, payload: Bytes) -> bool {
let mut disable_replay = false;
let mut retained = false;
let mut state = self.state.lock().expect("request body replay state lock");
if let RequestBodyReplayStatus::Collecting {
chunks,
@@ -577,7 +602,7 @@ impl RequestBodyReplayState {
drop(state);
self.release_reserved_bytes();
self.ready.notify_waiters();
return;
return false;
};
let accounted_bytes = payload.len().checked_add(std::mem::size_of::<Bytes>());
if next_len > self.budget_bytes
@@ -590,6 +615,7 @@ impl RequestBodyReplayState {
} else {
*buffered_len = next_len;
chunks.push(payload);
retained = true;
}
}
drop(state);
@@ -597,6 +623,7 @@ impl RequestBodyReplayState {
self.release_reserved_bytes();
self.ready.notify_waiters();
}
retained
}
fn try_reserve_bytes(&self, bytes: usize) -> bool {
@@ -951,9 +978,6 @@ pub(super) fn decode_request_body_frame(frame: TunnelFrame) -> Result<Bytes, std
Ok(frame.payload)
}
// Drain tunnel body frames on a detached task so the shared dispatcher is no
// longer coupled to upstream body polling. Redirect replay retains a bounded
// copy; crossing either replay budget only disables replay for this request.
fn prepare_request_body(
stream_id: u32,
body_rx: mpsc::Receiver<TunnelFrame>,
@@ -973,19 +997,20 @@ fn prepare_request_body(
None => ReplayableRequestBody::NonReplayable,
};
tokio::spawn(spool_request_body(
let spool_task = tokio::spawn(spool_request_body(
stream_id,
body_rx,
spool_tx,
replay_state,
body_size,
deadline,
frame_tx,
frame_tx.clone(),
));
PreparedRequestBody {
first_request_body: Some(build_spooled_request_body(spool_rx)),
first_request_body: Some(build_spooled_request_body(spool_rx, stream_id, frame_tx)),
replay_body,
spool_task: Some(spool_task),
}
}
@@ -1001,6 +1026,7 @@ fn prepare_bodyless_request_body(
} else {
ReplayableRequestBody::NonReplayable
},
spool_task: None,
}
}
@@ -1056,11 +1082,16 @@ async fn spool_request_body(
};
let Some(frame) = frame else {
let message = "tunnel request body closed before stream end".to_string();
if let Some(state) = &replay_state {
state.finish();
state.fail(message.clone());
}
let _ =
send_spool_event(&mut spool_tx, SpoolBodyEvent::End, replay_state.as_ref()).await;
let _ = send_spool_event(
&mut spool_tx,
SpoolBodyEvent::Error(message),
replay_state.as_ref(),
)
.await;
return;
};
@@ -1086,13 +1117,23 @@ async fn spool_request_body(
if !payload.is_empty() {
body_size.fetch_add(payload.len(), Ordering::Relaxed);
try_send_window_update(&frame_tx, stream_id, payload.len());
if let Some(state) = &replay_state {
state.push_chunk(payload.clone());
let credit_returned = replay_state
.as_ref()
.is_some_and(|state| state.push_chunk(payload.clone()));
if credit_returned
&& !send_window_update(&frame_tx, stream_id, payload.len()).await
{
if let Some(state) = &replay_state {
state.fail("tunnel flow-control update failed".to_string());
}
return;
}
if send_spool_event(
&mut spool_tx,
SpoolBodyEvent::Data(payload),
SpoolBodyEvent::Data {
payload,
credit_returned,
},
replay_state.as_ref(),
)
.await
@@ -1479,6 +1520,7 @@ where
}
let mut stream = response.into_body().into_data_stream();
let chunk_size = MAX_CHUNK_SIZE.min(response_window.initial_window_bytes as usize);
loop {
let chunk_result = if let Some(deadline) = response_body_deadline {
let Some(remaining) = remaining_timeout(deadline) else {
@@ -1531,7 +1573,7 @@ where
match chunk_result {
Ok(chunk) => {
if chunk.len() <= MAX_CHUNK_SIZE {
if chunk.len() <= chunk_size {
let (payload, extra_flags) = raw_payload(chunk);
if !acquire_response_credit(response_window, frame_tx, stream_id, payload.len())
.await
@@ -1561,7 +1603,7 @@ where
} else {
let mut offset = 0;
while offset < chunk.len() {
let end = (offset + MAX_CHUNK_SIZE).min(chunk.len());
let end = (offset + chunk_size).min(chunk.len());
let slice = chunk.slice(offset..end);
let (payload, extra_flags) = raw_payload(slice);
if !acquire_response_credit(
@@ -1735,6 +1777,7 @@ pub async fn handle_stream(
};
server.active_connections.fetch_add(1, Ordering::Release);
let _active_stream = ActiveStreamGuard(Arc::clone(&server));
let stream_io = StreamIo {
body_rx,
@@ -1745,7 +1788,6 @@ pub async fn handle_stream(
let connect_elapsed = handle_stream_inner(&state, &server, stream_id, meta, stream_io).await;
server.active_connections.fetch_sub(1, Ordering::Release);
if let Some(d) = connect_elapsed {
server.metrics.record_request(d);
}
@@ -1772,6 +1814,18 @@ async fn send_frame(tx: &FrameSender, frame: TunnelFrame) -> bool {
timeout_ms = FLOW_CONTROL_WAIT_TIMEOUT.as_millis() as u64,
"writer channel stalled for body frame, abandoning stream"
);
let reset = TunnelFrame::new(
stream_id,
MsgType::ResetStream,
0,
Bytes::from_static(b"{\"reason\":\"tunnel writer stalled\"}"),
);
if !matches!(
tokio::time::timeout(CONTROL_FRAME_SEND_TIMEOUT, tx.send(reset)).await,
Ok(Ok(()))
) {
tx.close();
}
false
}
Ok(Err(QueueSendError::Full(_))) => {
@@ -1781,7 +1835,10 @@ async fn send_frame(tx: &FrameSender, frame: TunnelFrame) -> bool {
} else {
match tokio::time::timeout(CONTROL_FRAME_SEND_TIMEOUT, tx.send(frame)).await {
Ok(Ok(())) => true,
Ok(Err(_)) => false,
Ok(Err(_)) => {
tx.close();
false
}
Err(_) => {
warn!(
stream_id,
@@ -1789,6 +1846,7 @@ async fn send_frame(tx: &FrameSender, frame: TunnelFrame) -> bool {
flags = flags,
"control frame send timeout (writer congested), abandoning stream"
);
tx.close();
false
}
}
@@ -2126,7 +2184,6 @@ async fn handle_stream_inner(
}
async fn send_error(tx: &FrameSender, stream_id: u32, msg: &str) {
// Error frames use best-effort delivery — don't block if writer is congested
let safe_message = safe_stream_error_message(msg);
let _ = send_frame(
tx,
@@ -2163,22 +2220,42 @@ fn build_streaming_request_body(
fn build_spooled_request_body(
spool_rx: mpsc::Receiver<SpoolBodyEvent>,
stream_id: u32,
frame_tx: FrameSender,
) -> upstream_client::UpstreamRequestBody {
let body_stream = stream::unfold((spool_rx, false), |(mut spool_rx, finished)| async move {
if finished {
return None;
}
let body_stream = stream::unfold(
(spool_rx, frame_tx, false),
move |(mut spool_rx, frame_tx, finished)| async move {
if finished {
return None;
}
match spool_rx.recv().await {
Some(SpoolBodyEvent::Data(payload)) => {
Some((Ok(BodyFrame::data(payload)), (spool_rx, false)))
match spool_rx.recv().await {
Some(SpoolBodyEvent::Data {
payload,
credit_returned,
}) => {
if !credit_returned
&& !send_window_update(&frame_tx, stream_id, payload.len()).await
{
return Some((
Err(io::Error::other("tunnel flow-control update failed")),
(spool_rx, frame_tx, true),
));
}
Some((Ok(BodyFrame::data(payload)), (spool_rx, frame_tx, false)))
}
Some(SpoolBodyEvent::Error(message)) => {
Some((Err(io::Error::other(message)), (spool_rx, frame_tx, true)))
}
Some(SpoolBodyEvent::End) => None,
None => Some((
Err(io::Error::other("tunnel request body ended unexpectedly")),
(spool_rx, frame_tx, true),
)),
}
Some(SpoolBodyEvent::Error(message)) => {
Some((Err(io::Error::other(message)), (spool_rx, true)))
}
Some(SpoolBodyEvent::End) | None => None,
}
});
},
);
upstream_client::stream_request_body(body_stream)
}
@@ -2249,6 +2326,105 @@ fn build_prefixed_request_body(
#[cfg(test)]
mod tests {
#[tokio::test(start_paused = true)]
async fn window_updates_wait_for_capacity_instead_of_disappearing() {
let (high_tx, mut high_rx) = aether_runtime::bounded_queue(1);
let (normal_tx, _normal_rx) = aether_runtime::bounded_queue(1);
let sender = FrameSender::from_test_queues(high_tx, normal_tx);
sender
.try_send(TunnelFrame::control(MsgType::Ping, Bytes::new()))
.unwrap();
let task = tokio::spawn(async move { send_window_update(&sender, 7, 1024).await });
tokio::time::sleep(Duration::from_secs(1)).await;
assert!(!task.is_finished());
high_rx.recv().await.unwrap();
assert!(task.await.unwrap());
let update = high_rx.recv().await.unwrap();
assert_eq!(update.msg_type, MsgType::WindowUpdate);
let payload: aether_contracts::tunnel::WindowUpdatePayload =
serde_json::from_slice(&update.payload).unwrap();
assert_eq!(payload.delta_bytes, 1024);
}
#[tokio::test(start_paused = true)]
async fn stalled_body_delivery_emits_a_reset() {
let (high_tx, mut high_rx) = aether_runtime::bounded_queue(4);
let (normal_tx, _normal_rx) = aether_runtime::bounded_queue(1);
let sender = FrameSender::from_test_queues(high_tx, normal_tx);
sender
.try_send(TunnelFrame::new(
7,
MsgType::ResponseBody,
0,
Bytes::from_static(b"first"),
))
.unwrap();
assert!(
!send_frame(
&sender,
TunnelFrame::new(7, MsgType::ResponseBody, 0, Bytes::from_static(b"second"))
)
.await
);
let reset = high_rx.recv().await.unwrap();
assert_eq!(reset.msg_type, MsgType::ResetStream);
assert_eq!(reset.stream_id, 7);
}
#[tokio::test]
async fn request_credit_follows_consumption_without_redirect_replay() {
let (body_tx, body_rx) = mpsc::channel(4);
let (high_tx, mut high_rx) = aether_runtime::bounded_queue(4);
let (normal_tx, _normal_rx) = aether_runtime::bounded_queue(4);
let sender = FrameSender::from_test_queues(high_tx, normal_tx);
let mut prepared = prepare_request_body(
7,
body_rx,
Arc::new(AtomicUsize::new(0)),
Instant::now() + Duration::from_secs(10),
false,
sender,
);
body_tx
.send(TunnelFrame::new(
7,
MsgType::RequestBody,
flags::END_STREAM,
Bytes::from_static(b"body"),
))
.await
.unwrap();
tokio::task::yield_now().await;
assert!(high_rx.try_recv().is_err());
let mut body = prepared.take_first_request_body();
assert!(body.frame().await.unwrap().is_ok());
assert_eq!(
high_rx.recv().await.unwrap().msg_type,
MsgType::WindowUpdate
);
assert!(body.frame().await.is_none());
}
#[tokio::test]
async fn dropping_prepared_body_cancels_its_spooler() {
let (body_tx, body_rx) = mpsc::channel(4);
let (high_tx, _high_rx) = aether_runtime::bounded_queue(4);
let (normal_tx, _normal_rx) = aether_runtime::bounded_queue(4);
let sender = FrameSender::from_test_queues(high_tx, normal_tx);
let prepared = prepare_request_body(
7,
body_rx,
Arc::new(AtomicUsize::new(0)),
Instant::now() + Duration::from_secs(3600),
false,
sender,
);
drop(prepared);
tokio::time::timeout(Duration::from_secs(1), body_tx.closed())
.await
.unwrap();
}
use std::collections::HashMap;
use std::net::SocketAddr;
use std::pin::Pin;
@@ -2379,7 +2555,7 @@ mod tests {
let (tx, rx) = mpsc::channel(4);
let (frame_tx, sent, writer_handle) = spawn_test_writer();
let body_size = Arc::new(AtomicUsize::new(0));
let prepared = prepare_request_body(
let mut prepared = prepare_request_body(
1,
rx,
Arc::clone(&body_size),
@@ -2389,6 +2565,7 @@ mod tests {
);
let mut body = prepared
.first_request_body
.take()
.expect("first request body should be present");
tx.send(TunnelFrame::new(
+52
View File
@@ -0,0 +1,52 @@
use std::future::Future;
use std::pin::Pin;
use std::task::{Context, Poll};
use tokio::task::{JoinError, JoinHandle};
pub(super) struct SessionTask<T>(JoinHandle<T>);
impl<T> SessionTask<T> {
pub(super) fn new(handle: JoinHandle<T>) -> Self {
Self(handle)
}
pub(super) fn abort(&self) {
self.0.abort();
}
pub(super) fn is_finished(&self) -> bool {
self.0.is_finished()
}
}
impl<T> Future for SessionTask<T> {
type Output = Result<T, JoinError>;
fn poll(mut self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll<Self::Output> {
Pin::new(&mut self.0).poll(context)
}
}
impl<T> Drop for SessionTask<T> {
fn drop(&mut self) {
self.0.abort();
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn dropping_a_session_task_aborts_its_child() {
let child = tokio::spawn(std::future::pending::<()>());
let abort = child.abort_handle();
drop(SessionTask::new(child));
tokio::time::timeout(std::time::Duration::from_secs(1), async {
while !abort.is_finished() {
tokio::task::yield_now().await;
}
})
.await
.unwrap();
}
}
+119 -5
View File
@@ -13,6 +13,7 @@ use aether_contracts::tunnel::{MsgType, HEADER_SIZE};
use aether_runtime::QueueSnapshot;
use aether_runtime::{bounded_queue, BoundedQueueSender, QueueSendError};
use futures_util::SinkExt;
use tokio::sync::watch;
use tokio::task::JoinHandle;
use tokio_tungstenite::tungstenite::Message;
use tracing::{debug, error, trace};
@@ -24,6 +25,8 @@ use aether_contracts::tunnel_security::SecureFrameCodec;
const HIGH_PRIORITY_QUEUE_CAPACITY: usize = 64;
const NORMAL_PRIORITY_QUEUE_CAPACITY: usize = 256;
const WRITE_TIMEOUT: Duration = Duration::from_secs(15);
const CLOSE_TIMEOUT: Duration = Duration::from_secs(1);
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum FramePriority {
@@ -43,9 +46,18 @@ pub struct FrameQueueSnapshots {
pub struct FrameSender {
high_tx: BoundedQueueSender<Frame>,
normal_tx: BoundedQueueSender<Frame>,
close_tx: watch::Sender<bool>,
}
impl FrameSender {
pub fn close(&self) {
let _ = self.close_tx.send(true);
}
pub(super) fn subscribe_close(&self) -> watch::Receiver<bool> {
self.close_tx.subscribe()
}
pub async fn send(&self, frame: Frame) -> Result<(), QueueSendError<Frame>> {
match classify_frame_priority(&frame) {
FramePriority::High => self.high_tx.send(frame).await,
@@ -73,7 +85,12 @@ impl FrameSender {
high_tx: BoundedQueueSender<Frame>,
normal_tx: BoundedQueueSender<Frame>,
) -> Self {
Self { high_tx, normal_tx }
let (close_tx, _) = watch::channel(false);
Self {
high_tx,
normal_tx,
close_tx,
}
}
}
@@ -113,15 +130,24 @@ where
{
let (high_tx, mut high_rx) = bounded_queue::<Frame>(HIGH_PRIORITY_QUEUE_CAPACITY);
let (normal_tx, mut normal_rx) = bounded_queue::<Frame>(NORMAL_PRIORITY_QUEUE_CAPACITY);
let tx = FrameSender { high_tx, normal_tx };
let (close_tx, mut close_rx) = watch::channel(false);
let tx = FrameSender {
high_tx,
normal_tx,
close_tx,
};
let handle = tokio::spawn(async move {
let mut ping_ticker = tokio::time::interval(ping_interval);
let mut high_open = true;
let mut normal_open = true;
let mut close_open = true;
ping_ticker.tick().await; // skip first immediate tick
loop {
if *close_rx.borrow() {
break;
}
if let Ok(frame) = high_rx.try_recv() {
if !write_frame(
&mut sink,
@@ -141,6 +167,10 @@ where
tokio::select! {
biased;
changed = close_rx.changed(), if close_open => {
if changed.is_err() { close_open = false; }
if *close_rx.borrow() { break; }
},
frame = high_rx.recv(), if high_open => {
match frame {
Some(frame) => {
@@ -152,7 +182,7 @@ where
}
}
_ = ping_ticker.tick(), if high_open || normal_open => {
if let Err(e) = sink.send(Message::Ping(vec![])).await {
if let Err(e) = send_message(&mut sink, Message::Ping(vec![])).await {
error!(error = %e, "failed to send WebSocket ping");
if let Some(metrics) = tunnel_metrics.as_deref() {
metrics.record_error("ws_ping_error", &e.to_string());
@@ -174,7 +204,7 @@ where
}
}
debug!("writer task exiting");
let _ = sink.close().await;
let _ = tokio::time::timeout(CLOSE_TIMEOUT, sink.close()).await;
});
(tx, handle)
@@ -228,7 +258,7 @@ where
None => frame.encode(),
};
let wire_len = data.len().max(HEADER_SIZE);
if let Err(e) = sink.send(Message::Binary(data.into())).await {
if let Err(e) = send_message(sink, Message::Binary(data.into())).await {
error!(
stream_id = stream_id,
msg_type = ?msg_type,
@@ -248,8 +278,92 @@ where
true
}
async fn send_message<S>(
sink: &mut S,
message: Message,
) -> Result<(), tokio_tungstenite::tungstenite::Error>
where
S: SinkExt<Message, Error = tokio_tungstenite::tungstenite::Error> + Unpin,
{
tokio::time::timeout(WRITE_TIMEOUT, sink.send(message))
.await
.map_err(|_| {
tokio_tungstenite::tungstenite::Error::Io(std::io::Error::new(
std::io::ErrorKind::TimedOut,
"tunnel WebSocket write timed out",
))
})?
}
#[cfg(test)]
mod tests {
#[tokio::test]
async fn dropping_last_sender_flushes_queued_body_and_end_frames() {
let sink = VecSink::default();
let sent = Arc::clone(&sink.sent);
let (sender, task) = spawn_writer(sink, Duration::from_secs(60));
sender
.send(Frame::new(
7,
MsgType::ResponseBody,
0,
bytes::Bytes::from_static(b"late"),
))
.await
.unwrap();
sender
.send(Frame::new(7, MsgType::StreamEnd, 0, bytes::Bytes::new()))
.await
.unwrap();
drop(sender);
task.await.unwrap();
let frames = sent.lock().unwrap();
assert_eq!(frames.len(), 2);
let Message::Binary(body) = &frames[0] else {
panic!("expected body")
};
assert_eq!(
Frame::decode(body.clone().into()).unwrap().payload,
b"late".as_slice()
);
}
struct StalledSink;
impl futures_util::Sink<Message> for StalledSink {
type Error = Error;
fn poll_ready(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<Result<(), Error>> {
Poll::Pending
}
fn start_send(self: Pin<&mut Self>, _: Message) -> Result<(), Error> {
Ok(())
}
fn poll_flush(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<Result<(), Error>> {
Poll::Pending
}
fn poll_close(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<Result<(), Error>> {
Poll::Pending
}
}
#[tokio::test(start_paused = true)]
async fn stalled_socket_write_and_close_are_bounded() {
let (sender, task) = spawn_writer(StalledSink, Duration::from_secs(60));
sender
.send(Frame::new(
1,
MsgType::ResponseBody,
0,
bytes::Bytes::from_static(b"data"),
))
.await
.unwrap();
tokio::time::timeout(Duration::from_secs(20), task)
.await
.expect("writer should time out")
.unwrap();
}
use std::pin::Pin;
use std::sync::{Arc, Mutex};
use std::task::{Context, Poll};
+62
View File
@@ -588,6 +588,68 @@ pub struct SettingsPayload {
pub drain_deadline_ms: u64,
}
impl SettingsPayload {
pub fn is_valid(&self) -> bool {
self.initial_stream_window_bytes > 0
&& u64::from(self.initial_stream_window_bytes)
<= MAX_TUNNEL_DECOMPRESSED_PAYLOAD_BYTES as u64
&& self.min_window_update_bytes > 0
&& self.min_window_update_bytes <= self.initial_stream_window_bytes
&& self.drain_deadline_ms > 0
}
pub fn negotiate(&self, initial_window_bytes: u32, drain_deadline_ms: u64) -> Self {
let window = self
.initial_stream_window_bytes
.min(initial_window_bytes)
.max(1);
Self {
initial_stream_window_bytes: window,
min_window_update_bytes: self.min_window_update_bytes.min((window / 4).max(1)),
drain_deadline_ms: self.drain_deadline_ms.min(drain_deadline_ms),
}
}
}
#[cfg(test)]
mod settings_tests {
use super::*;
#[test]
fn negotiation_bounds_window_updates_by_the_smaller_window() {
let settings = SettingsPayload {
initial_stream_window_bytes: 512 * 1024,
min_window_update_bytes: 128 * 1024,
drain_deadline_ms: 30_000,
};
let negotiated = settings.negotiate(4 * 1024 * 1024, 1000);
assert_eq!(negotiated.initial_stream_window_bytes, 512 * 1024);
assert_eq!(negotiated.min_window_update_bytes, 128 * 1024);
assert_eq!(negotiated.drain_deadline_ms, 1000);
assert!(negotiated.is_valid());
let tiny = settings.negotiate(1, 1);
assert_eq!(tiny.initial_stream_window_bytes, 1);
assert_eq!(tiny.min_window_update_bytes, 1);
assert!(tiny.is_valid());
}
#[test]
fn invalid_window_settings_are_rejected() {
let mut settings = SettingsPayload {
initial_stream_window_bytes: 1024,
min_window_update_bytes: 256,
drain_deadline_ms: 1,
};
settings.min_window_update_bytes = 1025;
assert!(!settings.is_valid());
settings.min_window_update_bytes = 0;
assert!(!settings.is_valid());
settings.initial_stream_window_bytes = u32::MAX;
settings.min_window_update_bytes = 1;
assert!(!settings.is_valid());
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct WindowUpdatePayload {
pub delta_bytes: u32,
@@ -0,0 +1,323 @@
<template>
<Card
variant="interactive"
class="flex min-w-0 flex-col cursor-pointer overflow-hidden"
@mousedown="$emit('mousedown', $event)"
@click="$emit('rowClick', $event, provider.id)"
>
<div class="flex items-start gap-2 p-4 pb-3">
<slot name="drag-handle" />
<div
class="flex h-10 w-10 shrink-0 items-center justify-center rounded-xl text-base font-semibold"
:class="provider.is_active ? 'bg-primary/10 text-primary' : 'bg-muted text-muted-foreground'"
aria-hidden="true"
>
{{ provider.name.slice(0, 1).toUpperCase() }}
</div>
<div class="min-w-0 flex-1 space-y-1">
<div class="flex items-center gap-1.5">
<button
type="button"
class="min-w-0 truncate rounded text-left text-sm font-semibold text-foreground hover:text-primary focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
:title="provider.name"
@click.stop="$emit('viewDetail', provider.id)"
>
{{ provider.name }}
</button>
<a
v-if="safeProviderWebsite"
:href="safeProviderWebsite"
target="_blank"
rel="noopener noreferrer"
class="shrink-0 text-muted-foreground transition-colors hover:text-primary"
:title="safeProviderWebsite"
@click.stop
>
<ExternalLink class="h-3.5 w-3.5" />
</a>
</div>
<div
v-if="editingDescriptionId === provider.id"
data-desc-editor
class="flex items-center gap-1"
@click.stop
>
<input
v-model="localDescriptionValue"
v-auto-focus
class="min-w-0 flex-1 rounded border border-border bg-background px-1.5 py-0.5 text-xs text-foreground focus:outline-none focus:ring-1 focus:ring-primary/50"
:placeholder="legacyT('输入备注...')"
:aria-label="legacyT('输入备注...')"
@keydown="handleDescriptionKeydown"
>
<button
type="button"
class="shrink-0 rounded p-0.5 text-primary hover:bg-muted"
:title="legacyT('保存')"
:aria-label="legacyT('保存')"
@click="handleSave"
>
<Check class="h-3.5 w-3.5" />
</button>
<button
type="button"
class="shrink-0 rounded p-0.5 text-muted-foreground hover:bg-muted"
:title="legacyT('取消')"
:aria-label="legacyT('取消')"
@click="handleCancel"
>
<X class="h-3.5 w-3.5" />
</button>
</div>
<button
v-else
type="button"
class="group/desc flex max-w-full items-center gap-1 rounded text-xs text-muted-foreground transition-colors hover:text-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
:title="provider.description || legacyT('添加备注')"
@click="handleStartEdit"
>
<span class="truncate">{{ provider.description || legacyT('添加备注') }}</span>
<Pencil class="h-3 w-3 shrink-0 opacity-0 transition-opacity group-hover/desc:opacity-60 group-focus-visible/desc:opacity-60" />
</button>
</div>
<Badge
:variant="provider.is_active ? 'success' : 'secondary'"
class="shrink-0 text-xs"
>
{{ legacyT(provider.is_active ? '活跃' : '停用') }}
</Badge>
</div>
<div class="flex flex-1 flex-col gap-4 px-4 pb-4">
<div class="space-y-2 rounded-xl border border-border/40 bg-muted/20 p-3">
<div class="flex flex-wrap items-center justify-between gap-2">
<span class="text-xs text-muted-foreground">{{ legacyT('余额监控') }}</span>
<Badge
variant="outline"
class="border-border/50 text-[10px] font-normal"
>
{{ formatBillingType(provider.billing_type || 'pay_as_you_go') }}
</Badge>
</div>
<ProviderBalanceCell
:provider="provider"
:is-balance-loading="isBalanceLoading"
:get-provider-balance="getProviderBalance"
:get-provider-balance-breakdown="getProviderBalanceBreakdown"
:get-provider-balance-error="getProviderBalanceError"
:get-provider-checkin="getProviderCheckin"
:get-provider-cookie-expired="getProviderCookieExpired"
:get-provider-balance-extra="getProviderBalanceExtra"
:format-balance-display="formatBalanceDisplay"
:format-reset-countdown="formatResetCountdown"
:get-quota-used-color-class="getQuotaUsedColorClass"
/>
</div>
<dl class="grid grid-cols-3 divide-x divide-border/50 text-center">
<div class="min-w-0 space-y-1 px-1">
<dt class="text-xs text-muted-foreground">
{{ legacyT('端点') }}
</dt>
<dd class="text-sm font-semibold tabular-nums">
{{ provider.active_endpoints }}<span class="ml-0.5 text-xs font-normal text-muted-foreground">/ {{ provider.total_endpoints }}</span>
</dd>
</div>
<div class="min-w-0 space-y-1 px-1">
<dt class="text-xs text-muted-foreground">
{{ legacyT(isKeyManagedProviderType(provider.provider_type) ? '密钥' : '账号') }}
</dt>
<dd class="text-sm font-semibold tabular-nums">
{{ provider.active_keys }}<span class="ml-0.5 text-xs font-normal text-muted-foreground">/ {{ provider.total_keys }}</span>
</dd>
</div>
<div class="min-w-0 space-y-1 px-1">
<dt class="text-xs text-muted-foreground">
{{ legacyT('模型') }}
</dt>
<dd class="text-sm font-semibold tabular-nums">
{{ provider.active_models }}<span class="ml-0.5 text-xs font-normal text-muted-foreground">/ {{ provider.total_models }}</span>
</dd>
</div>
</dl>
<div class="mt-auto space-y-2 border-t border-border/40 pt-3">
<div class="text-xs text-muted-foreground">
{{ legacyT('端点健康') }}
</div>
<div
v-if="provider.endpoint_health_details?.length"
class="grid grid-cols-3 gap-x-3 gap-y-2"
>
<div
v-for="endpoint in sortEndpoints(provider.endpoint_health_details)"
:key="endpoint.api_format"
class="flex min-w-0 flex-col gap-1.5"
:title="getEndpointTooltip(endpoint, locale)"
>
<div class="flex items-center justify-between gap-1 text-[10px] leading-none text-muted-foreground">
<span class="font-medium">{{ formatApiFormatShort(endpoint.api_format) }}</span>
<span class="tabular-nums">{{ getEndpointHealthLabel(endpoint) }}</span>
</div>
<div class="h-1.5 w-full overflow-hidden rounded-full bg-border dark:bg-border/80">
<div
class="h-full rounded-full transition-all duration-300"
:class="getEndpointDotColor(endpoint)"
:style="{ width: getEndpointHealthBarWidth(endpoint) }"
/>
</div>
</div>
</div>
<span
v-else
class="text-xs text-muted-foreground/60"
>{{ legacyT('暂无端点') }}</span>
</div>
</div>
<div
class="flex items-center justify-between gap-2 border-t border-border/40 bg-muted/10 px-3 py-2"
@click.stop
>
<Button
variant="ghost"
size="icon"
class="h-8 w-8 text-muted-foreground hover:text-primary"
:title="legacyT('查看详情')"
:aria-label="legacyT('查看详情')"
@click="$emit('viewDetail', provider.id)"
>
<Eye class="h-3.5 w-3.5" />
</Button>
<div class="flex shrink-0 items-center gap-0.5">
<Button
variant="ghost"
size="icon"
class="h-8 w-8 text-muted-foreground hover:text-foreground"
:title="legacyT('编辑提供商')"
:aria-label="legacyT('编辑提供商')"
@click="$emit('editProvider', provider)"
>
<Edit class="h-3.5 w-3.5" />
</Button>
<Button
variant="ghost"
size="icon"
class="h-8 w-8 text-muted-foreground hover:text-foreground"
:title="legacyT('扩展操作配置')"
:aria-label="legacyT('扩展操作配置')"
@click="$emit('openOpsConfig', provider)"
>
<KeyRound class="h-3.5 w-3.5" />
</Button>
<Button
variant="ghost"
size="icon"
class="h-8 w-8 text-muted-foreground hover:text-foreground"
:title="legacyT(provider.is_active ? '停用提供商' : '启用提供商')"
:aria-label="legacyT(provider.is_active ? '停用提供商' : '启用提供商')"
@click="$emit('toggleStatus', provider)"
>
<Power class="h-3.5 w-3.5" />
</Button>
<Button
variant="ghost"
size="icon"
class="h-8 w-8 text-muted-foreground hover:text-destructive"
:title="legacyT('删除提供商')"
:aria-label="legacyT('删除提供商')"
@click="$emit('deleteProvider', provider)"
>
<Trash2 class="h-3.5 w-3.5" />
</Button>
</div>
</div>
</Card>
</template>
<script setup lang="ts">
import { computed, ref, watch } from 'vue'
import { Check, Edit, ExternalLink, Eye, KeyRound, Pencil, Power, Trash2, X } from 'lucide-vue-next'
import Badge from '@/components/ui/badge.vue'
import Button from '@/components/ui/button.vue'
import Card from '@/components/ui/card.vue'
import ProviderBalanceCell from './ProviderBalanceCell.vue'
import { formatApiFormatShort, type ProviderWithEndpointsSummary } from '@/api/endpoints'
import type { BalanceExtraItem } from '@/features/providers/auth-templates'
import {
sortEndpoints,
getEndpointHealthLabel,
getEndpointHealthBarWidth,
getEndpointDotColor,
getEndpointTooltip,
} from '@/features/providers/composables/useEndpointStatus'
import { isKeyManagedProviderType } from '../utils/providerTypeUtils'
import { formatBillingType } from '@/utils/format'
import { safeExternalWebUrl } from '@/utils/navigationSecurity'
import { useI18n } from '@/i18n'
const props = defineProps<{
provider: ProviderWithEndpointsSummary
editingDescriptionId: string | null
isBalanceLoading: (providerId: string) => boolean
getProviderBalance: (providerId: string) => { available: number | null; currency: string } | null
getProviderBalanceBreakdown: (providerId: string) => { balance: number; points: number; currency: string } | null
getProviderBalanceError: (providerId: string) => { status: string; message: string } | null
getProviderCheckin: (providerId: string) => { success: boolean | null; message: string } | null
getProviderCookieExpired: (providerId: string) => { expired: boolean; message: string } | null
getProviderBalanceExtra: (providerId: string, architectureId?: string) => BalanceExtraItem[]
formatBalanceDisplay: (balance: { available: number | null; currency: string } | null) => string
formatResetCountdown: (resetsAt: number) => string
getQuotaUsedColorClass: (provider: ProviderWithEndpointsSummary) => string
}>()
const emit = defineEmits<{
'mousedown': [event: MouseEvent]
'rowClick': [event: MouseEvent, providerId: string]
'viewDetail': [providerId: string]
'editProvider': [provider: ProviderWithEndpointsSummary]
'openOpsConfig': [provider: ProviderWithEndpointsSummary]
'toggleStatus': [provider: ProviderWithEndpointsSummary]
'deleteProvider': [provider: ProviderWithEndpointsSummary]
'startEditDescription': [event: Event, provider: ProviderWithEndpointsSummary]
'saveDescription': [event: Event, provider: ProviderWithEndpointsSummary, value: string]
'cancelEditDescription': [event?: Event]
}>()
const { legacyT, locale } = useI18n()
const safeProviderWebsite = computed(() => safeExternalWebUrl(props.provider.website))
const localDescriptionValue = ref('')
const vAutoFocus = {
mounted: (element: HTMLElement) => element.focus(),
}
watch(() => props.editingDescriptionId, (providerId) => {
if (providerId === props.provider.id) {
localDescriptionValue.value = props.provider.description || ''
}
}, { immediate: true })
function handleStartEdit(event: Event) {
event.stopPropagation()
emit('startEditDescription', event, props.provider)
}
function handleSave(event: Event) {
event.stopPropagation()
emit('saveDescription', event, props.provider, localDescriptionValue.value)
}
function handleCancel(event: Event) {
event.stopPropagation()
emit('cancelEditDescription', event)
}
function handleDescriptionKeydown(event: KeyboardEvent) {
if (event.key === 'Enter') {
event.preventDefault()
handleSave(event)
} else if (event.key === 'Escape') {
handleCancel(event)
}
}
</script>
@@ -0,0 +1,36 @@
<template>
<Button
variant="ghost"
size="icon"
class="h-8 w-5 touch-none select-none cursor-grab text-muted-foreground/50 hover:text-primary active:cursor-grabbing disabled:cursor-default"
:disabled="disabled"
:title="legacyT('拖动调整当前页展示顺序,也可使用方向键移动')"
:aria-label="`${legacyT('调整展示顺序')}: ${providerName}`"
data-provider-drag-handle
@pointerdown.stop="$emit('pointerdown', $event)"
@keydown.stop="$emit('keydown', $event)"
@mousedown.stop
@click.stop
@dragstart.prevent
>
<GripVertical class="h-4 w-4" />
</Button>
</template>
<script setup lang="ts">
import { GripVertical } from 'lucide-vue-next'
import Button from '@/components/ui/button.vue'
import { useI18n } from '@/i18n'
defineProps<{
providerName: string
disabled: boolean
}>()
defineEmits<{
pointerdown: [event: PointerEvent]
keydown: [event: KeyboardEvent]
}>()
const { legacyT } = useI18n()
</script>
@@ -5,6 +5,7 @@
>
<!-- 第一行:名称 + 状态 + 操作 -->
<div class="flex items-start justify-between gap-3">
<slot name="drag-handle" />
<div class="flex-1 min-w-0 space-y-0.5">
<div class="flex items-center gap-1.5">
<span class="font-medium text-foreground truncate">{{ provider.name }}</span>
@@ -22,7 +22,7 @@
</div>
<!-- 状态筛选 -->
<div class="xl:hidden">
<div :class="{ 'xl:hidden': !cardView }">
<Select
:model-value="filterStatus"
@update:model-value="$emit('update:filterStatus', $event)"
@@ -43,7 +43,7 @@
</div>
<!-- API 格式筛选 -->
<div class="xl:hidden">
<div :class="{ 'xl:hidden': !cardView }">
<Select
:model-value="filterApiFormat"
@update:model-value="$emit('update:filterApiFormat', $event)"
@@ -64,7 +64,7 @@
</div>
<!-- 模型筛选 -->
<div class="xl:hidden">
<div :class="{ 'xl:hidden': !cardView }">
<Select
:model-value="filterModel"
@update:model-value="$emit('update:filterModel', $event)"
@@ -122,13 +122,32 @@
:loading="loading"
@click="$emit('refresh')"
/>
<Button
variant="ghost"
size="icon"
class="h-8 w-8"
:class="{ 'bg-primary/10 text-primary hover:bg-primary/15 hover:text-primary': cardView }"
:title="legacyT(cardView ? '切换到列表视图' : '切换到卡片视图')"
:aria-label="legacyT('卡片视图')"
:aria-pressed="cardView"
@click="$emit('toggleView')"
>
<List
v-if="cardView"
class="w-3.5 h-3.5"
/>
<LayoutGrid
v-else
class="w-3.5 h-3.5"
/>
</Button>
</div>
</div>
</div>
</template>
<script setup lang="ts">
import { Search, Plus, FilterX, Users } from 'lucide-vue-next'
import { Search, Plus, FilterX, Users, LayoutGrid, List } from 'lucide-vue-next'
import Button from '@/components/ui/button.vue'
import Input from '@/components/ui/input.vue'
import Select from '@/components/ui/select.vue'
@@ -150,6 +169,7 @@ defineProps<{
modelFilters: FilterOption[]
hasActiveFilters: boolean
loading: boolean
cardView: boolean
}>()
defineEmits<{
@@ -161,6 +181,7 @@ defineEmits<{
'batchProcess': []
'addProvider': []
'refresh': []
'toggleView': []
}>()
const { legacyT } = useI18n()
@@ -4,6 +4,13 @@
@mousedown="$emit('mousedown', $event)"
@click="$emit('rowClick', $event, provider.id)"
>
<TableCell
v-if="$slots['drag-handle']"
class="w-9 px-2 py-3.5"
@click.stop
>
<slot name="drag-handle" />
</TableCell>
<TableCell class="py-3.5">
<div class="space-y-0.5">
<div class="flex items-center gap-1.5">
@@ -5,6 +5,7 @@ import type { ProviderWithEndpointsSummary } from '@/api/endpoints'
import { createI18n } from '@/i18n'
import ProviderTableRow from '../ProviderTableRow.vue'
import ProviderMobileCard from '../ProviderMobileCard.vue'
import ProviderCard from '../ProviderCard.vue'
vi.mock('../ProviderBalanceCell.vue', () => ({
default: { render: () => null },
@@ -74,6 +75,7 @@ function mountProvider(component: Component, healthScore: number | null) {
describe.each([
['desktop provider row', ProviderTableRow],
['mobile provider card', ProviderMobileCard],
['grid provider card', ProviderCard],
] as const)('%s endpoint health', (_name, component) => {
it.each([
{ score: null, label: '-', width: '100%', color: 'bg-muted-foreground/40' },
@@ -0,0 +1,224 @@
import { computed, nextTick, onScopeDispose, ref, watch, type Ref } from 'vue'
import { useEventListener, useLocalStorage, useRafFn } from '@vueuse/core'
import { useI18n } from '@/i18n'
interface SortableProvider {
id: string
name: string
}
interface ProviderPointerDrag {
providerId: string
pointerId: number
startX: number
startY: number
handle: HTMLElement
scrollContainer: HTMLElement | null
}
export function useProviderDisplayOrder<Provider extends SortableProvider>(
providers: () => Provider[],
container: Ref<HTMLElement | null>,
) {
const { legacyT } = useI18n()
const savedOrder = useLocalStorage<string[]>('aether-provider-display-order', [])
const knownOrder = ref<string[]>([])
const draggingProviderId = ref<string | null>(null)
const dropTargetId = ref<string | null>(null)
const pointerPosition = ref({ clientX: 0, clientY: 0 })
const announcement = ref('')
let pointerDrag: ProviderPointerDrag | null = null
let suppressClickUntil = 0
const normalizedOrder = computed(() => Array.isArray(savedOrder.value)
? [...new Set(savedOrder.value.filter((providerId): providerId is string => typeof providerId === 'string'))]
: [])
const orderedProviders = computed(() => {
const ranks = new Map(normalizedOrder.value.map((providerId, index) => [providerId, index]))
return [...providers()].sort((first, second) => (
(ranks.get(first.id) ?? ranks.size) - (ranks.get(second.id) ?? ranks.size)
))
})
const draggingProvider = computed(() => orderedProviders.value.find(provider => provider.id === draggingProviderId.value))
const dragPreviewStyle = computed(() => ({
left: `${Math.max(8, Math.min(pointerPosition.value.clientX + 12, window.innerWidth - 208))}px`,
top: `${Math.max(8, Math.min(pointerPosition.value.clientY + 12, window.innerHeight - 48))}px`,
}))
function moveProvider(providerId: string, targetId: string) {
const visibleIds = orderedProviders.value.map(provider => provider.id)
const sourceIndex = visibleIds.indexOf(providerId)
const targetIndex = visibleIds.indexOf(targetId)
if (sourceIndex < 0 || targetIndex < 0 || sourceIndex === targetIndex) return
visibleIds.splice(sourceIndex, 1)
visibleIds.splice(targetIndex, 0, providerId)
const visibleSet = new Set(visibleIds)
const allIds = [...new Set([...normalizedOrder.value, ...knownOrder.value, ...visibleIds])]
let visibleIndex = 0
savedOrder.value = allIds.map(currentId => visibleSet.has(currentId) ? visibleIds[visibleIndex++] ?? currentId : currentId)
announcement.value = `${legacyT('展示顺序已更新')}: ${orderedProviders.value[targetIndex]?.name} (${targetIndex + 1}/${visibleIds.length})`
}
function updateDropTarget() {
const target = document.elementFromPoint(pointerPosition.value.clientX, pointerPosition.value.clientY)
?.closest<HTMLElement>('[data-provider-sort-id]')
const targetId = target?.dataset.providerSortId
dropTargetId.value = target && container.value?.contains(target)
&& targetId !== draggingProviderId.value
&& orderedProviders.value.some(provider => provider.id === targetId)
? targetId ?? null
: null
}
function findScrollContainer(handle: HTMLElement): HTMLElement | null {
let ancestor = handle.parentElement
while (ancestor && ancestor !== document.body) {
if (/(auto|scroll)/.test(getComputedStyle(ancestor).overflowY) && ancestor.scrollHeight > ancestor.clientHeight) {
return ancestor
}
ancestor = ancestor.parentElement
}
return document.scrollingElement as HTMLElement | null
}
const { pause, resume } = useRafFn(() => {
const scrollContainer = pointerDrag?.scrollContainer
if (scrollContainer) {
const bounds = scrollContainer.getBoundingClientRect()
const isDocument = scrollContainer === document.scrollingElement
const top = isDocument ? 0 : Math.max(0, bounds.top)
const bottom = isDocument ? window.innerHeight : Math.min(window.innerHeight, bounds.bottom)
const pointerY = pointerPosition.value.clientY
const pointerX = pointerPosition.value.clientX
if (isDocument || (pointerX >= bounds.left && pointerX <= bounds.right)) {
if (pointerY < top + 48) {
scrollContainer.scrollTop -= Math.min(12, (top + 48 - pointerY) / 4)
} else if (pointerY > bottom - 48) {
scrollContainer.scrollTop += Math.min(12, (pointerY - bottom + 48) / 4)
}
}
}
updateDropTarget()
}, { immediate: false })
function cancelDrag() {
const previous = pointerDrag
pointerDrag = null
pause()
if (draggingProviderId.value) suppressClickUntil = Date.now() + 250
draggingProviderId.value = null
dropTargetId.value = null
if (previous?.handle.hasPointerCapture?.(previous.pointerId)) {
previous.handle.releasePointerCapture(previous.pointerId)
}
}
function startDrag(providerId: string, event: PointerEvent) {
if (event.button !== 0 || event.isPrimary === false || orderedProviders.value.length < 2) return
if (!orderedProviders.value.some(provider => provider.id === providerId)) return
const handle = event.currentTarget
if (!(handle instanceof HTMLElement)) return
cancelDrag()
event.preventDefault()
handle.focus({ preventScroll: true })
pointerDrag = {
providerId,
pointerId: event.pointerId,
startX: event.clientX,
startY: event.clientY,
handle,
scrollContainer: findScrollContainer(handle),
}
pointerPosition.value = { clientX: event.clientX, clientY: event.clientY }
handle.setPointerCapture?.(event.pointerId)
}
function handlePointerMove(event: PointerEvent) {
if (!pointerDrag || pointerDrag.pointerId !== event.pointerId) return
pointerPosition.value = { clientX: event.clientX, clientY: event.clientY }
if (!draggingProviderId.value) {
if (Math.hypot(event.clientX - pointerDrag.startX, event.clientY - pointerDrag.startY) < 5) return
draggingProviderId.value = pointerDrag.providerId
resume()
}
event.preventDefault()
updateDropTarget()
}
function handlePointerUp(event: PointerEvent) {
if (!pointerDrag || pointerDrag.pointerId !== event.pointerId) return
const providerId = draggingProviderId.value
if (providerId) {
pointerPosition.value = { clientX: event.clientX, clientY: event.clientY }
updateDropTarget()
}
const targetId = dropTargetId.value
cancelDrag()
if (providerId && targetId) moveProvider(providerId, targetId)
}
function handleSortKeydown(providerId: string, event: KeyboardEvent) {
if (event.key === 'Escape') {
cancelDrag()
return
}
const directions: Record<string, number> = { ArrowUp: -1, ArrowLeft: -1, ArrowDown: 1, ArrowRight: 1 }
const direction = directions[event.key]
if (direction === undefined || pointerDrag) return
event.preventDefault()
const index = orderedProviders.value.findIndex(provider => provider.id === providerId)
const target = orderedProviders.value[index + direction]
if (index < 0 || !target) return
const handle = event.currentTarget as HTMLElement
moveProvider(providerId, target.id)
void nextTick(() => handle.focus({ preventScroll: true }))
}
function handleSortClick(event: MouseEvent) {
if (!draggingProviderId.value && Date.now() >= suppressClickUntil) return
suppressClickUntil = 0
event.preventDefault()
event.stopPropagation()
}
function sortItemClass(providerId: string) {
return {
'opacity-40': draggingProviderId.value === providerId,
'ring-2 ring-inset ring-primary/60 bg-primary/5': dropTargetId.value === providerId,
}
}
watch(() => providers().map(provider => provider.id), (providerIds) => {
knownOrder.value = [...new Set([...knownOrder.value, ...providerIds])]
cancelDrag()
}, { immediate: true })
useEventListener(window, 'pointermove', handlePointerMove, { passive: false })
useEventListener(window, 'pointerup', handlePointerUp)
useEventListener(window, 'pointercancel', (event) => {
if (pointerDrag?.pointerId === event.pointerId) cancelDrag()
})
useEventListener(window, 'lostpointercapture', (event) => {
if (pointerDrag?.pointerId === event.pointerId) cancelDrag()
})
useEventListener(window, 'blur', cancelDrag)
useEventListener(window, 'keydown', (event) => {
if (event.key === 'Escape') cancelDrag()
})
onScopeDispose(cancelDrag)
return {
orderedProviders,
draggingProvider,
dragPreviewStyle,
announcement,
startDrag,
cancelDrag,
handleSortKeydown,
handleSortClick,
sortItemClass,
}
}
+6
View File
@@ -1734,6 +1734,12 @@ const legacyExactEnglishMessages: Record<string, string> = {
'启用提供商': 'Enable provider',
'停用提供商': 'Disable provider',
'扩展操作配置': 'Ops configuration',
'卡片视图': 'Card view',
'切换到卡片视图': 'Switch to card view',
'切换到列表视图': 'Switch to list view',
'拖动调整当前页展示顺序,也可使用方向键移动': 'Drag to reorder the current page, or use the arrow keys',
'调整展示顺序': 'Adjust display order',
'展示顺序已更新': 'Display order updated',
'输入备注...': 'Enter notes...',
'添加备注': 'Add notes',
'配额': 'Quota',
+117 -5
View File
@@ -1,5 +1,10 @@
<template>
<div class="space-y-4">
<div
ref="providerListRef"
class="space-y-4"
:class="{ 'select-none [&_*]:!cursor-grabbing': draggingProvider }"
@click.capture="handleSortClick"
>
<ProviderDeleteProgressCard
:progress="providerDeleteProgress"
:stage-label="providerDeleteStageLabel"
@@ -25,6 +30,7 @@
:model-filters="modelFilters"
:has-active-filters="hasActiveFilters"
:loading="loading"
:card-view="cardView"
@update:search-query="searchQuery = $event"
@update:filter-status="filterStatus = $event"
@update:filter-api-format="filterApiFormat = $event"
@@ -33,6 +39,7 @@
@batch-process="openProviderBatchDialog"
@add-provider="openAddProviderDialog"
@refresh="loadProviders"
@toggle-view="cardView = !cardView"
/>
<!-- 加载状态 -->
@@ -54,6 +61,50 @@
/>
</div>
<div
v-else-if="cardView"
class="grid grid-cols-1 gap-4 p-4 sm:p-6 md:grid-cols-2 2xl:grid-cols-3"
>
<ProviderCard
v-for="provider in displayedProviders"
:key="provider.id"
:provider="provider"
:data-provider-sort-id="provider.id"
:class="sortItemClass(provider.id)"
:editing-description-id="editingDescriptionId"
:is-balance-loading="isBalanceLoading"
:get-provider-balance="getProviderBalance"
:get-provider-balance-breakdown="getProviderBalanceBreakdown"
:get-provider-balance-error="getProviderBalanceError"
:get-provider-checkin="getProviderCheckin"
:get-provider-cookie-expired="getProviderCookieExpired"
:get-provider-balance-extra="getProviderBalanceExtra"
:format-balance-display="formatBalanceDisplay"
:format-reset-countdown="formatResetCountdown"
:get-quota-used-color-class="getQuotaUsedColorClass"
@mousedown="handleMouseDown"
@row-click="handleRowClick"
@view-detail="openProviderDrawer"
@edit-provider="openEditProviderDialog"
@open-ops-config="openOpsConfigDialog"
@toggle-status="toggleProviderStatus"
@delete-provider="handleDeleteProvider"
@start-edit-description="startEditDescription"
@save-description="saveDescription"
@cancel-edit-description="cancelEditDescription"
>
<template #drag-handle>
<ProviderDragHandle
class="-ml-2 h-10 w-4"
:provider-name="provider.name"
:disabled="loading || displayedProviders.length < 2"
@pointerdown="startDrag(provider.id, $event)"
@keydown="handleSortKeydown(provider.id, $event)"
/>
</template>
</ProviderCard>
</div>
<!-- 桌面端表格 -->
<div
v-else
@@ -62,6 +113,9 @@
<Table>
<TableHeader>
<TableRow>
<TableHead class="w-9 px-2">
<span class="sr-only">{{ legacyT('调整展示顺序') }}</span>
</TableHead>
<TableHead class="w-[18%] min-w-[140px]">
{{ legacyT('提供商信息') }}
</TableHead>
@@ -131,6 +185,8 @@
v-for="provider in displayedProviders"
:key="provider.id"
:provider="provider"
:data-provider-sort-id="provider.id"
:class="sortItemClass(provider.id)"
:editing-description-id="editingDescriptionId"
:is-balance-loading="isBalanceLoading"
:get-provider-balance="getProviderBalance"
@@ -152,20 +208,31 @@
@start-edit-description="startEditDescription"
@save-description="saveDescription"
@cancel-edit-description="cancelEditDescription"
/>
>
<template #drag-handle>
<ProviderDragHandle
:provider-name="provider.name"
:disabled="loading || displayedProviders.length < 2"
@pointerdown="startDrag(provider.id, $event)"
@keydown="handleSortKeydown(provider.id, $event)"
/>
</template>
</ProviderTableRow>
</TableBody>
</Table>
</div>
<!-- 移动端卡片列表 -->
<div
v-if="!loading && providers.length > 0"
v-if="!cardView && !loading && providers.length > 0"
class="xl:hidden divide-y divide-border/40"
>
<ProviderMobileCard
v-for="provider in displayedProviders"
:key="provider.id"
:provider="provider"
:data-provider-sort-id="provider.id"
:class="sortItemClass(provider.id)"
:editing-description-id="editingDescriptionId"
:is-balance-loading="isBalanceLoading"
:get-provider-balance="getProviderBalance"
@@ -182,7 +249,17 @@
@start-edit-description="startEditDescription"
@save-description="saveDescription"
@cancel-edit-description="cancelEditDescription"
/>
>
<template #drag-handle>
<ProviderDragHandle
class="-ml-2 w-4"
:provider-name="provider.name"
:disabled="loading || displayedProviders.length < 2"
@pointerdown="startDrag(provider.id, $event)"
@keydown="handleSortKeydown(provider.id, $event)"
/>
</template>
</ProviderMobileCard>
</div>
<!-- 分页 -->
@@ -196,8 +273,25 @@
@update:page-size="pageSize = $event"
/>
</Card>
<span
class="sr-only"
role="status"
aria-live="polite"
aria-atomic="true"
>{{ announcement }}</span>
</div>
<Teleport to="body">
<div
v-if="draggingProvider"
class="pointer-events-none fixed z-[100] w-[200px] truncate rounded-xl border border-primary/40 bg-card px-3 py-2 text-sm font-medium text-foreground shadow-lg"
:style="dragPreviewStyle"
aria-hidden="true"
>
{{ draggingProvider.name }}
</div>
</Teleport>
<!-- 对话框 -->
<ProviderFormDialog
v-model="providerDialogOpen"
@@ -234,6 +328,7 @@
<script setup lang="ts">
import { ref, computed, watch, onMounted, onUnmounted, defineAsyncComponent } from 'vue'
import { useLocalStorage } from '@vueuse/core'
import Card from '@/components/ui/card.vue'
import Table from '@/components/ui/table.vue'
import TableHeader from '@/components/ui/table-header.vue'
@@ -248,6 +343,8 @@ import ProviderBatchActionDialog from '@/features/providers/components/ProviderB
import ProviderTableHeader from '@/features/providers/components/ProviderTableHeader.vue'
import ProviderTableRow from '@/features/providers/components/ProviderTableRow.vue'
import ProviderMobileCard from '@/features/providers/components/ProviderMobileCard.vue'
import ProviderCard from '@/features/providers/components/ProviderCard.vue'
import ProviderDragHandle from '@/features/providers/components/ProviderDragHandle.vue'
import ProviderDeleteProgressCard from '@/features/providers/components/ProviderDeleteProgressCard.vue'
import ProviderEmptyState from '@/features/providers/components/ProviderEmptyState.vue'
import { useToast } from '@/composables/useToast'
@@ -255,6 +352,7 @@ import { useConfirm } from '@/composables/useConfirm'
import { useRowClick } from '@/composables/useRowClick'
import { useProviderFilters } from '@/features/providers/composables/useProviderFilters'
import { useProviderBalance } from '@/features/providers/composables/useProviderBalance'
import { useProviderDisplayOrder } from '@/features/providers/composables/useProviderDisplayOrder'
import {
getProvidersSummary,
getProvider,
@@ -294,6 +392,7 @@ function showLegacyError(err: unknown, fallback: string, title = '错误') {
// 状态
const loading = ref(false)
const cardView = useLocalStorage('aether-provider-card-view', false, { flush: 'sync' })
const providers = ref<ProviderWithEndpointsSummary[]>([])
let providersRequestId = 0
const providerDialogOpen = ref(false)
@@ -475,7 +574,20 @@ function sortProvidersByActiveAndPriority(items: ProviderWithEndpointsSummary[])
})
}
const displayedProviders = computed(() => sortProvidersByActiveAndPriority(providers.value))
const providerListRef = ref<HTMLElement | null>(null)
const {
orderedProviders: displayedProviders,
draggingProvider,
dragPreviewStyle,
announcement,
startDrag,
cancelDrag,
handleSortKeydown,
handleSortClick,
sortItemClass,
} = useProviderDisplayOrder(() => sortProvidersByActiveAndPriority(providers.value), providerListRef)
watch([loading, cardView, queryParams], cancelDrag)
function startEditDescription(_event: Event, provider: ProviderWithEndpointsSummary) {
editingDescriptionId.value = provider.id
@@ -0,0 +1,539 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { createApp, nextTick, type App } from 'vue'
import type { ProviderWithEndpointsSummary } from '@/api/endpoints'
import { createI18n, setI18nLocale } from '@/i18n'
import ProviderManagement from '../ProviderManagement.vue'
const apiMocks = vi.hoisted(() => ({
getProvidersSummary: vi.fn(),
getGlobalModels: vi.fn(),
getProvider: vi.fn(),
updateProvider: vi.fn(),
}))
vi.mock('@/api/endpoints', async (importOriginal) => ({
...await importOriginal<typeof import('@/api/endpoints')>(),
...apiMocks,
}))
vi.mock('@/composables/useConfirm', () => ({
useConfirm: () => ({ confirmDanger: vi.fn().mockResolvedValue(false) }),
}))
vi.mock('@/composables/useToast', () => ({
useToast: () => ({ success: vi.fn(), error: vi.fn(), info: vi.fn() }),
}))
vi.mock('@/features/providers/composables/useProviderBalance', () => ({
useProviderBalance: () => ({
loadArchitectureSchemas: vi.fn(),
loadBalances: vi.fn(),
getProviderBalance: () => ({ available: 125, currency: 'USD' }),
getProviderBalanceBreakdown: () => null,
getProviderBalanceError: () => null,
isBalanceLoading: () => false,
getProviderCheckin: () => null,
getProviderCookieExpired: () => null,
formatBalanceDisplay: () => '$125.00',
formatResetCountdown: () => '',
getProviderBalanceExtra: () => [],
getQuotaUsedColorClass: () => '',
startTick: vi.fn(),
stopTick: vi.fn(),
}),
}))
vi.mock('@/features/providers/components', () => ({
ProviderFormDialog: { render: () => null },
ProviderAuthDialog: { render: () => null },
}))
vi.mock('@/features/providers/components/ProviderBatchActionDialog.vue', () => ({
default: { render: () => null },
}))
vi.mock('@/features/providers/components/ProviderDetailDrawer.vue', async () => {
const { h } = await import('vue')
return {
__esModule: true,
default: {
props: ['open', 'providerId'],
setup: (props: { open: boolean; providerId: string }) => () => props.open
? h('div', { 'data-provider-detail': props.providerId })
: null,
},
}
})
function createProvider(overrides: Partial<ProviderWithEndpointsSummary> = {}): ProviderWithEndpointsSummary {
return {
id: 'provider-1',
name: 'Provider One',
description: 'Primary provider',
provider_type: 'custom',
provider_priority: 10,
keep_priority_on_conversion: false,
enable_format_conversion: true,
is_active: true,
total_endpoints: 2,
active_endpoints: 1,
total_keys: 3,
active_keys: 2,
total_models: 4,
active_models: 3,
global_model_ids: ['model-1'],
avg_health_score: 0.8,
unhealthy_endpoints: 0,
api_formats: ['openai:chat'],
endpoint_health_details: [{
api_format: 'openai:chat',
health_score: 0.8,
is_active: true,
total_keys: 3,
active_keys: 2,
}],
ops_configured: true,
created_at: '2026-09-07T00:00:00Z',
updated_at: '2026-09-07T00:00:00Z',
...overrides,
}
}
let mountedApp: App | null = null
let mountedRoot: HTMLElement | null = null
async function settle() {
for (let index = 0; index < 8; index += 1) {
await Promise.resolve()
await nextTick()
}
}
async function mountView() {
const root = document.createElement('div')
document.body.appendChild(root)
mountedRoot = root
mountedApp = createApp(ProviderManagement)
mountedApp.use(createI18n())
mountedApp.mount(root)
await settle()
return root
}
function unmountView() {
mountedApp?.unmount()
mountedRoot?.remove()
mountedApp = null
mountedRoot = null
}
function findButton(root: HTMLElement, title: string): HTMLButtonElement {
const button = root.querySelector<HTMLButtonElement>(`button[title="${title}"]`)
expect(button, `Missing button: ${title}`).not.toBeNull()
return button!
}
beforeEach(() => {
vi.clearAllMocks()
apiMocks.getProvidersSummary.mockResolvedValue({
items: [createProvider()],
total: 40,
})
apiMocks.getGlobalModels.mockResolvedValue({ models: [{ id: 'model-1', name: 'Model One' }] })
apiMocks.getProvider.mockResolvedValue(createProvider())
apiMocks.updateProvider.mockResolvedValue(createProvider({ is_active: false }))
})
const originalElementFromPoint = Object.getOwnPropertyDescriptor(document, 'elementFromPoint')
afterEach(() => {
unmountView()
if (originalElementFromPoint) {
Object.defineProperty(document, 'elementFromPoint', originalElementFromPoint)
} else {
Reflect.deleteProperty(document, 'elementFromPoint')
}
})
describe('ProviderManagement card view', () => {
it('places the view toggle immediately after refresh and switches layouts without reloading data', async () => {
const root = await mountView()
const toggle = findButton(root, '切换到卡片视图')
const filters = [...root.querySelectorAll('[role="combobox"]')].slice(0, 3)
expect(toggle.previousElementSibling).toBe(findButton(root, '刷新'))
expect(toggle.getAttribute('aria-pressed')).toBe('false')
expect(root.querySelector('table')).not.toBeNull()
expect(root.querySelector('dl')).toBeNull()
for (const filter of filters) {
expect(filter.closest('.xl\\:hidden')).not.toBeNull()
}
const requests = apiMocks.getProvidersSummary.mock.calls.length
toggle.click()
await settle()
expect(root.querySelector('table')).toBeNull()
expect(root.querySelectorAll('dl')).toHaveLength(1)
expect(root.textContent).toContain('$125.00')
expect(root.textContent).toContain('Primary provider')
expect(root.textContent).toContain('80%')
expect(toggle.getAttribute('aria-pressed')).toBe('true')
for (const filter of filters) {
expect(filter.closest('.xl\\:hidden')).toBeNull()
}
findButton(root, '切换到列表视图').click()
await settle()
expect(root.querySelector('table')).not.toBeNull()
expect(root.querySelector('dl')).toBeNull()
expect(apiMocks.getProvidersSummary).toHaveBeenCalledTimes(requests)
})
it('remembers the chosen layout across remounts', async () => {
let root = await mountView()
findButton(root, '切换到卡片视图').click()
await settle()
expect(localStorage.getItem('aether-provider-card-view')).toBe('true')
unmountView()
root = await mountView()
expect(root.querySelector('table')).toBeNull()
expect(findButton(root, '切换到列表视图').getAttribute('aria-pressed')).toBe('true')
findButton(root, '切换到列表视图').click()
await settle()
expect(localStorage.getItem('aether-provider-card-view')).toBe('false')
})
it.each([
{ name: 'cards', initial: false, selected: true, title: '切换到卡片视图' },
{ name: 'table', initial: true, selected: false, title: '切换到列表视图' },
])('immediately saves $name and restores it while reloading data', async ({ initial, selected, title }) => {
localStorage.setItem('aether-provider-card-view', String(initial))
let root = await mountView()
findButton(root, title).click()
expect(localStorage.getItem('aether-provider-card-view')).toBe(String(selected))
unmountView()
let finishLoading!: (value: { items: ProviderWithEndpointsSummary[]; total: number }) => void
apiMocks.getProvidersSummary.mockReturnValue(new Promise((resolve) => {
finishLoading = resolve
}))
root = await mountView()
const restoredTitle = selected ? '切换到列表视图' : '切换到卡片视图'
expect(findButton(root, restoredTitle).getAttribute('aria-pressed')).toBe(String(selected))
expect(findButton(root, '刷新').disabled).toBe(true)
finishLoading({ items: [createProvider()], total: 1 })
await settle()
expect(root.querySelector('table') === null).toBe(selected)
expect(root.querySelector('dl') !== null).toBe(selected)
expect(localStorage.getItem('aether-provider-card-view')).toBe(String(selected))
})
it('keeps the current search and page when switching views', async () => {
const root = await mountView()
const search = root.querySelector<HTMLInputElement>('#provider-search')!
search.value = 'Provider'
search.dispatchEvent(new Event('input', { bubbles: true }))
await vi.waitFor(() => {
expect(apiMocks.getProvidersSummary).toHaveBeenLastCalledWith(
expect.objectContaining({ search: 'Provider' }),
expect.any(Object),
)
})
const secondPage = [...root.querySelectorAll<HTMLButtonElement>('button')]
.find(button => button.textContent?.trim() === '2')!
secondPage.click()
await settle()
const requests = apiMocks.getProvidersSummary.mock.calls.length
findButton(root, '切换到卡片视图').click()
await settle()
expect(search.value).toBe('Provider')
expect(root.querySelector('[aria-current="page"]')?.textContent?.trim()).toBe('2')
expect(apiMocks.getProvidersSummary).toHaveBeenCalledTimes(requests)
expect(apiMocks.getProvidersSummary).toHaveBeenLastCalledWith(
expect.objectContaining({ search: 'Provider', page: 2 }),
expect.any(Object),
)
})
it('supports note editing, status actions, and details from cards', async () => {
localStorage.setItem('aether-provider-card-view', 'true')
const root = await mountView()
findButton(root, 'Primary provider').click()
await settle()
const input = root.querySelector<HTMLInputElement>('[data-desc-editor] input')!
expect(input.value).toBe('Primary provider')
input.value = 'Updated note'
input.dispatchEvent(new Event('input', { bubbles: true }))
input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true }))
await settle()
expect(apiMocks.updateProvider).toHaveBeenCalledWith('provider-1', { description: 'Updated note' })
expect(root.textContent).toContain('Updated note')
expect(root.querySelector('[data-provider-detail]')).toBeNull()
findButton(root, '停用提供商').click()
await settle()
expect(apiMocks.updateProvider).toHaveBeenCalledWith('provider-1', { is_active: false })
expect(root.querySelector('[data-provider-detail]')).toBeNull()
findButton(root, 'Provider One').click()
await vi.waitFor(() => {
expect(root.querySelector('[data-provider-detail="provider-1"]')).not.toBeNull()
})
})
it('uses account labels and handles providers without endpoints', async () => {
localStorage.setItem('aether-provider-card-view', 'true')
apiMocks.getProvidersSummary.mockResolvedValue({
items: [createProvider({ provider_type: 'codex', is_active: false, endpoint_health_details: [] })],
total: 1,
})
const root = await mountView()
expect(root.querySelector('dl')?.textContent).toContain('账号')
expect(root.textContent).toContain('暂无端点')
expect(findButton(root, '启用提供商')).not.toBeNull()
})
it('does not display cards during loading or with an empty result', async () => {
localStorage.setItem('aether-provider-card-view', 'true')
let resolveRequest!: (value: { items: ProviderWithEndpointsSummary[]; total: number }) => void
apiMocks.getProvidersSummary.mockReturnValue(new Promise((resolve) => {
resolveRequest = resolve
}))
const root = await mountView()
expect(root.querySelector('dl')).toBeNull()
expect(findButton(root, '刷新').disabled).toBe(true)
resolveRequest({ items: [], total: 0 })
await settle()
expect(root.querySelector('dl')).toBeNull()
expect(root.textContent).toContain('暂无提供商,点击右上角添加')
expect(findButton(root, '刷新').disabled).toBe(false)
})
it('translates the view switch labels', async () => {
const root = await mountView()
setI18nLocale('en-US')
await settle()
expect(findButton(root, 'Switch to card view').getAttribute('aria-label')).toBe('Card view')
findButton(root, 'Switch to card view').click()
await settle()
expect(findButton(root, 'Switch to list view').getAttribute('aria-pressed')).toBe('true')
})
})
function mockSortableProviders() {
const providers = [1, 2, 3, 4].map(index => createProvider({
id: `provider-${index}`,
name: `Provider ${index}`,
provider_priority: index * 10,
}))
apiMocks.getProvidersSummary.mockResolvedValue({ items: providers, total: providers.length })
return providers
}
function providerElements(root: HTMLElement): HTMLElement[] {
const container = root.querySelector('table') ?? root
return [...container.querySelectorAll<HTMLElement>('[data-provider-sort-id]')]
}
function providerOrder(root: HTMLElement): string[] {
return providerElements(root).map(element => element.dataset.providerSortId!)
}
function pointerEvent(type: string, clientX: number, clientY: number, options: { button?: number; pointerType?: string } = {}) {
const event = new MouseEvent(type, { bubbles: true, cancelable: true, clientX, clientY, button: options.button ?? 0 })
Object.defineProperties(event, {
pointerId: { value: 1 },
isPrimary: { value: true },
pointerType: { value: options.pointerType ?? 'mouse' },
})
return event
}
function startProviderDrag(root: HTMLElement, sourceId: string, targetId: string, pointerType = 'mouse') {
const elements = providerElements(root)
const handle = elements.find(element => element.dataset.providerSortId === sourceId)!
.querySelector<HTMLButtonElement>('[data-provider-drag-handle]')!
const target = elements.find(element => element.dataset.providerSortId === targetId)!
const hitTest = vi.fn((): Element | null => target)
Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: hitTest })
handle.dispatchEvent(pointerEvent('pointerdown', 40, 100, { pointerType }))
window.dispatchEvent(pointerEvent('pointermove', 100, 200, { pointerType }))
return { handle, target, hitTest }
}
async function dropProvider(handle: HTMLButtonElement) {
window.dispatchEvent(pointerEvent('pointerup', 100, 200))
handle.click()
await settle()
}
describe('ProviderManagement shared display order', () => {
it('drags table rows, synchronizes both card layouts, and leaves scheduling priorities unchanged', async () => {
const providers = mockSortableProviders()
const root = await mountView()
const { handle, target } = startProviderDrag(root, 'provider-1', 'provider-3')
await settle()
expect(target.classList.contains('ring-2')).toBe(true)
expect(providerElements(root)[0]?.classList.contains('opacity-40')).toBe(true)
await dropProvider(handle)
const expected = ['provider-2', 'provider-3', 'provider-1', 'provider-4']
expect(providerOrder(root)).toEqual(expected)
const mobileOrder = [...root.querySelectorAll<HTMLElement>('[data-provider-sort-id]')]
.filter(element => !element.closest('table'))
.map(element => element.dataset.providerSortId)
expect(mobileOrder).toEqual(expected)
expect(root.querySelector('[data-provider-detail]')).toBeNull()
expect(apiMocks.updateProvider).not.toHaveBeenCalled()
expect(providers.map(provider => provider.provider_priority)).toEqual([10, 20, 30, 40])
findButton(root, '切换到卡片视图').click()
await settle()
expect(providerOrder(root)).toEqual(expected)
expect(apiMocks.getProvidersSummary).toHaveBeenCalledTimes(1)
})
it('supports touch dragging on card headers and restores the order after refresh and remount', async () => {
mockSortableProviders()
localStorage.setItem('aether-provider-card-view', 'true')
let root = await mountView()
const { handle } = startProviderDrag(root, 'provider-4', 'provider-1', 'touch')
await dropProvider(handle)
const expected = ['provider-4', 'provider-1', 'provider-2', 'provider-3']
expect(providerOrder(root)).toEqual(expected)
expect(JSON.parse(localStorage.getItem('aether-provider-display-order')!)).toEqual(expected)
findButton(root, '刷新').click()
await settle()
expect(providerOrder(root)).toEqual(expected)
findButton(root, '切换到列表视图').click()
await settle()
expect(providerOrder(root)).toEqual(expected)
unmountView()
root = await mountView()
expect(providerOrder(root)).toEqual(expected)
})
it.each(['escape', 'pointercancel', 'outside'] as const)('cancels a drag without saving when cancelled by %s', async (reason) => {
mockSortableProviders()
const root = await mountView()
const original = providerOrder(root)
const { handle, hitTest } = startProviderDrag(root, 'provider-1', 'provider-3')
if (reason === 'escape') {
window.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape', bubbles: true }))
} else if (reason === 'pointercancel') {
window.dispatchEvent(pointerEvent('pointercancel', 100, 200))
} else {
hitTest.mockReturnValue(null)
}
await dropProvider(handle)
expect(providerOrder(root)).toEqual(original)
expect(JSON.parse(localStorage.getItem('aether-provider-display-order')!)).toEqual([])
expect(root.querySelector('.opacity-40')).toBeNull()
expect(root.querySelector('[data-provider-detail]')).toBeNull()
})
it('does not reorder on a handle click or a secondary mouse button', async () => {
mockSortableProviders()
const root = await mountView()
const original = providerOrder(root)
const handle = providerElements(root)[0]!.querySelector<HTMLButtonElement>('[data-provider-drag-handle]')!
handle.dispatchEvent(pointerEvent('pointerdown', 40, 100))
window.dispatchEvent(pointerEvent('pointermove', 42, 101))
window.dispatchEvent(pointerEvent('pointerup', 42, 101))
handle.click()
handle.dispatchEvent(pointerEvent('pointerdown', 40, 100, { button: 2 }))
window.dispatchEvent(pointerEvent('pointermove', 100, 200))
window.dispatchEvent(pointerEvent('pointerup', 100, 200))
await settle()
expect(providerOrder(root)).toEqual(original)
expect(root.querySelector('[data-provider-detail]')).toBeNull()
})
it('moves only visible providers while preserving filtered-out positions', async () => {
const providers = mockSortableProviders()
const root = await mountView()
apiMocks.getProvidersSummary.mockResolvedValue({ items: [providers[0], providers[2]], total: 2 })
const search = root.querySelector<HTMLInputElement>('#provider-search')!
search.value = 'filtered'
search.dispatchEvent(new Event('input', { bubbles: true }))
await vi.waitFor(() => expect(providerOrder(root)).toEqual(['provider-1', 'provider-3']))
const { handle } = startProviderDrag(root, 'provider-1', 'provider-3')
await dropProvider(handle)
expect(providerOrder(root)).toEqual(['provider-3', 'provider-1'])
apiMocks.getProvidersSummary.mockResolvedValue({ items: providers, total: 4 })
findButton(root, '重置筛选').click()
await vi.waitFor(() => {
expect(providerOrder(root)).toEqual(['provider-3', 'provider-2', 'provider-1', 'provider-4'])
})
})
it('supports keyboard ordering and keeps focus on the moved handle', async () => {
mockSortableProviders()
const root = await mountView()
const handle = providerElements(root)[0]!.querySelector<HTMLButtonElement>('[data-provider-drag-handle]')!
handle.focus()
handle.dispatchEvent(new KeyboardEvent('keydown', { key: 'ArrowDown', bubbles: true, cancelable: true }))
await settle()
expect(providerOrder(root)).toEqual(['provider-2', 'provider-1', 'provider-3', 'provider-4'])
expect(document.activeElement).toBe(handle)
expect(root.querySelector('[role="status"]')?.textContent).toContain('展示顺序已更新')
expect(root.querySelector('[data-provider-detail]')).toBeNull()
handle.dispatchEvent(new KeyboardEvent('keydown', { key: 'ArrowLeft', bubbles: true, cancelable: true }))
await settle()
expect(providerOrder(root)).toEqual(['provider-1', 'provider-2', 'provider-3', 'provider-4'])
})
it('preserves saved ordering on other pages when reordering the current page', async () => {
const providers = mockSortableProviders()
apiMocks.getProvidersSummary.mockImplementation(async ({ page }: { page: number }) => ({
items: page === 1 ? providers.slice(0, 2) : providers.slice(2),
total: 40,
}))
const root = await mountView()
const firstDrag = startProviderDrag(root, 'provider-1', 'provider-2')
await dropProvider(firstDrag.handle)
const secondPage = [...root.querySelectorAll<HTMLButtonElement>('button')]
.find(button => button.textContent?.trim() === '2')!
secondPage.click()
await settle()
expect(providerOrder(root)).toEqual(['provider-3', 'provider-4'])
const secondDrag = startProviderDrag(root, 'provider-4', 'provider-3')
await dropProvider(secondDrag.handle)
expect(providerOrder(root)).toEqual(['provider-4', 'provider-3'])
const firstPage = [...root.querySelectorAll<HTMLButtonElement>('button')]
.find(button => button.textContent?.trim() === '1')!
firstPage.click()
await settle()
expect(providerOrder(root)).toEqual(['provider-2', 'provider-1'])
expect(JSON.parse(localStorage.getItem('aether-provider-display-order')!))
.toEqual(['provider-2', 'provider-1', 'provider-4', 'provider-3'])
})
it('ignores stale IDs and appends providers that are not in the saved order', async () => {
mockSortableProviders()
localStorage.setItem('aether-provider-display-order', JSON.stringify(['deleted-provider', 'provider-3', 'provider-1']))
const root = await mountView()
expect(providerOrder(root)).toEqual(['provider-3', 'provider-1', 'provider-2', 'provider-4'])
})
})