i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 01:10:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(tunnel/usage): proxy writer 双优先级队列、hub 连接压力感知选择、usage 请求记录级别控制及 trace 页面 proxy timing 增强

Browse Source
This commit is contained in:
fawney19
2026-04-16 20:14:50 +08:00
parent 65cd9dc3e5
commit 0dce667019
13 changed files with 1229 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
apps/aether-gateway/src/data/state/integrations.rs
View File

@@ -16,16 +16,37 @@ use aether_data_contracts::repository::settlement::{StoredUsageSettlement, Usage
use aether_data_contracts::repository::usage::{StoredRequestUsageAudit, UpsertUsageRecord};
use aether_data_contracts::repository::video_tasks::{StoredVideoTask, VideoTaskLookupKey};
use aether_usage_runtime::{
UsageBillingEventEnricher, UsageEvent, UsageRecordWriter, UsageRuntimeAccess,
UsageSettlementWriter,
UsageBillingEventEnricher, UsageEvent, UsageRecordWriter, UsageRequestRecordLevel,
UsageRuntimeAccess, UsageSettlementWriter,
};
use aether_video_tasks_core::StoredVideoTaskReadSide;
use async_trait::async_trait;
use serde_json::Value;
use super::GatewayDataState;
use crate::data::candidate_selection::MinimalCandidateSelectionRowSource;
use crate::provider_transport::ProviderTransportSnapshotSource;
const REQUEST_RECORD_LEVEL_KEY: &str = "request_record_level";
const LEGACY_REQUEST_LOG_LEVEL_KEY: &str = "request_log_level";
fn usage_request_record_level_from_value(value: Option<&Value>) -> UsageRequestRecordLevel {
let Some(value) = value.and_then(Value::as_str).map(str::trim) else {
return UsageRequestRecordLevel::Full;
};
if value.eq_ignore_ascii_case("basic")
|| value.eq_ignore_ascii_case("base")
|| value.eq_ignore_ascii_case("headers")
|| value.eq_ignore_ascii_case("minimal")
|| value.eq_ignore_ascii_case("none")
{
UsageRequestRecordLevel::Basic
} else {
UsageRequestRecordLevel::Full
}
}
#[async_trait]
impl RequestAuditReader for GatewayDataState {
async fn find_request_usage_audit_by_request_id(
@@ -161,6 +182,7 @@ impl UsageBillingEventEnricher for GatewayDataState {
}
}
#[async_trait]
impl UsageRuntimeAccess for GatewayDataState {
fn has_usage_writer(&self) -> bool {
GatewayDataState::has_usage_writer(self)
@@ -173,6 +195,16 @@ impl UsageRuntimeAccess for GatewayDataState {
fn usage_worker_runner(&self) -> Option<RedisStreamRunner> {
GatewayDataState::usage_worker_runner(self)
}
async fn request_record_level(&self) -> Result<UsageRequestRecordLevel, DataLayerError> {
let value = GatewayDataState::find_system_config_value(self, REQUEST_RECORD_LEVEL_KEY)
.await?
.or(
GatewayDataState::find_system_config_value(self, LEGACY_REQUEST_LOG_LEVEL_KEY)
.await?,
);
Ok(usage_request_record_level_from_value(value.as_ref()))
}
}
#[async_trait]
@@ -188,10 +220,11 @@ impl UsageRecordWriter for GatewayDataState {
#[cfg(test)]
mod tests {
use aether_billing::enrich_usage_event_with_billing;
use serde_json::Value;
use aether_usage_runtime::UsageRuntimeAccess;
use serde_json::{json, Value};
use super::GatewayDataState;
use crate::usage::{UsageEvent, UsageEventData, UsageEventType};
use crate::usage::{UsageEvent, UsageEventData, UsageEventType, UsageRequestRecordLevel};
#[tokio::test]
async fn enriches_completed_usage_event_with_billing_snapshot() {
@@ -255,4 +288,43 @@ mod tests {
Some("complete")
);
}
#[tokio::test]
async fn usage_runtime_access_reads_base_request_record_level_as_basic() {
let state = GatewayDataState::disabled().with_system_config_values_for_tests([(
"request_record_level".to_string(),
json!("base"),
)]);
let level = UsageRuntimeAccess::request_record_level(&state)
.await
.expect("request record level should read");
assert_eq!(level, UsageRequestRecordLevel::Basic);
}
#[tokio::test]
async fn usage_runtime_access_falls_back_to_legacy_request_log_level_alias() {
let state = GatewayDataState::disabled().with_system_config_values_for_tests([(
"request_log_level".to_string(),
json!("headers"),
)]);
let level = UsageRuntimeAccess::request_record_level(&state)
.await
.expect("legacy request log level should read");
assert_eq!(level, UsageRequestRecordLevel::Basic);
}
#[tokio::test]
async fn usage_runtime_access_defaults_missing_request_record_level_to_full() {
let state = GatewayDataState::disabled();
let level = UsageRuntimeAccess::request_record_level(&state)
.await
.expect("missing request record level should fall back");
assert_eq!(level, UsageRequestRecordLevel::Full);
}
}

85
apps/aether-gateway/src/handlers/admin/observability/monitoring/tests/trace.rs
View File

@@ -1,5 +1,5 @@
use super::super::test_support::{
request_context, sample_candidate, sample_endpoint, sample_key, sample_provider,
request_context, sample_candidate, sample_endpoint, sample_key, sample_provider, sample_usage,
};
use super::local_monitoring_response;
use crate::AppState;
@@ -10,6 +10,7 @@ use std::sync::Arc;
use aether_data::repository::candidates::InMemoryRequestCandidateRepository;
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
use aether_data::repository::usage::InMemoryUsageReadRepository;
#[tokio::test]
async fn admin_monitoring_trace_request_returns_local_payload() {
@@ -156,6 +157,88 @@ async fn admin_monitoring_trace_request_keeps_format_conversion_disabled_candida
);
}
#[tokio::test]
async fn admin_monitoring_trace_request_enriches_proxy_timing_from_usage_audit() {
let mut candidate = sample_candidate(
"cand-used",
"request-1",
1,
RequestCandidateStatus::Success,
Some(101),
Some(33),
Some(200),
);
candidate.extra_data = Some(json!({
"proxy": {
"node_id": "proxy-node-1",
"node_name": "edge-1",
"source": "provider"
}
}));
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![candidate]));
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider()],
vec![sample_endpoint()],
vec![sample_key()],
));
let mut usage = sample_usage(
"request-1",
"provider-1",
"OpenAI",
40,
0.02,
"completed",
Some(200),
100,
);
usage.candidate_id = Some("cand-used".to_string());
usage.response_headers = Some(json!({
"x-proxy-timing": "{\"connection_acquire_ms\":125,\"response_wait_ms\":475,\"ttfb_ms\":600}"
}));
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(vec![usage]));
let data_state =
crate::data::GatewayDataState::with_request_candidate_and_usage_repository_for_tests(
request_candidates,
usage_repository,
)
.with_provider_catalog_reader(provider_catalog);
let state = AppState::new()
.expect("state should build")
.with_data_state_for_tests(data_state);
let context = request_context(
http::Method::GET,
"/api/admin/monitoring/trace/request-1?attempted_only=true",
);
let response = local_monitoring_response(&state, &context)
.await
.expect("handler should not error")
.expect("route should be handled locally");
assert_eq!(response.status(), http::StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX)
.await
.expect("body should read");
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
assert_eq!(
payload["candidates"][0]["extra_data"]["first_byte_time_ms"],
json!(30)
);
assert_eq!(
payload["candidates"][0]["extra_data"]["proxy"]["ttfb_ms"],
json!(600)
);
assert_eq!(
payload["candidates"][0]["extra_data"]["proxy"]["timing"]["connection_acquire_ms"],
json!(125)
);
assert_eq!(
payload["candidates"][0]["extra_data"]["proxy"]["timing"]["response_wait_ms"],
json!(475)
);
}
#[tokio::test]
async fn admin_monitoring_trace_provider_stats_returns_local_payload() {
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![

6
apps/aether-gateway/src/handlers/admin/observability/monitoring/trace.rs
View File

@@ -51,9 +51,15 @@ pub(super) async fn build_admin_monitoring_trace_request_response(
attempted_only,
));
};
let usage = state
.data
.read_request_usage_audit(&request_id)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?;
Ok(build_admin_monitoring_trace_request_payload_response(
&trace,
usage.as_ref(),
))
}

145
apps/aether-gateway/src/tests/usage/local.rs
View File

@@ -371,6 +371,151 @@ async fn gateway_truncates_deep_request_echo_for_local_openai_chat_sync_usage()
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_strips_request_and_response_bodies_when_request_record_level_is_base() {
let usage_repository = Arc::new(InMemoryUsageReadRepository::default());
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
let upstream = Router::new().route(
"/api/internal/gateway/report-sync",
any(|_request: Request| async move { Json(json!({"ok": true})) }),
);
let execution_runtime = Router::new().route(
"/v1/execute/sync",
any(|_request: Request| async move {
Json(json!({
"request_id": "trace-openai-chat-local-report-sync-base-123",
"status_code": 200,
"headers": {
"content-type": "application/json"
},
"body": {
"json_body": {
"id": "chatcmpl-local-report-sync-base-123",
"object": "chat.completion",
"model": "gpt-5-upstream",
"choices": [{
"index": 0,
"message": {
"role": "assistant",
"content": "body should not be persisted"
}
}],
"usage": {
"prompt_tokens": 2,
"completion_tokens": 3,
"total_tokens": 5
}
}
},
"telemetry": {
"elapsed_ms": 25
}
}))
}),
);
let auth_repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
Some(hash_api_key("sk-client-openai-local-report-sync-base")),
sample_local_openai_auth_snapshot(
"api-key-openai-usage-local-base-1",
"user-openai-usage-local-base-1",
),
)]));
let candidate_selection_repository =
Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(vec![
sample_local_openai_candidate_row(),
]));
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_local_openai_provider()],
vec![sample_local_openai_endpoint()],
vec![sample_local_openai_key()],
));
let (_upstream_url, upstream_handle) = start_server(upstream).await;
let (execution_runtime_url, execution_runtime_handle) = start_server(execution_runtime).await;
let gateway_state = build_state_with_execution_runtime_override(execution_runtime_url)
.with_data_state_for_tests(
GatewayDataState::with_auth_candidate_selection_provider_catalog_request_candidates_and_usage_for_tests(
auth_repository,
candidate_selection_repository,
provider_catalog_repository,
Arc::clone(&request_candidate_repository),
Arc::clone(&usage_repository),
DEVELOPMENT_ENCRYPTION_KEY,
)
.with_system_config_values_for_tests([(
"request_record_level".to_string(),
json!("base"),
)]),
)
.with_usage_runtime_for_tests(UsageRuntimeConfig {
enabled: true,
..UsageRuntimeConfig::default()
});
let gateway = build_router_with_state(gateway_state);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.post(format!("{gateway_url}/v1/chat/completions"))
.header(http::header::CONTENT_TYPE, "application/json")
.header(
http::header::AUTHORIZATION,
"Bearer sk-client-openai-local-report-sync-base",
)
.header(
TRACE_ID_HEADER,
"trace-openai-chat-local-report-sync-base-123",
)
.body(
serde_json::to_string(&json!({
"model": "gpt-5",
"messages": [{
"role": "user",
"content": "request body should not be persisted"
}]
}))
.expect("request should encode"),
)
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let body_json: serde_json::Value = response.json().await.expect("body should parse");
assert_eq!(body_json["model"], "gpt-5-upstream");
let stored_usage = wait_for_usage_status(
usage_repository.as_ref(),
"trace-openai-chat-local-report-sync-base-123",
"completed",
)
.await;
assert_eq!(stored_usage.status, "completed");
assert_eq!(stored_usage.total_tokens, 5);
assert_eq!(stored_usage.response_time_ms, Some(25));
assert!(stored_usage.request_body.is_none());
assert!(stored_usage.request_body_ref.is_none());
assert!(stored_usage.provider_request_body.is_none());
assert!(stored_usage.provider_request_body_ref.is_none());
assert!(stored_usage.response_body.is_none());
assert!(stored_usage.response_body_ref.is_none());
assert!(stored_usage.client_response_body.is_none());
assert!(stored_usage.client_response_body_ref.is_none());
let stored_candidates = request_candidate_repository
.list_by_request_id("trace-openai-chat-local-report-sync-base-123")
.await
.expect("request candidate trace should read");
assert_eq!(stored_candidates.len(), 1);
assert_eq!(stored_candidates[0].status, RequestCandidateStatus::Success);
gateway_handle.abort();
execution_runtime_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_records_failed_usage_when_all_local_openai_chat_candidates_exhaust_after_retryable_sync_failure(
) {

346
apps/aether-gateway/src/tunnel/embedded/hub.rs
View File

@@ -3,7 +3,7 @@ use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering}
use std::sync::Arc;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use aether_runtime::{BoundedQueueSender, MetricKind, MetricSample, QueueSendError};
use aether_runtime::{BoundedQueueSender, MetricKind, MetricSample, QueueSendError, QueueSnapshot};
use axum::extract::ws::Message;
use bytes::Bytes;
use dashmap::DashMap;
@@ -16,6 +16,8 @@ use super::control_plane::ControlPlaneClient;
use super::protocol;
const MAX_REQUEST_BODY_FRAME_SIZE: usize = 32 * 1024;
const SOFT_AVOID_QUEUE_PRESSURE_PERCENT: u64 = 50;
const SOFT_AVOID_STREAM_PRESSURE_PERCENT: u64 = 85;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SendStatus {
@@ -75,6 +77,10 @@ impl BoundedOutbound {
let _ = self.close_tx.send(true);
true
}
pub fn snapshot(&self) -> QueueSnapshot {
self.tx.snapshot()
}
}
pub struct ProxyConn {
@@ -86,6 +92,7 @@ pub struct ProxyConn {
pub stream_count: AtomicUsize,
pub max_streams: usize,
draining: AtomicBool,
congested_total: AtomicU64,
}
impl ProxyConn {
@@ -106,6 +113,7 @@ impl ProxyConn {
stream_count: AtomicUsize::new(0),
max_streams,
draining: AtomicBool::new(false),
congested_total: AtomicU64::new(0),
}
}
@@ -180,6 +188,7 @@ impl ProxyConn {
let was_closing = self.outbound.is_closing();
let status = self.outbound.send(msg);
if status == SendStatus::Congested && !was_closing {
self.congested_total.fetch_add(1, Ordering::Relaxed);
warn!(
conn_id = self.id,
node_id = %self.node_id,
@@ -190,6 +199,62 @@ impl ProxyConn {
}
status
}
fn snapshot(&self) -> ProxyConnSnapshot {
let outbound = self.outbound.snapshot();
let stream_count = self.stream_count.load(Ordering::Relaxed);
let queue_pressure_percent = percent_u64(outbound.depth, outbound.capacity);
let stream_pressure_percent = percent_u64(stream_count, self.max_streams);
let soft_avoid = queue_pressure_percent >= SOFT_AVOID_QUEUE_PRESSURE_PERCENT
|| stream_pressure_percent >= SOFT_AVOID_STREAM_PRESSURE_PERCENT;
ProxyConnSnapshot {
conn_id: self.id,
available: self.is_available(),
closing: self.outbound.is_closing(),
draining: self.is_draining(),
stream_count,
max_streams: self.max_streams,
stream_pressure_percent,
outbound,
queue_pressure_percent,
soft_avoid,
congested_total: self.congested_total.load(Ordering::Relaxed),
}
}
}
#[derive(Debug, Clone, Copy)]
struct ProxyConnSnapshot {
conn_id: u64,
available: bool,
closing: bool,
draining: bool,
stream_count: usize,
max_streams: usize,
stream_pressure_percent: u64,
outbound: QueueSnapshot,
queue_pressure_percent: u64,
soft_avoid: bool,
congested_total: u64,
}
#[derive(Clone)]
struct ProxyConnCandidate {
conn: Arc<ProxyConn>,
snapshot: ProxyConnSnapshot,
}
impl ProxyConnCandidate {
fn rank_key(&self) -> (u8, u64, u64, usize, usize, u64) {
(
u8::from(self.snapshot.soft_avoid),
self.snapshot.queue_pressure_percent,
self.snapshot.stream_pressure_percent,
self.snapshot.outbound.depth,
self.snapshot.stream_count,
self.snapshot.conn_id,
)
}
}
#[derive(Debug, Clone)]
@@ -343,6 +408,9 @@ pub struct HubRouter {
next_local_stream_id: AtomicU64,
control_plane: ControlPlaneClient,
node_status_tx: mpsc::UnboundedSender<NodeStatusEvent>,
soft_avoid_selection_total: AtomicU64,
selection_retry_total: AtomicU64,
selection_unavailable_total: AtomicU64,
}
#[derive(Debug)]
@@ -391,6 +459,9 @@ impl HubRouter {
next_local_stream_id: AtomicU64::new(1),
control_plane,
node_status_tx,
soft_avoid_selection_total: AtomicU64::new(0),
selection_retry_total: AtomicU64::new(0),
selection_unavailable_total: AtomicU64::new(0),
})
}
@@ -477,28 +548,28 @@ impl HubRouter {
}
}
fn get_proxy_conn(&self, node_id: &str) -> Option<Arc<ProxyConn>> {
let map = self.proxy_conns.read();
let conns = map.get(node_id)?;
let result = conns
.iter()
.filter(|c| c.is_available())
.min_by_key(|c| c.stream_count.load(Ordering::Relaxed))
.cloned();
if result.is_none() && !conns.is_empty() {
warn!(
node_id = %node_id,
total_conns = conns.len(),
closing = conns.iter().filter(|c| c.outbound.is_closing()).count(),
draining = conns.iter().filter(|c| c.is_draining()).count(),
"no available proxy connection despite registered connections"
);
}
result
fn ranked_proxy_conn_candidates(&self, node_id: &str) -> Vec<ProxyConnCandidate> {
let conns = {
let map = self.proxy_conns.read();
map.get(node_id)
.map(|entries| entries.to_vec())
.unwrap_or_default()
};
let mut candidates = conns
.into_iter()
.filter_map(|conn| {
let snapshot = conn.snapshot();
snapshot
.available
.then_some(ProxyConnCandidate { conn, snapshot })
})
.collect::<Vec<_>>();
candidates.sort_by_key(|candidate| candidate.rank_key());
candidates
}
pub fn has_local_proxy(&self, node_id: &str) -> bool {
self.get_proxy_conn(node_id).is_some()
!self.ranked_proxy_conn_candidates(node_id).is_empty()
}
pub fn open_local_stream(
@@ -506,12 +577,49 @@ impl HubRouter {
node_id: &str,
meta: &protocol::RequestMeta,
) -> Result<Arc<LocalStream>, String> {
let proxy_conn = self
.get_proxy_conn(node_id)
.ok_or_else(|| format!("no proxy connection for node {node_id}"))?;
let proxy_stream_id = proxy_conn
.alloc_stream_id()
.ok_or_else(|| format!("stream limit reached for node {node_id}"))?;
let candidates = self.ranked_proxy_conn_candidates(node_id);
if candidates.is_empty() {
self.selection_unavailable_total
.fetch_add(1, Ordering::Relaxed);
self.warn_no_available_proxy_connection(node_id);
return Err(format!("no proxy connection for node {node_id}"));
}
let mut skipped_candidates = 0usize;
let mut selected_candidate = None;
let mut proxy_stream_id = None;
for candidate in candidates {
match candidate.conn.alloc_stream_id() {
Some(stream_id) => {
proxy_stream_id = Some(stream_id);
selected_candidate = Some(candidate);
break;
}
None => skipped_candidates = skipped_candidates.saturating_add(1),
}
}
let Some(candidate) = selected_candidate else {
self.selection_unavailable_total
.fetch_add(1, Ordering::Relaxed);
return Err(format!("stream limit reached for node {node_id}"));
};
if skipped_candidates > 0 {
self.selection_retry_total.fetch_add(1, Ordering::Relaxed);
}
if candidate.snapshot.soft_avoid {
self.soft_avoid_selection_total
.fetch_add(1, Ordering::Relaxed);
debug!(
node_id = %node_id,
conn_id = candidate.snapshot.conn_id,
queue_pressure_percent = candidate.snapshot.queue_pressure_percent,
stream_pressure_percent = candidate.snapshot.stream_pressure_percent,
"selected high-pressure proxy connection because no lower-pressure alternative was available"
);
}
let proxy_conn = candidate.conn;
let proxy_stream_id = proxy_stream_id.expect("selected candidate should carry a stream id");
// Encode frames before registering the stream so that encoding failures
// (practically impossible but theoretically possible) don't leak a stream
@@ -556,6 +664,7 @@ impl HubRouter {
proxy_stream_id = proxy_stream_id,
local_stream_id = local_stream_id,
stream_count = proxy_conn.stream_count.load(Ordering::Relaxed),
queue_depth = proxy_conn.outbound.snapshot().depth,
send_status = ?send_status,
"open_local_stream dispatched"
);
@@ -569,6 +678,28 @@ impl HubRouter {
}
}
fn warn_no_available_proxy_connection(&self, node_id: &str) {
let conns = {
let map = self.proxy_conns.read();
map.get(node_id)
.map(|entries| entries.to_vec())
.unwrap_or_default()
};
if conns.is_empty() {
return;
}
let snapshots = conns.iter().map(|conn| conn.snapshot()).collect::<Vec<_>>();
warn!(
node_id = %node_id,
total_conns = snapshots.len(),
available = snapshots.iter().filter(|snapshot| snapshot.available).count(),
closing = snapshots.iter().filter(|snapshot| snapshot.closing).count(),
draining = snapshots.iter().filter(|snapshot| snapshot.draining).count(),
soft_avoid = snapshots.iter().filter(|snapshot| snapshot.soft_avoid).count(),
"no available proxy connection despite registered connections"
);
}
pub fn push_local_request_body(
&self,
local_stream_id: u64,
@@ -873,15 +1004,72 @@ impl HubRouter {
}
pub fn stats(&self) -> HubStats {
let proxy_conns = self.proxy_conns.read();
let total_proxy = proxy_conns.values().map(|v| v.len()).sum();
let nodes = proxy_conns.len();
drop(proxy_conns);
let proxy_conns = self
.proxy_conns_by_id
.iter()
.map(|entry| entry.value().snapshot())
.collect::<Vec<_>>();
let total_proxy = proxy_conns.len();
let nodes = self.proxy_conns.read().len();
let available_proxy_connections = proxy_conns
.iter()
.filter(|snapshot| snapshot.available)
.count();
let closing_proxy_connections = proxy_conns
.iter()
.filter(|snapshot| snapshot.closing)
.count();
let draining_proxy_connections = proxy_conns
.iter()
.filter(|snapshot| snapshot.draining)
.count();
let soft_avoid_proxy_connections = proxy_conns
.iter()
.filter(|snapshot| snapshot.available && snapshot.soft_avoid)
.count();
let outbound_queue_depth_total = proxy_conns
.iter()
.map(|snapshot| snapshot.outbound.depth)
.sum();
let outbound_queue_depth_max = proxy_conns
.iter()
.map(|snapshot| snapshot.outbound.depth)
.max()
.unwrap_or(0);
let outbound_queue_capacity_total = proxy_conns
.iter()
.map(|snapshot| snapshot.outbound.capacity)
.sum();
let outbound_queue_rejected_full_total = proxy_conns
.iter()
.map(|snapshot| snapshot.outbound.rejected_full_total)
.sum();
let outbound_queue_rejected_closed_total = proxy_conns
.iter()
.map(|snapshot| snapshot.outbound.rejected_closed_total)
.sum();
let proxy_connection_congested_total = proxy_conns
.iter()
.map(|snapshot| snapshot.congested_total)
.sum();
HubStats {
proxy_connections: total_proxy,
available_proxy_connections,
closing_proxy_connections,
draining_proxy_connections,
soft_avoid_proxy_connections,
nodes,
active_streams: self.local_streams.len(),
outbound_queue_depth_total,
outbound_queue_depth_max,
outbound_queue_capacity_total,
outbound_queue_rejected_full_total,
outbound_queue_rejected_closed_total,
proxy_connection_congested_total,
soft_avoid_selection_total: self.soft_avoid_selection_total.load(Ordering::Relaxed),
selection_retry_total: self.selection_retry_total.load(Ordering::Relaxed),
selection_unavailable_total: self.selection_unavailable_total.load(Ordering::Relaxed),
}
}
}
@@ -893,11 +1081,31 @@ fn current_unix_secs() -> u64 {
.as_secs()
}
fn percent_u64(value: usize, total: usize) -> u64 {
if total == 0 {
return 0;
}
((value as u128) * 100 / (total as u128)) as u64
}
#[derive(serde::Serialize)]
pub struct HubStats {
pub proxy_connections: usize,
pub available_proxy_connections: usize,
pub closing_proxy_connections: usize,
pub draining_proxy_connections: usize,
pub soft_avoid_proxy_connections: usize,
pub nodes: usize,
pub active_streams: usize,
pub outbound_queue_depth_total: usize,
pub outbound_queue_depth_max: usize,
pub outbound_queue_capacity_total: usize,
pub outbound_queue_rejected_full_total: u64,
pub outbound_queue_rejected_closed_total: u64,
pub proxy_connection_congested_total: u64,
pub soft_avoid_selection_total: u64,
pub selection_retry_total: u64,
pub selection_unavailable_total: u64,
}
impl HubStats {
@@ -909,6 +1117,30 @@ impl HubStats {
MetricKind::Gauge,
self.proxy_connections as u64,
),
MetricSample::new(
"tunnel_proxy_connections_available",
"Current number of proxy connections available for new work.",
MetricKind::Gauge,
self.available_proxy_connections as u64,
),
MetricSample::new(
"tunnel_proxy_connections_closing",
"Current number of proxy connections marked closing.",
MetricKind::Gauge,
self.closing_proxy_connections as u64,
),
MetricSample::new(
"tunnel_proxy_connections_draining",
"Current number of proxy connections marked draining.",
MetricKind::Gauge,
self.draining_proxy_connections as u64,
),
MetricSample::new(
"tunnel_proxy_connections_soft_avoid",
"Current number of available proxy connections currently soft-avoided by the scheduler.",
MetricKind::Gauge,
self.soft_avoid_proxy_connections as u64,
),
MetricSample::new(
"tunnel_nodes",
"Current number of connected logical nodes.",
@@ -921,6 +1153,60 @@ impl HubStats {
MetricKind::Gauge,
self.active_streams as u64,
),
MetricSample::new(
"tunnel_proxy_outbound_queue_depth_total",
"Current aggregate depth across proxy outbound queues.",
MetricKind::Gauge,
self.outbound_queue_depth_total as u64,
),
MetricSample::new(
"tunnel_proxy_outbound_queue_depth_max",
"Current maximum depth observed on a single proxy outbound queue.",
MetricKind::Gauge,
self.outbound_queue_depth_max as u64,
),
MetricSample::new(
"tunnel_proxy_outbound_queue_capacity_total",
"Current aggregate capacity across proxy outbound queues.",
MetricKind::Gauge,
self.outbound_queue_capacity_total as u64,
),
MetricSample::new(
"tunnel_proxy_outbound_queue_rejected_full_total",
"Total proxy outbound queue sends rejected because a queue was full.",
MetricKind::Counter,
self.outbound_queue_rejected_full_total,
),
MetricSample::new(
"tunnel_proxy_outbound_queue_rejected_closed_total",
"Total proxy outbound queue sends rejected because a queue was closed.",
MetricKind::Counter,
self.outbound_queue_rejected_closed_total,
),
MetricSample::new(
"tunnel_proxy_connection_congested_total",
"Total number of times a proxy outbound queue became congested.",
MetricKind::Counter,
self.proxy_connection_congested_total,
),
MetricSample::new(
"tunnel_proxy_soft_avoid_selection_total",
"Total number of times the scheduler had to pick a high-pressure proxy connection.",
MetricKind::Counter,
self.soft_avoid_selection_total,
),
MetricSample::new(
"tunnel_proxy_selection_retry_total",
"Total number of times the scheduler retried a lower-ranked proxy connection after a race on stream allocation.",
MetricKind::Counter,
self.selection_retry_total,
),
MetricSample::new(
"tunnel_proxy_selection_unavailable_total",
"Total number of relay selections that failed because no proxy connection was available.",
MetricKind::Counter,
self.selection_unavailable_total,
),
]
}
}

3
apps/aether-gateway/src/usage/mod.rs
View File

@@ -6,7 +6,8 @@ pub(crate) mod write;
pub(crate) use aether_usage_runtime::UsageRuntime;
pub use aether_usage_runtime::UsageRuntimeConfig;
pub(crate) use aether_usage_runtime::{
now_ms, UsageEvent, UsageEventData, UsageEventType, UsageQueue, USAGE_EVENT_VERSION,
now_ms, UsageEvent, UsageEventData, UsageEventType, UsageQueue, UsageRequestRecordLevel,
USAGE_EVENT_VERSION,
};
pub(crate) use reporting::{
spawn_sync_report, submit_stream_report, submit_sync_report, GatewayStreamReportRequest,