i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 17:30:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(proxy): record tunnel stability metrics

Browse Source
This commit is contained in:
fawney19
2026-05-08 22:03:17 +08:00
parent 9a84a6ff6c
commit a703acd1fe
81 changed files with 7229 additions and 1110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
apps/aether-gateway/src/handlers/admin/provider/write/normalize.rs
View File

@@ -104,11 +104,7 @@ pub(crate) fn normalize_allow_auth_channel_mismatch_formats(
normalized.push(serde_json::Value::String(canonical));
}
}
if normalized.is_empty() {
Ok(None)
} else {
Ok(Some(serde_json::Value::Array(normalized)))
}
Ok(Some(serde_json::Value::Array(normalized)))
}
pub(crate) fn normalize_auth_type(value: Option<&str>) -> Result<String, String> {
@@ -179,9 +175,9 @@ fn normalize_json_like_object(
#[cfg(test)]
mod tests {
use super::{
normalize_api_format_json_object_keys, normalize_api_format_list, normalize_auth_type,
normalize_auth_type_by_format, normalize_pool_advanced_config,
normalize_provider_type_input, validate_vertex_api_formats,
normalize_allow_auth_channel_mismatch_formats, normalize_api_format_json_object_keys,
normalize_api_format_list, normalize_auth_type, normalize_auth_type_by_format,
normalize_pool_advanced_config, normalize_provider_type_input, validate_vertex_api_formats,
};
use serde_json::json;
@@ -280,6 +276,36 @@ mod tests {
);
}
#[test]
fn normalize_allow_auth_channel_mismatch_formats_preserves_explicit_empty_array() {
assert_eq!(
normalize_allow_auth_channel_mismatch_formats(
Some(Vec::new()),
"allow_auth_channel_mismatch_formats",
&["claude:messages".to_string()],
)
.expect("empty array should normalize"),
Some(json!([]))
);
}
#[test]
fn normalize_allow_auth_channel_mismatch_formats_normalizes_and_dedupes_values() {
assert_eq!(
normalize_allow_auth_channel_mismatch_formats(
Some(vec![
"claude:messages".to_string(),
"CLAUDE:MESSAGES".to_string(),
" claude:messages ".to_string(),
]),
"allow_auth_channel_mismatch_formats",
&["claude:messages".to_string()],
)
.expect("format list should normalize"),
Some(json!(["claude:messages"]))
);
}
#[test]
fn validate_vertex_api_formats_uses_canonical_message_formats() {
assert!(validate_vertex_api_formats(

67
apps/aether-gateway/src/handlers/admin/request/system/proxy_nodes.rs
View File

@@ -3,8 +3,10 @@ use crate::handlers::shared::unix_secs_to_rfc3339;
use crate::maintenance::{inspect_proxy_upgrade_rollout, ProxyUpgradeRolloutStatus};
use crate::GatewayError;
use aether_admin::system::{
build_admin_proxy_node_event_payload, build_admin_proxy_node_events_payload_response,
build_admin_proxy_node_payload, build_admin_proxy_nodes_data_unavailable_response,
build_admin_proxy_fleet_metrics_payload_response, build_admin_proxy_node_event_payload,
build_admin_proxy_node_events_payload_response,
build_admin_proxy_node_metrics_payload_response, build_admin_proxy_node_payload,
build_admin_proxy_nodes_data_unavailable_response,
build_admin_proxy_nodes_invalid_status_response, build_admin_proxy_nodes_list_payload_response,
build_admin_proxy_nodes_not_found_response,
};
@@ -84,6 +86,30 @@ impl<'a> AdminAppState<'a> {
pub(crate) async fn build_admin_proxy_node_events_response(
&self,
node_id: &str,
query: &aether_data::repository::proxy_nodes::ProxyNodeEventQuery,
) -> Result<Response<Body>, GatewayError> {
if !self.has_proxy_node_reader() {
return Ok(build_admin_proxy_nodes_data_unavailable_response());
}
if self.find_proxy_node(node_id).await?.is_none() {
return Ok(build_admin_proxy_nodes_not_found_response());
}
let items = self
.app
.list_proxy_node_events_filtered(node_id, query)
.await?
.into_iter()
.map(|event| build_admin_proxy_node_event_payload(&event))
.collect::<Vec<_>>();
Ok(build_admin_proxy_node_events_payload_response(items))
}
pub(crate) async fn build_admin_proxy_node_metrics_response(
&self,
node_id: &str,
step: aether_data::repository::proxy_nodes::ProxyNodeMetricsStep,
from_unix_secs: u64,
to_unix_secs: u64,
limit: usize,
) -> Result<Response<Body>, GatewayError> {
if !self.has_proxy_node_reader() {
@@ -93,12 +119,37 @@ impl<'a> AdminAppState<'a> {
return Ok(build_admin_proxy_nodes_not_found_response());
}
let items = self
.list_proxy_node_events(node_id, limit)
.await?
.into_iter()
.map(|event| build_admin_proxy_node_event_payload(&event))
.collect::<Vec<_>>();
Ok(build_admin_proxy_node_events_payload_response(items))
.app
.list_proxy_node_metrics(node_id, step, from_unix_secs, to_unix_secs, limit)
.await?;
Ok(build_admin_proxy_node_metrics_payload_response(
step,
from_unix_secs,
to_unix_secs,
items,
))
}
pub(crate) async fn build_admin_proxy_fleet_metrics_response(
&self,
step: aether_data::repository::proxy_nodes::ProxyNodeMetricsStep,
from_unix_secs: u64,
to_unix_secs: u64,
limit: usize,
) -> Result<Response<Body>, GatewayError> {
if !self.has_proxy_node_reader() {
return Ok(build_admin_proxy_nodes_data_unavailable_response());
}
let items = self
.app
.list_proxy_fleet_metrics(step, from_unix_secs, to_unix_secs, limit)
.await?;
Ok(build_admin_proxy_fleet_metrics_payload_response(
step,
from_unix_secs,
to_unix_secs,
items,
))
}
pub(crate) async fn unregister_proxy_node(

139
apps/aether-gateway/src/handlers/admin/system/proxy_nodes.rs
View File

@@ -10,12 +10,14 @@ use crate::maintenance::{
};
use crate::GatewayError;
use aether_admin::system::{
admin_proxy_node_event_node_id_from_path, build_admin_proxy_node_payload,
build_admin_proxy_nodes_data_unavailable_response, build_admin_proxy_nodes_not_found_response,
admin_proxy_node_event_node_id_from_path, admin_proxy_node_metrics_node_id_from_path,
build_admin_proxy_node_payload, build_admin_proxy_nodes_data_unavailable_response,
build_admin_proxy_nodes_not_found_response,
};
use aether_contracts::tunnel::{
TUNNEL_RELAY_FORWARDED_BY_HEADER, TUNNEL_RELAY_OWNER_INSTANCE_HEADER,
};
use aether_data::repository::proxy_nodes::{ProxyNodeEventQuery, ProxyNodeMetricsStep};
use axum::{
body::{Body, Bytes},
http,
@@ -148,6 +150,9 @@ const DEFAULT_PROXY_CONNECTIVITY_PROBE_URL: &str = "https://www.cloudflare.com/c
const PROXY_CONNECTIVITY_TIMEOUT_SECS: u64 = 10;
const TUNNEL_RELAY_ENVELOPE_CONTENT_TYPE: &str = "application/vnd.aether.tunnel-envelope";
const MAX_PROXY_CONNECTIVITY_RESPONSE_BYTES: usize = 64 * 1024;
const PROXY_NODE_METRICS_MAX_POINTS: usize = 50_000;
const PROXY_NODE_METRICS_1M_MAX_WINDOW_SECS: u64 = 30 * 24 * 60 * 60;
const PROXY_NODE_METRICS_1H_MAX_WINDOW_SECS: u64 = 365 * 24 * 60 * 60;
#[cfg(test)]
fn manual_proxy_connectivity_probe_url_override() -> &'static std::sync::RwLock<Option<String>> {
@@ -244,13 +249,53 @@ pub(crate) async fn maybe_build_local_admin_proxy_nodes_response(
return Ok(Some(build_admin_proxy_nodes_not_found_response()));
};
let limit = query_param_value(request_context.query_string(), "limit")
.and_then(|value| value.parse::<usize>().ok())
.filter(|value| *value > 0 && *value <= 200)
.unwrap_or(50);
let query = match parse_proxy_node_event_query(request_context.query_string()) {
Ok(query) => query,
Err(response) => return Ok(Some(response)),
};
return Ok(Some(
state
.build_admin_proxy_node_events_response(node_id, limit)
.build_admin_proxy_node_events_response(node_id, &query)
.await?,
));
}
if decision.route_kind.as_deref() == Some("list_node_metrics")
&& request_context.method() == http::Method::GET
{
let Some(node_id) = admin_proxy_node_metrics_node_id_from_path(request_context.path())
else {
return Ok(Some(build_admin_proxy_nodes_not_found_response()));
};
let (step, from_unix_secs, to_unix_secs, limit) =
match parse_proxy_node_metrics_query(request_context.query_string()) {
Ok(query) => query,
Err(response) => return Ok(Some(response)),
};
return Ok(Some(
state
.build_admin_proxy_node_metrics_response(
node_id,
step,
from_unix_secs,
to_unix_secs,
limit,
)
.await?,
));
}
if decision.route_kind.as_deref() == Some("list_fleet_metrics")
&& request_context.method() == http::Method::GET
{
let (step, from_unix_secs, to_unix_secs, limit) =
match parse_proxy_node_metrics_query(request_context.query_string()) {
Ok(query) => query,
Err(response) => return Ok(Some(response)),
};
return Ok(Some(
state
.build_admin_proxy_fleet_metrics_response(step, from_unix_secs, to_unix_secs, limit)
.await?,
));
}
@@ -1976,6 +2021,86 @@ fn validate_optional_object(value: Option<&Value>, field: &str) -> Result<(), Re
Ok(())
}
fn parse_proxy_node_event_query(
query: Option<&str>,
) -> Result<ProxyNodeEventQuery, Response<Body>> {
let limit = query_param_value(query, "limit")
.map(|value| parse_query_u64("limit", &value))
.transpose()?
.and_then(|value| usize::try_from(value).ok())
.filter(|value| *value > 0 && *value <= 200)
.unwrap_or(50);
let from_unix_secs = query_param_value(query, "from")
.map(|value| parse_query_u64("from", &value))
.transpose()?;
let to_unix_secs = query_param_value(query, "to")
.map(|value| parse_query_u64("to", &value))
.transpose()?;
if from_unix_secs
.zip(to_unix_secs)
.is_some_and(|(from, to)| from > to)
{
return Err(bad_request_response("from 不能大于 to"));
}
let event_type = query_param_value(query, "event_type")
.map(|value| value.trim().to_ascii_lowercase())
.filter(|value| !value.is_empty());
Ok(ProxyNodeEventQuery {
limit,
from_unix_secs,
to_unix_secs,
event_type,
})
}
fn parse_proxy_node_metrics_query(
query: Option<&str>,
) -> Result<(ProxyNodeMetricsStep, u64, u64, usize), Response<Body>> {
let step = match query_param_value(query, "step")
.unwrap_or_else(|| "1m".to_string())
.trim()
{
"1m" => ProxyNodeMetricsStep::OneMinute,
"1h" => ProxyNodeMetricsStep::OneHour,
_ => return Err(bad_request_response("step 仅支持 1m 或 1h")),
};
let from_unix_secs = query_param_value(query, "from")
.ok_or_else(|| bad_request_response("from 为必填 Unix 秒时间戳"))?;
let from_unix_secs = parse_query_u64("from", &from_unix_secs)?;
let to_unix_secs = query_param_value(query, "to")
.ok_or_else(|| bad_request_response("to 为必填 Unix 秒时间戳"))?;
let to_unix_secs = parse_query_u64("to", &to_unix_secs)?;
if from_unix_secs > to_unix_secs {
return Err(bad_request_response("from 不能大于 to"));
}
let window_secs = to_unix_secs.saturating_sub(from_unix_secs);
let max_window_secs = match step {
ProxyNodeMetricsStep::OneMinute => PROXY_NODE_METRICS_1M_MAX_WINDOW_SECS,
ProxyNodeMetricsStep::OneHour => PROXY_NODE_METRICS_1H_MAX_WINDOW_SECS,
};
if window_secs > max_window_secs {
return Err(bad_request_response(match step {
ProxyNodeMetricsStep::OneMinute => "1m 最大查询窗口为 30 天",
ProxyNodeMetricsStep::OneHour => "1h 最大查询窗口为 365 天",
}));
}
let points = window_secs / step.bucket_size_secs() + 1;
let limit = usize::try_from(points)
.ok()
.filter(|value| *value > 0 && *value <= PROXY_NODE_METRICS_MAX_POINTS)
.ok_or_else(|| bad_request_response("查询点数过多"))?;
Ok((step, from_unix_secs, to_unix_secs, limit))
}
fn parse_query_u64(field: &str, value: &str) -> Result<u64, Response<Body>> {
value
.parse::<u64>()
.map_err(|_| bad_request_response(format!("{field} 必须是非负 Unix 秒时间戳")))
}
fn bad_request_response(detail: impl Into<String>) -> Response<Body> {
(
http::StatusCode::BAD_REQUEST,