i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-10-08 02:17:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(security): harden gateway boundaries and usage policies

Browse Source 
Consolidate subscription usage policy enforcement, privacy-safe persistence, and gateway security hardening into one reviewable change.

Includes bounded HTTP and execution envelopes, header and protocol guards, DNS and relay validation, authentication and secret projection hardening, secure backup/install paths, and regression coverage.
This commit is contained in:
elky
2026-09-04 03:45:52 +08:00
parent ddcbeb3ae9
commit 579f2c7cc1
1019 changed files with 190437 additions and 26080 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/aether-gateway/src/handlers/admin/model/external_cache.rs
+259 -26
View File
@@ -10,6 +10,7 @@ use axum::http;
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use std::collections::BTreeMap;
use std::net::{IpAddr, SocketAddr};
use std::time::Duration;
use tracing::warn;
@@ -23,6 +24,12 @@ pub(in crate::handlers::admin) const ADMIN_EXTERNAL_MODELS_PROXY_NODE_CONFIG_KEY
"external_models_proxy_node_id";
const ADMIN_EXTERNAL_MODELS_CONNECT_TIMEOUT_MS: u64 = 10_000;
const ADMIN_EXTERNAL_MODELS_TOTAL_TIMEOUT_MS: u64 = 300_000;
const ADMIN_EXTERNAL_MODELS_RESPONSE_LIMIT_BYTES: usize = 8 * 1024 * 1024;
// Keep the cache envelope bounded independently of the upstream body limit.
// Normalization adds a small amount of metadata, while a corrupted/shared
// runtime KV value must never be allowed to drive an unbounded serde
// allocation during cache reads.
const ADMIN_EXTERNAL_MODELS_CACHE_MAX_BYTES: usize = 16 * 1024 * 1024;
pub(crate) const ADMIN_EXTERNAL_MODELS_CONFIG_MUTATION_LOCK_KEY: &str =
"admin:external_models_proxy_node_config:mutation";
const ADMIN_EXTERNAL_MODELS_CONFIG_MUTATION_LOCK_TTL: Duration = Duration::from_secs(10 * 60);
@@ -34,6 +41,13 @@ struct AdminExternalModelsCacheEnvelope {
payload: Value,
}
#[derive(Debug)]
struct ResolvedAdminExternalModelsSource {
url: url::Url,
host: String,
addresses: Vec<SocketAddr>,
}
#[cfg(test)]
pub(crate) struct AdminExternalModelsSourceUrlEnvGuard {
previous: Option<String>,
@@ -76,6 +90,104 @@ fn admin_external_models_source_url() -> String {
.unwrap_or_else(|| ADMIN_EXTERNAL_MODELS_SOURCE_URL_DEFAULT.to_string())
}
fn parse_admin_external_models_source_url(
raw_url: &str,
allow_insecure_test_target: bool,
) -> Result<(url::Url, String, u16), GatewayError> {
let url = url::Url::parse(raw_url)
.map_err(|_| GatewayError::Internal("external models source URL is invalid".to_string()))?;
let allowed_scheme =
url.scheme() == "https" || (allow_insecure_test_target && url.scheme() == "http");
if !allowed_scheme
|| !url.username().is_empty()
|| url.password().is_some()
|| url.query().is_some()
|| url.fragment().is_some()
{
return Err(GatewayError::Internal(
"external models source must be an HTTPS URL without credentials, query, or fragment"
.to_string(),
));
}
let host = url.host_str().map(ToOwned::to_owned).ok_or_else(|| {
GatewayError::Internal("external models source is missing a host".to_string())
})?;
let port = url.port_or_known_default().ok_or_else(|| {
GatewayError::Internal("external models source is missing a port".to_string())
})?;
Ok((url, host, port))
}
fn validate_admin_external_models_source_addresses(
addresses: &[SocketAddr],
allow_insecure_test_target: bool,
) -> Result<(), GatewayError> {
if addresses.is_empty() {
return Err(GatewayError::Internal(
"external models source DNS resolution returned no addresses".to_string(),
));
}
if !allow_insecure_test_target
&& addresses
.iter()
.any(|address| aether_http::is_private_or_reserved_ip(address.ip()))
{
return Err(GatewayError::Internal(
"external models source resolves to a private or reserved address".to_string(),
));
}
Ok(())
}
async fn resolve_admin_external_models_source(
raw_url: &str,
allow_insecure_test_target: bool,
) -> Result<ResolvedAdminExternalModelsSource, GatewayError> {
let (url, host, port) =
parse_admin_external_models_source_url(raw_url, allow_insecure_test_target)?;
let addresses = if let Ok(ip) = host.parse::<IpAddr>() {
vec![SocketAddr::new(ip, port)]
} else {
aether_http::lookup_host_with_limits(
host.as_str(),
port,
aether_http::DEFAULT_DNS_LOOKUP_TIMEOUT,
)
.await
.map_err(|_| {
GatewayError::Internal("external models source DNS resolution failed".to_string())
})?
};
validate_admin_external_models_source_addresses(&addresses, allow_insecure_test_target)?;
Ok(ResolvedAdminExternalModelsSource {
url,
host,
addresses,
})
}
fn build_admin_external_models_direct_client(
source: &ResolvedAdminExternalModelsSource,
) -> Result<reqwest::Client, GatewayError> {
let mut builder = aether_http::apply_http_client_config(
reqwest::Client::builder()
.no_proxy()
.redirect(reqwest::redirect::Policy::none()),
&aether_http::HttpClientConfig {
connect_timeout_ms: Some(ADMIN_EXTERNAL_MODELS_CONNECT_TIMEOUT_MS),
request_timeout_ms: Some(ADMIN_EXTERNAL_MODELS_TOTAL_TIMEOUT_MS),
http2_adaptive_window: true,
..aether_http::HttpClientConfig::default()
},
);
if source.host.parse::<IpAddr>().is_err() {
builder = builder.resolve_to_addrs(&source.host, &source.addresses);
}
builder.build().map_err(|_| {
GatewayError::Internal("external models HTTP client initialization failed".to_string())
})
}
fn normalize_admin_external_models_payload(payload: serde_json::Value) -> serde_json::Value {
mark_external_models_official_providers(&payload).unwrap_or(payload)
}
@@ -116,6 +228,11 @@ async fn store_admin_external_models_cache(
};
let serialized =
serde_json::to_string(&envelope).map_err(|err| GatewayError::Internal(err.to_string()))?;
if serialized.len() > ADMIN_EXTERNAL_MODELS_CACHE_MAX_BYTES {
return Err(GatewayError::Internal(
"external models cache envelope exceeds the allowed size".to_string(),
));
}
state
.as_ref()
.runtime_kv_setex(
@@ -127,6 +244,13 @@ async fn store_admin_external_models_cache(
Ok(())
}
fn parse_admin_external_models_cache(raw: &str) -> Option<AdminExternalModelsCacheEnvelope> {
if raw.len() > ADMIN_EXTERNAL_MODELS_CACHE_MAX_BYTES {
return None;
}
serde_json::from_str::<AdminExternalModelsCacheEnvelope>(raw).ok()
}
fn normalize_admin_external_models_proxy_node_id(
value: Option<&Value>,
) -> Result<Option<String>, GatewayError> {
@@ -345,7 +469,8 @@ async fn fetch_admin_external_models_from_source(
request_id: &str,
proxy_node_id: Option<&str>,
) -> Result<serde_json::Value, GatewayError> {
let url = admin_external_models_source_url();
let source_url = admin_external_models_source_url();
let source = resolve_admin_external_models_source(&source_url, cfg!(test)).await?;
if let Some(node_id) = proxy_node_id {
let Some(proxy) = state.resolve_admin_proxy_node_snapshot(Some(node_id)).await else {
warn!(
@@ -374,7 +499,7 @@ async fn fetch_admin_external_models_from_source(
),
(
EXECUTION_REQUEST_FOLLOW_REDIRECTS_HEADER.to_string(),
"true".to_string(),
"false".to_string(),
),
]);
let plan = ExecutionPlan {
@@ -385,7 +510,7 @@ async fn fetch_admin_external_models_from_source(
endpoint_id: String::new(),
key_id: String::new(),
method: http::Method::GET.as_str().to_string(),
url,
url: source.url.to_string(),
headers,
content_type: None,
content_encoding: None,
@@ -409,8 +534,12 @@ async fn fetch_admin_external_models_from_source(
..ExecutionTimeouts::default()
}),
};
let bounded_plan = crate::execution_runtime::transport::with_upstream_response_body_limit(
&plan,
ADMIN_EXTERNAL_MODELS_RESPONSE_LIMIT_BYTES,
);
let result = match state
.execute_execution_runtime_sync_plan(Some(request_id), &plan)
.execute_execution_runtime_sync_plan(Some(request_id), &bounded_plan)
.await
{
Ok(result) => result,
@@ -444,19 +573,44 @@ async fn fetch_admin_external_models_from_source(
return Ok(normalize_admin_external_models_payload(payload));
}
let response = state
.http_client()
.get(&url)
let client = build_admin_external_models_direct_client(&source)?;
let response = client
.get(source.url)
.header(reqwest::header::ACCEPT, "application/json")
.header(
reqwest::header::USER_AGENT,
"aether-gateway/external-models",
)
.send()
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?;
let response = response
.error_for_status()
.map_err(|err| GatewayError::Internal(err.to_string()))?;
let payload = response
.json::<serde_json::Value>()
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?;
.map_err(|err| {
let error_message = err.to_string();
let transport_error_kind =
classify_admin_external_models_transport_error(&error_message);
warn!(
request_id = %request_id,
transport_error_kind,
"external models direct request failed"
);
GatewayError::Internal("external models source request failed".to_string())
})?;
if !response.status().is_success() {
return Err(GatewayError::Internal(format!(
"external models source returned HTTP {}",
response.status().as_u16()
)));
}
let body = aether_http::read_response_bytes_with_limit(
response,
ADMIN_EXTERNAL_MODELS_RESPONSE_LIMIT_BYTES,
)
.await
.map_err(|_| {
GatewayError::Internal("external models source response read failed".to_string())
})?;
let payload = serde_json::from_slice::<serde_json::Value>(&body).map_err(|_| {
GatewayError::Internal("external models source returned invalid JSON".to_string())
})?;
Ok(normalize_admin_external_models_payload(payload))
}
@@ -470,8 +624,8 @@ pub(crate) async fn read_admin_external_models_cache(
.runtime_kv_get(ADMIN_EXTERNAL_MODELS_CACHE_KEY)
.await?
{
match serde_json::from_str::<AdminExternalModelsCacheEnvelope>(&raw) {
Ok(envelope)
match parse_admin_external_models_cache(&raw) {
Some(envelope)
if envelope.schema_version == ADMIN_EXTERNAL_MODELS_CACHE_VERSION
&& envelope.proxy_node_id == proxy_node_id =>
{
@@ -479,25 +633,23 @@ pub(crate) async fn read_admin_external_models_cache(
envelope.payload,
)));
}
Ok(_) => {}
Err(err) => {
warn!(error = %err, "failed to parse cached external models payload");
}
Some(_) => {}
None => warn!("failed to parse cached external models payload"),
}
}
match fetch_admin_external_models_from_source(state, request_id, proxy_node_id.as_deref()).await
{
Ok(payload) => {
if let Err(err) =
if let Err(_) =
store_admin_external_models_cache(state, proxy_node_id.as_deref(), &payload).await
{
warn!(error = ?err, "failed to store fetched external models cache");
warn!("failed to store fetched external models cache");
}
Ok(Some(payload))
}
Err(err) => {
warn!(error = ?err, "failed to fetch external models catalog");
Err(_) => {
warn!("failed to fetch external models catalog");
Ok(None)
}
}
@@ -518,7 +670,9 @@ mod tests {
use super::{
admin_external_models_source_url, classify_admin_external_models_transport_error,
normalize_admin_external_models_payload, normalize_admin_external_models_proxy_node_id,
read_admin_external_models_cache, set_admin_external_models_source_url_for_tests,
parse_admin_external_models_cache, parse_admin_external_models_source_url,
read_admin_external_models_cache, resolve_admin_external_models_source,
set_admin_external_models_source_url_for_tests, ADMIN_EXTERNAL_MODELS_CACHE_MAX_BYTES,
};
use crate::handlers::admin::request::AdminAppState;
use crate::tests::{start_server, AppState};
@@ -590,6 +744,51 @@ mod tests {
);
}
#[test]
fn production_external_models_source_requires_safe_https_url_shape() {
assert!(
parse_admin_external_models_source_url("https://models.dev/api.json", false).is_ok()
);
for source_url in [
"http://models.dev/api.json",
"file:///etc/passwd",
"https://user:[email protected]/api.json",
"https://models.dev/api.json?next=http://169.254.169.254",
"https://models.dev/api.json#fragment",
] {
assert!(
parse_admin_external_models_source_url(source_url, false).is_err(),
"source URL should be rejected: {source_url}"
);
}
}
#[test]
fn external_models_cache_parser_rejects_oversized_runtime_values() {
let oversized = "x".repeat(ADMIN_EXTERNAL_MODELS_CACHE_MAX_BYTES + 1);
assert!(parse_admin_external_models_cache(&oversized).is_none());
let valid = r#"{"schema_version":2,"proxy_node_id":null,"payload":{}}"#;
assert!(parse_admin_external_models_cache(valid).is_some());
}
#[tokio::test]
async fn production_external_models_source_rejects_private_ip_literals() {
for source_url in [
"https://127.0.0.1/api.json",
"https://169.254.169.254/latest/meta-data",
"https://[::1]/api.json",
] {
assert!(
resolve_admin_external_models_source(source_url, false)
.await
.is_err(),
"private source URL should be rejected: {source_url}"
);
}
}
#[tokio::test]
async fn read_external_models_fetches_remote_payload_when_cache_missing() {
let upstream = Router::new().route(
@@ -623,4 +822,38 @@ mod tests {
upstream_handle.abort();
}
#[tokio::test]
async fn direct_external_models_fetch_does_not_follow_redirects() {
let upstream = Router::new()
.route(
"/redirect",
get(|| async { axum::response::Redirect::temporary("/api.json") }),
)
.route(
"/api.json",
get(|| async {
Json(json!({
"openai": {
"name": "redirected payload",
"models": {}
}
}))
}),
);
let (upstream_url, upstream_handle) = start_server(upstream).await;
let _guard =
set_admin_external_models_source_url_for_tests(&format!("{upstream_url}/redirect"));
let state = AppState::new().expect("gateway should build");
let payload = read_admin_external_models_cache(
&AdminAppState::new(&state),
"external-models-redirect",
)
.await
.expect("external models read should not fail");
assert!(payload.is_none(), "redirected payload must not be accepted");
upstream_handle.abort();
}
}
apps/aether-gateway/src/handlers/admin/model/global_models/routes/core/writes.rs
+84 -6
View File
@@ -27,6 +27,36 @@ use axum::{
Json,
};
use serde_json::json;
use std::collections::HashSet;
const MAX_ADMIN_GLOBAL_MODEL_BATCH_ITEMS: usize = 100;
fn normalize_admin_global_model_batch_ids(
ids: Vec<String>,
field_name: &str,
) -> Result<Vec<String>, String> {
if ids.len() > MAX_ADMIN_GLOBAL_MODEL_BATCH_ITEMS {
return Err(format!(
"{field_name} 最多 {MAX_ADMIN_GLOBAL_MODEL_BATCH_ITEMS} 个"
));
}
let mut seen = HashSet::with_capacity(ids.len());
let mut normalized = Vec::with_capacity(ids.len());
for id in ids {
let trimmed = id.trim();
if trimmed.is_empty() {
// Keep the original value so batch-delete retains its existing per-item failure.
normalized.push(id);
continue;
}
let trimmed = trimmed.to_string();
if seen.insert(trimmed.clone()) {
normalized.push(trimmed);
}
}
Ok(normalized)
}
pub(super) async fn maybe_build_local_admin_global_models_write_response(
state: &AdminAppState<'_>,
@@ -212,17 +242,20 @@ async fn build_batch_delete_global_models_response(
Ok(payload) => payload,
Err(response) => return Ok(response),
};
let ids = match normalize_admin_global_model_batch_ids(payload.ids, "ids") {
Ok(ids) => ids,
Err(detail) => return Ok(bad_request_response(detail)),
};
let mut success_count = 0usize;
let mut failed = Vec::new();
for id in payload.ids {
let trimmed = id.trim();
if trimmed.is_empty() {
for id in ids {
if id.trim().is_empty() {
failed.push(json!({"id": id, "error": "not found"}));
continue;
}
let Some(existing) = state.get_admin_global_model_by_id(trimmed).await? else {
failed.push(json!({"id": trimmed, "error": "not found"}));
let Some(existing) = state.get_admin_global_model_by_id(&id).await? else {
failed.push(json!({"id": id, "error": "not found"}));
continue;
};
if state.delete_admin_global_model(&existing.id).await? {
@@ -245,6 +278,46 @@ async fn build_batch_delete_global_models_response(
))
}
#[cfg(test)]
mod batch_boundary_tests {
use super::{normalize_admin_global_model_batch_ids, MAX_ADMIN_GLOBAL_MODEL_BATCH_ITEMS};
#[test]
fn global_model_batch_ids_are_bounded_and_deduplicated() {
assert_eq!(
normalize_admin_global_model_batch_ids(
vec![
"model-2".to_string(),
"model-1".to_string(),
" model-2 ".to_string(),
" ".to_string(),
],
"ids",
)
.expect("valid ids"),
vec![
"model-2".to_string(),
"model-1".to_string(),
" ".to_string(),
]
);
assert!(normalize_admin_global_model_batch_ids(
(0..=MAX_ADMIN_GLOBAL_MODEL_BATCH_ITEMS)
.map(|index| format!("model-{index}"))
.collect(),
"ids",
)
.is_err());
assert!(normalize_admin_global_model_batch_ids(
(0..MAX_ADMIN_GLOBAL_MODEL_BATCH_ITEMS)
.map(|index| format!("provider-{index}"))
.collect(),
"provider_ids",
)
.is_ok());
}
}
async fn build_assign_to_providers_response(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
@@ -259,10 +332,15 @@ async fn build_assign_to_providers_response(
Ok(payload) => payload,
Err(response) => return Ok(response),
};
let provider_ids =
match normalize_admin_global_model_batch_ids(payload.provider_ids, "provider_ids") {
Ok(provider_ids) => provider_ids,
Err(detail) => return Ok(bad_request_response(detail)),
};
let payload: serde_json::Value = match build_admin_assign_global_model_to_providers_payload(
state,
&global_model_id,
payload.provider_ids,
provider_ids,
payload.create_models.unwrap_or(false),
)
.await