i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-10-07 01:47:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(security): harden gateway boundaries and usage policies

Browse Source 
Consolidate subscription usage policy enforcement, privacy-safe persistence, and gateway security hardening into one reviewable change.

Includes bounded HTTP and execution envelopes, header and protocol guards, DNS and relay validation, authentication and secret projection hardening, secure backup/install paths, and regression coverage.
This commit is contained in:
elky
2026-09-04 03:45:52 +08:00
parent ddcbeb3ae9
commit 579f2c7cc1
1019 changed files with 190437 additions and 26080 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/aether-admin/src/observability/monitoring.rs
+42 -101
View File
@@ -1,5 +1,10 @@
use aether_data_contracts::repository::{
candidates::{DecisionTrace, DecisionTraceCandidate, RequestCandidateStatus},
candidates::{
sanitize_request_candidate_api_formats, sanitize_request_candidate_error_type,
sanitize_request_candidate_extra_data, sanitize_request_candidate_required_capabilities,
sanitize_request_candidate_skip_reason, DecisionTrace, DecisionTraceCandidate,
RequestCandidateStatus,
},
provider_catalog::StoredProviderCatalogKey,
usage::StoredRequestUsageAudit,
};
@@ -9,7 +14,6 @@ use axum::{
response::{IntoResponse, Response},
Json,
};
use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
use serde_json::{json, Value};
use std::collections::BTreeMap;
@@ -329,7 +333,19 @@ pub fn build_admin_monitoring_trace_request_candidate_payload_with_key_accounts(
usage: Option<&StoredRequestUsageAudit>,
key_accounts: &BTreeMap<String, AdminMonitoringKeyAccountDisplay>,
) -> Value {
let mut item = item.clone();
item.sanitize_sensitive_diagnostics();
let candidate = &item.candidate;
let sanitized_extra_data =
build_admin_monitoring_trace_candidate_extra_data(candidate.extra_data.as_ref(), usage);
let sanitized_extra_data_ref =
(!sanitized_extra_data.is_null()).then_some(&sanitized_extra_data);
let sanitized_key_api_formats =
sanitize_request_candidate_api_formats(item.provider_key_api_formats.clone());
let sanitized_key_capabilities =
sanitize_request_candidate_required_capabilities(item.provider_key_capabilities.clone());
let sanitized_required_capabilities =
sanitize_request_candidate_required_capabilities(candidate.required_capabilities.clone());
let key_account = candidate
.key_id
.as_deref()
@@ -349,32 +365,32 @@ pub fn build_admin_monitoring_trace_request_candidate_payload_with_key_accounts(
"endpoint_name": item.endpoint_api_format,
"endpoint_api_family": item.endpoint_api_family,
"endpoint_kind": item.endpoint_kind,
"endpoint_format_acceptance_config": item.endpoint_format_acceptance_config,
"endpoint_format_acceptance_config": serde_json::Value::Null,
"key_id": candidate.key_id,
"key_name": item.provider_key_name,
"key_account_label": key_account.and_then(|item| item.label.clone()),
"key_preview": serde_json::Value::Null,
"key_auth_type": item.provider_key_auth_type,
"key_api_formats": item.provider_key_api_formats,
"key_api_formats": sanitized_key_api_formats,
"key_internal_priority": item.provider_key_internal_priority,
"key_global_priority_by_format": item.provider_key_global_priority_by_format,
"key_global_priority_by_format": serde_json::Value::Null,
"key_oauth_plan_type": key_account.and_then(|item| item.oauth_plan_type.clone()),
"key_capabilities": item.provider_key_capabilities,
"required_capabilities": candidate.required_capabilities,
"key_capabilities": sanitized_key_capabilities,
"required_capabilities": sanitized_required_capabilities,
"status": candidate.status,
"skip_reason": candidate.skip_reason,
"skip_reason": sanitize_request_candidate_skip_reason(candidate.skip_reason.clone()),
"is_cached": candidate.is_cached,
"status_code": candidate.status_code,
"error_type": candidate.error_type,
"error_message": candidate.error_message,
"error_type": sanitize_request_candidate_error_type(candidate.error_type.clone()),
"error_message": serde_json::Value::Null,
"latency_ms": candidate.latency_ms,
"concurrent_requests": candidate.concurrent_requests,
"ranking": build_admin_monitoring_trace_candidate_ranking(candidate.extra_data.as_ref()),
"image_progress": candidate.extra_data.as_ref()
"ranking": build_admin_monitoring_trace_candidate_ranking(sanitized_extra_data_ref),
"image_progress": sanitized_extra_data_ref
.and_then(|value| value.get("image_progress"))
.cloned()
.unwrap_or(Value::Null),
"extra_data": build_admin_monitoring_trace_candidate_extra_data(candidate.extra_data.as_ref(), usage),
"extra_data": sanitized_extra_data,
"created_at": unix_ms_to_rfc3339(candidate.created_at_unix_ms),
"started_at": candidate.started_at_unix_ms.and_then(unix_ms_to_rfc3339),
"finished_at": candidate.finished_at_unix_ms.and_then(unix_ms_to_rfc3339),
@@ -502,11 +518,8 @@ fn build_admin_monitoring_trace_candidate_extra_data(
existing: Option<&Value>,
usage: Option<&StoredRequestUsageAudit>,
) -> Value {
let mut extra_data = match existing {
Some(Value::Object(object)) => Some(object.clone()),
Some(other) => return other.clone(),
None => None,
};
let mut extra_data = sanitize_request_candidate_extra_data(existing.cloned())
.and_then(|value| value.as_object().cloned());
if let Some(usage) = usage {
let extra_object = extra_data.get_or_insert_with(serde_json::Map::new);
@@ -534,9 +547,6 @@ fn build_admin_monitoring_trace_candidate_extra_data(
if let Some(response) = admin_monitoring_trace_response_data(
"upstream_response",
usage.status_code,
usage.response_headers.as_ref(),
usage.response_body.as_ref(),
usage.response_body_ref.as_deref(),
usage.response_body_state,
) {
merge_admin_monitoring_trace_response(extra_object, "upstream_response", response);
@@ -563,92 +573,25 @@ fn build_admin_monitoring_trace_candidate_extra_data(
}
}
match extra_data {
Some(object) => Value::Object(object),
None => Value::Null,
}
sanitize_request_candidate_extra_data(extra_data.map(Value::Object)).unwrap_or(Value::Null)
}
fn admin_monitoring_trace_response_data(
source: &str,
status_code: Option<u16>,
headers: Option<&Value>,
body: Option<&Value>,
body_ref: Option<&str>,
body_state: Option<aether_data_contracts::repository::usage::UsageBodyCaptureState>,
) -> Option<Value> {
if status_code.is_none()
&& headers.is_none()
&& body.is_none()
&& body_ref.is_none()
&& body_state.is_none()
{
if status_code.is_none() && body_state.is_none() {
return None;
}
let body = admin_monitoring_trace_response_body(headers, body);
Some(json!({
"source": source,
"status_code": status_code,
"headers": headers.cloned().unwrap_or(Value::Null),
"body": body.unwrap_or(Value::Null),
"body_ref": body_ref,
"body_state": body_state.map(|state| state.as_str()),
}))
}
fn admin_monitoring_trace_response_body(
headers: Option<&Value>,
body: Option<&Value>,
) -> Option<Value> {
let body = body?;
admin_monitoring_decode_connect_json_error_body(headers, body).or_else(|| Some(body.clone()))
}
fn admin_monitoring_decode_connect_json_error_body(
headers: Option<&Value>,
body: &Value,
) -> Option<Value> {
if !admin_monitoring_headers_indicate_connect_json(headers) {
return None;
}
let body_base64 = match body {
Value::String(value) => Some(value.as_str()),
Value::Object(object) => object
.get("encoding")
.and_then(Value::as_str)
.is_some_and(|value| value.eq_ignore_ascii_case("base64"))
.then(|| object.get("data").and_then(Value::as_str))
.flatten(),
_ => None,
}?
.trim();
if body_base64.is_empty() {
return None;
}
let body_bytes = BASE64_STANDARD.decode(body_base64).ok()?;
aether_ai_formats::api::extract_provider_private_stream_error_body(None, &body_bytes)
}
fn admin_monitoring_headers_indicate_connect_json(headers: Option<&Value>) -> bool {
headers
.and_then(Value::as_object)
.and_then(|object| {
object.iter().find_map(|(key, value)| {
key.eq_ignore_ascii_case("content-type")
.then(|| value.as_str())
.flatten()
})
})
.map(str::trim)
.is_some_and(|value| {
let value = value.to_ascii_lowercase();
value.contains("application/connect+json") || value.contains("+connect+json")
})
}
fn merge_admin_monitoring_trace_response(
extra_object: &mut serde_json::Map<String, Value>,
key: &str,
@@ -707,27 +650,25 @@ fn admin_monitoring_trace_request_path_and_query(
fn admin_monitoring_usage_request_path(usage: &StoredRequestUsageAudit) -> Option<String> {
admin_monitoring_usage_metadata_string(usage, "request_path")
.and_then(|value| aether_ai_formats::api::sanitize_request_path(&value))
}
fn admin_monitoring_usage_request_query_string(usage: &StoredRequestUsageAudit) -> Option<String> {
admin_monitoring_usage_metadata_string(usage, "request_query_string")
.map(|value| value.trim_start_matches('?').to_string())
.filter(|value| !value.is_empty())
.and_then(|value| aether_ai_formats::api::sanitize_request_query_string(&value))
}
fn admin_monitoring_usage_request_path_and_query(
usage: &StoredRequestUsageAudit,
) -> Option<String> {
admin_monitoring_usage_metadata_string(usage, "request_path_and_query").or_else(|| {
let path = admin_monitoring_usage_metadata_string(usage, "request_path")?;
let query = admin_monitoring_usage_metadata_string(usage, "request_query_string")
.map(|value| value.trim_start_matches('?').to_string())
.filter(|value| !value.is_empty());
Some(match query {
Some(query) if !path.contains('?') => format!("{path}?{query}"),
_ => path,
admin_monitoring_usage_metadata_string(usage, "request_path_and_query")
.and_then(|value| aether_ai_formats::api::sanitize_request_path_and_query(&value, None))
.or_else(|| {
let path = admin_monitoring_usage_metadata_string(usage, "request_path")?;
let query = admin_monitoring_usage_metadata_string(usage, "request_query_string")
.and_then(|value| aether_ai_formats::api::sanitize_request_query_string(&value));
aether_ai_formats::api::sanitize_request_path_and_query(&path, query.as_deref())
})
})
}
fn admin_monitoring_usage_metadata_string(
crates/aether-admin/src/observability/usage.rs
+208 -32
View File
@@ -24,6 +24,29 @@ use url::form_urlencoded;
pub const ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL: &str = "Admin usage data unavailable";
fn admin_usage_safe_error_message(item: &StoredRequestUsageAudit) -> Option<String> {
if item
.error_message
.as_deref()
.is_none_or(|value| value.trim().is_empty())
&& item.status_code.is_none_or(|status| status < 400)
{
return None;
}
item.error_category
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
.or_else(|| {
item.status_code
.filter(|status| *status >= 400)
.map(|status| format!("http_{status}"))
})
.or_else(|| Some("request_failed".to_string()))
}
pub fn admin_usage_data_unavailable_response(detail: &'static str) -> Response<Body> {
(
http::StatusCode::SERVICE_UNAVAILABLE,
@@ -339,6 +362,77 @@ fn admin_usage_strip_trace_metadata(metadata: &mut serde_json::Map<String, Value
metadata.remove("trace_id");
}
/// Projects historical observability metadata without exposing credentials or
/// URL query/fragment values. This is shared by all admin observability views
/// because older rows may predate the current persistence policy.
pub fn admin_usage_safe_metadata_value(value: &Value) -> Value {
admin_usage_safe_metadata_value_inner(value, false)
}
fn admin_usage_safe_metadata_value_inner(value: &Value, in_realtime_session: bool) -> Value {
match value {
Value::Object(object) => Value::Object(
object
.iter()
.filter_map(|(key, value)| {
let compact = key
.chars()
.filter(|ch| ch.is_ascii_alphanumeric())
.map(|ch| ch.to_ascii_lowercase())
.collect::<String>();
// The generic `token` key filter protects historical rows that may
// predate the persistence projection. Realtime audio usage counters
// are explicitly safe numeric metrics, but only when they occur in the
// structured realtime-session object. Do not broaden this exception to
// arbitrary token-shaped fields or string values.
let safe_realtime_counter = in_realtime_session
&& matches!(compact.as_str(), "inputaudiotokens" | "outputaudiotokens")
&& value.as_u64().is_some();
if (compact.contains("authorization")
|| compact.contains("credential")
|| compact.contains("secret")
|| compact.contains("token")
|| compact.contains("apikey")
|| compact.contains("password")
|| matches!(compact.as_str(), "cookie" | "setcookie" | "headers"))
&& !safe_realtime_counter
{
return None;
}
let child_in_realtime_session = compact == "realtimesession";
Some((
key.clone(),
admin_usage_safe_metadata_value_inner(value, child_in_realtime_session),
))
})
.collect(),
),
Value::Array(items) => Value::Array(
items
.iter()
.map(|item| admin_usage_safe_metadata_value_inner(item, false))
.collect(),
),
Value::String(value) => {
if let Ok(mut url) = url::Url::parse(value.trim()) {
if !url.username().is_empty()
|| url.password().is_some()
|| url.query().is_some()
|| url.fragment().is_some()
{
let _ = url.set_username("");
let _ = url.set_password(None);
url.set_query(None);
url.set_fragment(None);
return Value::String(url.to_string());
}
}
Value::String(value.clone())
}
other => other.clone(),
}
}
fn admin_usage_string_field<'a>(value: &'a Value, field: &str) -> Option<&'a str> {
value
.get(field)
@@ -1219,7 +1313,7 @@ fn admin_usage_active_request_json(
"updated_at": unix_secs_to_rfc3339(item.updated_at_unix_secs),
"response_time_updated_at": admin_usage_response_time_updated_at(item),
"status_code": item.status_code,
"error_message": item.error_message,
"error_message": admin_usage_safe_error_message(item),
"provider": item.provider_name,
"api_key_name": api_key_name,
"provider_key_name": provider_key_name,
@@ -1334,7 +1428,7 @@ pub fn admin_usage_record_json(
"cache_creation_price_per_1m": cache_creation_price_per_1m,
"cache_read_price_per_1m": cache_read_price_per_1m,
"status_code": item.status_code,
"error_message": item.error_message,
"error_message": admin_usage_safe_error_message(item),
"status": item.status,
"request_type": item.request_type,
"has_fallback": admin_usage_has_fallback(item),
@@ -2464,7 +2558,11 @@ pub fn build_admin_usage_detail_payload(
auth_api_key_reader_available,
provider_key_name,
);
let mut metadata = match item.request_metadata.clone() {
let mut metadata = match item
.request_metadata
.as_ref()
.map(admin_usage_safe_metadata_value)
{
Some(Value::Object(object)) => Value::Object(object),
Some(value) => json!({ "request_metadata": value }),
None => json!({}),
@@ -2626,7 +2724,8 @@ mod tests {
admin_usage_has_fallback, admin_usage_is_failed, admin_usage_is_success,
admin_usage_matches_search, admin_usage_matches_status, admin_usage_matches_username,
admin_usage_record_json, admin_usage_resolve_request_capture_body,
admin_usage_total_tokens, admin_usage_upstream_is_stream, build_admin_usage_detail_payload,
admin_usage_safe_metadata_value, admin_usage_total_tokens, admin_usage_upstream_is_stream,
build_admin_usage_detail_payload,
};
use aether_data_contracts::repository::usage::{StoredRequestUsageAudit, UsageBodyField};
@@ -2688,6 +2787,35 @@ mod tests {
assert!(admin_usage_matches_status(&item, Some("completed")));
}
#[test]
fn admin_usage_summaries_do_not_return_historical_raw_error_text() {
let item = StoredRequestUsageAudit {
error_category: Some("authentication_error".to_string()),
..sample_usage(
"failed",
Some(401),
Some(
"upstream said Authorization: Bearer live-secret at https://api.example?key=secret",
),
)
};
let record = admin_usage_record_json(
&item,
&BTreeMap::new(),
&BTreeMap::new(),
false,
false,
None,
);
let active = admin_usage_active_request_json(&item, None, None, None);
assert_eq!(record["error_message"], "authentication_error");
assert_eq!(active["error_message"], "authentication_error");
assert!(!record.to_string().contains("live-secret"));
assert!(!active.to_string().contains("live-secret"));
}
#[test]
fn client_requested_stream_prefers_request_metadata_flag() {
let item = StoredRequestUsageAudit {
@@ -2842,6 +2970,35 @@ mod tests {
assert_eq!(payload["metadata"]["realtime_session"], realtime_session);
}
#[test]
fn admin_usage_safe_metadata_keeps_only_numeric_realtime_audio_counters() {
let projected = admin_usage_safe_metadata_value(&json!({
"input_audio_tokens": 99,
"refresh_token": "should-be-removed",
"realtime_session": {
"input_audio_tokens": 7,
"output_audio_tokens": 3,
"input_audio_tokens_text": "should-be-removed",
"refresh_token": 11,
"nested": {
"output_audio_tokens": 5
}
}
}));
assert!(projected.get("input_audio_tokens").is_none());
assert!(projected.get("refresh_token").is_none());
assert_eq!(projected["realtime_session"]["input_audio_tokens"], 7);
assert_eq!(projected["realtime_session"]["output_audio_tokens"], 3);
assert!(projected["realtime_session"]
.get("input_audio_tokens_text")
.is_none());
assert!(projected["realtime_session"].get("refresh_token").is_none());
assert!(projected["realtime_session"]["nested"]
.get("output_audio_tokens")
.is_none());
}
#[test]
fn admin_usage_record_infers_client_family_from_user_agent() {
let item = StoredRequestUsageAudit {
@@ -3313,6 +3470,46 @@ mod tests {
assert_eq!(payload["has_client_response_body"], true);
}
#[test]
fn detail_payload_sanitizes_historical_request_metadata() {
let item = StoredRequestUsageAudit {
request_metadata: Some(json!({
"trace_id": "internal-trace",
"authorization": "Bearer live-secret",
"nested": {
"refresh_token": "refresh-secret",
"endpoint_url": "https://user:[email protected]/v1?key=secret#fragment",
"safe_count": 2
}
})),
..sample_usage("failed", Some(500), Some("failed"))
};
let payload = build_admin_usage_detail_payload(
&item,
&BTreeMap::new(),
&BTreeMap::new(),
false,
false,
None,
false,
None,
&BTreeMap::new(),
);
let metadata = &payload["metadata"];
assert!(metadata.get("authorization").is_none());
assert!(metadata["nested"].get("refresh_token").is_none());
assert_eq!(
metadata["nested"]["endpoint_url"],
"https://api.example.test/v1"
);
assert_eq!(metadata["nested"]["safe_count"], 2);
let encoded = metadata.to_string();
for secret in ["live-secret", "refresh-secret", "password", "key=secret"] {
assert!(!encoded.contains(secret), "leaked {secret}");
}
}
#[test]
fn detail_payload_separates_upstream_client_and_summary_errors() {
let item = StoredRequestUsageAudit {
@@ -3505,14 +3702,8 @@ mod tests {
&BTreeMap::new(),
);
assert_eq!(
payload["client_error"]["message"],
"没有可用提供商支持模型 gpt-5.4 的流式请求"
);
assert_eq!(
payload["failure_summary"]["message"],
"没有可用提供商支持模型 gpt-5.4 的流式请求"
);
assert!(payload["client_error"]["message"].is_null());
assert!(payload["failure_summary"].is_null());
assert_eq!(
payload["scheduling_failure"]["title"],
"本地调度失败：没有可调度候选"
@@ -3522,10 +3713,7 @@ mod tests {
"candidate_list_empty"
);
assert!(payload["scheduling_failure"]["reason_summary"].is_null());
assert_eq!(
payload["scheduling_failure"]["message"],
"没有可用提供商支持模型 gpt-5.4 的流式请求"
);
assert!(payload["scheduling_failure"]["message"].is_null());
assert_eq!(payload["scheduling_failure"]["no_upstream_attempt"], true);
}
@@ -3562,14 +3750,8 @@ mod tests {
&BTreeMap::new(),
);
assert_eq!(
payload["client_error"]["message"],
"没有可用提供商支持模型 gpt-5.4 的流式请求"
);
assert_eq!(
payload["failure_summary"]["message"],
"没有可用提供商支持模型 gpt-5.4 的流式请求"
);
assert!(payload["client_error"]["message"].is_null());
assert!(payload["failure_summary"].is_null());
assert_eq!(
payload["scheduling_failure"]["title"],
"本地调度失败：所有候选均被跳过"
@@ -3578,14 +3760,8 @@ mod tests {
payload["scheduling_failure"]["reason"],
"all_candidates_skipped"
);
assert_eq!(
payload["scheduling_failure"]["reason_summary"],
"provider_quota_blocked 2 次"
);
assert_eq!(
payload["scheduling_failure"]["message"],
"没有可用提供商支持模型 gpt-5.4 的流式请求"
);
assert!(payload["scheduling_failure"]["reason_summary"].is_null());
assert!(payload["scheduling_failure"]["message"].is_null());
assert_eq!(payload["scheduling_failure"]["no_upstream_attempt"], true);
}