mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-03 01:40:21 +08:00
feat: 引入 aether-runtime/cache/data/http/testkit 基础 crate，完善并发门控与审计系统
新增 crate： - aether-runtime: 服务运行时基础设施（并发门控、分布式并发、指标、队列、优雅关闭、tracing） - aether-cache: 通用 TTL 缓存与命名空间抽象 - aether-data: 数据访问层（PostgreSQL/Redis 后端、repository 模式） - aether-http: HTTP 客户端封装（重试、配置） - aether-testkit: 集成测试工具集（gateway/executor/hub/proxy fixture、等待、负载测试） gateway 扩展： - 引入 audit 模块（shadow 执行审计、决策链路追踪、请求审计 bundle） - 引入 cache 模块（AuthContext 缓存、direct-plan bypass 缓存） - 引入 data 模块（auth/candidates/config/usage/video_tasks 数据访问） - 集成 ConcurrencyGate/DistributedConcurrencyGate 请求门控 - 新增本地 auth 拒绝、过载响应构建器 - 补充 control/auth_cache/video/concurrency 集成测试 aether-proxy 扩展： - AppState 集成 stream_gate / distributed_stream_gate 并发门控 - 新增 ProxyAdmissionError 及准入拒绝流程 - stream_handler 补充门控饱和/不可用场景测试 - 配置与注册客户端逻辑完善 aether-hub 扩展： - main.rs 引入运行时初始化、指标端点、健康检查 - local_relay 重构为 lib.rs 暴露公共接口
This commit is contained in:
fawney19
205
crates/aether-gateway/src/audit/http.rs
Normal file
205
crates/aether-gateway/src/audit/http.rs
Normal file
@@ -0,0 +1,205 @@
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use axum::extract::{Path, Query, State};
|
||||
use axum::response::IntoResponse;
|
||||
use axum::Json;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::json;
|
||||
|
||||
use crate::gateway::{AppState, GatewayError};
|
||||
|
||||
const DEFAULT_RECENT_LIMIT: usize = 20;
|
||||
const MAX_RECENT_LIMIT: usize = 200;
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(crate) struct ListRecentShadowResultsQuery {
|
||||
pub(crate) limit: Option<usize>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub(crate) struct ShadowResultStatusCounts {
|
||||
pub(crate) pending: usize,
|
||||
pub(crate) r#match: usize,
|
||||
pub(crate) mismatch: usize,
|
||||
pub(crate) error: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub(crate) struct ListRecentShadowResultsResponse {
|
||||
pub(crate) items: Vec<aether_data::repository::shadow_results::StoredShadowResult>,
|
||||
pub(crate) limit_applied: usize,
|
||||
pub(crate) counts: ShadowResultStatusCounts,
|
||||
}
|
||||
|
||||
pub(crate) async fn list_recent_shadow_results(
|
||||
State(state): State<AppState>,
|
||||
Query(query): Query<ListRecentShadowResultsQuery>,
|
||||
) -> Result<Json<ListRecentShadowResultsResponse>, GatewayError> {
|
||||
let limit = query
|
||||
.limit
|
||||
.unwrap_or(DEFAULT_RECENT_LIMIT)
|
||||
.clamp(1, MAX_RECENT_LIMIT);
|
||||
let items = state.list_recent_shadow_results(limit).await?;
|
||||
|
||||
let mut counts = ShadowResultStatusCounts {
|
||||
pending: 0,
|
||||
r#match: 0,
|
||||
mismatch: 0,
|
||||
error: 0,
|
||||
};
|
||||
for item in &items {
|
||||
match item.match_status {
|
||||
aether_data::repository::shadow_results::ShadowResultMatchStatus::Pending => {
|
||||
counts.pending += 1
|
||||
}
|
||||
aether_data::repository::shadow_results::ShadowResultMatchStatus::Match => {
|
||||
counts.r#match += 1
|
||||
}
|
||||
aether_data::repository::shadow_results::ShadowResultMatchStatus::Mismatch => {
|
||||
counts.mismatch += 1
|
||||
}
|
||||
aether_data::repository::shadow_results::ShadowResultMatchStatus::Error => {
|
||||
counts.error += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Json(ListRecentShadowResultsResponse {
|
||||
items,
|
||||
limit_applied: limit,
|
||||
counts,
|
||||
}))
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(crate) struct GetRequestCandidateTraceQuery {
|
||||
pub(crate) attempted_only: Option<bool>,
|
||||
}
|
||||
|
||||
pub(crate) async fn get_request_candidate_trace(
|
||||
State(state): State<AppState>,
|
||||
Path(request_id): Path<String>,
|
||||
Query(query): Query<GetRequestCandidateTraceQuery>,
|
||||
) -> Result<Json<crate::gateway::data::RequestCandidateTrace>, axum::response::Response> {
|
||||
let attempted_only = query.attempted_only.unwrap_or(false);
|
||||
let trace = state
|
||||
.read_request_candidate_trace(&request_id, attempted_only)
|
||||
.await
|
||||
.map_err(IntoResponse::into_response)?;
|
||||
|
||||
match trace {
|
||||
Some(trace) => Ok(Json(trace)),
|
||||
None => Err((
|
||||
axum::http::StatusCode::NOT_FOUND,
|
||||
Json(json!({
|
||||
"error": {
|
||||
"message": "Request not found",
|
||||
}
|
||||
})),
|
||||
)
|
||||
.into_response()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn get_decision_trace(
|
||||
State(state): State<AppState>,
|
||||
Path(request_id): Path<String>,
|
||||
Query(query): Query<GetRequestCandidateTraceQuery>,
|
||||
) -> Result<Json<crate::gateway::data::DecisionTrace>, axum::response::Response> {
|
||||
let attempted_only = query.attempted_only.unwrap_or(false);
|
||||
let trace = state
|
||||
.read_decision_trace(&request_id, attempted_only)
|
||||
.await
|
||||
.map_err(IntoResponse::into_response)?;
|
||||
|
||||
match trace {
|
||||
Some(trace) => Ok(Json(trace)),
|
||||
None => Err((
|
||||
axum::http::StatusCode::NOT_FOUND,
|
||||
Json(json!({
|
||||
"error": {
|
||||
"message": "Decision trace not found",
|
||||
}
|
||||
})),
|
||||
)
|
||||
.into_response()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn get_request_usage_audit(
|
||||
State(state): State<AppState>,
|
||||
Path(request_id): Path<String>,
|
||||
) -> Result<Json<crate::gateway::data::RequestUsageAudit>, axum::response::Response> {
|
||||
let usage = state
|
||||
.read_request_usage_audit(&request_id)
|
||||
.await
|
||||
.map_err(IntoResponse::into_response)?;
|
||||
|
||||
match usage {
|
||||
Some(usage) => Ok(Json(usage)),
|
||||
None => Err((
|
||||
axum::http::StatusCode::NOT_FOUND,
|
||||
Json(json!({
|
||||
"error": {
|
||||
"message": "Request usage not found",
|
||||
}
|
||||
})),
|
||||
)
|
||||
.into_response()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn get_request_audit_bundle(
|
||||
State(state): State<AppState>,
|
||||
Path(request_id): Path<String>,
|
||||
Query(query): Query<GetRequestCandidateTraceQuery>,
|
||||
) -> Result<Json<crate::gateway::data::RequestAuditBundle>, axum::response::Response> {
|
||||
let attempted_only = query.attempted_only.unwrap_or(false);
|
||||
let bundle = state
|
||||
.read_request_audit_bundle(&request_id, attempted_only, current_unix_secs())
|
||||
.await
|
||||
.map_err(IntoResponse::into_response)?;
|
||||
|
||||
match bundle {
|
||||
Some(bundle) => Ok(Json(bundle)),
|
||||
None => Err((
|
||||
axum::http::StatusCode::NOT_FOUND,
|
||||
Json(json!({
|
||||
"error": {
|
||||
"message": "Request audit bundle not found",
|
||||
}
|
||||
})),
|
||||
)
|
||||
.into_response()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn get_auth_api_key_snapshot(
|
||||
State(state): State<AppState>,
|
||||
Path((user_id, api_key_id)): Path<(String, String)>,
|
||||
) -> Result<Json<crate::gateway::data::StoredGatewayAuthApiKeySnapshot>, axum::response::Response> {
|
||||
let snapshot = state
|
||||
.read_auth_api_key_snapshot(&user_id, &api_key_id, current_unix_secs())
|
||||
.await
|
||||
.map_err(IntoResponse::into_response)?;
|
||||
|
||||
match snapshot {
|
||||
Some(snapshot) => Ok(Json(snapshot)),
|
||||
None => Err((
|
||||
axum::http::StatusCode::NOT_FOUND,
|
||||
Json(json!({
|
||||
"error": {
|
||||
"message": "Auth snapshot not found",
|
||||
}
|
||||
})),
|
||||
)
|
||||
.into_response()),
|
||||
}
|
||||
}
|
||||
|
||||
fn current_unix_secs() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
}
|
||||
10
crates/aether-gateway/src/audit/mod.rs
Normal file
10
crates/aether-gateway/src/audit/mod.rs
Normal file
@@ -0,0 +1,10 @@
|
||||
mod http;
|
||||
mod shadow;
|
||||
|
||||
pub(crate) use http::get_auth_api_key_snapshot;
|
||||
pub(crate) use http::get_decision_trace;
|
||||
pub(crate) use http::get_request_audit_bundle;
|
||||
pub(crate) use http::get_request_candidate_trace;
|
||||
pub(crate) use http::get_request_usage_audit;
|
||||
pub(crate) use http::list_recent_shadow_results;
|
||||
pub(crate) use shadow::record_shadow_result_non_blocking;
|
||||
271
crates/aether-gateway/src/audit/shadow.rs
Normal file
271
crates/aether-gateway/src/audit/shadow.rs
Normal file
@@ -0,0 +1,271 @@
|
||||
use std::collections::hash_map::DefaultHasher;
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use aether_data::repository::shadow_results::{RecordShadowResultSample, ShadowResultSampleOrigin};
|
||||
use axum::body::Body;
|
||||
use axum::http::header::CONTENT_TYPE;
|
||||
use axum::http::Response;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::gateway::constants::{
|
||||
CONTROL_CANDIDATE_ID_HEADER, CONTROL_REQUEST_ID_HEADER, EXECUTION_PATH_CONTROL_EXECUTE_STREAM,
|
||||
EXECUTION_PATH_CONTROL_EXECUTE_SYNC,
|
||||
};
|
||||
use crate::gateway::{AppState, GatewayControlDecision};
|
||||
|
||||
pub(crate) fn record_shadow_result_non_blocking(
|
||||
state: AppState,
|
||||
trace_id: &str,
|
||||
method: &http::Method,
|
||||
path_and_query: &str,
|
||||
control_decision: Option<&GatewayControlDecision>,
|
||||
execution_path: &'static str,
|
||||
response: &Response<Body>,
|
||||
) {
|
||||
let Some(decision) =
|
||||
control_decision.filter(|decision| decision.route_class.as_deref() == Some("ai_public"))
|
||||
else {
|
||||
return;
|
||||
};
|
||||
if !state.has_shadow_result_data_writer() {
|
||||
return;
|
||||
}
|
||||
|
||||
let route_family = decision.route_family.clone();
|
||||
let route_kind = decision.route_kind.clone();
|
||||
let status_code = response.status().as_u16();
|
||||
let content_type = response
|
||||
.headers()
|
||||
.get(CONTENT_TYPE)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.unwrap_or_default()
|
||||
.to_string();
|
||||
let candidate_id = response
|
||||
.headers()
|
||||
.get(CONTROL_CANDIDATE_ID_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned);
|
||||
let request_id = response
|
||||
.headers()
|
||||
.get(CONTROL_REQUEST_ID_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned);
|
||||
let now_unix_secs = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
|
||||
let sample = RecordShadowResultSample {
|
||||
trace_id: trace_id.to_string(),
|
||||
request_fingerprint: build_request_fingerprint(
|
||||
method,
|
||||
path_and_query,
|
||||
route_family.as_deref(),
|
||||
route_kind.as_deref(),
|
||||
),
|
||||
request_id,
|
||||
route_family,
|
||||
route_kind,
|
||||
candidate_id,
|
||||
origin: sample_origin_for_execution_path(execution_path),
|
||||
result_digest: build_result_digest(status_code, &content_type),
|
||||
status_code: Some(status_code),
|
||||
error_message: (status_code >= 400)
|
||||
.then(|| format!("gateway response status {status_code}")),
|
||||
recorded_at_unix_secs: now_unix_secs,
|
||||
};
|
||||
let trace_id = trace_id.to_string();
|
||||
|
||||
tokio::spawn(async move {
|
||||
if let Err(err) = state.record_shadow_result_sample(sample).await {
|
||||
warn!(trace_id = %trace_id, error = ?err, "gateway failed to record shadow result");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn build_request_fingerprint(
|
||||
method: &http::Method,
|
||||
path_and_query: &str,
|
||||
route_family: Option<&str>,
|
||||
route_kind: Option<&str>,
|
||||
) -> String {
|
||||
let mut hasher = DefaultHasher::new();
|
||||
method.as_str().hash(&mut hasher);
|
||||
path_and_query.hash(&mut hasher);
|
||||
route_family.unwrap_or_default().hash(&mut hasher);
|
||||
route_kind.unwrap_or_default().hash(&mut hasher);
|
||||
format!("{:x}", hasher.finish())
|
||||
}
|
||||
|
||||
fn build_result_digest(status_code: u16, content_type: &str) -> String {
|
||||
let mut hasher = DefaultHasher::new();
|
||||
status_code.hash(&mut hasher);
|
||||
content_type.hash(&mut hasher);
|
||||
format!("{:x}", hasher.finish())
|
||||
}
|
||||
|
||||
fn sample_origin_for_execution_path(execution_path: &str) -> ShadowResultSampleOrigin {
|
||||
match execution_path {
|
||||
EXECUTION_PATH_CONTROL_EXECUTE_SYNC | EXECUTION_PATH_CONTROL_EXECUTE_STREAM => {
|
||||
ShadowResultSampleOrigin::Python
|
||||
}
|
||||
_ => ShadowResultSampleOrigin::Rust,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use aether_data::repository::shadow_results::{
|
||||
InMemoryShadowResultRepository, ShadowResultMatchStatus, ShadowResultReadRepository,
|
||||
};
|
||||
use axum::body::Body;
|
||||
use axum::http::header::CONTENT_TYPE;
|
||||
use axum::http::{Method, Response, StatusCode};
|
||||
|
||||
use super::record_shadow_result_non_blocking;
|
||||
use crate::gateway::constants::{
|
||||
CONTROL_REQUEST_ID_HEADER, EXECUTION_PATH_CONTROL_EXECUTE_SYNC,
|
||||
EXECUTION_PATH_EXECUTOR_SYNC,
|
||||
};
|
||||
use crate::gateway::{AppState, GatewayControlDecision};
|
||||
|
||||
fn sample_decision() -> GatewayControlDecision {
|
||||
GatewayControlDecision {
|
||||
public_path: "/v1/chat/completions".to_string(),
|
||||
public_query_string: Some("stream=true".to_string()),
|
||||
route_class: Some("ai_public".to_string()),
|
||||
route_family: Some("openai".to_string()),
|
||||
route_kind: Some("chat".to_string()),
|
||||
auth_endpoint_signature: Some("openai:chat".to_string()),
|
||||
executor_candidate: true,
|
||||
auth_context: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn records_shadow_result_for_ai_public_response() {
|
||||
let repository = Arc::new(InMemoryShadowResultRepository::default());
|
||||
let state = AppState::new_with_executor(
|
||||
"http://127.0.0.1:18084",
|
||||
Some("http://127.0.0.1:18085".to_string()),
|
||||
Some("http://127.0.0.1:18086".to_string()),
|
||||
)
|
||||
.expect("app state should build")
|
||||
.with_shadow_result_data_writer_for_tests(repository.clone());
|
||||
let response = Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header(CONTENT_TYPE, "application/json")
|
||||
.header(CONTROL_REQUEST_ID_HEADER, "req-shadow-123")
|
||||
.body(Body::from("{}"))
|
||||
.expect("response should build");
|
||||
|
||||
record_shadow_result_non_blocking(
|
||||
state,
|
||||
"trace-shadow-123",
|
||||
&Method::POST,
|
||||
"/v1/chat/completions?stream=true",
|
||||
Some(&sample_decision()),
|
||||
EXECUTION_PATH_EXECUTOR_SYNC,
|
||||
&response,
|
||||
);
|
||||
|
||||
for _ in 0..30 {
|
||||
if repository
|
||||
.list_recent(1)
|
||||
.await
|
||||
.map(|rows| !rows.is_empty())
|
||||
.unwrap_or(false)
|
||||
{
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
|
||||
let stored = repository
|
||||
.list_recent(1)
|
||||
.await
|
||||
.expect("list should succeed")
|
||||
.into_iter()
|
||||
.next()
|
||||
.expect("stored result should exist");
|
||||
assert_eq!(stored.trace_id, "trace-shadow-123");
|
||||
assert_eq!(stored.request_id.as_deref(), Some("req-shadow-123"));
|
||||
assert_eq!(stored.route_family.as_deref(), Some("openai"));
|
||||
assert_eq!(stored.route_kind.as_deref(), Some("chat"));
|
||||
assert_eq!(stored.match_status, ShadowResultMatchStatus::Pending);
|
||||
assert_eq!(stored.status_code, Some(200));
|
||||
assert!(stored.rust_result_digest.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn merges_rust_and_python_shadow_samples_into_match() {
|
||||
let repository = Arc::new(InMemoryShadowResultRepository::default());
|
||||
let state = AppState::new_with_executor(
|
||||
"http://127.0.0.1:18084",
|
||||
Some("http://127.0.0.1:18085".to_string()),
|
||||
Some("http://127.0.0.1:18086".to_string()),
|
||||
)
|
||||
.expect("app state should build")
|
||||
.with_shadow_result_data_repository_for_tests(repository.clone());
|
||||
let response = Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header(CONTENT_TYPE, "application/json")
|
||||
.header(CONTROL_REQUEST_ID_HEADER, "req-shadow-compare-123")
|
||||
.body(Body::from("{}"))
|
||||
.expect("response should build");
|
||||
|
||||
record_shadow_result_non_blocking(
|
||||
state.clone(),
|
||||
"trace-shadow-compare-123",
|
||||
&Method::POST,
|
||||
"/v1/chat/completions?stream=true",
|
||||
Some(&sample_decision()),
|
||||
EXECUTION_PATH_EXECUTOR_SYNC,
|
||||
&response,
|
||||
);
|
||||
record_shadow_result_non_blocking(
|
||||
state,
|
||||
"trace-shadow-compare-123",
|
||||
&Method::POST,
|
||||
"/v1/chat/completions?stream=true",
|
||||
Some(&sample_decision()),
|
||||
EXECUTION_PATH_CONTROL_EXECUTE_SYNC,
|
||||
&response,
|
||||
);
|
||||
|
||||
for _ in 0..30 {
|
||||
if repository
|
||||
.list_recent(1)
|
||||
.await
|
||||
.map(|rows| {
|
||||
rows.first()
|
||||
.map(|row| row.match_status == ShadowResultMatchStatus::Match)
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.unwrap_or(false)
|
||||
{
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
|
||||
let stored = repository
|
||||
.list_recent(1)
|
||||
.await
|
||||
.expect("list should succeed")
|
||||
.into_iter()
|
||||
.next()
|
||||
.expect("stored result should exist");
|
||||
assert_eq!(stored.request_id.as_deref(), Some("req-shadow-compare-123"));
|
||||
assert_eq!(stored.match_status, ShadowResultMatchStatus::Match);
|
||||
assert!(stored.rust_result_digest.is_some());
|
||||
assert!(stored.python_result_digest.is_some());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user