feat: 引入 aether-runtime/cache/data/http/testkit 基础 crate,完善并发门控与审计系统

新增 crate:
- aether-runtime: 服务运行时基础设施(并发门控、分布式并发、指标、队列、优雅关闭、tracing)
- aether-cache: 通用 TTL 缓存与命名空间抽象
- aether-data: 数据访问层(PostgreSQL/Redis 后端、repository 模式)
- aether-http: HTTP 客户端封装(重试、配置)
- aether-testkit: 集成测试工具集(gateway/executor/hub/proxy fixture、等待、负载测试)

gateway 扩展:
- 引入 audit 模块(shadow 执行审计、决策链路追踪、请求审计 bundle)
- 引入 cache 模块(AuthContext 缓存、direct-plan bypass 缓存)
- 引入 data 模块(auth/candidates/config/usage/video_tasks 数据访问)
- 集成 ConcurrencyGate/DistributedConcurrencyGate 请求门控
- 新增本地 auth 拒绝、过载响应构建器
- 补充 control/auth_cache/video/concurrency 集成测试

aether-proxy 扩展:
- AppState 集成 stream_gate / distributed_stream_gate 并发门控
- 新增 ProxyAdmissionError 及准入拒绝流程
- stream_handler 补充门控饱和/不可用场景测试
- 配置与注册客户端逻辑完善

aether-hub 扩展:
- main.rs 引入运行时初始化、指标端点、健康检查
- local_relay 重构为 lib.rs 暴露公共接口
This commit is contained in:
fawney19
2026-03-24 15:12:56 +08:00
parent eaf8475f9e
commit b5a0070023
157 changed files with 22097 additions and 448 deletions

View File

@@ -7,7 +7,11 @@ repository.workspace = true
description = "Rust ingress gateway for Aether phase 3a transparent proxy"
[dependencies]
aether-cache.workspace = true
aether-contracts.workspace = true
aether-data.workspace = true
aether-http.workspace = true
aether-runtime.workspace = true
async-stream.workspace = true
axum = { version = "0.8" }
base64.workspace = true
@@ -21,7 +25,6 @@ serde_json.workspace = true
thiserror.workspace = true
tokio.workspace = true
tokio-util.workspace = true
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
tracing.workspace = true
url.workspace = true
uuid.workspace = true

View File

@@ -0,0 +1,205 @@
use std::time::{SystemTime, UNIX_EPOCH};
use axum::extract::{Path, Query, State};
use axum::response::IntoResponse;
use axum::Json;
use serde::{Deserialize, Serialize};
use serde_json::json;
use crate::gateway::{AppState, GatewayError};
const DEFAULT_RECENT_LIMIT: usize = 20;
const MAX_RECENT_LIMIT: usize = 200;
#[derive(Debug, Deserialize)]
pub(crate) struct ListRecentShadowResultsQuery {
pub(crate) limit: Option<usize>,
}
#[derive(Debug, Serialize)]
pub(crate) struct ShadowResultStatusCounts {
pub(crate) pending: usize,
pub(crate) r#match: usize,
pub(crate) mismatch: usize,
pub(crate) error: usize,
}
#[derive(Debug, Serialize)]
pub(crate) struct ListRecentShadowResultsResponse {
pub(crate) items: Vec<aether_data::repository::shadow_results::StoredShadowResult>,
pub(crate) limit_applied: usize,
pub(crate) counts: ShadowResultStatusCounts,
}
pub(crate) async fn list_recent_shadow_results(
State(state): State<AppState>,
Query(query): Query<ListRecentShadowResultsQuery>,
) -> Result<Json<ListRecentShadowResultsResponse>, GatewayError> {
let limit = query
.limit
.unwrap_or(DEFAULT_RECENT_LIMIT)
.clamp(1, MAX_RECENT_LIMIT);
let items = state.list_recent_shadow_results(limit).await?;
let mut counts = ShadowResultStatusCounts {
pending: 0,
r#match: 0,
mismatch: 0,
error: 0,
};
for item in &items {
match item.match_status {
aether_data::repository::shadow_results::ShadowResultMatchStatus::Pending => {
counts.pending += 1
}
aether_data::repository::shadow_results::ShadowResultMatchStatus::Match => {
counts.r#match += 1
}
aether_data::repository::shadow_results::ShadowResultMatchStatus::Mismatch => {
counts.mismatch += 1
}
aether_data::repository::shadow_results::ShadowResultMatchStatus::Error => {
counts.error += 1
}
}
}
Ok(Json(ListRecentShadowResultsResponse {
items,
limit_applied: limit,
counts,
}))
}
#[derive(Debug, Deserialize)]
pub(crate) struct GetRequestCandidateTraceQuery {
pub(crate) attempted_only: Option<bool>,
}
pub(crate) async fn get_request_candidate_trace(
State(state): State<AppState>,
Path(request_id): Path<String>,
Query(query): Query<GetRequestCandidateTraceQuery>,
) -> Result<Json<crate::gateway::data::RequestCandidateTrace>, axum::response::Response> {
let attempted_only = query.attempted_only.unwrap_or(false);
let trace = state
.read_request_candidate_trace(&request_id, attempted_only)
.await
.map_err(IntoResponse::into_response)?;
match trace {
Some(trace) => Ok(Json(trace)),
None => Err((
axum::http::StatusCode::NOT_FOUND,
Json(json!({
"error": {
"message": "Request not found",
}
})),
)
.into_response()),
}
}
pub(crate) async fn get_decision_trace(
State(state): State<AppState>,
Path(request_id): Path<String>,
Query(query): Query<GetRequestCandidateTraceQuery>,
) -> Result<Json<crate::gateway::data::DecisionTrace>, axum::response::Response> {
let attempted_only = query.attempted_only.unwrap_or(false);
let trace = state
.read_decision_trace(&request_id, attempted_only)
.await
.map_err(IntoResponse::into_response)?;
match trace {
Some(trace) => Ok(Json(trace)),
None => Err((
axum::http::StatusCode::NOT_FOUND,
Json(json!({
"error": {
"message": "Decision trace not found",
}
})),
)
.into_response()),
}
}
pub(crate) async fn get_request_usage_audit(
State(state): State<AppState>,
Path(request_id): Path<String>,
) -> Result<Json<crate::gateway::data::RequestUsageAudit>, axum::response::Response> {
let usage = state
.read_request_usage_audit(&request_id)
.await
.map_err(IntoResponse::into_response)?;
match usage {
Some(usage) => Ok(Json(usage)),
None => Err((
axum::http::StatusCode::NOT_FOUND,
Json(json!({
"error": {
"message": "Request usage not found",
}
})),
)
.into_response()),
}
}
pub(crate) async fn get_request_audit_bundle(
State(state): State<AppState>,
Path(request_id): Path<String>,
Query(query): Query<GetRequestCandidateTraceQuery>,
) -> Result<Json<crate::gateway::data::RequestAuditBundle>, axum::response::Response> {
let attempted_only = query.attempted_only.unwrap_or(false);
let bundle = state
.read_request_audit_bundle(&request_id, attempted_only, current_unix_secs())
.await
.map_err(IntoResponse::into_response)?;
match bundle {
Some(bundle) => Ok(Json(bundle)),
None => Err((
axum::http::StatusCode::NOT_FOUND,
Json(json!({
"error": {
"message": "Request audit bundle not found",
}
})),
)
.into_response()),
}
}
pub(crate) async fn get_auth_api_key_snapshot(
State(state): State<AppState>,
Path((user_id, api_key_id)): Path<(String, String)>,
) -> Result<Json<crate::gateway::data::StoredGatewayAuthApiKeySnapshot>, axum::response::Response> {
let snapshot = state
.read_auth_api_key_snapshot(&user_id, &api_key_id, current_unix_secs())
.await
.map_err(IntoResponse::into_response)?;
match snapshot {
Some(snapshot) => Ok(Json(snapshot)),
None => Err((
axum::http::StatusCode::NOT_FOUND,
Json(json!({
"error": {
"message": "Auth snapshot not found",
}
})),
)
.into_response()),
}
}
fn current_unix_secs() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}

View File

@@ -0,0 +1,10 @@
mod http;
mod shadow;
pub(crate) use http::get_auth_api_key_snapshot;
pub(crate) use http::get_decision_trace;
pub(crate) use http::get_request_audit_bundle;
pub(crate) use http::get_request_candidate_trace;
pub(crate) use http::get_request_usage_audit;
pub(crate) use http::list_recent_shadow_results;
pub(crate) use shadow::record_shadow_result_non_blocking;

View File

@@ -0,0 +1,271 @@
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
use std::time::{SystemTime, UNIX_EPOCH};
use aether_data::repository::shadow_results::{RecordShadowResultSample, ShadowResultSampleOrigin};
use axum::body::Body;
use axum::http::header::CONTENT_TYPE;
use axum::http::Response;
use tracing::warn;
use crate::gateway::constants::{
CONTROL_CANDIDATE_ID_HEADER, CONTROL_REQUEST_ID_HEADER, EXECUTION_PATH_CONTROL_EXECUTE_STREAM,
EXECUTION_PATH_CONTROL_EXECUTE_SYNC,
};
use crate::gateway::{AppState, GatewayControlDecision};
pub(crate) fn record_shadow_result_non_blocking(
state: AppState,
trace_id: &str,
method: &http::Method,
path_and_query: &str,
control_decision: Option<&GatewayControlDecision>,
execution_path: &'static str,
response: &Response<Body>,
) {
let Some(decision) =
control_decision.filter(|decision| decision.route_class.as_deref() == Some("ai_public"))
else {
return;
};
if !state.has_shadow_result_data_writer() {
return;
}
let route_family = decision.route_family.clone();
let route_kind = decision.route_kind.clone();
let status_code = response.status().as_u16();
let content_type = response
.headers()
.get(CONTENT_TYPE)
.and_then(|value| value.to_str().ok())
.unwrap_or_default()
.to_string();
let candidate_id = response
.headers()
.get(CONTROL_CANDIDATE_ID_HEADER)
.and_then(|value| value.to_str().ok())
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned);
let request_id = response
.headers()
.get(CONTROL_REQUEST_ID_HEADER)
.and_then(|value| value.to_str().ok())
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned);
let now_unix_secs = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs();
let sample = RecordShadowResultSample {
trace_id: trace_id.to_string(),
request_fingerprint: build_request_fingerprint(
method,
path_and_query,
route_family.as_deref(),
route_kind.as_deref(),
),
request_id,
route_family,
route_kind,
candidate_id,
origin: sample_origin_for_execution_path(execution_path),
result_digest: build_result_digest(status_code, &content_type),
status_code: Some(status_code),
error_message: (status_code >= 400)
.then(|| format!("gateway response status {status_code}")),
recorded_at_unix_secs: now_unix_secs,
};
let trace_id = trace_id.to_string();
tokio::spawn(async move {
if let Err(err) = state.record_shadow_result_sample(sample).await {
warn!(trace_id = %trace_id, error = ?err, "gateway failed to record shadow result");
}
});
}
fn build_request_fingerprint(
method: &http::Method,
path_and_query: &str,
route_family: Option<&str>,
route_kind: Option<&str>,
) -> String {
let mut hasher = DefaultHasher::new();
method.as_str().hash(&mut hasher);
path_and_query.hash(&mut hasher);
route_family.unwrap_or_default().hash(&mut hasher);
route_kind.unwrap_or_default().hash(&mut hasher);
format!("{:x}", hasher.finish())
}
fn build_result_digest(status_code: u16, content_type: &str) -> String {
let mut hasher = DefaultHasher::new();
status_code.hash(&mut hasher);
content_type.hash(&mut hasher);
format!("{:x}", hasher.finish())
}
fn sample_origin_for_execution_path(execution_path: &str) -> ShadowResultSampleOrigin {
match execution_path {
EXECUTION_PATH_CONTROL_EXECUTE_SYNC | EXECUTION_PATH_CONTROL_EXECUTE_STREAM => {
ShadowResultSampleOrigin::Python
}
_ => ShadowResultSampleOrigin::Rust,
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use aether_data::repository::shadow_results::{
InMemoryShadowResultRepository, ShadowResultMatchStatus, ShadowResultReadRepository,
};
use axum::body::Body;
use axum::http::header::CONTENT_TYPE;
use axum::http::{Method, Response, StatusCode};
use super::record_shadow_result_non_blocking;
use crate::gateway::constants::{
CONTROL_REQUEST_ID_HEADER, EXECUTION_PATH_CONTROL_EXECUTE_SYNC,
EXECUTION_PATH_EXECUTOR_SYNC,
};
use crate::gateway::{AppState, GatewayControlDecision};
fn sample_decision() -> GatewayControlDecision {
GatewayControlDecision {
public_path: "/v1/chat/completions".to_string(),
public_query_string: Some("stream=true".to_string()),
route_class: Some("ai_public".to_string()),
route_family: Some("openai".to_string()),
route_kind: Some("chat".to_string()),
auth_endpoint_signature: Some("openai:chat".to_string()),
executor_candidate: true,
auth_context: None,
}
}
#[tokio::test]
async fn records_shadow_result_for_ai_public_response() {
let repository = Arc::new(InMemoryShadowResultRepository::default());
let state = AppState::new_with_executor(
"http://127.0.0.1:18084",
Some("http://127.0.0.1:18085".to_string()),
Some("http://127.0.0.1:18086".to_string()),
)
.expect("app state should build")
.with_shadow_result_data_writer_for_tests(repository.clone());
let response = Response::builder()
.status(StatusCode::OK)
.header(CONTENT_TYPE, "application/json")
.header(CONTROL_REQUEST_ID_HEADER, "req-shadow-123")
.body(Body::from("{}"))
.expect("response should build");
record_shadow_result_non_blocking(
state,
"trace-shadow-123",
&Method::POST,
"/v1/chat/completions?stream=true",
Some(&sample_decision()),
EXECUTION_PATH_EXECUTOR_SYNC,
&response,
);
for _ in 0..30 {
if repository
.list_recent(1)
.await
.map(|rows| !rows.is_empty())
.unwrap_or(false)
{
break;
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
let stored = repository
.list_recent(1)
.await
.expect("list should succeed")
.into_iter()
.next()
.expect("stored result should exist");
assert_eq!(stored.trace_id, "trace-shadow-123");
assert_eq!(stored.request_id.as_deref(), Some("req-shadow-123"));
assert_eq!(stored.route_family.as_deref(), Some("openai"));
assert_eq!(stored.route_kind.as_deref(), Some("chat"));
assert_eq!(stored.match_status, ShadowResultMatchStatus::Pending);
assert_eq!(stored.status_code, Some(200));
assert!(stored.rust_result_digest.is_some());
}
#[tokio::test]
async fn merges_rust_and_python_shadow_samples_into_match() {
let repository = Arc::new(InMemoryShadowResultRepository::default());
let state = AppState::new_with_executor(
"http://127.0.0.1:18084",
Some("http://127.0.0.1:18085".to_string()),
Some("http://127.0.0.1:18086".to_string()),
)
.expect("app state should build")
.with_shadow_result_data_repository_for_tests(repository.clone());
let response = Response::builder()
.status(StatusCode::OK)
.header(CONTENT_TYPE, "application/json")
.header(CONTROL_REQUEST_ID_HEADER, "req-shadow-compare-123")
.body(Body::from("{}"))
.expect("response should build");
record_shadow_result_non_blocking(
state.clone(),
"trace-shadow-compare-123",
&Method::POST,
"/v1/chat/completions?stream=true",
Some(&sample_decision()),
EXECUTION_PATH_EXECUTOR_SYNC,
&response,
);
record_shadow_result_non_blocking(
state,
"trace-shadow-compare-123",
&Method::POST,
"/v1/chat/completions?stream=true",
Some(&sample_decision()),
EXECUTION_PATH_CONTROL_EXECUTE_SYNC,
&response,
);
for _ in 0..30 {
if repository
.list_recent(1)
.await
.map(|rows| {
rows.first()
.map(|row| row.match_status == ShadowResultMatchStatus::Match)
.unwrap_or(false)
})
.unwrap_or(false)
{
break;
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
let stored = repository
.list_recent(1)
.await
.expect("list should succeed")
.into_iter()
.next()
.expect("stored result should exist");
assert_eq!(stored.request_id.as_deref(), Some("req-shadow-compare-123"));
assert_eq!(stored.match_status, ShadowResultMatchStatus::Match);
assert!(stored.rust_result_digest.is_some());
assert!(stored.python_result_digest.is_some());
}
}

View File

@@ -0,0 +1,31 @@
use std::time::Duration;
use aether_cache::ExpiringMap;
use crate::gateway::GatewayControlAuthContext;
#[derive(Debug, Default)]
pub(crate) struct AuthContextCache {
entries: ExpiringMap<String, GatewayControlAuthContext>,
}
impl AuthContextCache {
pub(crate) fn get_fresh(
&self,
cache_key: &str,
ttl: Duration,
) -> Option<GatewayControlAuthContext> {
self.entries.get_fresh(&cache_key.to_string(), ttl)
}
pub(crate) fn insert(
&self,
cache_key: String,
auth_context: GatewayControlAuthContext,
ttl: Duration,
max_entries: usize,
) {
self.entries
.insert(cache_key, auth_context, ttl, max_entries);
}
}

View File

@@ -0,0 +1,18 @@
use std::time::Duration;
use aether_cache::ExpiringMap;
#[derive(Debug, Default)]
pub(crate) struct DirectPlanBypassCache {
entries: ExpiringMap<String, ()>,
}
impl DirectPlanBypassCache {
pub(crate) fn should_skip(&self, cache_key: &str, ttl: Duration) -> bool {
self.entries.contains_fresh(&cache_key.to_string(), ttl)
}
pub(crate) fn mark(&self, cache_key: String, ttl: Duration, max_entries: usize) {
self.entries.insert(cache_key, (), ttl, max_entries);
}
}

View File

@@ -0,0 +1,5 @@
mod auth_context;
mod direct_plan_bypass;
pub(crate) use auth_context::AuthContextCache;
pub(crate) use direct_plan_bypass::DirectPlanBypassCache;

View File

@@ -11,10 +11,15 @@ pub(crate) const EXECUTION_PATH_EXECUTOR_SYNC: &str = "executor_sync";
pub(crate) const EXECUTION_PATH_EXECUTOR_STREAM: &str = "executor_stream";
pub(crate) const EXECUTION_PATH_CONTROL_EXECUTE_SYNC: &str = "control_execute_sync";
pub(crate) const EXECUTION_PATH_CONTROL_EXECUTE_STREAM: &str = "control_execute_stream";
pub(crate) const EXECUTION_PATH_LOCAL_AUTH_DENIED: &str = "local_auth_denied";
pub(crate) const EXECUTION_PATH_LOCAL_OVERLOADED: &str = "local_overloaded";
pub(crate) const EXECUTION_PATH_DISTRIBUTED_OVERLOADED: &str = "distributed_overloaded";
pub(crate) const CONTROL_ROUTE_CLASS_HEADER: &str = "x-aether-control-route-class";
pub(crate) const CONTROL_ROUTE_FAMILY_HEADER: &str = "x-aether-control-route-family";
pub(crate) const CONTROL_ROUTE_KIND_HEADER: &str = "x-aether-control-route-kind";
pub(crate) const CONTROL_EXECUTOR_HEADER: &str = "x-aether-control-executor-candidate";
pub(crate) const CONTROL_REQUEST_ID_HEADER: &str = "x-aether-control-request-id";
pub(crate) const CONTROL_CANDIDATE_ID_HEADER: &str = "x-aether-control-candidate-id";
pub(crate) const CONTROL_ENDPOINT_SIGNATURE_HEADER: &str = "x-aether-control-endpoint-signature";
pub(crate) const CONTROL_EXECUTED_HEADER: &str = "x-aether-control-executed";
pub(crate) const CONTROL_ACTION_HEADER: &str = "x-aether-control-action";

View File

@@ -1,4 +1,4 @@
use std::time::{Duration, Instant};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use axum::body::{Body, Bytes};
use axum::http::{Response, StatusCode, Uri};
@@ -10,7 +10,7 @@ use crate::gateway::constants::*;
use crate::gateway::headers::{
collect_control_headers, header_equals, header_value_str, header_value_u64, is_json_request,
};
use crate::gateway::{build_client_response, AppState, CachedAuthContextEntry, GatewayError};
use crate::gateway::{build_client_response, AppState, GatewayError};
const AUTH_CONTEXT_CACHE_TTL: Duration = Duration::from_secs(60);
const AUTH_CONTEXT_CACHE_MAX_ENTRIES: usize = 256;
@@ -72,6 +72,15 @@ pub(crate) struct GatewayControlAuthContext {
pub(crate) api_key_id: String,
pub(crate) balance_remaining: Option<f64>,
pub(crate) access_allowed: bool,
#[serde(skip)]
pub(crate) local_rejection: Option<GatewayLocalAuthRejection>,
}
#[derive(Debug, Clone)]
pub(crate) enum GatewayLocalAuthRejection {
InvalidApiKey,
LockedApiKey,
BalanceDenied { remaining: Option<f64> },
}
#[derive(Debug, Clone)]
@@ -116,10 +125,31 @@ pub(crate) async fn resolve_control_route(
};
decision.public_query_string = uri.query().map(ToOwned::to_owned);
if let Some(auth_context) = resolve_data_backed_auth_context(
state,
headers,
decision.auth_endpoint_signature.as_deref(),
)
.await?
{
if let Some(cache_key) = decision
.auth_endpoint_signature
.as_deref()
.and_then(|signature| build_auth_context_cache_key(headers, uri, signature))
{
put_cached_auth_context(state, cache_key, auth_context.clone());
}
decision.auth_context = Some(auth_context);
}
if state.executor_base_url.is_some() && decision.executor_candidate {
return Ok(Some(decision));
}
if decision.auth_context.is_some() {
return Ok(Some(decision));
}
match fetch_auth_context(
state,
control_base_url,
@@ -170,6 +200,13 @@ pub(crate) async fn resolve_executor_auth_context(
return Ok(Some(auth_context));
}
if let Some(auth_context) =
resolve_data_backed_auth_context(state, headers, Some(auth_endpoint_signature)).await?
{
put_cached_auth_context(state, cache_key, auth_context.clone());
return Ok(Some(auth_context));
}
Ok(None)
}
@@ -190,6 +227,19 @@ pub(crate) fn cache_executor_auth_context(
put_cached_auth_context(state, cache_key, auth_context);
}
pub(crate) fn trusted_auth_local_rejection(
decision: Option<&GatewayControlDecision>,
_headers: &http::HeaderMap,
) -> Option<GatewayLocalAuthRejection> {
let decision = decision?;
if decision.route_class.as_deref() != Some("ai_public") {
return None;
}
let auth_context = decision.auth_context.as_ref()?;
auth_context.local_rejection.clone()
}
async fn fetch_auth_context(
state: &AppState,
control_base_url: &str,
@@ -334,13 +384,9 @@ fn build_auth_context_cache_key(
}
fn get_cached_auth_context(state: &AppState, cache_key: &str) -> Option<GatewayControlAuthContext> {
let mut cache = state.auth_context_cache.lock().ok()?;
let entry = cache.get(cache_key)?.clone();
if entry.cached_at.elapsed() > AUTH_CONTEXT_CACHE_TTL {
cache.remove(cache_key);
return None;
}
Some(entry.auth_context)
state
.auth_context_cache
.get_fresh(cache_key, AUTH_CONTEXT_CACHE_TTL)
}
fn put_cached_auth_context(
@@ -348,28 +394,100 @@ fn put_cached_auth_context(
cache_key: String,
auth_context: GatewayControlAuthContext,
) {
let Ok(mut cache) = state.auth_context_cache.lock() else {
return;
state.auth_context_cache.insert(
cache_key,
auth_context,
AUTH_CONTEXT_CACHE_TTL,
AUTH_CONTEXT_CACHE_MAX_ENTRIES,
);
}
async fn resolve_data_backed_auth_context(
state: &AppState,
headers: &http::HeaderMap,
auth_endpoint_signature: Option<&str>,
) -> Result<Option<GatewayControlAuthContext>, GatewayError> {
let Some(signature) = auth_endpoint_signature
.map(str::trim)
.filter(|value| !value.is_empty())
else {
return Ok(None);
};
let _ = signature;
let Some(user_id) =
header_value_str(headers, TRUSTED_AUTH_USER_ID_HEADER).filter(|value| !value.is_empty())
else {
return Ok(None);
};
let Some(api_key_id) =
header_value_str(headers, TRUSTED_AUTH_API_KEY_ID_HEADER).filter(|value| !value.is_empty())
else {
return Ok(None);
};
cache.retain(|_, entry| entry.cached_at.elapsed() <= AUTH_CONTEXT_CACHE_TTL);
if cache.len() >= AUTH_CONTEXT_CACHE_MAX_ENTRIES {
if let Some(oldest_key) = cache
.iter()
.min_by_key(|(_, entry)| entry.cached_at)
.map(|(key, _)| key.clone())
{
cache.remove(&oldest_key);
}
}
let snapshot = state
.read_auth_api_key_snapshot(&user_id, &api_key_id, current_unix_secs())
.await?;
let Some(snapshot) = snapshot else {
return Ok(None);
};
cache.insert(
cache_key,
CachedAuthContextEntry {
auth_context,
cached_at: Instant::now(),
},
);
let header_access_allowed = header_value_str(headers, TRUSTED_AUTH_ACCESS_ALLOWED_HEADER)
.as_deref()
.and_then(parse_bool_header);
let invalid_api_key = !snapshot.user_is_active
|| snapshot.user_is_deleted
|| !snapshot.api_key_is_active
|| snapshot
.api_key_expires_at_unix_secs
.is_some_and(|expires_at| expires_at < current_unix_secs());
let locked_api_key = snapshot.api_key_is_locked && !snapshot.api_key_is_standalone;
let access_allowed = header_access_allowed
.map(|value| value && snapshot.currently_usable)
.unwrap_or(snapshot.currently_usable);
let local_rejection = if invalid_api_key {
Some(GatewayLocalAuthRejection::InvalidApiKey)
} else if locked_api_key {
Some(GatewayLocalAuthRejection::LockedApiKey)
} else if header_access_allowed.is_some_and(|value| !value) && snapshot.currently_usable {
Some(GatewayLocalAuthRejection::BalanceDenied {
remaining: header_value_str(headers, TRUSTED_AUTH_BALANCE_HEADER)
.as_deref()
.and_then(parse_f64_header),
})
} else {
None
};
Ok(Some(GatewayControlAuthContext {
user_id: snapshot.user_id,
api_key_id: snapshot.api_key_id,
balance_remaining: header_value_str(headers, TRUSTED_AUTH_BALANCE_HEADER)
.as_deref()
.and_then(parse_f64_header),
access_allowed,
local_rejection,
}))
}
fn parse_bool_header(value: &str) -> Option<bool> {
match value.trim().to_ascii_lowercase().as_str() {
"true" | "1" | "yes" => Some(true),
"false" | "0" | "no" => Some(false),
_ => None,
}
}
fn parse_f64_header(value: &str) -> Option<f64> {
value.trim().parse::<f64>().ok()
}
fn current_unix_secs() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
fn classify_control_route(

View File

@@ -0,0 +1,155 @@
use aether_data::repository::auth::{AuthApiKeyLookupKey, StoredAuthApiKeySnapshot};
use aether_data::DataLayerError;
use super::state::GatewayDataState;
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
pub(crate) struct StoredGatewayAuthApiKeySnapshot {
pub(crate) user_id: String,
pub(crate) username: String,
pub(crate) email: Option<String>,
pub(crate) user_role: String,
pub(crate) user_auth_source: String,
pub(crate) user_is_active: bool,
pub(crate) user_is_deleted: bool,
pub(crate) user_allowed_providers: Option<Vec<String>>,
pub(crate) user_allowed_api_formats: Option<Vec<String>>,
pub(crate) user_allowed_models: Option<Vec<String>>,
pub(crate) api_key_id: String,
pub(crate) api_key_name: Option<String>,
pub(crate) api_key_is_active: bool,
pub(crate) api_key_is_locked: bool,
pub(crate) api_key_is_standalone: bool,
pub(crate) api_key_rate_limit: Option<i32>,
pub(crate) api_key_concurrent_limit: Option<i32>,
pub(crate) api_key_expires_at_unix_secs: Option<u64>,
pub(crate) api_key_allowed_providers: Option<Vec<String>>,
pub(crate) api_key_allowed_api_formats: Option<Vec<String>>,
pub(crate) api_key_allowed_models: Option<Vec<String>>,
pub(crate) currently_usable: bool,
}
impl StoredGatewayAuthApiKeySnapshot {
fn from_stored(snapshot: StoredAuthApiKeySnapshot, now_unix_secs: u64) -> Self {
let currently_usable = snapshot.is_currently_usable(now_unix_secs);
Self {
user_id: snapshot.user_id,
username: snapshot.username,
email: snapshot.email,
user_role: snapshot.user_role,
user_auth_source: snapshot.user_auth_source,
user_is_active: snapshot.user_is_active,
user_is_deleted: snapshot.user_is_deleted,
user_allowed_providers: snapshot.user_allowed_providers,
user_allowed_api_formats: snapshot.user_allowed_api_formats,
user_allowed_models: snapshot.user_allowed_models,
api_key_id: snapshot.api_key_id,
api_key_name: snapshot.api_key_name,
api_key_is_active: snapshot.api_key_is_active,
api_key_is_locked: snapshot.api_key_is_locked,
api_key_is_standalone: snapshot.api_key_is_standalone,
api_key_rate_limit: snapshot.api_key_rate_limit,
api_key_concurrent_limit: snapshot.api_key_concurrent_limit,
api_key_expires_at_unix_secs: snapshot.api_key_expires_at_unix_secs,
api_key_allowed_providers: snapshot.api_key_allowed_providers,
api_key_allowed_api_formats: snapshot.api_key_allowed_api_formats,
api_key_allowed_models: snapshot.api_key_allowed_models,
currently_usable,
}
}
}
pub(crate) async fn read_auth_api_key_snapshot(
state: &GatewayDataState,
user_id: &str,
api_key_id: &str,
now_unix_secs: u64,
) -> Result<Option<StoredGatewayAuthApiKeySnapshot>, DataLayerError> {
let snapshot = state
.find_auth_api_key_snapshot(AuthApiKeyLookupKey::UserApiKeyIds {
user_id,
api_key_id,
})
.await?;
Ok(snapshot
.map(|snapshot| StoredGatewayAuthApiKeySnapshot::from_stored(snapshot, now_unix_secs)))
}
#[cfg(test)]
mod tests {
use super::super::GatewayDataState;
use super::{read_auth_api_key_snapshot, StoredGatewayAuthApiKeySnapshot};
use aether_data::repository::auth::{
InMemoryAuthApiKeySnapshotRepository, StoredAuthApiKeySnapshot,
};
use std::sync::Arc;
fn sample_snapshot(api_key_id: &str, user_id: &str) -> StoredAuthApiKeySnapshot {
StoredAuthApiKeySnapshot::new(
user_id.to_string(),
"alice".to_string(),
Some("alice@example.com".to_string()),
"user".to_string(),
"local".to_string(),
true,
false,
Some(serde_json::json!(["openai"])),
Some(serde_json::json!(["openai:chat"])),
Some(serde_json::json!(["gpt-4.1"])),
api_key_id.to_string(),
Some("default".to_string()),
true,
false,
false,
Some(60),
Some(5),
Some(200),
Some(serde_json::json!(["openai"])),
Some(serde_json::json!(["openai:chat"])),
Some(serde_json::json!(["gpt-4.1"])),
)
.expect("snapshot should build")
}
#[tokio::test]
async fn reads_trusted_auth_snapshot_and_derives_usability() {
let repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
Some("hash-1".to_string()),
sample_snapshot("key-1", "user-1"),
)]));
let state = GatewayDataState::with_auth_api_key_reader_for_tests(repository);
let snapshot = read_auth_api_key_snapshot(&state, "user-1", "key-1", 150)
.await
.expect("read should succeed")
.expect("snapshot should exist");
assert_eq!(
snapshot,
StoredGatewayAuthApiKeySnapshot {
user_id: "user-1".to_string(),
username: "alice".to_string(),
email: Some("alice@example.com".to_string()),
user_role: "user".to_string(),
user_auth_source: "local".to_string(),
user_is_active: true,
user_is_deleted: false,
user_allowed_providers: Some(vec!["openai".to_string()]),
user_allowed_api_formats: Some(vec!["openai:chat".to_string()]),
user_allowed_models: Some(vec!["gpt-4.1".to_string()]),
api_key_id: "key-1".to_string(),
api_key_name: Some("default".to_string()),
api_key_is_active: true,
api_key_is_locked: false,
api_key_is_standalone: false,
api_key_rate_limit: Some(60),
api_key_concurrent_limit: Some(5),
api_key_expires_at_unix_secs: Some(200),
api_key_allowed_providers: Some(vec!["openai".to_string()]),
api_key_allowed_api_formats: Some(vec!["openai:chat".to_string()]),
api_key_allowed_models: Some(vec!["gpt-4.1".to_string()]),
currently_usable: true,
}
);
}
}

View File

@@ -0,0 +1,213 @@
use aether_data::repository::candidates::{RequestCandidateStatus, StoredRequestCandidate};
use aether_data::DataLayerError;
use super::state::GatewayDataState;
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum RequestCandidateFinalStatus {
Success,
Failed,
Cancelled,
Streaming,
Pending,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
pub(crate) struct RequestCandidateTrace {
pub(crate) request_id: String,
pub(crate) total_candidates: usize,
pub(crate) final_status: RequestCandidateFinalStatus,
pub(crate) total_latency_ms: u64,
pub(crate) candidates: Vec<StoredRequestCandidate>,
}
pub(crate) async fn read_request_candidate_trace(
state: &GatewayDataState,
request_id: &str,
attempted_only: bool,
) -> Result<Option<RequestCandidateTrace>, DataLayerError> {
let all_candidates = state
.list_request_candidates_by_request_id(request_id)
.await?;
if all_candidates.is_empty() {
return Ok(None);
}
let candidates = if attempted_only {
all_candidates
.iter()
.filter(|candidate| {
candidate
.status
.is_attempted(candidate.started_at_unix_secs)
})
.cloned()
.collect::<Vec<_>>()
} else {
all_candidates.clone()
};
let total_latency_ms = candidates
.iter()
.filter(|candidate| {
matches!(
candidate.status,
RequestCandidateStatus::Success
| RequestCandidateStatus::Failed
| RequestCandidateStatus::Cancelled
) && candidate.latency_ms.is_some()
})
.map(|candidate| candidate.latency_ms.unwrap_or(0))
.sum();
let final_status_source = if attempted_only && candidates.is_empty() {
&all_candidates
} else {
&candidates
};
Ok(Some(RequestCandidateTrace {
request_id: request_id.to_string(),
total_candidates: candidates.len(),
final_status: derive_final_status(final_status_source),
total_latency_ms,
candidates,
}))
}
fn derive_final_status(candidates: &[StoredRequestCandidate]) -> RequestCandidateFinalStatus {
let has_success = candidates.iter().any(|candidate| {
candidate.status == RequestCandidateStatus::Success
|| matches!(candidate.status_code, Some(status_code) if (200..300).contains(&status_code))
});
if has_success {
return RequestCandidateFinalStatus::Success;
}
if candidates
.iter()
.any(|candidate| candidate.status == RequestCandidateStatus::Streaming)
{
return RequestCandidateFinalStatus::Streaming;
}
if candidates
.iter()
.any(|candidate| candidate.status == RequestCandidateStatus::Pending)
{
return RequestCandidateFinalStatus::Pending;
}
let has_cancelled = candidates
.iter()
.any(|candidate| candidate.status == RequestCandidateStatus::Cancelled);
let has_failed = candidates
.iter()
.any(|candidate| candidate.status == RequestCandidateStatus::Failed);
if has_cancelled && !has_failed {
return RequestCandidateFinalStatus::Cancelled;
}
RequestCandidateFinalStatus::Failed
}
#[cfg(test)]
mod tests {
use super::super::GatewayDataState;
use super::{derive_final_status, read_request_candidate_trace, RequestCandidateFinalStatus};
use aether_data::repository::candidates::{
InMemoryRequestCandidateRepository, RequestCandidateStatus, StoredRequestCandidate,
};
use std::sync::Arc;
fn sample_candidate(
id: &str,
request_id: &str,
candidate_index: i32,
status: RequestCandidateStatus,
started_at_unix_secs: Option<i64>,
latency_ms: Option<i32>,
status_code: Option<i32>,
) -> StoredRequestCandidate {
StoredRequestCandidate::new(
id.to_string(),
request_id.to_string(),
Some("user-1".to_string()),
Some("api-key-1".to_string()),
Some("alice".to_string()),
Some("default".to_string()),
candidate_index,
0,
Some("provider-1".to_string()),
Some("endpoint-1".to_string()),
Some("provider-key-1".to_string()),
status,
None,
false,
status_code,
None,
None,
latency_ms,
Some(1),
None,
None,
100 + i64::from(candidate_index),
started_at_unix_secs,
started_at_unix_secs.map(|value| value + 1),
)
.expect("candidate should build")
}
#[test]
fn derive_final_status_prefers_success() {
let candidates = vec![sample_candidate(
"cand-1",
"req-1",
0,
RequestCandidateStatus::Success,
Some(100),
Some(25),
Some(200),
)];
assert_eq!(
derive_final_status(&candidates),
RequestCandidateFinalStatus::Success
);
}
#[tokio::test]
async fn read_request_candidate_trace_filters_attempted_rows() {
let repository = Arc::new(InMemoryRequestCandidateRepository::seed(vec![
sample_candidate(
"cand-1",
"req-1",
0,
RequestCandidateStatus::Pending,
None,
None,
None,
),
sample_candidate(
"cand-2",
"req-1",
1,
RequestCandidateStatus::Failed,
Some(101),
Some(33),
Some(502),
),
]));
let state = GatewayDataState::with_request_candidate_reader_for_tests(repository);
let trace = read_request_candidate_trace(&state, "req-1", true)
.await
.expect("trace should succeed")
.expect("trace should exist");
assert_eq!(trace.total_candidates, 1);
assert_eq!(trace.candidates[0].id, "cand-2");
assert_eq!(trace.final_status, RequestCandidateFinalStatus::Failed);
assert_eq!(trace.total_latency_ms, 33);
}
}

View File

@@ -0,0 +1,41 @@
use aether_data::postgres::PostgresPoolConfig;
use aether_data::DataLayerConfig;
#[derive(Debug, Clone, Default)]
pub struct GatewayDataConfig {
postgres: Option<PostgresPoolConfig>,
}
impl GatewayDataConfig {
pub fn disabled() -> Self {
Self::default()
}
pub fn from_postgres_config(postgres: PostgresPoolConfig) -> Self {
Self {
postgres: Some(postgres),
}
}
pub fn from_postgres_url(database_url: impl Into<String>, require_ssl: bool) -> Self {
let mut postgres = PostgresPoolConfig::default();
postgres.database_url = database_url.into();
postgres.require_ssl = require_ssl;
Self::from_postgres_config(postgres)
}
pub fn postgres(&self) -> Option<&PostgresPoolConfig> {
self.postgres.as_ref()
}
pub fn is_enabled(&self) -> bool {
self.postgres.is_some()
}
pub fn to_data_layer_config(&self) -> DataLayerConfig {
DataLayerConfig {
postgres: self.postgres.clone(),
redis: None,
}
}
}

View File

@@ -0,0 +1,267 @@
use std::collections::{BTreeMap, BTreeSet};
use aether_data::repository::candidates::StoredRequestCandidate;
use aether_data::repository::provider_catalog::{
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
};
use aether_data::DataLayerError;
use super::candidates::RequestCandidateFinalStatus;
use super::state::GatewayDataState;
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
pub(crate) struct DecisionTraceCandidate {
#[serde(flatten)]
pub(crate) candidate: StoredRequestCandidate,
pub(crate) provider_name: Option<String>,
pub(crate) provider_website: Option<String>,
pub(crate) provider_type: Option<String>,
pub(crate) endpoint_api_format: Option<String>,
pub(crate) endpoint_api_family: Option<String>,
pub(crate) endpoint_kind: Option<String>,
pub(crate) provider_key_name: Option<String>,
pub(crate) provider_key_auth_type: Option<String>,
pub(crate) provider_key_capabilities: Option<serde_json::Value>,
pub(crate) provider_key_is_active: Option<bool>,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
pub(crate) struct DecisionTrace {
pub(crate) request_id: String,
pub(crate) total_candidates: usize,
pub(crate) final_status: RequestCandidateFinalStatus,
pub(crate) total_latency_ms: u64,
pub(crate) candidates: Vec<DecisionTraceCandidate>,
}
pub(crate) async fn read_decision_trace(
state: &GatewayDataState,
request_id: &str,
attempted_only: bool,
) -> Result<Option<DecisionTrace>, DataLayerError> {
let Some(trace) = state
.read_request_candidate_trace(request_id, attempted_only)
.await?
else {
return Ok(None);
};
let provider_ids = unique_ids(
trace
.candidates
.iter()
.filter_map(|item| item.provider_id.as_ref()),
);
let endpoint_ids = unique_ids(
trace
.candidates
.iter()
.filter_map(|item| item.endpoint_id.as_ref()),
);
let key_ids = unique_ids(
trace
.candidates
.iter()
.filter_map(|item| item.key_id.as_ref()),
);
let provider_map = state
.list_provider_catalog_providers_by_ids(&provider_ids)
.await?
.into_iter()
.map(|item| (item.id.clone(), item))
.collect::<BTreeMap<_, _>>();
let endpoint_map = state
.list_provider_catalog_endpoints_by_ids(&endpoint_ids)
.await?
.into_iter()
.map(|item| (item.id.clone(), item))
.collect::<BTreeMap<_, _>>();
let key_map = state
.list_provider_catalog_keys_by_ids(&key_ids)
.await?
.into_iter()
.map(|item| (item.id.clone(), item))
.collect::<BTreeMap<_, _>>();
Ok(Some(DecisionTrace {
request_id: trace.request_id,
total_candidates: trace.total_candidates,
final_status: trace.final_status,
total_latency_ms: trace.total_latency_ms,
candidates: trace
.candidates
.into_iter()
.map(|candidate| enrich_candidate(candidate, &provider_map, &endpoint_map, &key_map))
.collect(),
}))
}
fn enrich_candidate(
candidate: StoredRequestCandidate,
provider_map: &BTreeMap<String, StoredProviderCatalogProvider>,
endpoint_map: &BTreeMap<String, StoredProviderCatalogEndpoint>,
key_map: &BTreeMap<String, StoredProviderCatalogKey>,
) -> DecisionTraceCandidate {
let provider = candidate
.provider_id
.as_ref()
.and_then(|provider_id| provider_map.get(provider_id));
let endpoint = candidate
.endpoint_id
.as_ref()
.and_then(|endpoint_id| endpoint_map.get(endpoint_id));
let provider_key = candidate
.key_id
.as_ref()
.and_then(|key_id| key_map.get(key_id));
DecisionTraceCandidate {
provider_name: provider.map(|item| item.name.clone()),
provider_website: provider.and_then(|item| item.website.clone()),
provider_type: provider.map(|item| item.provider_type.clone()),
endpoint_api_format: endpoint.map(|item| item.api_format.clone()),
endpoint_api_family: endpoint.and_then(|item| item.api_family.clone()),
endpoint_kind: endpoint.and_then(|item| item.endpoint_kind.clone()),
provider_key_name: provider_key
.map(|item| item.name.clone())
.or_else(|| candidate.api_key_name.clone()),
provider_key_auth_type: provider_key.map(|item| item.auth_type.clone()),
provider_key_capabilities: provider_key.and_then(|item| item.capabilities.clone()),
provider_key_is_active: provider_key.map(|item| item.is_active),
candidate,
}
}
fn unique_ids<'a>(items: impl Iterator<Item = &'a String>) -> Vec<String> {
items
.cloned()
.collect::<BTreeSet<_>>()
.into_iter()
.collect()
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use aether_data::repository::candidates::{
InMemoryRequestCandidateRepository, RequestCandidateStatus, StoredRequestCandidate,
};
use aether_data::repository::provider_catalog::{
InMemoryProviderCatalogReadRepository, StoredProviderCatalogEndpoint,
StoredProviderCatalogKey, StoredProviderCatalogProvider,
};
use super::{read_decision_trace, DecisionTrace, DecisionTraceCandidate};
use crate::gateway::data::candidates::RequestCandidateFinalStatus;
use crate::gateway::data::GatewayDataState;
fn sample_candidate(request_id: &str) -> StoredRequestCandidate {
StoredRequestCandidate::new(
"cand-1".to_string(),
request_id.to_string(),
Some("user-1".to_string()),
Some("api-key-1".to_string()),
Some("alice".to_string()),
Some("default".to_string()),
0,
0,
Some("provider-1".to_string()),
Some("endpoint-1".to_string()),
Some("provider-key-1".to_string()),
RequestCandidateStatus::Failed,
None,
false,
Some(502),
Some("bad_gateway".to_string()),
Some("upstream failed".to_string()),
Some(37),
Some(1),
None,
Some(serde_json::json!({"cache_1h": true})),
100,
Some(101),
Some(102),
)
.expect("candidate should build")
}
fn sample_provider() -> StoredProviderCatalogProvider {
StoredProviderCatalogProvider::new(
"provider-1".to_string(),
"OpenAI".to_string(),
Some("https://openai.com".to_string()),
"custom".to_string(),
)
.expect("provider should build")
}
fn sample_endpoint() -> StoredProviderCatalogEndpoint {
StoredProviderCatalogEndpoint::new(
"endpoint-1".to_string(),
"provider-1".to_string(),
"openai:chat".to_string(),
Some("openai".to_string()),
Some("chat".to_string()),
true,
)
.expect("endpoint should build")
}
fn sample_key() -> StoredProviderCatalogKey {
StoredProviderCatalogKey::new(
"provider-key-1".to_string(),
"provider-1".to_string(),
"prod-key".to_string(),
"api_key".to_string(),
Some(serde_json::json!({"cache_1h": true})),
true,
)
.expect("key should build")
}
#[tokio::test]
async fn enriches_request_candidate_trace_with_provider_catalog_metadata() {
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![
sample_candidate("req-1"),
]));
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider()],
vec![sample_endpoint()],
vec![sample_key()],
));
let state = GatewayDataState::with_decision_trace_readers_for_tests(
request_candidates,
provider_catalog,
);
let trace = read_decision_trace(&state, "req-1", true)
.await
.expect("trace should read")
.expect("trace should exist");
assert_eq!(
trace,
DecisionTrace {
request_id: "req-1".to_string(),
total_candidates: 1,
final_status: RequestCandidateFinalStatus::Failed,
total_latency_ms: 37,
candidates: vec![DecisionTraceCandidate {
candidate: sample_candidate("req-1"),
provider_name: Some("OpenAI".to_string()),
provider_website: Some("https://openai.com".to_string()),
provider_type: Some("custom".to_string()),
endpoint_api_format: Some("openai:chat".to_string()),
endpoint_api_family: Some("openai".to_string()),
endpoint_kind: Some("chat".to_string()),
provider_key_name: Some("prod-key".to_string()),
provider_key_auth_type: Some("api_key".to_string()),
provider_key_capabilities: Some(serde_json::json!({"cache_1h": true})),
provider_key_is_active: Some(true),
}],
}
);
}
}

View File

@@ -0,0 +1,80 @@
use aether_data::repository::video_tasks::{StoredVideoTask, VideoTaskStatus};
use serde_json::json;
use crate::gateway::video_tasks::LocalVideoTaskReadResponse;
pub(super) fn map_gemini_video_task_to_read_response(
task: StoredVideoTask,
) -> LocalVideoTaskReadResponse {
match task.status {
VideoTaskStatus::Cancelled => LocalVideoTaskReadResponse {
status_code: 404,
body_json: json!({"detail": "Video task was cancelled"}),
},
VideoTaskStatus::Deleted => LocalVideoTaskReadResponse {
status_code: 404,
body_json: json!({"detail": "Video task not found"}),
},
VideoTaskStatus::Completed => LocalVideoTaskReadResponse {
status_code: 200,
body_json: build_gemini_completed_body(task),
},
VideoTaskStatus::Failed | VideoTaskStatus::Expired => LocalVideoTaskReadResponse {
status_code: 200,
body_json: build_gemini_failed_body(task),
},
_ => LocalVideoTaskReadResponse {
status_code: 200,
body_json: build_gemini_pending_body(task),
},
}
}
fn build_gemini_completed_body(task: StoredVideoTask) -> serde_json::Value {
let operation_name = operation_name(&task);
let short_id = task.short_id.unwrap_or_default();
json!({
"name": operation_name,
"done": true,
"response": {
"generateVideoResponse": {
"generatedSamples": [
{
"video": {
"uri": format!("/v1beta/files/aev_{short_id}:download?alt=media"),
"mimeType": "video/mp4"
}
}
]
}
}
})
}
fn build_gemini_failed_body(task: StoredVideoTask) -> serde_json::Value {
json!({
"name": operation_name(&task),
"done": true,
"error": {
"code": task.error_code.unwrap_or_else(|| "UNKNOWN".to_string()),
"message": task
.error_message
.unwrap_or_else(|| "Video generation failed".to_string()),
}
})
}
fn build_gemini_pending_body(task: StoredVideoTask) -> serde_json::Value {
json!({
"name": operation_name(&task),
"done": false,
"metadata": {}
})
}
fn operation_name(task: &StoredVideoTask) -> String {
let model = task.model.clone().unwrap_or_else(|| "unknown".to_string());
let short_id = task.short_id.clone().unwrap_or_else(|| task.id.clone());
format!("models/{model}/operations/{short_id}")
}

View File

@@ -0,0 +1,21 @@
mod auth;
mod candidates;
mod config;
mod decision_trace;
mod gemini;
mod openai;
mod request_audit;
mod state;
mod usage;
mod video_tasks;
#[cfg(test)]
mod tests;
pub(crate) use auth::StoredGatewayAuthApiKeySnapshot;
pub(crate) use candidates::RequestCandidateTrace;
pub use config::GatewayDataConfig;
pub(crate) use decision_trace::DecisionTrace;
pub(crate) use request_audit::RequestAuditBundle;
pub(crate) use state::GatewayDataState;
pub(crate) use usage::RequestUsageAudit;

View File

@@ -0,0 +1,69 @@
use aether_data::repository::video_tasks::{StoredVideoTask, VideoTaskStatus};
use serde_json::{json, Value};
use crate::gateway::video_tasks::LocalVideoTaskReadResponse;
pub(super) fn map_openai_video_task_to_read_response(
task: StoredVideoTask,
) -> LocalVideoTaskReadResponse {
match task.status {
VideoTaskStatus::Cancelled => LocalVideoTaskReadResponse {
status_code: 404,
body_json: json!({"detail": "Video task was cancelled"}),
},
VideoTaskStatus::Deleted => LocalVideoTaskReadResponse {
status_code: 404,
body_json: json!({"detail": "Video task not found"}),
},
status => LocalVideoTaskReadResponse {
status_code: 200,
body_json: build_openai_video_task_body(task, status),
},
}
}
fn build_openai_video_task_body(task: StoredVideoTask, status: VideoTaskStatus) -> Value {
let mut body = json!({
"id": task.id,
"object": "video",
"status": map_openai_video_status(status),
"progress": task.progress_percent,
"created_at": task.created_at_unix_secs,
});
if let Some(model) = task.model {
body["model"] = Value::String(model);
}
if let Some(prompt) = task.prompt {
body["prompt"] = Value::String(prompt);
}
if let Some(size) = task.size {
body["size"] = Value::String(size);
}
if let Some(video_url) = task.video_url {
body["video_url"] = Value::String(video_url);
}
if matches!(
status,
VideoTaskStatus::Failed | VideoTaskStatus::Expired | VideoTaskStatus::Cancelled
) {
body["error"] = json!({
"code": task.error_code.unwrap_or_else(|| "unknown".to_string()),
"message": task
.error_message
.unwrap_or_else(|| "Video generation failed".to_string()),
});
}
body
}
fn map_openai_video_status(status: VideoTaskStatus) -> &'static str {
match status {
VideoTaskStatus::Pending | VideoTaskStatus::Submitted | VideoTaskStatus::Queued => "queued",
VideoTaskStatus::Processing => "processing",
VideoTaskStatus::Completed => "completed",
VideoTaskStatus::Failed | VideoTaskStatus::Cancelled | VideoTaskStatus::Expired => "failed",
VideoTaskStatus::Deleted => "deleted",
}
}

View File

@@ -0,0 +1,56 @@
use aether_data::DataLayerError;
use super::auth::StoredGatewayAuthApiKeySnapshot;
use super::decision_trace::DecisionTrace;
use super::state::GatewayDataState;
use super::usage::RequestUsageAudit;
#[derive(Debug, Clone, PartialEq, serde::Serialize)]
pub(crate) struct RequestAuditBundle {
pub(crate) request_id: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) usage: Option<RequestUsageAudit>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) decision_trace: Option<DecisionTrace>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) auth_snapshot: Option<StoredGatewayAuthApiKeySnapshot>,
}
pub(crate) async fn read_request_audit_bundle(
state: &GatewayDataState,
request_id: &str,
attempted_only: bool,
now_unix_secs: u64,
) -> Result<Option<RequestAuditBundle>, DataLayerError> {
let usage = state.read_request_usage_audit(request_id).await?;
let decision_trace = state
.read_decision_trace(request_id, attempted_only)
.await?;
let auth_snapshot = if let Some(usage) = usage.as_ref() {
match (
usage.usage.user_id.as_deref(),
usage.usage.api_key_id.as_deref(),
) {
(Some(user_id), Some(api_key_id)) => {
state
.read_auth_api_key_snapshot(user_id, api_key_id, now_unix_secs)
.await?
}
_ => None,
}
} else {
None
};
if usage.is_none() && decision_trace.is_none() && auth_snapshot.is_none() {
return Ok(None);
}
Ok(Some(RequestAuditBundle {
request_id: request_id.to_string(),
usage,
decision_trace,
auth_snapshot,
}))
}

View File

@@ -0,0 +1,454 @@
use std::fmt;
use std::sync::Arc;
use aether_data::repository::auth::{
AuthApiKeyLookupKey, AuthApiKeyReadRepository, StoredAuthApiKeySnapshot,
};
use aether_data::repository::candidates::{RequestCandidateReadRepository, StoredRequestCandidate};
use aether_data::repository::provider_catalog::{
ProviderCatalogReadRepository, StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
StoredProviderCatalogProvider,
};
use aether_data::repository::shadow_results::{
merge_shadow_result_sample, RecordShadowResultSample, ShadowResultLookupKey,
ShadowResultReadRepository, ShadowResultWriteRepository, StoredShadowResult,
};
use aether_data::repository::usage::{StoredRequestUsageAudit, UsageReadRepository};
use aether_data::repository::video_tasks::{
StoredVideoTask, VideoTaskLookupKey, VideoTaskReadRepository,
};
use aether_data::{DataBackends, DataLayerError};
use super::auth::{read_auth_api_key_snapshot, StoredGatewayAuthApiKeySnapshot};
use super::candidates::{read_request_candidate_trace, RequestCandidateTrace};
use super::config::GatewayDataConfig;
use super::decision_trace::{read_decision_trace, DecisionTrace};
use super::request_audit::{read_request_audit_bundle, RequestAuditBundle};
use super::usage::{read_request_usage_audit, RequestUsageAudit};
use super::video_tasks::read_video_task_response;
use crate::gateway::video_tasks::LocalVideoTaskReadResponse;
#[derive(Clone, Default)]
pub(crate) struct GatewayDataState {
config: GatewayDataConfig,
backends: Option<DataBackends>,
auth_api_key_reader: Option<Arc<dyn AuthApiKeyReadRepository>>,
request_candidate_reader: Option<Arc<dyn RequestCandidateReadRepository>>,
provider_catalog_reader: Option<Arc<dyn ProviderCatalogReadRepository>>,
usage_reader: Option<Arc<dyn UsageReadRepository>>,
video_task_reader: Option<Arc<dyn VideoTaskReadRepository>>,
shadow_result_reader: Option<Arc<dyn ShadowResultReadRepository>>,
shadow_result_writer: Option<Arc<dyn ShadowResultWriteRepository>>,
}
impl fmt::Debug for GatewayDataState {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("GatewayDataState")
.field("config", &self.config)
.field("has_backends", &self.backends.is_some())
.field(
"has_auth_api_key_reader",
&self.auth_api_key_reader.is_some(),
)
.field(
"has_request_candidate_reader",
&self.request_candidate_reader.is_some(),
)
.field(
"has_provider_catalog_reader",
&self.provider_catalog_reader.is_some(),
)
.field("has_usage_reader", &self.usage_reader.is_some())
.field("has_video_task_reader", &self.video_task_reader.is_some())
.field(
"has_shadow_result_reader",
&self.shadow_result_reader.is_some(),
)
.field(
"has_shadow_result_writer",
&self.shadow_result_writer.is_some(),
)
.finish()
}
}
impl GatewayDataState {
pub(crate) fn disabled() -> Self {
Self::default()
}
pub(crate) fn from_config(config: GatewayDataConfig) -> Result<Self, DataLayerError> {
if !config.is_enabled() {
return Ok(Self {
config,
backends: None,
auth_api_key_reader: None,
request_candidate_reader: None,
provider_catalog_reader: None,
usage_reader: None,
video_task_reader: None,
shadow_result_reader: None,
shadow_result_writer: None,
});
}
let backends = DataBackends::from_config(config.to_data_layer_config())?;
let auth_api_key_reader = backends.read().auth_api_keys();
let request_candidate_reader = backends.read().request_candidates();
let provider_catalog_reader = backends.read().provider_catalog();
let usage_reader = backends.read().usage();
let video_task_reader = backends.read().video_tasks();
let shadow_result_reader = backends.read().shadow_results();
let shadow_result_writer = backends.write().shadow_results();
Ok(Self {
config,
backends: Some(backends),
auth_api_key_reader,
request_candidate_reader,
provider_catalog_reader,
usage_reader,
video_task_reader,
shadow_result_reader,
shadow_result_writer,
})
}
pub(crate) fn has_backends(&self) -> bool {
self.backends.is_some()
}
pub(crate) fn has_auth_api_key_reader(&self) -> bool {
self.auth_api_key_reader.is_some()
}
pub(crate) fn has_request_candidate_reader(&self) -> bool {
self.request_candidate_reader.is_some()
}
pub(crate) fn has_provider_catalog_reader(&self) -> bool {
self.provider_catalog_reader.is_some()
}
pub(crate) fn has_usage_reader(&self) -> bool {
self.usage_reader.is_some()
}
pub(crate) fn has_video_task_reader(&self) -> bool {
self.video_task_reader.is_some()
}
pub(crate) fn has_shadow_result_writer(&self) -> bool {
self.shadow_result_writer.is_some()
}
pub(crate) fn has_shadow_result_reader(&self) -> bool {
self.shadow_result_reader.is_some()
}
pub(super) async fn find_video_task(
&self,
key: VideoTaskLookupKey<'_>,
) -> Result<Option<StoredVideoTask>, DataLayerError> {
match &self.video_task_reader {
Some(repository) => repository.find(key).await,
None => Ok(None),
}
}
pub(super) async fn find_auth_api_key_snapshot(
&self,
key: AuthApiKeyLookupKey<'_>,
) -> Result<Option<StoredAuthApiKeySnapshot>, DataLayerError> {
match &self.auth_api_key_reader {
Some(repository) => repository.find_api_key_snapshot(key).await,
None => Ok(None),
}
}
pub(super) async fn list_request_candidates_by_request_id(
&self,
request_id: &str,
) -> Result<Vec<StoredRequestCandidate>, DataLayerError> {
match &self.request_candidate_reader {
Some(repository) => repository.list_by_request_id(request_id).await,
None => Ok(Vec::new()),
}
}
pub(super) async fn list_provider_catalog_providers_by_ids(
&self,
provider_ids: &[String],
) -> Result<Vec<StoredProviderCatalogProvider>, DataLayerError> {
match &self.provider_catalog_reader {
Some(repository) => repository.list_providers_by_ids(provider_ids).await,
None => Ok(Vec::new()),
}
}
pub(super) async fn list_provider_catalog_endpoints_by_ids(
&self,
endpoint_ids: &[String],
) -> Result<Vec<StoredProviderCatalogEndpoint>, DataLayerError> {
match &self.provider_catalog_reader {
Some(repository) => repository.list_endpoints_by_ids(endpoint_ids).await,
None => Ok(Vec::new()),
}
}
pub(super) async fn list_provider_catalog_keys_by_ids(
&self,
key_ids: &[String],
) -> Result<Vec<StoredProviderCatalogKey>, DataLayerError> {
match &self.provider_catalog_reader {
Some(repository) => repository.list_keys_by_ids(key_ids).await,
None => Ok(Vec::new()),
}
}
pub(super) async fn find_request_usage_by_request_id(
&self,
request_id: &str,
) -> Result<Option<StoredRequestUsageAudit>, DataLayerError> {
match &self.usage_reader {
Some(repository) => repository.find_by_request_id(request_id).await,
None => Ok(None),
}
}
pub(crate) async fn read_request_candidate_trace(
&self,
request_id: &str,
attempted_only: bool,
) -> Result<Option<RequestCandidateTrace>, DataLayerError> {
read_request_candidate_trace(self, request_id, attempted_only).await
}
pub(crate) async fn read_decision_trace(
&self,
request_id: &str,
attempted_only: bool,
) -> Result<Option<DecisionTrace>, DataLayerError> {
read_decision_trace(self, request_id, attempted_only).await
}
pub(crate) async fn read_request_usage_audit(
&self,
request_id: &str,
) -> Result<Option<RequestUsageAudit>, DataLayerError> {
read_request_usage_audit(self, request_id).await
}
pub(crate) async fn read_request_audit_bundle(
&self,
request_id: &str,
attempted_only: bool,
now_unix_secs: u64,
) -> Result<Option<RequestAuditBundle>, DataLayerError> {
read_request_audit_bundle(self, request_id, attempted_only, now_unix_secs).await
}
pub(crate) async fn read_auth_api_key_snapshot(
&self,
user_id: &str,
api_key_id: &str,
now_unix_secs: u64,
) -> Result<Option<StoredGatewayAuthApiKeySnapshot>, DataLayerError> {
read_auth_api_key_snapshot(self, user_id, api_key_id, now_unix_secs).await
}
pub(crate) async fn read_video_task_response(
&self,
route_family: Option<&str>,
request_path: &str,
) -> Result<Option<LocalVideoTaskReadResponse>, DataLayerError> {
read_video_task_response(self, route_family, request_path).await
}
#[cfg(test)]
pub(crate) async fn write_shadow_result(
&self,
result: aether_data::repository::shadow_results::UpsertShadowResult,
) -> Result<Option<StoredShadowResult>, DataLayerError> {
match &self.shadow_result_writer {
Some(repository) => repository.upsert(result).await.map(Some),
None => Ok(None),
}
}
pub(crate) async fn record_shadow_result_sample(
&self,
sample: RecordShadowResultSample,
) -> Result<Option<StoredShadowResult>, DataLayerError> {
let Some(writer) = &self.shadow_result_writer else {
return Ok(None);
};
let existing = match &self.shadow_result_reader {
Some(reader) => {
reader
.find(ShadowResultLookupKey::TraceFingerprint {
trace_id: &sample.trace_id,
request_fingerprint: &sample.request_fingerprint,
})
.await?
}
None => None,
};
let merged = merge_shadow_result_sample(existing.as_ref(), sample);
writer.upsert(merged).await.map(Some)
}
pub(crate) async fn list_recent_shadow_results(
&self,
limit: usize,
) -> Result<Vec<StoredShadowResult>, DataLayerError> {
match &self.shadow_result_reader {
Some(repository) => repository.list_recent(limit).await,
None => Ok(Vec::new()),
}
}
#[cfg(test)]
pub(crate) fn with_video_task_reader_for_tests(
repository: Arc<dyn VideoTaskReadRepository>,
) -> Self {
Self {
config: GatewayDataConfig::disabled(),
backends: None,
auth_api_key_reader: None,
request_candidate_reader: None,
provider_catalog_reader: None,
usage_reader: None,
video_task_reader: Some(repository),
shadow_result_reader: None,
shadow_result_writer: None,
}
}
#[cfg(test)]
pub(crate) fn with_request_candidate_reader_for_tests(
repository: Arc<dyn RequestCandidateReadRepository>,
) -> Self {
Self {
config: GatewayDataConfig::disabled(),
backends: None,
auth_api_key_reader: None,
request_candidate_reader: Some(repository),
provider_catalog_reader: None,
usage_reader: None,
video_task_reader: None,
shadow_result_reader: None,
shadow_result_writer: None,
}
}
#[cfg(test)]
pub(crate) fn with_usage_reader_for_tests(repository: Arc<dyn UsageReadRepository>) -> Self {
Self {
config: GatewayDataConfig::disabled(),
backends: None,
auth_api_key_reader: None,
request_candidate_reader: None,
provider_catalog_reader: None,
usage_reader: Some(repository),
video_task_reader: None,
shadow_result_reader: None,
shadow_result_writer: None,
}
}
#[cfg(test)]
pub(crate) fn with_auth_api_key_reader_for_tests(
repository: Arc<dyn AuthApiKeyReadRepository>,
) -> Self {
Self {
config: GatewayDataConfig::disabled(),
backends: None,
auth_api_key_reader: Some(repository),
request_candidate_reader: None,
provider_catalog_reader: None,
usage_reader: None,
video_task_reader: None,
shadow_result_reader: None,
shadow_result_writer: None,
}
}
#[cfg(test)]
pub(crate) fn with_decision_trace_readers_for_tests(
request_candidate_repository: Arc<dyn RequestCandidateReadRepository>,
provider_catalog_repository: Arc<dyn ProviderCatalogReadRepository>,
) -> Self {
Self {
config: GatewayDataConfig::disabled(),
backends: None,
auth_api_key_reader: None,
request_candidate_reader: Some(request_candidate_repository),
provider_catalog_reader: Some(provider_catalog_repository),
usage_reader: None,
video_task_reader: None,
shadow_result_reader: None,
shadow_result_writer: None,
}
}
#[cfg(test)]
pub(crate) fn with_request_audit_readers_for_tests(
auth_api_key_repository: Arc<dyn AuthApiKeyReadRepository>,
request_candidate_repository: Arc<dyn RequestCandidateReadRepository>,
provider_catalog_repository: Arc<dyn ProviderCatalogReadRepository>,
usage_repository: Arc<dyn UsageReadRepository>,
) -> Self {
Self {
config: GatewayDataConfig::disabled(),
backends: None,
auth_api_key_reader: Some(auth_api_key_repository),
request_candidate_reader: Some(request_candidate_repository),
provider_catalog_reader: Some(provider_catalog_repository),
usage_reader: Some(usage_repository),
video_task_reader: None,
shadow_result_reader: None,
shadow_result_writer: None,
}
}
#[cfg(test)]
pub(crate) fn with_shadow_result_writer_for_tests(
repository: Arc<dyn ShadowResultWriteRepository>,
) -> Self {
Self {
config: GatewayDataConfig::disabled(),
backends: None,
auth_api_key_reader: None,
request_candidate_reader: None,
provider_catalog_reader: None,
usage_reader: None,
video_task_reader: None,
shadow_result_reader: None,
shadow_result_writer: Some(repository),
}
}
#[cfg(test)]
pub(crate) fn with_shadow_result_repository_for_tests<T>(repository: Arc<T>) -> Self
where
T: aether_data::repository::shadow_results::ShadowResultRepository + 'static,
{
let shadow_result_reader: Arc<dyn ShadowResultReadRepository> = repository.clone();
let shadow_result_writer: Arc<dyn ShadowResultWriteRepository> = repository;
Self {
config: GatewayDataConfig::disabled(),
backends: None,
auth_api_key_reader: None,
request_candidate_reader: None,
provider_catalog_reader: None,
usage_reader: None,
video_task_reader: None,
shadow_result_reader: Some(shadow_result_reader),
shadow_result_writer: Some(shadow_result_writer),
}
}
}

View File

@@ -0,0 +1,683 @@
use std::sync::Arc;
use aether_data::repository::auth::{
InMemoryAuthApiKeySnapshotRepository, StoredAuthApiKeySnapshot,
};
use aether_data::repository::candidates::{
InMemoryRequestCandidateRepository, RequestCandidateStatus, StoredRequestCandidate,
};
use aether_data::repository::provider_catalog::{
InMemoryProviderCatalogReadRepository, StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
StoredProviderCatalogProvider,
};
use aether_data::repository::shadow_results::{
InMemoryShadowResultRepository, RecordShadowResultSample, ShadowResultLookupKey,
ShadowResultMatchStatus, ShadowResultReadRepository, ShadowResultSampleOrigin,
UpsertShadowResult,
};
use aether_data::repository::usage::{InMemoryUsageReadRepository, StoredRequestUsageAudit};
use aether_data::repository::video_tasks::{
InMemoryVideoTaskRepository, UpsertVideoTask, VideoTaskLookupKey, VideoTaskStatus,
VideoTaskWriteRepository,
};
use super::{GatewayDataConfig, GatewayDataState};
use crate::gateway::AppState;
#[test]
fn disabled_gateway_data_state_has_no_backends() {
let state = GatewayDataState::from_config(GatewayDataConfig::disabled())
.expect("disabled config should build");
assert!(!state.has_backends());
assert!(!state.has_auth_api_key_reader());
assert!(!state.has_request_candidate_reader());
assert!(!state.has_provider_catalog_reader());
assert!(!state.has_usage_reader());
assert!(!state.has_video_task_reader());
assert!(!state.has_shadow_result_reader());
assert!(!state.has_shadow_result_writer());
}
#[tokio::test]
async fn postgres_gateway_data_state_builds_video_task_reader() {
let state = GatewayDataState::from_config(GatewayDataConfig::from_postgres_url(
"postgres://localhost/aether",
false,
))
.expect("postgres-backed state should build");
assert!(state.has_backends());
assert!(state.has_auth_api_key_reader());
assert!(state.has_request_candidate_reader());
assert!(state.has_provider_catalog_reader());
assert!(state.has_usage_reader());
assert!(state.has_video_task_reader());
assert!(state.has_shadow_result_reader());
assert!(state.has_shadow_result_writer());
}
#[tokio::test]
async fn data_state_find_uses_configured_read_repository() {
let repository = Arc::new(InMemoryVideoTaskRepository::default());
repository
.upsert(UpsertVideoTask {
id: "task-1".to_string(),
short_id: Some("short-task-1".to_string()),
user_id: Some("user-1".to_string()),
external_task_id: Some("ext-task-1".to_string()),
provider_api_format: Some("openai:video".to_string()),
model: Some("sora-2".to_string()),
prompt: Some("hello".to_string()),
size: Some("1280x720".to_string()),
status: VideoTaskStatus::Queued,
progress_percent: 0,
created_at_unix_secs: 100,
updated_at_unix_secs: 100,
error_code: None,
error_message: None,
video_url: None,
})
.await
.expect("upsert should succeed");
let state = GatewayDataState::with_video_task_reader_for_tests(repository);
let task = state
.find_video_task(VideoTaskLookupKey::Id("task-1"))
.await
.expect("find should succeed");
assert_eq!(task.expect("task should exist").id, "task-1");
}
#[tokio::test]
async fn app_state_wires_gateway_data_state_from_config() {
let state = AppState::new_with_executor(
"http://127.0.0.1:18084",
Some("http://127.0.0.1:18085".to_string()),
Some("http://127.0.0.1:18086".to_string()),
)
.expect("app state should build")
.with_data_config(GatewayDataConfig::from_postgres_url(
"postgres://localhost/aether",
false,
))
.expect("data config should wire");
assert!(state.data.has_backends());
assert!(state.data.has_auth_api_key_reader());
assert!(state.data.has_request_candidate_reader());
assert!(state.data.has_provider_catalog_reader());
assert!(state.data.has_usage_reader());
assert!(state.data.has_video_task_reader());
assert!(state.data.has_shadow_result_reader());
assert!(state.data.has_shadow_result_writer());
}
fn sample_auth_snapshot(api_key_id: &str, user_id: &str) -> StoredAuthApiKeySnapshot {
StoredAuthApiKeySnapshot::new(
user_id.to_string(),
"alice".to_string(),
Some("alice@example.com".to_string()),
"user".to_string(),
"local".to_string(),
true,
false,
Some(serde_json::json!(["openai"])),
Some(serde_json::json!(["openai:chat"])),
Some(serde_json::json!(["gpt-4.1"])),
api_key_id.to_string(),
Some("default".to_string()),
true,
false,
false,
Some(60),
Some(5),
Some(200),
Some(serde_json::json!(["openai"])),
Some(serde_json::json!(["openai:chat"])),
Some(serde_json::json!(["gpt-4.1"])),
)
.expect("auth snapshot should build")
}
#[tokio::test]
async fn data_state_reads_auth_api_key_snapshot_from_reader() {
let repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
Some("hash-1".to_string()),
sample_auth_snapshot("key-1", "user-1"),
)]));
let state = GatewayDataState::with_auth_api_key_reader_for_tests(repository);
let snapshot = state
.read_auth_api_key_snapshot("user-1", "key-1", 150)
.await
.expect("read should succeed")
.expect("snapshot should exist");
assert_eq!(snapshot.user_id, "user-1");
assert_eq!(snapshot.api_key_id, "key-1");
assert_eq!(snapshot.username, "alice");
assert_eq!(
snapshot.api_key_allowed_models,
Some(vec!["gpt-4.1".to_string()])
);
assert!(snapshot.currently_usable);
}
fn sample_provider_catalog_provider() -> StoredProviderCatalogProvider {
StoredProviderCatalogProvider::new(
"provider-1".to_string(),
"OpenAI".to_string(),
Some("https://openai.com".to_string()),
"custom".to_string(),
)
.expect("provider should build")
}
fn sample_provider_catalog_endpoint() -> StoredProviderCatalogEndpoint {
StoredProviderCatalogEndpoint::new(
"endpoint-1".to_string(),
"provider-1".to_string(),
"openai:chat".to_string(),
Some("openai".to_string()),
Some("chat".to_string()),
true,
)
.expect("endpoint should build")
}
fn sample_provider_catalog_key() -> StoredProviderCatalogKey {
StoredProviderCatalogKey::new(
"provider-key-1".to_string(),
"provider-1".to_string(),
"prod-key".to_string(),
"api_key".to_string(),
Some(serde_json::json!({"cache_1h": true})),
true,
)
.expect("key should build")
}
fn sample_request_usage(request_id: &str) -> StoredRequestUsageAudit {
StoredRequestUsageAudit::new(
"usage-1".to_string(),
request_id.to_string(),
Some("user-1".to_string()),
Some("api-key-1".to_string()),
Some("alice".to_string()),
Some("default".to_string()),
"OpenAI".to_string(),
"gpt-4.1".to_string(),
Some("gpt-4.1-mini".to_string()),
Some("provider-1".to_string()),
Some("endpoint-1".to_string()),
Some("provider-key-1".to_string()),
Some("chat".to_string()),
Some("openai:chat".to_string()),
Some("openai".to_string()),
Some("chat".to_string()),
Some("openai:chat".to_string()),
Some("openai".to_string()),
Some("chat".to_string()),
true,
false,
120,
40,
160,
0.24,
0.36,
Some(200),
None,
None,
Some(450),
Some(120),
"completed".to_string(),
"settled".to_string(),
100,
101,
Some(102),
)
.expect("usage should build")
}
#[tokio::test]
async fn data_state_reads_decision_trace_with_provider_catalog_metadata() {
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![
StoredRequestCandidate::new(
"cand-1".to_string(),
"req-1".to_string(),
Some("user-1".to_string()),
Some("api-key-1".to_string()),
Some("alice".to_string()),
Some("default".to_string()),
0,
0,
Some("provider-1".to_string()),
Some("endpoint-1".to_string()),
Some("provider-key-1".to_string()),
RequestCandidateStatus::Failed,
None,
false,
Some(502),
None,
None,
Some(37),
Some(1),
None,
Some(serde_json::json!({"cache_1h": true})),
100,
Some(101),
Some(102),
)
.expect("candidate should build"),
]));
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider_catalog_provider()],
vec![sample_provider_catalog_endpoint()],
vec![sample_provider_catalog_key()],
));
let state = GatewayDataState::with_decision_trace_readers_for_tests(
request_candidates,
provider_catalog,
);
let trace = state
.read_decision_trace("req-1", true)
.await
.expect("trace should read")
.expect("trace should exist");
assert_eq!(trace.request_id, "req-1");
assert_eq!(trace.total_candidates, 1);
assert_eq!(trace.candidates[0].provider_name.as_deref(), Some("OpenAI"));
assert_eq!(
trace.candidates[0].endpoint_api_format.as_deref(),
Some("openai:chat")
);
assert_eq!(
trace.candidates[0].provider_key_auth_type.as_deref(),
Some("api_key")
);
assert_eq!(
trace.candidates[0].provider_key_capabilities,
Some(serde_json::json!({"cache_1h": true}))
);
}
#[tokio::test]
async fn data_state_reads_request_usage_audit_from_reader() {
let repository = Arc::new(InMemoryUsageReadRepository::seed(vec![
sample_request_usage("req-usage-1"),
]));
let state = GatewayDataState::with_usage_reader_for_tests(repository);
let usage = state
.read_request_usage_audit("req-usage-1")
.await
.expect("read should succeed")
.expect("usage should exist");
assert_eq!(usage.usage.request_id, "req-usage-1");
assert_eq!(usage.usage.provider_name, "OpenAI");
assert_eq!(usage.usage.total_tokens, 160);
assert_eq!(usage.usage.total_cost_usd, 0.24);
assert!(usage.usage.has_format_conversion);
}
#[tokio::test]
async fn data_state_reads_request_audit_bundle_from_multiple_readers() {
let auth_repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
Some("hash-1".to_string()),
sample_auth_snapshot("api-key-1", "user-1"),
)]));
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![
StoredRequestCandidate::new(
"cand-1".to_string(),
"req-usage-1".to_string(),
Some("user-1".to_string()),
Some("api-key-1".to_string()),
Some("alice".to_string()),
Some("default".to_string()),
0,
0,
Some("provider-1".to_string()),
Some("endpoint-1".to_string()),
Some("provider-key-1".to_string()),
RequestCandidateStatus::Success,
None,
false,
Some(200),
None,
None,
Some(37),
Some(1),
None,
Some(serde_json::json!({"cache_1h": true})),
100,
Some(101),
Some(102),
)
.expect("candidate should build"),
]));
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider_catalog_provider()],
vec![sample_provider_catalog_endpoint()],
vec![sample_provider_catalog_key()],
));
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(vec![
sample_request_usage("req-usage-1"),
]));
let state = GatewayDataState::with_request_audit_readers_for_tests(
auth_repository,
request_candidates,
provider_catalog,
usage_repository,
);
let bundle = state
.read_request_audit_bundle("req-usage-1", true, 150)
.await
.expect("bundle should read")
.expect("bundle should exist");
assert_eq!(bundle.request_id, "req-usage-1");
assert_eq!(
bundle
.usage
.as_ref()
.and_then(|usage| usage.usage.target_model.as_deref()),
Some("gpt-4.1-mini")
);
assert_eq!(
bundle
.decision_trace
.as_ref()
.and_then(|trace| trace.candidates.first())
.and_then(|candidate| candidate.provider_name.as_deref()),
Some("OpenAI")
);
assert_eq!(
bundle
.auth_snapshot
.as_ref()
.map(|snapshot| snapshot.currently_usable),
Some(true)
);
}
#[tokio::test]
async fn maps_openai_video_task_repository_row_into_read_response() {
let repository = Arc::new(InMemoryVideoTaskRepository::default());
repository
.upsert(UpsertVideoTask {
id: "task-1".to_string(),
short_id: Some("short-task-1".to_string()),
user_id: Some("user-1".to_string()),
external_task_id: Some("ext-task-1".to_string()),
provider_api_format: Some("openai:video".to_string()),
model: Some("sora-2".to_string()),
prompt: Some("hello".to_string()),
size: Some("1280x720".to_string()),
status: VideoTaskStatus::Processing,
progress_percent: 45,
created_at_unix_secs: 100,
updated_at_unix_secs: 120,
error_code: None,
error_message: None,
video_url: None,
})
.await
.expect("upsert should succeed");
let state = GatewayDataState::with_video_task_reader_for_tests(repository);
let response = state
.read_video_task_response(Some("openai"), "/v1/videos/task-1")
.await
.expect("read should succeed")
.expect("read response should exist");
assert_eq!(response.status_code, 200);
assert_eq!(response.body_json["id"], "task-1");
assert_eq!(response.body_json["status"], "processing");
assert_eq!(response.body_json["created_at"], 100);
}
#[tokio::test]
async fn maps_gemini_video_task_repository_row_into_read_response() {
let repository = Arc::new(InMemoryVideoTaskRepository::default());
repository
.upsert(UpsertVideoTask {
id: "task-1".to_string(),
short_id: Some("localshort123".to_string()),
user_id: Some("user-1".to_string()),
external_task_id: Some("operations/ext-task-1".to_string()),
provider_api_format: Some("gemini:video".to_string()),
model: Some("veo-3".to_string()),
prompt: Some("hello".to_string()),
size: Some("720p".to_string()),
status: VideoTaskStatus::Completed,
progress_percent: 100,
created_at_unix_secs: 100,
updated_at_unix_secs: 120,
error_code: None,
error_message: None,
video_url: None,
})
.await
.expect("upsert should succeed");
let state = GatewayDataState::with_video_task_reader_for_tests(repository);
let response = state
.read_video_task_response(
Some("gemini"),
"/v1beta/models/veo-3/operations/localshort123",
)
.await
.expect("read should succeed")
.expect("read response should exist");
assert_eq!(response.status_code, 200);
assert_eq!(
response.body_json["name"],
"models/veo-3/operations/localshort123"
);
assert_eq!(response.body_json["done"], true);
}
#[tokio::test]
async fn data_state_write_uses_configured_shadow_result_writer() {
let repository = Arc::new(InMemoryShadowResultRepository::default());
let state = GatewayDataState::with_shadow_result_writer_for_tests(repository.clone());
let written = state
.write_shadow_result(UpsertShadowResult {
trace_id: "trace-1".to_string(),
request_fingerprint: "fp-1".to_string(),
request_id: Some("req-1".to_string()),
route_family: Some("openai".to_string()),
route_kind: Some("chat".to_string()),
candidate_id: None,
rust_result_digest: Some("rust-digest".to_string()),
python_result_digest: None,
match_status: ShadowResultMatchStatus::Pending,
status_code: Some(200),
error_message: None,
created_at_unix_secs: 100,
updated_at_unix_secs: 100,
})
.await
.expect("write should succeed");
assert!(written.is_some());
let stored = repository
.find(ShadowResultLookupKey::TraceFingerprint {
trace_id: "trace-1",
request_fingerprint: "fp-1",
})
.await
.expect("find should succeed");
assert_eq!(
stored.expect("stored result should exist").match_status,
ShadowResultMatchStatus::Pending
);
}
#[tokio::test]
async fn data_state_records_shadow_result_samples_and_merges_match_status() {
let repository = Arc::new(InMemoryShadowResultRepository::default());
let state = GatewayDataState::with_shadow_result_repository_for_tests(repository);
let first = state
.record_shadow_result_sample(RecordShadowResultSample {
trace_id: "trace-1".to_string(),
request_fingerprint: "fp-1".to_string(),
request_id: Some("req-1".to_string()),
route_family: Some("openai".to_string()),
route_kind: Some("chat".to_string()),
candidate_id: None,
origin: ShadowResultSampleOrigin::Rust,
result_digest: "digest-1".to_string(),
status_code: Some(200),
error_message: None,
recorded_at_unix_secs: 100,
})
.await
.expect("first record should succeed")
.expect("first stored result should exist");
assert_eq!(first.match_status, ShadowResultMatchStatus::Pending);
let second = state
.record_shadow_result_sample(RecordShadowResultSample {
trace_id: "trace-1".to_string(),
request_fingerprint: "fp-1".to_string(),
request_id: Some("req-1".to_string()),
route_family: Some("openai".to_string()),
route_kind: Some("chat".to_string()),
candidate_id: None,
origin: ShadowResultSampleOrigin::Python,
result_digest: "digest-1".to_string(),
status_code: Some(200),
error_message: None,
recorded_at_unix_secs: 200,
})
.await
.expect("second record should succeed")
.expect("second stored result should exist");
assert_eq!(second.match_status, ShadowResultMatchStatus::Match);
assert_eq!(second.created_at_unix_secs, 100);
assert_eq!(second.updated_at_unix_secs, 200);
assert_eq!(second.request_id.as_deref(), Some("req-1"));
}
#[tokio::test]
async fn data_state_lists_recent_shadow_results_from_reader() {
let repository = Arc::new(InMemoryShadowResultRepository::default());
let state = GatewayDataState::with_shadow_result_repository_for_tests(repository.clone());
state
.record_shadow_result_sample(RecordShadowResultSample {
trace_id: "trace-1".to_string(),
request_fingerprint: "fp-1".to_string(),
request_id: Some("req-shadow-1".to_string()),
route_family: Some("openai".to_string()),
route_kind: Some("chat".to_string()),
candidate_id: None,
origin: ShadowResultSampleOrigin::Rust,
result_digest: "digest-1".to_string(),
status_code: Some(200),
error_message: None,
recorded_at_unix_secs: 100,
})
.await
.expect("record should succeed");
let recent = state
.list_recent_shadow_results(5)
.await
.expect("list recent should succeed");
assert_eq!(recent.len(), 1);
assert_eq!(recent[0].trace_id, "trace-1");
assert_eq!(recent[0].request_id.as_deref(), Some("req-shadow-1"));
}
fn sample_request_candidate(
id: &str,
request_id: &str,
candidate_index: i32,
status: RequestCandidateStatus,
started_at_unix_secs: Option<i64>,
latency_ms: Option<i32>,
status_code: Option<i32>,
) -> StoredRequestCandidate {
StoredRequestCandidate::new(
id.to_string(),
request_id.to_string(),
Some("user-1".to_string()),
Some("api-key-1".to_string()),
Some("alice".to_string()),
Some("default".to_string()),
candidate_index,
0,
Some("provider-1".to_string()),
Some("endpoint-1".to_string()),
Some("provider-key-1".to_string()),
status,
None,
false,
status_code,
None,
None,
latency_ms,
Some(1),
None,
None,
100 + i64::from(candidate_index),
started_at_unix_secs,
started_at_unix_secs.map(|value| value + 1),
)
.expect("candidate should build")
}
#[tokio::test]
async fn data_state_reads_request_candidate_trace_from_reader() {
let repository = Arc::new(InMemoryRequestCandidateRepository::seed(vec![
sample_request_candidate(
"cand-1",
"req-1",
0,
RequestCandidateStatus::Pending,
None,
None,
None,
),
sample_request_candidate(
"cand-2",
"req-1",
1,
RequestCandidateStatus::Success,
Some(101),
Some(42),
Some(200),
),
]));
let state = GatewayDataState::with_request_candidate_reader_for_tests(repository);
let trace = state
.read_request_candidate_trace("req-1", true)
.await
.expect("trace should succeed")
.expect("trace should exist");
assert_eq!(trace.request_id, "req-1");
assert_eq!(trace.total_candidates, 1);
assert_eq!(
trace.final_status,
super::candidates::RequestCandidateFinalStatus::Success
);
assert_eq!(trace.total_latency_ms, 42);
assert_eq!(trace.candidates[0].id, "cand-2");
}

View File

@@ -0,0 +1,20 @@
use aether_data::repository::usage::StoredRequestUsageAudit;
use aether_data::DataLayerError;
use super::state::GatewayDataState;
#[derive(Debug, Clone, PartialEq, serde::Serialize)]
pub(crate) struct RequestUsageAudit {
#[serde(flatten)]
pub(crate) usage: StoredRequestUsageAudit,
}
pub(crate) async fn read_request_usage_audit(
state: &GatewayDataState,
request_id: &str,
) -> Result<Option<RequestUsageAudit>, DataLayerError> {
Ok(state
.find_request_usage_by_request_id(request_id)
.await?
.map(|usage| RequestUsageAudit { usage }))
}

View File

@@ -0,0 +1,65 @@
use aether_data::repository::video_tasks::VideoTaskLookupKey;
use aether_data::DataLayerError;
use super::gemini::map_gemini_video_task_to_read_response;
use super::openai::map_openai_video_task_to_read_response;
use super::state::GatewayDataState;
use crate::gateway::video_tasks::{
extract_gemini_short_id_from_path, extract_openai_task_id_from_path, LocalVideoTaskReadResponse,
};
pub(super) async fn read_video_task_response(
state: &GatewayDataState,
route_family: Option<&str>,
request_path: &str,
) -> Result<Option<LocalVideoTaskReadResponse>, DataLayerError> {
match route_family {
Some("openai") => read_openai_video_task_response(state, request_path).await,
Some("gemini") => read_gemini_video_task_response(state, request_path).await,
_ => Ok(None),
}
}
async fn read_openai_video_task_response(
state: &GatewayDataState,
request_path: &str,
) -> Result<Option<LocalVideoTaskReadResponse>, DataLayerError> {
let Some(task_id) = extract_openai_task_id_from_path(request_path) else {
return Ok(None);
};
let Some(task) = state
.find_video_task(VideoTaskLookupKey::Id(task_id))
.await?
else {
return Ok(None);
};
if !matches!(task.provider_api_format.as_deref(), Some("openai:video")) {
return Ok(None);
}
Ok(Some(map_openai_video_task_to_read_response(task)))
}
async fn read_gemini_video_task_response(
state: &GatewayDataState,
request_path: &str,
) -> Result<Option<LocalVideoTaskReadResponse>, DataLayerError> {
let Some(short_id) = extract_gemini_short_id_from_path(request_path) else {
return Ok(None);
};
let Some(task) = state
.find_video_task(VideoTaskLookupKey::ShortId(short_id))
.await?
else {
return Ok(None);
};
if !matches!(task.provider_api_format.as_deref(), Some("gemini:video")) {
return Ok(None);
}
Ok(Some(map_gemini_video_task_to_read_response(task)))
}

View File

@@ -2,7 +2,7 @@ use std::collections::hash_map::DefaultHasher;
use std::collections::BTreeMap;
use std::hash::{Hash, Hasher};
use std::io::Error as IoError;
use std::time::{Duration, Instant};
use std::time::Duration;
use aether_contracts::{
ExecutionPlan, ExecutionResult, ExecutionTelemetry, ExecutionTimeouts, ProxySnapshot,
@@ -24,8 +24,8 @@ use crate::gateway::headers::{
should_skip_upstream_passthrough_header,
};
use crate::gateway::{
build_client_response, build_client_response_from_parts, cache_executor_auth_context,
local_finalize::maybe_build_local_core_sync_finalize_response,
attach_control_metadata_headers, build_client_response, build_client_response_from_parts,
cache_executor_auth_context, local_finalize::maybe_build_local_core_sync_finalize_response,
local_stream::maybe_build_local_stream_rewriter, resolve_executor_auth_context, AppState,
GatewayControlAuthContext, GatewayControlDecision, GatewayError,
};

View File

@@ -161,34 +161,17 @@ pub(crate) fn build_direct_plan_bypass_cache_key(
}
pub(crate) fn should_skip_direct_plan(state: &AppState, cache_key: &str) -> bool {
let Ok(mut cache) = state.direct_plan_bypass_cache.lock() else {
return false;
};
let Some(cached_at) = cache.get(cache_key).copied() else {
return false;
};
if cached_at.elapsed() > DIRECT_PLAN_BYPASS_TTL {
cache.remove(cache_key);
return false;
}
true
state
.direct_plan_bypass_cache
.should_skip(cache_key, DIRECT_PLAN_BYPASS_TTL)
}
pub(crate) fn mark_direct_plan_bypass(state: &AppState, cache_key: String) {
let Ok(mut cache) = state.direct_plan_bypass_cache.lock() else {
return;
};
cache.retain(|_, cached_at| cached_at.elapsed() <= DIRECT_PLAN_BYPASS_TTL);
if cache.len() >= DIRECT_PLAN_BYPASS_MAX_ENTRIES {
if let Some(oldest_key) = cache
.iter()
.min_by_key(|(_, cached_at)| *cached_at)
.map(|(key, _)| key.clone())
{
cache.remove(&oldest_key);
}
}
cache.insert(cache_key, Instant::now());
state.direct_plan_bypass_cache.mark(
cache_key,
DIRECT_PLAN_BYPASS_TTL,
DIRECT_PLAN_BYPASS_MAX_ENTRIES,
);
}
pub(crate) fn resolve_direct_executor_stream_plan_kind(

View File

@@ -1,5 +1,6 @@
use base64::Engine as _;
use futures_util::TryStreamExt;
use tracing::debug;
use super::super::submission::{
maybe_build_local_core_error_response, resolve_core_error_background_report_kind,
@@ -26,6 +27,8 @@ pub(super) async fn execute_executor_stream(
report_kind: Option<String>,
report_context: Option<serde_json::Value>,
) -> Result<Option<Response<Body>>, GatewayError> {
let request_id = plan.request_id.as_str();
let candidate_id = plan.candidate_id.as_deref();
let response = match state
.client
.post(format!("{executor_base_url}/v1/execute/stream"))
@@ -42,10 +45,10 @@ pub(super) async fn execute_executor_stream(
};
if response.status() != http::StatusCode::OK {
return Ok(Some(build_client_response(
response,
trace_id,
Some(decision),
return Ok(Some(attach_control_metadata_headers(
build_client_response(response, trace_id, Some(decision))?,
Some(request_id),
candidate_id,
)?));
}
@@ -116,22 +119,30 @@ pub(super) async fn execute_executor_stream(
payload,
);
}
return Ok(Some(response));
return Ok(Some(attach_control_metadata_headers(
response,
Some(request_id),
candidate_id,
)?));
}
let response = submit_sync_finalize(state, control_base_url, trace_id, payload).await?;
return Ok(Some(build_client_response(
response,
trace_id,
Some(decision),
return Ok(Some(attach_control_metadata_headers(
build_client_response(response, trace_id, Some(decision))?,
Some(request_id),
candidate_id,
)?));
}
return Ok(Some(build_executor_error_response(
trace_id,
decision,
plan_kind,
status_code,
headers,
error_body,
return Ok(Some(attach_control_metadata_headers(
build_executor_error_response(
trace_id,
decision,
plan_kind,
status_code,
headers,
error_body,
)?,
Some(request_id),
candidate_id,
)?));
}
@@ -216,15 +227,19 @@ pub(super) async fn execute_executor_stream(
payload,
);
}
return Ok(Some(response));
return Ok(Some(attach_control_metadata_headers(
response,
Some(request_id),
candidate_id,
)?));
}
let response =
submit_sync_finalize(state, control_base_url, trace_id, payload)
.await?;
return Ok(Some(build_client_response(
response,
trace_id,
Some(decision),
return Ok(Some(attach_control_metadata_headers(
build_client_response(response, trace_id, Some(decision))?,
Some(request_id),
candidate_id,
)?));
}
StreamPrefetchInspection::NeedMore => {}
@@ -278,6 +293,7 @@ pub(super) async fn execute_executor_stream(
let mut buffered_body = prefetched_body_for_report;
let mut telemetry: Option<ExecutionTelemetry> = initial_telemetry;
let reached_eof = initial_reached_eof;
let mut downstream_dropped = false;
if !reached_eof {
loop {
@@ -334,6 +350,7 @@ pub(super) async fn execute_executor_stream(
trace_id = %trace_id_owned,
"gateway stream downstream dropped; stopping executor stream forwarding"
);
downstream_dropped = true;
break;
}
}
@@ -354,7 +371,12 @@ pub(super) async fn execute_executor_stream(
}
}
if let Some(rewriter) = local_stream_rewriter.as_mut() {
if downstream_dropped {
debug!(
trace_id = %trace_id_owned,
"gateway skipped local stream flush after downstream disconnect"
);
} else if let Some(rewriter) = local_stream_rewriter.as_mut() {
match rewriter.finish() {
Ok(flushed_chunk) if !flushed_chunk.is_empty() => {
buffered_body.extend_from_slice(&flushed_chunk);
@@ -363,6 +385,7 @@ pub(super) async fn execute_executor_stream(
trace_id = %trace_id_owned,
"gateway stream downstream dropped while flushing local stream rewrite"
);
downstream_dropped = true;
}
}
Ok(_) => {}
@@ -374,6 +397,14 @@ pub(super) async fn execute_executor_stream(
drop(tx);
if downstream_dropped {
debug!(
trace_id = %trace_id_owned,
"gateway skipped stream report because downstream disconnected before completion"
);
return;
}
if let Some(report_kind) = report_kind_owned {
let report = GatewayStreamReportRequest {
trace_id: trace_id_owned.clone(),
@@ -407,6 +438,21 @@ pub(super) async fn execute_executor_stream(
}
};
headers.insert(
CONTROL_REQUEST_ID_HEADER.to_string(),
request_id.to_string(),
);
if let Some(candidate_id) = candidate_id
.map(str::trim)
.filter(|value| !value.is_empty())
{
headers.insert(
CONTROL_CANDIDATE_ID_HEADER.to_string(),
candidate_id.to_string(),
);
}
Ok(Some(build_client_response_from_parts(
status_code,
&headers,

View File

@@ -251,6 +251,17 @@ async fn maybe_build_local_video_task_read_response(
let read_response = state
.video_tasks
.read_response(decision.route_family.as_deref(), parts.uri.path());
let read_response = match read_response {
Some(read_response) => Some(read_response),
None => {
state
.read_data_backed_video_task_response(
decision.route_family.as_deref(),
parts.uri.path(),
)
.await?
}
};
let Some(read_response) = read_response else {
return Ok(None);
};

View File

@@ -39,6 +39,8 @@ pub(super) async fn execute_executor_sync(
report_kind: Option<String>,
report_context: Option<serde_json::Value>,
) -> Result<Option<Response<Body>>, GatewayError> {
let plan_request_id = plan.request_id.as_str();
let plan_candidate_id = plan.candidate_id.as_deref();
let response = match state
.client
.post(format!("{executor_base_url}/v1/execute/sync"))
@@ -55,10 +57,10 @@ pub(super) async fn execute_executor_sync(
};
if response.status() != http::StatusCode::OK {
return Ok(Some(build_client_response(
response,
trace_id,
Some(decision),
return Ok(Some(attach_control_metadata_headers(
build_client_response(response, trace_id, Some(decision))?,
Some(plan_request_id),
plan_candidate_id,
)?));
}
@@ -66,6 +68,10 @@ pub(super) async fn execute_executor_sync(
.json()
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?;
let request_id = (!result.request_id.trim().is_empty())
.then_some(result.request_id.as_str())
.or(Some(plan_request_id));
let candidate_id = result.candidate_id.as_deref().or(plan_candidate_id);
let mut headers = result.headers.clone();
let (body_bytes, body_json, body_base64) = decode_execution_result_body(&result, &mut headers)?;
let has_body_bytes = body_base64.is_some();
@@ -119,7 +125,11 @@ pub(super) async fn execute_executor_sync(
payload,
);
}
return Ok(Some(outcome.response));
return Ok(Some(attach_control_metadata_headers(
outcome.response,
request_id,
candidate_id,
)?));
}
if let Some(outcome) = maybe_build_local_video_success_outcome(
trace_id,
@@ -145,7 +155,11 @@ pub(super) async fn execute_executor_sync(
);
}
}
return Ok(Some(outcome.response));
return Ok(Some(attach_control_metadata_headers(
outcome.response,
request_id,
candidate_id,
)?));
}
if let Some(response) =
maybe_build_local_sync_finalize_response(trace_id, decision, &payload)?
@@ -172,7 +186,11 @@ pub(super) async fn execute_executor_sync(
payload,
);
}
return Ok(Some(response));
return Ok(Some(attach_control_metadata_headers(
response,
request_id,
candidate_id,
)?));
}
if let Some(response) =
maybe_build_local_video_error_response(trace_id, decision, &payload)?
@@ -196,7 +214,11 @@ pub(super) async fn execute_executor_sync(
payload,
);
}
return Ok(Some(response));
return Ok(Some(attach_control_metadata_headers(
response,
request_id,
candidate_id,
)?));
}
if let Some(response) = maybe_build_local_core_error_response(trace_id, decision, &payload)?
{
@@ -219,13 +241,17 @@ pub(super) async fn execute_executor_sync(
payload,
);
}
return Ok(Some(response));
return Ok(Some(attach_control_metadata_headers(
response,
request_id,
candidate_id,
)?));
}
let response = submit_sync_finalize(state, control_base_url, trace_id, payload).await?;
return Ok(Some(build_client_response(
response,
trace_id,
Some(decision),
return Ok(Some(attach_control_metadata_headers(
build_client_response(response, trace_id, Some(decision))?,
request_id,
candidate_id,
)?));
}
@@ -249,6 +275,23 @@ pub(super) async fn execute_executor_sync(
);
}
if let Some(request_id) = request_id.map(str::trim).filter(|value| !value.is_empty()) {
headers.insert(
CONTROL_REQUEST_ID_HEADER.to_string(),
request_id.to_string(),
);
}
if let Some(candidate_id) = candidate_id
.map(str::trim)
.filter(|value| !value.is_empty())
{
headers.insert(
CONTROL_CANDIDATE_ID_HEADER.to_string(),
candidate_id.to_string(),
);
}
Ok(Some(build_client_response_from_parts(
result.status_code,
&headers,

View File

@@ -1,7 +1,13 @@
#[path = "audit/mod.rs"]
mod audit;
#[path = "cache/mod.rs"]
mod cache;
#[path = "constants.rs"]
mod constants;
#[path = "control.rs"]
mod control;
#[path = "data/mod.rs"]
mod data;
#[path = "error.rs"]
mod error;
#[path = "executor.rs"]
@@ -22,42 +28,57 @@ mod response;
mod video_tasks;
use aether_contracts::ExecutionResult;
use aether_http::{build_http_client, HttpClientConfig};
use aether_runtime::{
prometheus_response, service_up_sample, AdmissionPermit, ConcurrencyError, ConcurrencyGate,
ConcurrencySnapshot, DistributedConcurrencyError, DistributedConcurrencyGate,
DistributedConcurrencySnapshot, MetricKind, MetricLabel, MetricSample,
};
use axum::http::header::{HeaderName, HeaderValue};
use axum::routing::{any, get};
use axum::Router;
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use std::sync::Arc;
use std::time::Duration;
use tokio::task::JoinHandle;
use tracing::warn;
use cache::{AuthContextCache, DirectPlanBypassCache};
pub(crate) use audit::record_shadow_result_non_blocking;
use audit::{
get_auth_api_key_snapshot, get_decision_trace, get_request_audit_bundle,
get_request_candidate_trace, get_request_usage_audit, list_recent_shadow_results,
};
pub(crate) use control::{
cache_executor_auth_context, maybe_execute_via_control, resolve_control_route,
resolve_executor_auth_context, GatewayControlAuthContext, GatewayControlDecision,
resolve_executor_auth_context, trusted_auth_local_rejection, GatewayControlAuthContext,
GatewayControlDecision, GatewayLocalAuthRejection,
};
pub use data::GatewayDataConfig;
use data::GatewayDataState;
pub(crate) use error::GatewayError;
pub(crate) use executor::{maybe_execute_via_executor_stream, maybe_execute_via_executor_sync};
use handlers::{health, proxy_request};
pub(crate) use response::{build_client_response, build_client_response_from_parts};
pub(crate) use response::{
attach_control_metadata_headers, build_client_response, build_client_response_from_parts,
build_local_auth_rejection_response, build_local_overloaded_response,
};
pub(crate) use video_tasks::VideoTaskService;
pub use video_tasks::VideoTaskTruthSourceMode;
#[derive(Debug, Clone)]
pub(crate) struct CachedAuthContextEntry {
pub(crate) auth_context: GatewayControlAuthContext,
pub(crate) cached_at: Instant,
}
#[derive(Debug, Clone)]
pub struct AppState {
upstream_base_url: String,
control_base_url: Option<String>,
executor_base_url: Option<String>,
data: Arc<GatewayDataState>,
video_tasks: Arc<VideoTaskService>,
video_task_poller: Option<VideoTaskPollerConfig>,
request_gate: Option<Arc<ConcurrencyGate>>,
distributed_request_gate: Option<Arc<DistributedConcurrencyGate>>,
client: reqwest::Client,
auth_context_cache: Arc<Mutex<HashMap<String, CachedAuthContextEntry>>>,
direct_plan_bypass_cache: Arc<Mutex<HashMap<String, Instant>>>,
auth_context_cache: Arc<AuthContextCache>,
direct_plan_bypass_cache: Arc<DirectPlanBypassCache>,
}
#[derive(Debug, Clone, Copy)]
@@ -79,11 +100,12 @@ impl AppState {
control_base_url: Option<String>,
executor_base_url: Option<String>,
) -> Result<Self, reqwest::Error> {
let client = reqwest::Client::builder()
.http2_adaptive_window(true)
.connect_timeout(std::time::Duration::from_secs(10))
.timeout(std::time::Duration::from_secs(300))
.build()?;
let client = build_http_client(&HttpClientConfig {
connect_timeout_ms: Some(10_000),
request_timeout_ms: Some(300_000),
http2_adaptive_window: true,
..HttpClientConfig::default()
})?;
Ok(Self {
upstream_base_url: normalize_upstream_base_url(upstream_base_url.into()),
control_base_url: control_base_url
@@ -92,16 +114,27 @@ impl AppState {
executor_base_url: executor_base_url
.map(normalize_upstream_base_url)
.filter(|value| !value.is_empty()),
data: Arc::new(GatewayDataState::disabled()),
video_tasks: Arc::new(VideoTaskService::new(
VideoTaskTruthSourceMode::PythonSyncReport,
)),
video_task_poller: None,
request_gate: None,
distributed_request_gate: None,
client,
auth_context_cache: Arc::new(Mutex::new(HashMap::new())),
direct_plan_bypass_cache: Arc::new(Mutex::new(HashMap::new())),
auth_context_cache: Arc::new(AuthContextCache::default()),
direct_plan_bypass_cache: Arc::new(DirectPlanBypassCache::default()),
})
}
pub fn with_data_config(
mut self,
config: GatewayDataConfig,
) -> Result<Self, aether_data::DataLayerError> {
self.data = Arc::new(GatewayDataState::from_config(config)?);
Ok(self)
}
pub fn with_video_task_truth_source_mode(mut self, mode: VideoTaskTruthSourceMode) -> Self {
self.video_tasks = Arc::new(VideoTaskService::new(mode));
self
@@ -115,6 +148,308 @@ impl AppState {
self
}
pub fn with_request_concurrency_limit(mut self, limit: usize) -> Self {
self.request_gate = Some(Arc::new(ConcurrencyGate::new(
"gateway_requests",
limit.max(1),
)));
self
}
pub fn with_distributed_request_concurrency_gate(
mut self,
gate: DistributedConcurrencyGate,
) -> Self {
self.distributed_request_gate = Some(Arc::new(gate));
self
}
pub fn has_data_backends(&self) -> bool {
self.data.has_backends()
}
pub(crate) fn request_concurrency_snapshot(&self) -> Option<ConcurrencySnapshot> {
self.request_gate.as_ref().map(|gate| gate.snapshot())
}
pub(crate) async fn distributed_request_concurrency_snapshot(
&self,
) -> Result<Option<DistributedConcurrencySnapshot>, DistributedConcurrencyError> {
match self.distributed_request_gate.as_ref() {
Some(gate) => gate.snapshot().await.map(Some),
None => Ok(None),
}
}
pub(crate) async fn metric_samples(&self) -> Vec<MetricSample> {
let mut samples = vec![service_up_sample("aether-gateway")];
if let Some(snapshot) = self.request_concurrency_snapshot() {
samples.extend(snapshot.to_metric_samples("gateway_requests"));
}
if let Some(gate) = self.distributed_request_gate.as_ref() {
match gate.snapshot().await {
Ok(snapshot) => {
samples.extend(snapshot.to_metric_samples("gateway_requests_distributed"));
}
Err(_) => samples.push(
MetricSample::new(
"concurrency_unavailable",
"Whether the distributed concurrency gate is currently unavailable.",
MetricKind::Gauge,
1,
)
.with_labels(vec![MetricLabel::new(
"gate",
"gateway_requests_distributed",
)]),
),
}
}
samples
}
pub(crate) async fn try_acquire_request_permit(
&self,
) -> Result<Option<AdmissionPermit>, RequestAdmissionError> {
let local = self
.request_gate
.as_ref()
.map(|gate| gate.try_acquire())
.transpose()
.map_err(RequestAdmissionError::Local)?;
let distributed = match self.distributed_request_gate.as_ref() {
Some(gate) => Some(
gate.try_acquire()
.await
.map_err(RequestAdmissionError::Distributed)?,
),
None => None,
};
Ok(AdmissionPermit::from_parts(local, distributed))
}
pub fn has_auth_api_key_data_reader(&self) -> bool {
self.data.has_auth_api_key_reader()
}
pub fn has_video_task_data_reader(&self) -> bool {
self.data.has_video_task_reader()
}
pub fn has_request_candidate_data_reader(&self) -> bool {
self.data.has_request_candidate_reader()
}
pub fn has_provider_catalog_data_reader(&self) -> bool {
self.data.has_provider_catalog_reader()
}
pub fn has_usage_data_reader(&self) -> bool {
self.data.has_usage_reader()
}
pub fn has_shadow_result_data_writer(&self) -> bool {
self.data.has_shadow_result_writer()
}
pub fn has_shadow_result_data_reader(&self) -> bool {
self.data.has_shadow_result_reader()
}
pub(crate) async fn read_data_backed_video_task_response(
&self,
route_family: Option<&str>,
request_path: &str,
) -> Result<Option<video_tasks::LocalVideoTaskReadResponse>, GatewayError> {
self.data
.read_video_task_response(route_family, request_path)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))
}
pub(crate) async fn read_request_candidate_trace(
&self,
request_id: &str,
attempted_only: bool,
) -> Result<Option<data::RequestCandidateTrace>, GatewayError> {
self.data
.read_request_candidate_trace(request_id, attempted_only)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))
}
pub(crate) async fn read_decision_trace(
&self,
request_id: &str,
attempted_only: bool,
) -> Result<Option<data::DecisionTrace>, GatewayError> {
self.data
.read_decision_trace(request_id, attempted_only)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))
}
pub(crate) async fn read_request_usage_audit(
&self,
request_id: &str,
) -> Result<Option<data::RequestUsageAudit>, GatewayError> {
self.data
.read_request_usage_audit(request_id)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))
}
pub(crate) async fn read_request_audit_bundle(
&self,
request_id: &str,
attempted_only: bool,
now_unix_secs: u64,
) -> Result<Option<data::RequestAuditBundle>, GatewayError> {
self.data
.read_request_audit_bundle(request_id, attempted_only, now_unix_secs)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))
}
pub(crate) async fn read_auth_api_key_snapshot(
&self,
user_id: &str,
api_key_id: &str,
now_unix_secs: u64,
) -> Result<Option<data::StoredGatewayAuthApiKeySnapshot>, GatewayError> {
self.data
.read_auth_api_key_snapshot(user_id, api_key_id, now_unix_secs)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))
}
pub(crate) async fn record_shadow_result_sample(
&self,
sample: aether_data::repository::shadow_results::RecordShadowResultSample,
) -> Result<Option<aether_data::repository::shadow_results::StoredShadowResult>, GatewayError>
{
self.data
.record_shadow_result_sample(sample)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))
}
pub(crate) async fn list_recent_shadow_results(
&self,
limit: usize,
) -> Result<Vec<aether_data::repository::shadow_results::StoredShadowResult>, GatewayError>
{
self.data
.list_recent_shadow_results(limit)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))
}
#[cfg(test)]
pub(crate) fn with_video_task_data_reader_for_tests(
mut self,
repository: Arc<dyn aether_data::repository::video_tasks::VideoTaskReadRepository>,
) -> Self {
self.data = Arc::new(GatewayDataState::with_video_task_reader_for_tests(
repository,
));
self
}
#[cfg(test)]
pub(crate) fn with_request_candidate_data_reader_for_tests(
mut self,
repository: Arc<dyn aether_data::repository::candidates::RequestCandidateReadRepository>,
) -> Self {
self.data = Arc::new(GatewayDataState::with_request_candidate_reader_for_tests(
repository,
));
self
}
#[cfg(test)]
pub(crate) fn with_decision_trace_data_readers_for_tests(
mut self,
request_candidate_repository: Arc<
dyn aether_data::repository::candidates::RequestCandidateReadRepository,
>,
provider_catalog_repository: Arc<
dyn aether_data::repository::provider_catalog::ProviderCatalogReadRepository,
>,
) -> Self {
self.data = Arc::new(GatewayDataState::with_decision_trace_readers_for_tests(
request_candidate_repository,
provider_catalog_repository,
));
self
}
#[cfg(test)]
pub(crate) fn with_request_audit_data_readers_for_tests(
mut self,
auth_api_key_repository: Arc<dyn aether_data::repository::auth::AuthApiKeyReadRepository>,
request_candidate_repository: Arc<
dyn aether_data::repository::candidates::RequestCandidateReadRepository,
>,
provider_catalog_repository: Arc<
dyn aether_data::repository::provider_catalog::ProviderCatalogReadRepository,
>,
usage_repository: Arc<dyn aether_data::repository::usage::UsageReadRepository>,
) -> Self {
self.data = Arc::new(GatewayDataState::with_request_audit_readers_for_tests(
auth_api_key_repository,
request_candidate_repository,
provider_catalog_repository,
usage_repository,
));
self
}
#[cfg(test)]
pub(crate) fn with_auth_api_key_data_reader_for_tests(
mut self,
repository: Arc<dyn aether_data::repository::auth::AuthApiKeyReadRepository>,
) -> Self {
self.data = Arc::new(GatewayDataState::with_auth_api_key_reader_for_tests(
repository,
));
self
}
#[cfg(test)]
pub(crate) fn with_usage_data_reader_for_tests(
mut self,
repository: Arc<dyn aether_data::repository::usage::UsageReadRepository>,
) -> Self {
self.data = Arc::new(GatewayDataState::with_usage_reader_for_tests(repository));
self
}
#[cfg(test)]
pub(crate) fn with_shadow_result_data_writer_for_tests(
mut self,
repository: Arc<dyn aether_data::repository::shadow_results::ShadowResultWriteRepository>,
) -> Self {
self.data = Arc::new(GatewayDataState::with_shadow_result_writer_for_tests(
repository,
));
self
}
#[cfg(test)]
pub(crate) fn with_shadow_result_data_repository_for_tests<T>(
mut self,
repository: Arc<T>,
) -> Self
where
T: aether_data::repository::shadow_results::ShadowResultRepository + 'static,
{
self.data = Arc::new(GatewayDataState::with_shadow_result_repository_for_tests(
repository,
));
self
}
pub fn with_video_task_store_path(
mut self,
path: impl Into<std::path::PathBuf>,
@@ -250,11 +585,48 @@ pub fn build_router_with_endpoints(
pub fn build_router_with_state(state: AppState) -> Router {
Router::new()
.route("/_gateway/health", get(health))
.route("/_gateway/metrics", get(metrics))
.route(
"/_gateway/audit/auth/users/{user_id}/api-keys/{api_key_id}",
get(get_auth_api_key_snapshot),
)
.route(
"/_gateway/audit/decision-trace/{request_id}",
get(get_decision_trace),
)
.route(
"/_gateway/audit/request-candidates/{request_id}",
get(get_request_candidate_trace),
)
.route(
"/_gateway/audit/request-audit/{request_id}",
get(get_request_audit_bundle),
)
.route(
"/_gateway/audit/request-usage/{request_id}",
get(get_request_usage_audit),
)
.route(
"/_gateway/audit/shadow-results/recent",
get(list_recent_shadow_results),
)
.route("/", any(proxy_request))
.route("/{*path}", any(proxy_request))
.with_state(state)
}
async fn metrics(
axum::extract::State(state): axum::extract::State<AppState>,
) -> impl axum::response::IntoResponse {
prometheus_response(&state.metric_samples().await)
}
#[derive(Debug)]
pub(crate) enum RequestAdmissionError {
Local(ConcurrencyError),
Distributed(DistributedConcurrencyError),
}
pub async fn serve_tcp(
bind: &str,
upstream_base_url: &str,

View File

@@ -1,4 +1,43 @@
use super::*;
use aether_data::repository::auth::{
InMemoryAuthApiKeySnapshotRepository, StoredAuthApiKeySnapshot,
};
fn sample_currently_usable_auth_snapshot(
api_key_id: &str,
user_id: &str,
) -> StoredAuthApiKeySnapshot {
StoredAuthApiKeySnapshot::new(
user_id.to_string(),
"alice".to_string(),
Some("alice@example.com".to_string()),
"user".to_string(),
"local".to_string(),
true,
false,
Some(serde_json::json!(["openai"])),
Some(serde_json::json!(["openai:chat"])),
Some(serde_json::json!(["gpt-5"])),
api_key_id.to_string(),
Some("default".to_string()),
true,
false,
false,
Some(60),
Some(5),
Some(4_102_444_800),
Some(serde_json::json!(["openai"])),
Some(serde_json::json!(["openai:chat"])),
Some(serde_json::json!(["gpt-5"])),
)
.expect("auth snapshot should build")
}
fn sample_locked_auth_snapshot(api_key_id: &str, user_id: &str) -> StoredAuthApiKeySnapshot {
let mut snapshot = sample_currently_usable_auth_snapshot(api_key_id, user_id);
snapshot.api_key_is_locked = true;
snapshot
}
#[tokio::test]
async fn gateway_reuses_cached_auth_context_for_direct_executor_plans() {
@@ -395,3 +434,368 @@ async fn gateway_reuses_cached_auth_context_when_falling_back_to_control_execute
executor_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_uses_data_backed_trusted_auth_context_for_direct_executor_plans() {
#[derive(Debug, Clone)]
struct SeenPlanSyncRequest {
auth_context_present: bool,
auth_context_user_id: String,
auth_context_balance_remaining: String,
auth_context_access_allowed: bool,
}
let seen_plan = Arc::new(Mutex::new(None::<SeenPlanSyncRequest>));
let seen_plan_clone = Arc::clone(&seen_plan);
let upstream = Router::new()
.route(
"/api/internal/gateway/plan-sync",
any(move |request: Request| {
let seen_plan_inner = Arc::clone(&seen_plan_clone);
async move {
let raw_body = to_bytes(request.into_body(), usize::MAX)
.await
.expect("body should read");
let payload: serde_json::Value =
serde_json::from_slice(&raw_body).expect("plan payload should parse");
*seen_plan_inner.lock().expect("mutex should lock") =
Some(SeenPlanSyncRequest {
auth_context_present: payload
.get("auth_context")
.is_some_and(|value| !value.is_null()),
auth_context_user_id: payload
.get("auth_context")
.and_then(|value| value.get("user_id"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
auth_context_balance_remaining: payload
.get("auth_context")
.and_then(|value| value.get("balance_remaining"))
.and_then(|value| value.as_f64())
.map(|value| value.to_string())
.unwrap_or_default(),
auth_context_access_allowed: payload
.get("auth_context")
.and_then(|value| value.get("access_allowed"))
.and_then(|value| value.as_bool())
.unwrap_or(false),
});
Json(json!({
"action": "executor_sync",
"plan_kind": "openai_chat_sync",
"plan": {
"request_id": "req-openai-chat-trusted-123",
"provider_name": "openai",
"provider_id": "provider-openai-chat-trusted-123",
"endpoint_id": "endpoint-openai-chat-trusted-123",
"key_id": "key-openai-chat-trusted-123",
"method": "POST",
"url": "https://api.openai.example/v1/chat/completions",
"headers": {
"authorization": "Bearer upstream-key",
"content-type": "application/json"
},
"body": {
"json_body": {
"model": "gpt-5",
"messages": []
}
},
"stream": false,
"client_api_format": "openai:chat",
"provider_api_format": "openai:chat",
"model_name": "gpt-5"
},
"report_kind": "openai_chat_sync_success",
"report_context": {
"user_id": "user-chat-trusted-123",
"api_key_id": "key-chat-trusted-123"
}
}))
}
}),
)
.route(
"/api/internal/gateway/report-sync",
any(|_request: Request| async move { Json(json!({"ok": true})) }),
);
let executor = Router::new().route(
"/v1/execute/sync",
any(|_request: Request| async move {
Json(json!({
"request_id": "req-openai-chat-trusted-123",
"status_code": 200,
"headers": {
"content-type": "application/json"
},
"body": {
"json_body": {
"id": "chatcmpl-trusted-123",
"object": "chat.completion",
"model": "gpt-5",
"choices": [],
"usage": {
"prompt_tokens": 1,
"completion_tokens": 2,
"total_tokens": 3
}
}
}
}))
}),
);
let repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
Some("hash-1".to_string()),
sample_currently_usable_auth_snapshot("key-chat-trusted-123", "user-chat-trusted-123"),
)]));
let (upstream_url, upstream_handle) = start_server(upstream).await;
let (executor_url, executor_handle) = start_server(executor).await;
let gateway = build_router_with_state(
AppState::new_with_executor(
upstream_url.clone(),
Some(upstream_url.clone()),
Some(executor_url.clone()),
)
.expect("gateway state should build")
.with_auth_api_key_data_reader_for_tests(repository),
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.post(format!("{gateway_url}/v1/chat/completions"))
.header(http::header::CONTENT_TYPE, "application/json")
.header(TRACE_ID_HEADER, "trace-openai-chat-trusted-1")
.header(TRUSTED_AUTH_USER_ID_HEADER, "user-chat-trusted-123")
.header(TRUSTED_AUTH_API_KEY_ID_HEADER, "key-chat-trusted-123")
.header(TRUSTED_AUTH_BALANCE_HEADER, "7.5")
.body("{\"model\":\"gpt-5\",\"messages\":[]}")
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let seen_plan_request = seen_plan
.lock()
.expect("mutex should lock")
.clone()
.expect("plan-sync should be captured");
assert!(seen_plan_request.auth_context_present);
assert_eq!(
seen_plan_request.auth_context_user_id,
"user-chat-trusted-123"
);
assert_eq!(seen_plan_request.auth_context_balance_remaining, "7.5");
assert!(seen_plan_request.auth_context_access_allowed);
gateway_handle.abort();
executor_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_locally_denies_explicit_trusted_balance_failure_before_direct_executor_plan() {
let seen_plan = Arc::new(Mutex::new(0usize));
let seen_plan_clone = Arc::clone(&seen_plan);
let seen_executor = Arc::new(Mutex::new(0usize));
let seen_executor_clone = Arc::clone(&seen_executor);
let upstream = Router::new()
.route(
"/api/internal/gateway/plan-sync",
any(move |_request: Request| {
let seen_plan_inner = Arc::clone(&seen_plan_clone);
async move {
*seen_plan_inner.lock().expect("mutex should lock") += 1;
Json(json!({
"action": "executor_sync",
"plan_kind": "openai_chat_sync"
}))
}
}),
)
.route(
"/api/internal/gateway/report-sync",
any(|_request: Request| async move { Json(json!({"ok": true})) }),
);
let executor = Router::new().route(
"/v1/execute/sync",
any(move |_request: Request| {
let seen_executor_inner = Arc::clone(&seen_executor_clone);
async move {
*seen_executor_inner.lock().expect("mutex should lock") += 1;
Json(json!({
"request_id": "req-openai-chat-trusted-denied-123",
"status_code": 200,
"headers": {
"content-type": "application/json"
},
"body": {
"json_body": {
"id": "chatcmpl-trusted-denied-123",
"object": "chat.completion",
"model": "gpt-5",
"choices": []
}
}
}))
}
}),
);
let repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
Some("hash-1".to_string()),
sample_currently_usable_auth_snapshot("key-chat-trusted-123", "user-chat-trusted-123"),
)]));
let (upstream_url, upstream_handle) = start_server(upstream).await;
let (executor_url, executor_handle) = start_server(executor).await;
let gateway = build_router_with_state(
AppState::new_with_executor(
upstream_url.clone(),
Some(upstream_url.clone()),
Some(executor_url.clone()),
)
.expect("gateway state should build")
.with_auth_api_key_data_reader_for_tests(repository),
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.post(format!("{gateway_url}/v1/chat/completions"))
.header(http::header::CONTENT_TYPE, "application/json")
.header(TRACE_ID_HEADER, "trace-openai-chat-trusted-denied-1")
.header(TRUSTED_AUTH_USER_ID_HEADER, "user-chat-trusted-123")
.header(TRUSTED_AUTH_API_KEY_ID_HEADER, "key-chat-trusted-123")
.header(TRUSTED_AUTH_BALANCE_HEADER, "0")
.header(TRUSTED_AUTH_ACCESS_ALLOWED_HEADER, "false")
.body("{\"model\":\"gpt-5\",\"messages\":[]}")
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::TOO_MANY_REQUESTS);
assert_eq!(
response
.headers()
.get(EXECUTION_PATH_HEADER)
.and_then(|value| value.to_str().ok()),
Some(EXECUTION_PATH_LOCAL_AUTH_DENIED)
);
let payload: serde_json::Value = response.json().await.expect("response json should parse");
assert_eq!(payload["error"]["type"], "balance_exceeded");
assert_eq!(payload["error"]["details"]["remaining"], 0.0);
assert_eq!(*seen_plan.lock().expect("mutex should lock"), 0);
assert_eq!(*seen_executor.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
executor_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_locally_denies_locked_trusted_snapshot_before_direct_executor_plan() {
let seen_plan = Arc::new(Mutex::new(0usize));
let seen_plan_clone = Arc::clone(&seen_plan);
let seen_executor = Arc::new(Mutex::new(0usize));
let seen_executor_clone = Arc::clone(&seen_executor);
let upstream = Router::new()
.route(
"/api/internal/gateway/plan-sync",
any(move |_request: Request| {
let seen_plan_inner = Arc::clone(&seen_plan_clone);
async move {
*seen_plan_inner.lock().expect("mutex should lock") += 1;
Json(json!({
"action": "executor_sync",
"plan_kind": "openai_chat_sync"
}))
}
}),
)
.route(
"/api/internal/gateway/report-sync",
any(|_request: Request| async move { Json(json!({"ok": true})) }),
);
let executor = Router::new().route(
"/v1/execute/sync",
any(move |_request: Request| {
let seen_executor_inner = Arc::clone(&seen_executor_clone);
async move {
*seen_executor_inner.lock().expect("mutex should lock") += 1;
Json(json!({
"request_id": "req-openai-chat-trusted-locked-123",
"status_code": 200,
"headers": {
"content-type": "application/json"
},
"body": {
"json_body": {
"id": "chatcmpl-trusted-locked-123",
"object": "chat.completion",
"model": "gpt-5",
"choices": []
}
}
}))
}
}),
);
let repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
Some("hash-1".to_string()),
sample_locked_auth_snapshot("key-chat-trusted-123", "user-chat-trusted-123"),
)]));
let (upstream_url, upstream_handle) = start_server(upstream).await;
let (executor_url, executor_handle) = start_server(executor).await;
let gateway = build_router_with_state(
AppState::new_with_executor(
upstream_url.clone(),
Some(upstream_url.clone()),
Some(executor_url.clone()),
)
.expect("gateway state should build")
.with_auth_api_key_data_reader_for_tests(repository),
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.post(format!("{gateway_url}/v1/chat/completions"))
.header(http::header::CONTENT_TYPE, "application/json")
.header(TRACE_ID_HEADER, "trace-openai-chat-trusted-locked-1")
.header(TRUSTED_AUTH_USER_ID_HEADER, "user-chat-trusted-123")
.header(TRUSTED_AUTH_API_KEY_ID_HEADER, "key-chat-trusted-123")
.body("{\"model\":\"gpt-5\",\"messages\":[]}")
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::FORBIDDEN);
assert_eq!(
response
.headers()
.get(EXECUTION_PATH_HEADER)
.and_then(|value| value.to_str().ok()),
Some(EXECUTION_PATH_LOCAL_AUTH_DENIED)
);
let payload: serde_json::Value = response.json().await.expect("response json should parse");
assert_eq!(payload["error"]["type"], "http_error");
assert_eq!(
payload["error"]["message"],
"该密钥已被管理员锁定,请联系管理员"
);
assert_eq!(*seen_plan.lock().expect("mutex should lock"), 0);
assert_eq!(*seen_executor.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
executor_handle.abort();
upstream_handle.abort();
}

View File

@@ -0,0 +1,840 @@
use std::sync::Arc;
use aether_data::repository::auth::{
InMemoryAuthApiKeySnapshotRepository, StoredAuthApiKeySnapshot,
};
use aether_data::repository::candidates::{
InMemoryRequestCandidateRepository, RequestCandidateStatus, StoredRequestCandidate,
};
use aether_data::repository::provider_catalog::{
InMemoryProviderCatalogReadRepository, StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
StoredProviderCatalogProvider,
};
use aether_data::repository::shadow_results::{
InMemoryShadowResultRepository, ShadowResultMatchStatus, ShadowResultReadRepository,
};
use aether_data::repository::usage::{InMemoryUsageReadRepository, StoredRequestUsageAudit};
use serde_json::Value;
use super::*;
#[tokio::test]
async fn gateway_records_shadow_result_for_ai_public_proxy_response() {
let repository = Arc::new(InMemoryShadowResultRepository::default());
let upstream = Router::new()
.route(
"/api/internal/gateway/resolve",
any(|_request: Request| async move {
Json(json!({
"action": "proxy_public",
"route_class": "ai_public",
"route_family": "openai",
"route_kind": "chat",
"auth_endpoint_signature": "openai:chat",
"executor_candidate": false,
"public_path": "/v1/chat/completions"
}))
}),
)
.route(
"/v1/chat/completions",
any(|_request: Request| async move {
let mut response = Response::builder()
.status(StatusCode::OK)
.body(Body::from("{\"id\":\"chatcmpl-shadow\"}"))
.expect("response should build");
response.headers_mut().insert(
http::header::CONTENT_TYPE,
HeaderValue::from_static("application/json"),
);
response
}),
);
let (upstream_url, upstream_handle) = start_server(upstream).await;
let gateway_state = AppState::new(upstream_url.clone(), Some(upstream_url))
.expect("gateway state should build")
.with_shadow_result_data_writer_for_tests(repository.clone());
let gateway = build_router_with_state(gateway_state);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.post(format!("{gateway_url}/v1/chat/completions"))
.header(http::header::CONTENT_TYPE, "application/json")
.body("{\"model\":\"gpt-4.1\",\"messages\":[]}")
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response
.headers()
.get(CONTROL_ROUTE_CLASS_HEADER)
.and_then(|value| value.to_str().ok()),
Some("ai_public")
);
let response_trace_id = response
.headers()
.get(TRACE_ID_HEADER)
.and_then(|value| value.to_str().ok())
.expect("trace id should exist")
.to_string();
assert_eq!(
response.text().await.expect("body should read"),
"{\"id\":\"chatcmpl-shadow\"}"
);
for _ in 0..50 {
if repository
.list_recent(1)
.await
.map(|rows| !rows.is_empty())
.unwrap_or(false)
{
break;
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
let stored = repository
.list_recent(1)
.await
.expect("list should succeed")
.into_iter()
.next()
.expect("stored result should exist");
assert_eq!(stored.trace_id, response_trace_id);
assert!(stored.request_id.is_none());
assert_eq!(stored.route_family.as_deref(), Some("openai"));
assert_eq!(stored.route_kind.as_deref(), Some("chat"));
assert_eq!(stored.match_status, ShadowResultMatchStatus::Pending);
assert_eq!(stored.status_code, Some(200));
assert!(stored.rust_result_digest.is_some());
assert!(stored.python_result_digest.is_none());
gateway_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_records_candidate_id_in_shadow_result_for_direct_executor_response() {
let repository = Arc::new(InMemoryShadowResultRepository::default());
let upstream = Router::new().route(
"/api/internal/gateway/plan-sync",
any(|_request: Request| async move {
Json(json!({
"action": "executor_sync",
"plan_kind": "openai_chat_sync",
"plan": {
"request_id": "req-shadow-direct-123",
"candidate_id": "cand-shadow-direct-123",
"provider_name": "openai",
"provider_id": "provider-shadow-direct-123",
"endpoint_id": "endpoint-shadow-direct-123",
"key_id": "key-shadow-direct-123",
"method": "POST",
"url": "https://api.openai.example/v1/chat/completions",
"headers": {
"authorization": "Bearer upstream-key",
"content-type": "application/json"
},
"body": {
"json_body": {
"model": "gpt-5",
"messages": []
}
},
"stream": false,
"client_api_format": "openai:chat",
"provider_api_format": "openai:chat",
"model_name": "gpt-5"
}
}))
}),
);
let executor = Router::new().route(
"/v1/execute/sync",
any(|_request: Request| async move {
Json(json!({
"request_id": "req-shadow-direct-123",
"candidate_id": "cand-shadow-direct-123",
"status_code": 200,
"headers": {
"content-type": "application/json"
},
"body": {
"json_body": {
"id": "chatcmpl-shadow-direct-123",
"object": "chat.completion",
"model": "gpt-5",
"choices": []
}
}
}))
}),
);
let (upstream_url, upstream_handle) = start_server(upstream).await;
let (executor_url, executor_handle) = start_server(executor).await;
let gateway_state =
AppState::new_with_executor(upstream_url.clone(), Some(upstream_url), Some(executor_url))
.expect("gateway state should build")
.with_shadow_result_data_repository_for_tests(repository.clone());
let gateway = build_router_with_state(gateway_state);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.post(format!("{gateway_url}/v1/chat/completions"))
.header(http::header::CONTENT_TYPE, "application/json")
.body("{\"model\":\"gpt-5\",\"messages\":[]}")
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response
.headers()
.get(CONTROL_CANDIDATE_ID_HEADER)
.and_then(|value| value.to_str().ok()),
Some("cand-shadow-direct-123")
);
for _ in 0..50 {
if repository
.list_recent(1)
.await
.map(|rows| !rows.is_empty())
.unwrap_or(false)
{
break;
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
let stored = repository
.list_recent(1)
.await
.expect("list should succeed")
.into_iter()
.next()
.expect("stored result should exist");
assert_eq!(stored.request_id.as_deref(), Some("req-shadow-direct-123"));
assert_eq!(
stored.candidate_id.as_deref(),
Some("cand-shadow-direct-123")
);
assert_eq!(stored.route_family.as_deref(), Some("openai"));
assert_eq!(stored.route_kind.as_deref(), Some("chat"));
assert_eq!(stored.match_status, ShadowResultMatchStatus::Pending);
gateway_handle.abort();
executor_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_exposes_request_id_header_for_direct_executor_response() {
let auth_repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
Some("hash-1".to_string()),
sample_auth_snapshot("api-key-1", "user-1"),
)]));
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![
sample_request_candidate(
"cand-1",
"req-direct-audit-123",
0,
RequestCandidateStatus::Success,
Some(101),
Some(37),
Some(200),
),
]));
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider_catalog_provider()],
vec![sample_provider_catalog_endpoint()],
vec![sample_provider_catalog_key()],
));
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(vec![
sample_request_usage("req-direct-audit-123"),
]));
let upstream = Router::new().route(
"/api/internal/gateway/plan-sync",
any(|_request: Request| async move {
Json(json!({
"action": "executor_sync",
"plan_kind": "openai_chat_sync",
"plan": {
"request_id": "req-direct-audit-123",
"candidate_id": "cand-direct-audit-123",
"provider_name": "openai",
"provider_id": "provider-direct-audit-123",
"endpoint_id": "endpoint-direct-audit-123",
"key_id": "key-direct-audit-123",
"method": "POST",
"url": "https://api.openai.example/v1/chat/completions",
"headers": {
"authorization": "Bearer upstream-key",
"content-type": "application/json"
},
"body": {
"json_body": {
"model": "gpt-5",
"messages": []
}
},
"stream": false,
"client_api_format": "openai:chat",
"provider_api_format": "openai:chat",
"model_name": "gpt-5"
}
}))
}),
);
let executor = Router::new().route(
"/v1/execute/sync",
any(|_request: Request| async move {
Json(json!({
"request_id": "req-direct-audit-123",
"candidate_id": "cand-direct-audit-123",
"status_code": 200,
"headers": {
"content-type": "application/json"
},
"body": {
"json_body": {
"id": "chatcmpl-direct-audit-123",
"object": "chat.completion",
"model": "gpt-5",
"choices": []
}
}
}))
}),
);
let (upstream_url, upstream_handle) = start_server(upstream).await;
let (executor_url, executor_handle) = start_server(executor).await;
let gateway_state =
AppState::new_with_executor(upstream_url.clone(), Some(upstream_url), Some(executor_url))
.expect("gateway state should build")
.with_request_audit_data_readers_for_tests(
auth_repository,
request_candidates,
provider_catalog,
usage_repository,
);
let gateway = build_router_with_state(gateway_state);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.post(format!("{gateway_url}/v1/chat/completions"))
.header(http::header::CONTENT_TYPE, "application/json")
.body("{\"model\":\"gpt-5\",\"messages\":[]}")
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let request_id = response
.headers()
.get(CONTROL_REQUEST_ID_HEADER)
.and_then(|value| value.to_str().ok())
.expect("request id header should exist")
.to_string();
assert_eq!(request_id, "req-direct-audit-123");
let audit_response = reqwest::Client::new()
.get(format!(
"{gateway_url}/_gateway/audit/request-audit/{request_id}?attempted_only=true"
))
.send()
.await
.expect("request audit should succeed");
assert_eq!(audit_response.status(), StatusCode::OK);
let payload: Value = audit_response.json().await.expect("payload should parse");
assert_eq!(payload["request_id"], "req-direct-audit-123");
assert_eq!(payload["usage"]["provider_name"], "OpenAI");
assert_eq!(payload["decision_trace"]["total_candidates"], 1);
assert_eq!(payload["auth_snapshot"]["api_key_id"], "api-key-1");
gateway_handle.abort();
executor_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_exposes_recent_shadow_results_via_internal_audit_endpoint() {
let repository = Arc::new(InMemoryShadowResultRepository::default());
let upstream = Router::new()
.route(
"/api/internal/gateway/resolve",
any(|_request: Request| async move {
Json(json!({
"action": "proxy_public",
"route_class": "ai_public",
"route_family": "openai",
"route_kind": "chat",
"auth_endpoint_signature": "openai:chat",
"executor_candidate": false,
"public_path": "/v1/chat/completions"
}))
}),
)
.route(
"/v1/chat/completions",
any(|_request: Request| async move {
let mut response = Response::builder()
.status(StatusCode::OK)
.body(Body::from("{\"id\":\"chatcmpl-shadow-read\"}"))
.expect("response should build");
response.headers_mut().insert(
http::header::CONTENT_TYPE,
HeaderValue::from_static("application/json"),
);
response
}),
);
let (upstream_url, upstream_handle) = start_server(upstream).await;
let gateway_state = AppState::new(upstream_url.clone(), Some(upstream_url))
.expect("gateway state should build")
.with_shadow_result_data_repository_for_tests(repository.clone());
let gateway = build_router_with_state(gateway_state);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let write_response = reqwest::Client::new()
.post(format!("{gateway_url}/v1/chat/completions"))
.header(http::header::CONTENT_TYPE, "application/json")
.body("{\"model\":\"gpt-4.1\",\"messages\":[]}")
.send()
.await
.expect("request should succeed");
assert_eq!(write_response.status(), StatusCode::OK);
for _ in 0..50 {
if repository
.list_recent(1)
.await
.map(|rows| !rows.is_empty())
.unwrap_or(false)
{
break;
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
let response = reqwest::Client::new()
.get(format!(
"{gateway_url}/_gateway/audit/shadow-results/recent?limit=5"
))
.send()
.await
.expect("audit request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let payload: Value = response.json().await.expect("payload should parse");
assert_eq!(payload["limit_applied"], 5);
assert_eq!(payload["counts"]["pending"], 1);
assert_eq!(payload["counts"]["match"], 0);
assert_eq!(
payload["items"].as_array().map(|items| items.len()),
Some(1)
);
assert!(payload["items"][0]["request_id"].is_null());
assert_eq!(payload["items"][0]["route_family"], "openai");
assert_eq!(payload["items"][0]["route_kind"], "chat");
assert_eq!(payload["items"][0]["match_status"], "Pending");
gateway_handle.abort();
upstream_handle.abort();
}
fn sample_request_candidate(
id: &str,
request_id: &str,
candidate_index: i32,
status: RequestCandidateStatus,
started_at_unix_secs: Option<i64>,
latency_ms: Option<i32>,
status_code: Option<i32>,
) -> StoredRequestCandidate {
StoredRequestCandidate::new(
id.to_string(),
request_id.to_string(),
Some("user-1".to_string()),
Some("api-key-1".to_string()),
Some("alice".to_string()),
Some("default".to_string()),
candidate_index,
0,
Some("provider-1".to_string()),
Some("endpoint-1".to_string()),
Some("provider-key-1".to_string()),
status,
None,
false,
status_code,
None,
None,
latency_ms,
Some(1),
None,
None,
100 + i64::from(candidate_index),
started_at_unix_secs,
started_at_unix_secs.map(|value| value + 1),
)
.expect("candidate should build")
}
fn sample_auth_snapshot(api_key_id: &str, user_id: &str) -> StoredAuthApiKeySnapshot {
StoredAuthApiKeySnapshot::new(
user_id.to_string(),
"alice".to_string(),
Some("alice@example.com".to_string()),
"user".to_string(),
"local".to_string(),
true,
false,
Some(serde_json::json!(["openai"])),
Some(serde_json::json!(["openai:chat"])),
Some(serde_json::json!(["gpt-4.1"])),
api_key_id.to_string(),
Some("default".to_string()),
true,
false,
false,
Some(60),
Some(5),
Some(4_102_444_800),
Some(serde_json::json!(["openai"])),
Some(serde_json::json!(["openai:chat"])),
Some(serde_json::json!(["gpt-4.1"])),
)
.expect("auth snapshot should build")
}
fn sample_provider_catalog_provider() -> StoredProviderCatalogProvider {
StoredProviderCatalogProvider::new(
"provider-1".to_string(),
"OpenAI".to_string(),
Some("https://openai.com".to_string()),
"custom".to_string(),
)
.expect("provider should build")
}
fn sample_provider_catalog_endpoint() -> StoredProviderCatalogEndpoint {
StoredProviderCatalogEndpoint::new(
"endpoint-1".to_string(),
"provider-1".to_string(),
"openai:chat".to_string(),
Some("openai".to_string()),
Some("chat".to_string()),
true,
)
.expect("endpoint should build")
}
fn sample_provider_catalog_key() -> StoredProviderCatalogKey {
StoredProviderCatalogKey::new(
"provider-key-1".to_string(),
"provider-1".to_string(),
"prod-key".to_string(),
"api_key".to_string(),
Some(serde_json::json!({"cache_1h": true})),
true,
)
.expect("key should build")
}
fn sample_request_usage(request_id: &str) -> StoredRequestUsageAudit {
StoredRequestUsageAudit::new(
"usage-1".to_string(),
request_id.to_string(),
Some("user-1".to_string()),
Some("api-key-1".to_string()),
Some("alice".to_string()),
Some("default".to_string()),
"OpenAI".to_string(),
"gpt-4.1".to_string(),
Some("gpt-4.1-mini".to_string()),
Some("provider-1".to_string()),
Some("endpoint-1".to_string()),
Some("provider-key-1".to_string()),
Some("chat".to_string()),
Some("openai:chat".to_string()),
Some("openai".to_string()),
Some("chat".to_string()),
Some("openai:chat".to_string()),
Some("openai".to_string()),
Some("chat".to_string()),
true,
false,
120,
40,
160,
0.24,
0.36,
Some(200),
None,
None,
Some(450),
Some(120),
"completed".to_string(),
"settled".to_string(),
100,
101,
Some(102),
)
.expect("usage should build")
}
#[tokio::test]
async fn gateway_exposes_request_usage_via_internal_audit_endpoint() {
let repository = Arc::new(InMemoryUsageReadRepository::seed(vec![
sample_request_usage("req-usage-2"),
]));
let gateway_state = AppState::new("http://127.0.0.1:18091", None)
.expect("gateway state should build")
.with_usage_data_reader_for_tests(repository);
let gateway = build_router_with_state(gateway_state);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.get(format!(
"{gateway_url}/_gateway/audit/request-usage/req-usage-2"
))
.send()
.await
.expect("audit request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let payload: Value = response.json().await.expect("payload should parse");
assert_eq!(payload["request_id"], "req-usage-2");
assert_eq!(payload["provider_name"], "OpenAI");
assert_eq!(payload["api_format"], "openai:chat");
assert_eq!(payload["total_tokens"], 160);
assert_eq!(payload["total_cost_usd"], 0.24);
assert_eq!(payload["status"], "completed");
assert_eq!(payload["billing_status"], "settled");
gateway_handle.abort();
}
#[tokio::test]
async fn gateway_exposes_request_audit_bundle_via_internal_audit_endpoint() {
let auth_repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
Some("hash-1".to_string()),
sample_auth_snapshot("api-key-1", "user-1"),
)]));
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![
sample_request_candidate(
"cand-1",
"req-audit-1",
0,
RequestCandidateStatus::Success,
Some(101),
Some(37),
Some(200),
),
]));
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider_catalog_provider()],
vec![sample_provider_catalog_endpoint()],
vec![sample_provider_catalog_key()],
));
let usage_repository = Arc::new(InMemoryUsageReadRepository::seed(vec![
sample_request_usage("req-audit-1"),
]));
let gateway_state = AppState::new("http://127.0.0.1:18092", None)
.expect("gateway state should build")
.with_request_audit_data_readers_for_tests(
auth_repository,
request_candidates,
provider_catalog,
usage_repository,
);
let gateway = build_router_with_state(gateway_state);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.get(format!(
"{gateway_url}/_gateway/audit/request-audit/req-audit-1?attempted_only=true"
))
.send()
.await
.expect("audit request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let payload: Value = response.json().await.expect("payload should parse");
assert_eq!(payload["request_id"], "req-audit-1");
assert_eq!(payload["usage"]["provider_name"], "OpenAI");
assert_eq!(payload["usage"]["total_tokens"], 160);
assert_eq!(payload["decision_trace"]["total_candidates"], 1);
assert_eq!(
payload["decision_trace"]["candidates"][0]["provider_key_name"],
"prod-key"
);
assert_eq!(payload["auth_snapshot"]["api_key_id"], "api-key-1");
assert_eq!(payload["auth_snapshot"]["currently_usable"], true);
gateway_handle.abort();
}
#[tokio::test]
async fn gateway_exposes_request_candidate_trace_via_internal_audit_endpoint() {
let repository = Arc::new(InMemoryRequestCandidateRepository::seed(vec![
sample_request_candidate(
"cand-1",
"req-trace-1",
0,
RequestCandidateStatus::Pending,
None,
None,
None,
),
sample_request_candidate(
"cand-2",
"req-trace-1",
1,
RequestCandidateStatus::Failed,
Some(101),
Some(37),
Some(502),
),
]));
let gateway_state = AppState::new("http://127.0.0.1:19081", None)
.expect("gateway state should build")
.with_request_candidate_data_reader_for_tests(repository);
let gateway = build_router_with_state(gateway_state);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.get(format!(
"{gateway_url}/_gateway/audit/request-candidates/req-trace-1?attempted_only=true"
))
.send()
.await
.expect("audit request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let payload: Value = response.json().await.expect("payload should parse");
assert_eq!(payload["request_id"], "req-trace-1");
assert_eq!(payload["total_candidates"], 1);
assert_eq!(payload["final_status"], "failed");
assert_eq!(payload["total_latency_ms"], 37);
assert_eq!(
payload["candidates"].as_array().map(|items| items.len()),
Some(1)
);
assert_eq!(payload["candidates"][0]["id"], "cand-2");
assert_eq!(payload["candidates"][0]["status"], "failed");
gateway_handle.abort();
}
#[tokio::test]
async fn gateway_exposes_decision_trace_via_internal_audit_endpoint() {
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![
sample_request_candidate(
"cand-1",
"req-trace-2",
0,
RequestCandidateStatus::Failed,
Some(101),
Some(37),
Some(502),
),
]));
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider_catalog_provider()],
vec![sample_provider_catalog_endpoint()],
vec![sample_provider_catalog_key()],
));
let gateway_state = AppState::new("http://127.0.0.1:19083", None)
.expect("gateway state should build")
.with_decision_trace_data_readers_for_tests(request_candidates, provider_catalog);
let gateway = build_router_with_state(gateway_state);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.get(format!(
"{gateway_url}/_gateway/audit/decision-trace/req-trace-2?attempted_only=true"
))
.send()
.await
.expect("audit request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let payload: Value = response.json().await.expect("payload should parse");
assert_eq!(payload["request_id"], "req-trace-2");
assert_eq!(payload["total_candidates"], 1);
assert_eq!(payload["candidates"][0]["provider_name"], "OpenAI");
assert_eq!(
payload["candidates"][0]["provider_website"],
"https://openai.com"
);
assert_eq!(
payload["candidates"][0]["endpoint_api_format"],
"openai:chat"
);
assert_eq!(payload["candidates"][0]["provider_key_name"], "prod-key");
assert_eq!(
payload["candidates"][0]["provider_key_auth_type"],
"api_key"
);
assert_eq!(
payload["candidates"][0]["provider_key_capabilities"]["cache_1h"],
true
);
gateway_handle.abort();
}
#[tokio::test]
async fn gateway_exposes_auth_api_key_snapshot_via_internal_audit_endpoint() {
let repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
Some("hash-1".to_string()),
sample_auth_snapshot("key-1", "user-1"),
)]));
let gateway_state = AppState::new("http://127.0.0.1:19082", None)
.expect("gateway state should build")
.with_auth_api_key_data_reader_for_tests(repository);
let gateway = build_router_with_state(gateway_state);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.get(format!(
"{gateway_url}/_gateway/audit/auth/users/user-1/api-keys/key-1"
))
.send()
.await
.expect("audit request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let payload: Value = response.json().await.expect("payload should parse");
assert_eq!(payload["user_id"], "user-1");
assert_eq!(payload["api_key_id"], "key-1");
assert_eq!(payload["username"], "alice");
assert_eq!(payload["user_role"], "user");
assert_eq!(payload["api_key_name"], "default");
assert_eq!(payload["currently_usable"], true);
assert_eq!(payload["user_allowed_providers"][0], "openai");
assert_eq!(payload["api_key_allowed_api_formats"][0], "openai:chat");
gateway_handle.abort();
}

View File

@@ -0,0 +1,187 @@
use std::sync::atomic::{AtomicUsize, Ordering};
use super::*;
#[tokio::test]
async fn gateway_rejects_second_in_flight_stream_request_with_distributed_overload() {
let upstream_hits = Arc::new(AtomicUsize::new(0));
let upstream_hits_clone = Arc::clone(&upstream_hits);
let upstream = Router::new().route(
"/{*path}",
any(move |_request: Request| {
let upstream_hits = Arc::clone(&upstream_hits_clone);
async move {
upstream_hits.fetch_add(1, Ordering::SeqCst);
let stream = async_stream::stream! {
yield Ok::<_, Infallible>(Bytes::from_static(b"chunk-1"));
futures_util::future::pending::<()>().await;
};
Response::builder()
.status(StatusCode::OK)
.body(Body::from_stream(stream))
.expect("response should build")
}
}),
);
let (upstream_url, upstream_handle) = start_server(upstream).await;
let distributed_gate = aether_runtime::DistributedConcurrencyGate::new_in_memory(
"gateway_requests_distributed",
1,
);
let gateway_a = build_router_with_state(
AppState::new(upstream_url.clone(), None)
.expect("gateway state should build")
.with_distributed_request_concurrency_gate(distributed_gate.clone()),
);
let gateway_b = build_router_with_state(
AppState::new(upstream_url, None)
.expect("gateway state should build")
.with_distributed_request_concurrency_gate(distributed_gate),
);
let (gateway_a_url, gateway_a_handle) = start_server(gateway_a).await;
let (gateway_b_url, gateway_b_handle) = start_server(gateway_b).await;
let client = reqwest::Client::new();
let first_response = client
.get(format!("{gateway_a_url}/v1/messages"))
.send()
.await
.expect("first request should succeed");
wait_until(500, || upstream_hits.load(Ordering::SeqCst) == 1).await;
let second_response = client
.get(format!("{gateway_b_url}/v1/messages"))
.send()
.await
.expect("second request should complete");
assert_eq!(second_response.status(), StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(
second_response
.headers()
.get(EXECUTION_PATH_HEADER)
.and_then(|value| value.to_str().ok()),
Some(EXECUTION_PATH_DISTRIBUTED_OVERLOADED)
);
assert_eq!(
second_response
.json::<serde_json::Value>()
.await
.expect("json body should decode")["error"]["details"]["gate"],
"gateway_requests_distributed"
);
assert_eq!(upstream_hits.load(Ordering::SeqCst), 1);
drop(first_response);
gateway_a_handle.abort();
gateway_b_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_rejects_second_in_flight_stream_request_with_local_overload() {
let upstream_hits = Arc::new(AtomicUsize::new(0));
let upstream_hits_clone = Arc::clone(&upstream_hits);
let upstream = Router::new().route(
"/{*path}",
any(move |_request: Request| {
let upstream_hits = Arc::clone(&upstream_hits_clone);
async move {
upstream_hits.fetch_add(1, Ordering::SeqCst);
let stream = async_stream::stream! {
yield Ok::<_, Infallible>(Bytes::from_static(b"chunk-1"));
futures_util::future::pending::<()>().await;
};
Response::builder()
.status(StatusCode::OK)
.body(Body::from_stream(stream))
.expect("response should build")
}
}),
);
let (upstream_url, upstream_handle) = start_server(upstream).await;
let gateway = build_router_with_state(
AppState::new(upstream_url, None)
.expect("gateway state should build")
.with_request_concurrency_limit(1),
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let client = reqwest::Client::new();
let first_response = client
.get(format!("{gateway_url}/v1/messages"))
.send()
.await
.expect("first request should succeed");
wait_until(500, || upstream_hits.load(Ordering::SeqCst) == 1).await;
let second_response = client
.get(format!("{gateway_url}/v1/messages"))
.send()
.await
.expect("second request should complete");
assert_eq!(second_response.status(), StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(
second_response
.headers()
.get(EXECUTION_PATH_HEADER)
.and_then(|value| value.to_str().ok()),
Some(EXECUTION_PATH_LOCAL_OVERLOADED)
);
assert_eq!(
second_response
.json::<serde_json::Value>()
.await
.expect("json body should decode")["error"]["type"],
"overloaded"
);
assert_eq!(upstream_hits.load(Ordering::SeqCst), 1);
drop(first_response);
gateway_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_exposes_request_concurrency_metrics() {
let gateway = build_router_with_state(
AppState::new("http://127.0.0.1:1", None)
.expect("gateway state should build")
.with_request_concurrency_limit(3)
.with_distributed_request_concurrency_gate(
aether_runtime::DistributedConcurrencyGate::new_in_memory(
"gateway_requests_distributed",
5,
),
),
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.get(format!("{gateway_url}/_gateway/metrics"))
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response
.headers()
.get(http::header::CONTENT_TYPE)
.and_then(|value| value.to_str().ok()),
Some("text/plain; version=0.0.4; charset=utf-8")
);
let body = response.text().await.expect("body should read");
assert!(body.contains("service_up{service=\"aether-gateway\"} 1"));
assert!(body.contains("concurrency_in_flight{gate=\"gateway_requests\"} 0"));
assert!(body.contains("concurrency_available_permits{gate=\"gateway_requests\"} 3"));
assert!(body.contains("concurrency_in_flight{gate=\"gateway_requests_distributed\"} 0"));
assert!(body.contains("concurrency_available_permits{gate=\"gateway_requests_distributed\"} 5"));
gateway_handle.abort();
}

View File

@@ -1,4 +1,43 @@
use super::*;
use aether_data::repository::auth::{
InMemoryAuthApiKeySnapshotRepository, StoredAuthApiKeySnapshot,
};
fn sample_currently_usable_auth_snapshot(
api_key_id: &str,
user_id: &str,
) -> StoredAuthApiKeySnapshot {
StoredAuthApiKeySnapshot::new(
user_id.to_string(),
"alice".to_string(),
Some("alice@example.com".to_string()),
"user".to_string(),
"local".to_string(),
true,
false,
Some(serde_json::json!(["openai"])),
Some(serde_json::json!(["openai:chat"])),
Some(serde_json::json!(["gpt-5"])),
api_key_id.to_string(),
Some("default".to_string()),
true,
false,
false,
Some(60),
Some(5),
Some(4_102_444_800),
Some(serde_json::json!(["openai"])),
Some(serde_json::json!(["openai:chat"])),
Some(serde_json::json!(["gpt-5"])),
)
.expect("auth snapshot should build")
}
fn sample_expired_auth_snapshot(api_key_id: &str, user_id: &str) -> StoredAuthApiKeySnapshot {
let mut snapshot = sample_currently_usable_auth_snapshot(api_key_id, user_id);
snapshot.api_key_expires_at_unix_secs = Some(1);
snapshot
}
#[tokio::test]
async fn gateway_consults_control_api_for_ai_routes_and_propagates_decision_headers() {
@@ -205,3 +244,284 @@ async fn gateway_consults_control_api_for_ai_routes_and_propagates_decision_head
gateway_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_uses_data_backed_trusted_auth_context_without_calling_control_auth_endpoint() {
#[derive(Debug, Clone)]
struct SeenPublicRequest {
trusted_user_id: String,
trusted_api_key_id: String,
trusted_balance_remaining: String,
trusted_access_allowed: String,
}
let auth_context_hits = Arc::new(Mutex::new(0usize));
let auth_context_hits_clone = Arc::clone(&auth_context_hits);
let seen_public = Arc::new(Mutex::new(None::<SeenPublicRequest>));
let seen_public_clone = Arc::clone(&seen_public);
let upstream = Router::new()
.route(
"/api/internal/gateway/auth-context",
any(move |_request: Request| {
let auth_context_hits_inner = Arc::clone(&auth_context_hits_clone);
async move {
*auth_context_hits_inner.lock().expect("mutex should lock") += 1;
Json(json!({
"auth_context": {
"user_id": "user-from-control",
"api_key_id": "key-from-control",
"balance_remaining": 99.0,
"access_allowed": true
}
}))
}
}),
)
.route(
"/v1/chat/completions",
any(move |request: Request| {
let seen_public_inner = Arc::clone(&seen_public_clone);
async move {
*seen_public_inner.lock().expect("mutex should lock") =
Some(SeenPublicRequest {
trusted_user_id: request
.headers()
.get(TRUSTED_AUTH_USER_ID_HEADER)
.and_then(|value| value.to_str().ok())
.unwrap_or_default()
.to_string(),
trusted_api_key_id: request
.headers()
.get(TRUSTED_AUTH_API_KEY_ID_HEADER)
.and_then(|value| value.to_str().ok())
.unwrap_or_default()
.to_string(),
trusted_balance_remaining: request
.headers()
.get(TRUSTED_AUTH_BALANCE_HEADER)
.and_then(|value| value.to_str().ok())
.unwrap_or_default()
.to_string(),
trusted_access_allowed: request
.headers()
.get(TRUSTED_AUTH_ACCESS_ALLOWED_HEADER)
.and_then(|value| value.to_str().ok())
.unwrap_or_default()
.to_string(),
});
(
StatusCode::OK,
[(GATEWAY_HEADER, "python-upstream")],
Body::from("proxied"),
)
}
}),
);
let repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
Some("hash-1".to_string()),
sample_currently_usable_auth_snapshot("key-123", "user-123"),
)]));
let (upstream_url, upstream_handle) = start_server(upstream).await;
let gateway = build_router_with_state(
AppState::new(upstream_url.clone(), Some(upstream_url))
.expect("gateway state should build")
.with_auth_api_key_data_reader_for_tests(repository),
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.post(format!("{gateway_url}/v1/chat/completions"))
.header(TRACE_ID_HEADER, "trace-control-data-auth-1")
.header(TRUSTED_AUTH_USER_ID_HEADER, "user-123")
.header(TRUSTED_AUTH_API_KEY_ID_HEADER, "key-123")
.header(TRUSTED_AUTH_BALANCE_HEADER, "42.5")
.body("{\"hello\":\"world\"}")
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(*auth_context_hits.lock().expect("mutex should lock"), 0);
let seen_public_request = seen_public
.lock()
.expect("mutex should lock")
.clone()
.expect("public request should be captured");
assert_eq!(seen_public_request.trusted_user_id, "user-123");
assert_eq!(seen_public_request.trusted_api_key_id, "key-123");
assert_eq!(seen_public_request.trusted_balance_remaining, "42.5");
assert_eq!(seen_public_request.trusted_access_allowed, "true");
gateway_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_locally_denies_explicit_trusted_balance_failure_without_hitting_control_or_upstream(
) {
let auth_context_hits = Arc::new(Mutex::new(0usize));
let auth_context_hits_clone = Arc::clone(&auth_context_hits);
let public_hits = Arc::new(Mutex::new(0usize));
let public_hits_clone = Arc::clone(&public_hits);
let upstream = Router::new()
.route(
"/api/internal/gateway/auth-context",
any(move |_request: Request| {
let auth_context_hits_inner = Arc::clone(&auth_context_hits_clone);
async move {
*auth_context_hits_inner.lock().expect("mutex should lock") += 1;
Json(json!({
"auth_context": {
"user_id": "user-from-control",
"api_key_id": "key-from-control",
"balance_remaining": 99.0,
"access_allowed": true
}
}))
}
}),
)
.route(
"/v1/chat/completions",
any(move |_request: Request| {
let public_hits_inner = Arc::clone(&public_hits_clone);
async move {
*public_hits_inner.lock().expect("mutex should lock") += 1;
(StatusCode::OK, Body::from("unexpected upstream hit"))
}
}),
);
let repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
Some("hash-1".to_string()),
sample_currently_usable_auth_snapshot("key-123", "user-123"),
)]));
let (upstream_url, upstream_handle) = start_server(upstream).await;
let gateway = build_router_with_state(
AppState::new(upstream_url.clone(), Some(upstream_url))
.expect("gateway state should build")
.with_auth_api_key_data_reader_for_tests(repository),
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.post(format!("{gateway_url}/v1/chat/completions"))
.header(http::header::CONTENT_TYPE, "application/json")
.header(TRACE_ID_HEADER, "trace-control-balance-denied-1")
.header(TRUSTED_AUTH_USER_ID_HEADER, "user-123")
.header(TRUSTED_AUTH_API_KEY_ID_HEADER, "key-123")
.header(TRUSTED_AUTH_BALANCE_HEADER, "0")
.header(TRUSTED_AUTH_ACCESS_ALLOWED_HEADER, "false")
.body("{\"model\":\"gpt-5\",\"messages\":[]}")
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::TOO_MANY_REQUESTS);
assert_eq!(
response
.headers()
.get(EXECUTION_PATH_HEADER)
.and_then(|value| value.to_str().ok()),
Some(EXECUTION_PATH_LOCAL_AUTH_DENIED)
);
assert_eq!(
response
.headers()
.get(CONTROL_ROUTE_CLASS_HEADER)
.and_then(|value| value.to_str().ok()),
Some("ai_public")
);
let payload: serde_json::Value = response.json().await.expect("response json should parse");
assert_eq!(payload["error"]["type"], "balance_exceeded");
assert_eq!(payload["error"]["message"], "余额不足(剩余: $0.00");
assert_eq!(payload["error"]["details"]["balance_type"], "USD");
assert_eq!(payload["error"]["details"]["remaining"], 0.0);
assert_eq!(*auth_context_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_locally_denies_invalid_trusted_snapshot_without_hitting_control_or_upstream() {
let auth_context_hits = Arc::new(Mutex::new(0usize));
let auth_context_hits_clone = Arc::clone(&auth_context_hits);
let public_hits = Arc::new(Mutex::new(0usize));
let public_hits_clone = Arc::clone(&public_hits);
let upstream = Router::new()
.route(
"/api/internal/gateway/auth-context",
any(move |_request: Request| {
let auth_context_hits_inner = Arc::clone(&auth_context_hits_clone);
async move {
*auth_context_hits_inner.lock().expect("mutex should lock") += 1;
Json(json!({
"auth_context": {
"user_id": "user-from-control",
"api_key_id": "key-from-control",
"access_allowed": true
}
}))
}
}),
)
.route(
"/v1/chat/completions",
any(move |_request: Request| {
let public_hits_inner = Arc::clone(&public_hits_clone);
async move {
*public_hits_inner.lock().expect("mutex should lock") += 1;
(StatusCode::OK, Body::from("unexpected upstream hit"))
}
}),
);
let repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
Some("hash-1".to_string()),
sample_expired_auth_snapshot("key-123", "user-123"),
)]));
let (upstream_url, upstream_handle) = start_server(upstream).await;
let gateway = build_router_with_state(
AppState::new(upstream_url.clone(), Some(upstream_url))
.expect("gateway state should build")
.with_auth_api_key_data_reader_for_tests(repository),
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.post(format!("{gateway_url}/v1/chat/completions"))
.header(http::header::CONTENT_TYPE, "application/json")
.header(TRACE_ID_HEADER, "trace-control-invalid-trusted-1")
.header(TRUSTED_AUTH_USER_ID_HEADER, "user-123")
.header(TRUSTED_AUTH_API_KEY_ID_HEADER, "key-123")
.body("{\"model\":\"gpt-5\",\"messages\":[]}")
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
assert_eq!(
response
.headers()
.get(EXECUTION_PATH_HEADER)
.and_then(|value| value.to_str().ok()),
Some(EXECUTION_PATH_LOCAL_AUTH_DENIED)
);
let payload: serde_json::Value = response.json().await.expect("response json should parse");
assert_eq!(payload["error"]["type"], "http_error");
assert_eq!(payload["error"]["message"], "无效的API密钥");
assert_eq!(*auth_context_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
upstream_handle.abort();
}

View File

@@ -10,6 +10,8 @@ pub(super) use http::StatusCode;
pub(super) use serde_json::json;
mod ai_execute;
mod audit;
mod concurrency;
mod control;
mod files;
mod proxy;

View File

@@ -1,4 +1,7 @@
use super::*;
use aether_data::repository::video_tasks::{
InMemoryVideoTaskRepository, UpsertVideoTask, VideoTaskWriteRepository,
};
mod error;
mod gemini_sync_create;
@@ -7,6 +10,186 @@ mod openai_sync_create;
mod openai_sync_task;
mod stream;
#[tokio::test]
async fn gateway_reads_openai_video_task_via_data_read_side_without_hitting_public_route() {
let public_hits = Arc::new(Mutex::new(0usize));
let public_hits_clone = Arc::clone(&public_hits);
let upstream = Router::new()
.route(
"/api/internal/gateway/resolve",
any(|_request: Request| async move {
Json(json!({
"action": "proxy_public",
"route_class": "ai_public",
"route_family": "openai",
"route_kind": "video",
"auth_endpoint_signature": "openai:video",
"executor_candidate": true,
"auth_context": {
"user_id": "user-video-db-123",
"api_key_id": "key-video-db-123",
"access_allowed": true
},
"public_path": "/v1/videos/task-db-123"
}))
}),
)
.route(
"/v1/videos/task-db-123",
any(move |_request: Request| {
let public_hits_inner = Arc::clone(&public_hits_clone);
async move {
*public_hits_inner.lock().expect("mutex should lock") += 1;
(StatusCode::IM_A_TEAPOT, Body::from("public-route-hit"))
}
}),
);
let (upstream_url, upstream_handle) = start_server(upstream).await;
let repository = Arc::new(InMemoryVideoTaskRepository::default());
repository
.upsert(UpsertVideoTask {
id: "task-db-123".to_string(),
short_id: Some("short-task-db-123".to_string()),
user_id: Some("user-video-db-123".to_string()),
external_task_id: Some("ext-video-db-123".to_string()),
provider_api_format: Some("openai:video".to_string()),
model: Some("sora-2".to_string()),
prompt: Some("hello from db".to_string()),
size: Some("1280x720".to_string()),
status: aether_data::repository::video_tasks::VideoTaskStatus::Processing,
progress_percent: 45,
created_at_unix_secs: 123,
updated_at_unix_secs: 124,
error_code: None,
error_message: None,
video_url: None,
})
.await
.expect("upsert should succeed");
let gateway = build_router_with_state(
AppState::new_with_executor(
upstream_url.clone(),
Some(upstream_url.clone()),
Some(upstream_url.clone()),
)
.expect("gateway state should build")
.with_video_task_data_reader_for_tests(repository),
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.get(format!("{gateway_url}/v1/videos/task-db-123"))
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let body: serde_json::Value = response.json().await.expect("body should parse");
assert_eq!(body["id"], "task-db-123");
assert_eq!(body["status"], "processing");
assert_eq!(body["progress"], 45);
assert_eq!(body["created_at"], 123);
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_reads_gemini_video_task_via_data_read_side_without_hitting_public_route() {
let public_hits = Arc::new(Mutex::new(0usize));
let public_hits_clone = Arc::clone(&public_hits);
let upstream = Router::new()
.route(
"/api/internal/gateway/resolve",
any(|_request: Request| async move {
Json(json!({
"action": "proxy_public",
"route_class": "ai_public",
"route_family": "gemini",
"route_kind": "video",
"auth_endpoint_signature": "gemini:video",
"executor_candidate": true,
"auth_context": {
"user_id": "user-video-db-123",
"api_key_id": "key-video-db-123",
"access_allowed": true
},
"public_path": "/v1beta/models/veo-3/operations/localshort123"
}))
}),
)
.route(
"/v1beta/models/veo-3/operations/localshort123",
any(move |_request: Request| {
let public_hits_inner = Arc::clone(&public_hits_clone);
async move {
*public_hits_inner.lock().expect("mutex should lock") += 1;
(StatusCode::IM_A_TEAPOT, Body::from("public-route-hit"))
}
}),
);
let (upstream_url, upstream_handle) = start_server(upstream).await;
let repository = Arc::new(InMemoryVideoTaskRepository::default());
repository
.upsert(UpsertVideoTask {
id: "task-db-456".to_string(),
short_id: Some("localshort123".to_string()),
user_id: Some("user-video-db-123".to_string()),
external_task_id: Some("operations/ext-video-db-123".to_string()),
provider_api_format: Some("gemini:video".to_string()),
model: Some("veo-3".to_string()),
prompt: Some("hello from gemini db".to_string()),
size: Some("720p".to_string()),
status: aether_data::repository::video_tasks::VideoTaskStatus::Completed,
progress_percent: 100,
created_at_unix_secs: 223,
updated_at_unix_secs: 224,
error_code: None,
error_message: None,
video_url: None,
})
.await
.expect("upsert should succeed");
let gateway = build_router_with_state(
AppState::new_with_executor(
upstream_url.clone(),
Some(upstream_url.clone()),
Some(upstream_url.clone()),
)
.expect("gateway state should build")
.with_video_task_data_reader_for_tests(repository),
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.get(format!(
"{gateway_url}/v1beta/models/veo-3/operations/localshort123"
))
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let body: serde_json::Value = response.json().await.expect("body should parse");
assert_eq!(body["name"], "models/veo-3/operations/localshort123");
assert_eq!(body["done"], true);
assert_eq!(
body["response"]["generateVideoResponse"]["generatedSamples"][0]["video"]["uri"],
"/v1beta/files/aev_localshort123:download?alt=media"
);
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_executes_video_get_route_via_control_sync_endpoint() {
#[derive(Debug, Clone)]

View File

@@ -1,5 +1,6 @@
use std::time::Instant;
use aether_runtime::{maybe_hold_axum_response_permit, AdmissionPermit};
use axum::body::{to_bytes, Body};
use axum::extract::{ConnectInfo, Request, State};
use axum::http::header::{HeaderName, HeaderValue};
@@ -15,16 +16,42 @@ use crate::gateway::headers::{
extract_or_generate_trace_id, header_value_str, is_json_request, should_skip_request_header,
};
use crate::gateway::{
build_client_response, maybe_execute_via_control, maybe_execute_via_executor_stream,
maybe_execute_via_executor_sync, resolve_control_route, AppState, GatewayControlDecision,
GatewayError,
build_client_response, build_local_auth_rejection_response, build_local_overloaded_response,
maybe_execute_via_control, maybe_execute_via_executor_stream, maybe_execute_via_executor_sync,
record_shadow_result_non_blocking, resolve_control_route, trusted_auth_local_rejection,
AppState, GatewayControlDecision, GatewayError,
};
pub(crate) async fn health(State(state): State<AppState>) -> impl IntoResponse {
let request_concurrency = state.request_concurrency_snapshot().map(|snapshot| {
json!({
"limit": snapshot.limit,
"in_flight": snapshot.in_flight,
"available_permits": snapshot.available_permits,
"high_watermark": snapshot.high_watermark,
"rejected": snapshot.rejected,
})
});
let distributed_request_concurrency = state
.distributed_request_concurrency_snapshot()
.await
.ok()
.flatten()
.map(|snapshot| {
json!({
"limit": snapshot.limit,
"in_flight": snapshot.in_flight,
"available_permits": snapshot.available_permits,
"high_watermark": snapshot.high_watermark,
"rejected": snapshot.rejected,
})
});
Json(json!({
"status": "ok",
"component": "aether-gateway",
"control_api_enabled": state.control_base_url.is_some(),
"request_concurrency": request_concurrency,
"distributed_request_concurrency": distributed_request_concurrency,
}))
}
@@ -34,6 +61,66 @@ pub(crate) async fn proxy_request(
request: Request,
) -> Result<Response<Body>, GatewayError> {
let started_at = Instant::now();
let mut request_permit = match state.try_acquire_request_permit().await {
Ok(permit) => permit,
Err(crate::gateway::RequestAdmissionError::Local(
aether_runtime::ConcurrencyError::Saturated { gate, limit },
)) => {
let trace_id = extract_or_generate_trace_id(request.headers());
let response = build_local_overloaded_response(&trace_id, None, gate, limit)?;
return Ok(finalize_gateway_response(
&state,
response,
&trace_id,
&remote_addr,
request.method(),
request
.uri()
.path_and_query()
.map(|value| value.as_str())
.unwrap_or("/"),
None,
EXECUTION_PATH_LOCAL_OVERLOADED,
&started_at,
None,
));
}
Err(crate::gateway::RequestAdmissionError::Local(
aether_runtime::ConcurrencyError::Closed { gate },
)) => {
return Err(GatewayError::Internal(format!(
"gateway request concurrency gate {gate} is closed"
)));
}
Err(crate::gateway::RequestAdmissionError::Distributed(
aether_runtime::DistributedConcurrencyError::Saturated { gate, limit },
))
| Err(crate::gateway::RequestAdmissionError::Distributed(
aether_runtime::DistributedConcurrencyError::Unavailable { gate, limit, .. },
)) => {
let trace_id = extract_or_generate_trace_id(request.headers());
let response = build_local_overloaded_response(&trace_id, None, gate, limit)?;
return Ok(finalize_gateway_response(
&state,
response,
&trace_id,
&remote_addr,
request.method(),
request
.uri()
.path_and_query()
.map(|value| value.as_str())
.unwrap_or("/"),
None,
EXECUTION_PATH_DISTRIBUTED_OVERLOADED,
&started_at,
None,
));
}
Err(crate::gateway::RequestAdmissionError::Distributed(
aether_runtime::DistributedConcurrencyError::InvalidConfiguration(message),
)) => return Err(GatewayError::Internal(message)),
};
let (parts, body) = request.into_parts();
let method = parts.method.clone();
let path_and_query = parts
@@ -46,6 +133,23 @@ pub(crate) async fn proxy_request(
let trace_id = extract_or_generate_trace_id(&parts.headers);
let control_decision =
resolve_control_route(&state, &method, &parts.uri, &parts.headers, &trace_id).await?;
if let Some(rejection) = trusted_auth_local_rejection(control_decision.as_ref(), &parts.headers)
{
let response =
build_local_auth_rejection_response(&trace_id, control_decision.as_ref(), &rejection)?;
return Ok(finalize_gateway_response(
&state,
response,
&trace_id,
&remote_addr,
&method,
path_and_query,
control_decision.as_ref(),
EXECUTION_PATH_LOCAL_AUTH_DENIED,
&started_at,
request_permit.take(),
));
}
let upstream_path_and_query = control_decision
.as_ref()
.map(|decision| decision.proxy_path_and_query())
@@ -152,6 +256,7 @@ pub(crate) async fn proxy_request(
.await?
{
return Ok(finalize_gateway_response(
&state,
executor_response,
&trace_id,
&remote_addr,
@@ -160,6 +265,7 @@ pub(crate) async fn proxy_request(
control_decision.as_ref(),
EXECUTION_PATH_EXECUTOR_STREAM,
&started_at,
request_permit.take(),
));
}
}
@@ -173,6 +279,7 @@ pub(crate) async fn proxy_request(
.await?
{
return Ok(finalize_gateway_response(
&state,
executor_response,
&trace_id,
&remote_addr,
@@ -181,6 +288,7 @@ pub(crate) async fn proxy_request(
control_decision.as_ref(),
EXECUTION_PATH_EXECUTOR_SYNC,
&started_at,
request_permit.take(),
));
}
if parts.method != http::Method::POST {
@@ -194,6 +302,7 @@ pub(crate) async fn proxy_request(
.await?
{
return Ok(finalize_gateway_response(
&state,
executor_response,
&trace_id,
&remote_addr,
@@ -202,6 +311,7 @@ pub(crate) async fn proxy_request(
control_decision.as_ref(),
EXECUTION_PATH_EXECUTOR_STREAM,
&started_at,
request_permit.take(),
));
}
}
@@ -216,6 +326,7 @@ pub(crate) async fn proxy_request(
.await?
{
return Ok(finalize_gateway_response(
&state,
control_response,
&trace_id,
&remote_addr,
@@ -228,6 +339,7 @@ pub(crate) async fn proxy_request(
EXECUTION_PATH_CONTROL_EXECUTE_SYNC
},
&started_at,
request_permit.take(),
));
}
}
@@ -263,6 +375,7 @@ pub(crate) async fn proxy_request(
let response = build_client_response(upstream_response, &trace_id, control_decision.as_ref())?;
Ok(finalize_gateway_response(
&state,
response,
&trace_id,
&remote_addr,
@@ -275,6 +388,7 @@ pub(crate) async fn proxy_request(
EXECUTION_PATH_PUBLIC_PROXY_PASSTHROUGH
},
&started_at,
request_permit.take(),
))
}
@@ -292,6 +406,7 @@ fn request_wants_stream(parts: &http::request::Parts, body: &axum::body::Bytes)
}
fn finalize_gateway_response(
state: &AppState,
mut response: Response<Body>,
trace_id: &str,
remote_addr: &std::net::SocketAddr,
@@ -300,6 +415,7 @@ fn finalize_gateway_response(
control_decision: Option<&GatewayControlDecision>,
execution_path: &'static str,
started_at: &Instant,
request_permit: Option<AdmissionPermit>,
) -> Response<Body> {
response.headers_mut().insert(
HeaderName::from_static(EXECUTION_PATH_HEADER),
@@ -321,5 +437,15 @@ fn finalize_gateway_response(
"gateway completed request"
);
response
record_shadow_result_non_blocking(
state.clone(),
trace_id,
method,
path_and_query,
control_decision,
execution_path,
&response,
);
maybe_hold_axum_response_permit(response, request_permit)
}

View File

@@ -2,5 +2,5 @@ mod gateway;
pub use gateway::{
build_router, build_router_with_control, build_router_with_endpoints, build_router_with_state,
serve_tcp, serve_tcp_with_endpoints, AppState, VideoTaskTruthSourceMode,
serve_tcp, serve_tcp_with_endpoints, AppState, GatewayDataConfig, VideoTaskTruthSourceMode,
};

View File

@@ -1,7 +1,14 @@
use clap::{Parser, ValueEnum};
use clap::{Args as ClapArgs, Parser, ValueEnum};
use tracing::info;
use aether_gateway::{build_router_with_state, AppState, VideoTaskTruthSourceMode};
use aether_data::postgres::PostgresPoolConfig;
use aether_gateway::{
build_router_with_state, AppState, GatewayDataConfig, VideoTaskTruthSourceMode,
};
use aether_runtime::{
init_service_runtime, DistributedConcurrencyGate, RedisDistributedConcurrencyConfig,
ServiceRuntimeConfig,
};
#[derive(Copy, Clone, Debug, Eq, PartialEq, ValueEnum)]
enum VideoTaskTruthSourceArg {
@@ -20,6 +27,85 @@ impl From<VideoTaskTruthSourceArg> for VideoTaskTruthSourceMode {
}
}
#[derive(ClapArgs, Debug, Clone)]
struct GatewayDataArgs {
#[arg(long, env = "AETHER_GATEWAY_DATA_POSTGRES_URL")]
postgres_url: Option<String>,
#[arg(
long,
env = "AETHER_GATEWAY_DATA_POSTGRES_MIN_CONNECTIONS",
default_value_t = 1
)]
postgres_min_connections: u32,
#[arg(
long,
env = "AETHER_GATEWAY_DATA_POSTGRES_MAX_CONNECTIONS",
default_value_t = 20
)]
postgres_max_connections: u32,
#[arg(
long,
env = "AETHER_GATEWAY_DATA_POSTGRES_ACQUIRE_TIMEOUT_MS",
default_value_t = 5_000
)]
postgres_acquire_timeout_ms: u64,
#[arg(
long,
env = "AETHER_GATEWAY_DATA_POSTGRES_IDLE_TIMEOUT_MS",
default_value_t = 60_000
)]
postgres_idle_timeout_ms: u64,
#[arg(
long,
env = "AETHER_GATEWAY_DATA_POSTGRES_MAX_LIFETIME_MS",
default_value_t = 1_800_000
)]
postgres_max_lifetime_ms: u64,
#[arg(
long,
env = "AETHER_GATEWAY_DATA_POSTGRES_STATEMENT_CACHE_CAPACITY",
default_value_t = 100
)]
postgres_statement_cache_capacity: usize,
#[arg(
long,
env = "AETHER_GATEWAY_DATA_POSTGRES_REQUIRE_SSL",
default_value_t = false
)]
postgres_require_ssl: bool,
}
impl GatewayDataArgs {
fn to_config(&self) -> GatewayDataConfig {
let Some(database_url) = self
.postgres_url
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
else {
return GatewayDataConfig::disabled();
};
GatewayDataConfig::from_postgres_config(PostgresPoolConfig {
database_url: database_url.to_string(),
min_connections: self.postgres_min_connections,
max_connections: self.postgres_max_connections,
acquire_timeout_ms: self.postgres_acquire_timeout_ms,
idle_timeout_ms: self.postgres_idle_timeout_ms,
max_lifetime_ms: self.postgres_max_lifetime_ms,
statement_cache_capacity: self.postgres_statement_cache_capacity,
require_ssl: self.postgres_require_ssl,
})
}
}
#[derive(Parser, Debug)]
#[command(
name = "aether-gateway",
@@ -66,16 +152,50 @@ struct Args {
#[arg(long, env = "AETHER_GATEWAY_VIDEO_TASK_STORE_PATH")]
video_task_store_path: Option<String>,
#[arg(long, env = "AETHER_GATEWAY_MAX_IN_FLIGHT_REQUESTS")]
max_in_flight_requests: Option<usize>,
#[arg(long, env = "AETHER_GATEWAY_DISTRIBUTED_REQUEST_LIMIT")]
distributed_request_limit: Option<usize>,
#[arg(long, env = "AETHER_GATEWAY_DISTRIBUTED_REQUEST_REDIS_URL")]
distributed_request_redis_url: Option<String>,
#[arg(long, env = "AETHER_GATEWAY_DISTRIBUTED_REQUEST_REDIS_KEY_PREFIX")]
distributed_request_redis_key_prefix: Option<String>,
#[arg(
long,
env = "AETHER_GATEWAY_DISTRIBUTED_REQUEST_LEASE_TTL_MS",
default_value_t = 30_000
)]
distributed_request_lease_ttl_ms: u64,
#[arg(
long,
env = "AETHER_GATEWAY_DISTRIBUTED_REQUEST_RENEW_INTERVAL_MS",
default_value_t = 10_000
)]
distributed_request_renew_interval_ms: u64,
#[arg(
long,
env = "AETHER_GATEWAY_DISTRIBUTED_REQUEST_COMMAND_TIMEOUT_MS",
default_value_t = 1_000
)]
distributed_request_command_timeout_ms: u64,
#[command(flatten)]
data: GatewayDataArgs,
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::try_from_default_env()
.unwrap_or_else(|_| "aether_gateway=info".into()),
)
.init();
init_service_runtime(ServiceRuntimeConfig::new(
"aether-gateway",
"aether_gateway=info",
))?;
let args = Args::parse();
let control_url = args
@@ -97,13 +217,24 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
video_task_poller_interval_ms = args.video_task_poller_interval_ms,
video_task_poller_batch_size = args.video_task_poller_batch_size,
video_task_store_path = args.video_task_store_path.as_deref().unwrap_or("-"),
max_in_flight_requests = args.max_in_flight_requests.unwrap_or_default(),
distributed_request_limit = args.distributed_request_limit.unwrap_or_default(),
distributed_request_redis_url = args
.distributed_request_redis_url
.as_deref()
.unwrap_or("-"),
data_postgres_url = args.data.postgres_url.as_deref().unwrap_or("-"),
data_postgres_require_ssl = args.data.postgres_require_ssl,
"aether-gateway started"
);
let data_config = args.data.to_config();
let mut state = AppState::new_with_executor(
args.upstream,
control_url.map(ToOwned::to_owned),
executor_url.map(ToOwned::to_owned),
)?
.with_data_config(data_config)?
.with_video_task_truth_source_mode(args.video_task_truth_source_mode.into());
if matches!(
args.video_task_truth_source_mode,
@@ -122,6 +253,44 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
{
state = state.with_video_task_store_path(path)?;
}
if let Some(limit) = args.max_in_flight_requests.filter(|limit| *limit > 0) {
state = state.with_request_concurrency_limit(limit);
}
if let Some(limit) = args.distributed_request_limit.filter(|limit| *limit > 0) {
let redis_url = args
.distributed_request_redis_url
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
.ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"AETHER_GATEWAY_DISTRIBUTED_REQUEST_REDIS_URL is required when distributed request limit is enabled",
)
})?;
state =
state.with_distributed_request_concurrency_gate(DistributedConcurrencyGate::new_redis(
"gateway_requests_distributed",
limit,
RedisDistributedConcurrencyConfig {
url: redis_url.to_string(),
key_prefix: args
.distributed_request_redis_key_prefix
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned),
lease_ttl_ms: args.distributed_request_lease_ttl_ms.max(1),
renew_interval_ms: args.distributed_request_renew_interval_ms.max(1),
command_timeout_ms: Some(args.distributed_request_command_timeout_ms.max(1)),
},
)?);
}
info!(
has_data_backends = state.has_data_backends(),
has_video_task_data_reader = state.has_video_task_data_reader(),
"aether-gateway data layer configured"
);
let background_tasks = state.spawn_background_tasks();
let listener = tokio::net::TcpListener::bind(&args.bind).await?;
let router = build_router_with_state(state);

View File

@@ -3,10 +3,14 @@ use std::collections::BTreeMap;
use axum::body::Body;
use axum::http::header::{HeaderName, HeaderValue};
use axum::http::Response;
use axum::http::StatusCode;
use serde_json::json;
use crate::gateway::constants::*;
use crate::gateway::headers::should_skip_response_header;
use crate::gateway::{insert_header_if_missing, GatewayControlDecision, GatewayError};
use crate::gateway::{
insert_header_if_missing, GatewayControlDecision, GatewayError, GatewayLocalAuthRejection,
};
pub(crate) fn build_client_response(
upstream_response: reqwest::Response,
@@ -90,3 +94,143 @@ pub(crate) fn build_client_response_from_parts(
}
Ok(response)
}
pub(crate) fn insert_candidate_id_header_if_present(
headers: &mut http::HeaderMap,
candidate_id: Option<&str>,
) -> Result<(), GatewayError> {
let Some(candidate_id) = candidate_id
.map(str::trim)
.filter(|value| !value.is_empty())
else {
return Ok(());
};
insert_header_if_missing(headers, CONTROL_CANDIDATE_ID_HEADER, candidate_id)
}
pub(crate) fn insert_request_id_header_if_present(
headers: &mut http::HeaderMap,
request_id: Option<&str>,
) -> Result<(), GatewayError> {
let Some(request_id) = request_id.map(str::trim).filter(|value| !value.is_empty()) else {
return Ok(());
};
insert_header_if_missing(headers, CONTROL_REQUEST_ID_HEADER, request_id)
}
pub(crate) fn attach_control_metadata_headers(
mut response: Response<Body>,
request_id: Option<&str>,
candidate_id: Option<&str>,
) -> Result<Response<Body>, GatewayError> {
insert_request_id_header_if_present(response.headers_mut(), request_id)?;
insert_candidate_id_header_if_present(response.headers_mut(), candidate_id)?;
Ok(response)
}
pub(crate) fn build_local_balance_denied_response(
trace_id: &str,
control_decision: Option<&GatewayControlDecision>,
balance_remaining: Option<f64>,
) -> Result<Response<Body>, GatewayError> {
let message = match balance_remaining {
Some(remaining) => format!("余额不足(剩余: ${remaining:.2}"),
None => "余额不足".to_string(),
};
let payload = json!({
"error": {
"type": "balance_exceeded",
"message": message,
"details": {
"balance_type": "USD",
"remaining": balance_remaining,
}
}
});
let body =
serde_json::to_vec(&payload).map_err(|err| GatewayError::Internal(err.to_string()))?;
let headers = BTreeMap::from([("content-type".to_string(), "application/json".to_string())]);
build_client_response_from_parts(
StatusCode::TOO_MANY_REQUESTS.as_u16(),
&headers,
Body::from(body),
trace_id,
control_decision,
)
}
pub(crate) fn build_local_http_error_response(
trace_id: &str,
control_decision: Option<&GatewayControlDecision>,
status_code: StatusCode,
message: &str,
) -> Result<Response<Body>, GatewayError> {
let payload = json!({
"error": {
"type": "http_error",
"message": message,
}
});
let body =
serde_json::to_vec(&payload).map_err(|err| GatewayError::Internal(err.to_string()))?;
let headers = BTreeMap::from([("content-type".to_string(), "application/json".to_string())]);
build_client_response_from_parts(
status_code.as_u16(),
&headers,
Body::from(body),
trace_id,
control_decision,
)
}
pub(crate) fn build_local_auth_rejection_response(
trace_id: &str,
control_decision: Option<&GatewayControlDecision>,
rejection: &GatewayLocalAuthRejection,
) -> Result<Response<Body>, GatewayError> {
match rejection {
GatewayLocalAuthRejection::InvalidApiKey => build_local_http_error_response(
trace_id,
control_decision,
StatusCode::UNAUTHORIZED,
"无效的API密钥",
),
GatewayLocalAuthRejection::LockedApiKey => build_local_http_error_response(
trace_id,
control_decision,
StatusCode::FORBIDDEN,
"该密钥已被管理员锁定,请联系管理员",
),
GatewayLocalAuthRejection::BalanceDenied { remaining } => {
build_local_balance_denied_response(trace_id, control_decision, *remaining)
}
}
}
pub(crate) fn build_local_overloaded_response(
trace_id: &str,
control_decision: Option<&GatewayControlDecision>,
gate: &str,
limit: usize,
) -> Result<Response<Body>, GatewayError> {
let payload = json!({
"error": {
"type": "overloaded",
"message": "服务繁忙,请稍后重试",
"details": {
"gate": gate,
"limit": limit,
}
}
});
let body =
serde_json::to_vec(&payload).map_err(|err| GatewayError::Internal(err.to_string()))?;
let headers = BTreeMap::from([("content-type".to_string(), "application/json".to_string())]);
build_client_response_from_parts(
StatusCode::SERVICE_UNAVAILABLE.as_u16(),
&headers,
Body::from(body),
trace_id,
control_decision,
)
}

View File

@@ -2682,6 +2682,7 @@ mod tests {
api_key_id: "key-123".to_string(),
balance_remaining: None,
access_allowed: true,
local_rejection: None,
}
}