i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-01 17:00:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: 移除 shadow results 相关模块和接口

Browse Source
This commit is contained in:
fawney19
2026-04-14 09:46:16 +08:00
parent 5fd3240fcf
commit 62e0a0338d
30 changed files with 115 additions and 1991 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
apps/aether-gateway/src/api/ops.rs
View File

@@ -5,10 +5,7 @@ use crate::async_task::{
cancel_video_task, get_video_task_detail, get_video_task_stats, get_video_task_video,
list_video_tasks,
};
use crate::audit::{
get_auth_api_key_snapshot, get_decision_trace, get_request_candidate_trace,
list_recent_shadow_results,
};
use crate::audit::{get_auth_api_key_snapshot, get_decision_trace, get_request_candidate_trace};
use crate::hooks::{get_request_audit_bundle, get_request_usage_audit};
use crate::router::metrics;
use crate::state::AppState;
@@ -53,8 +50,4 @@ pub(crate) fn mount_operational_routes(router: Router<AppState>) -> Router<AppSt
"/_gateway/audit/request-usage/{request_id}",
get(get_request_usage_audit),
)
.route(
"/_gateway/audit/shadow-results/recent",
get(list_recent_shadow_results),
)
}

63
apps/aether-gateway/src/audit/http.rs
View File

@@ -8,69 +8,6 @@ use serde_json::json;
use crate::{AppState, GatewayError};
const DEFAULT_RECENT_LIMIT: usize = 20;
const MAX_RECENT_LIMIT: usize = 200;
#[derive(Debug, Deserialize)]
pub(crate) struct ListRecentShadowResultsQuery {
pub(crate) limit: Option<usize>,
}
#[derive(Debug, Serialize)]
pub(crate) struct ShadowResultStatusCounts {
pub(crate) pending: usize,
pub(crate) r#match: usize,
pub(crate) mismatch: usize,
pub(crate) error: usize,
}
#[derive(Debug, Serialize)]
pub(crate) struct ListRecentShadowResultsResponse {
pub(crate) items: Vec<aether_data::repository::shadow_results::StoredShadowResult>,
pub(crate) limit_applied: usize,
pub(crate) counts: ShadowResultStatusCounts,
}
pub(crate) async fn list_recent_shadow_results(
State(state): State<AppState>,
Query(query): Query<ListRecentShadowResultsQuery>,
) -> Result<Json<ListRecentShadowResultsResponse>, GatewayError> {
let limit = query
.limit
.unwrap_or(DEFAULT_RECENT_LIMIT)
.clamp(1, MAX_RECENT_LIMIT);
let items = state.list_recent_shadow_results(limit).await?;
let mut counts = ShadowResultStatusCounts {
pending: 0,
r#match: 0,
mismatch: 0,
error: 0,
};
for item in &items {
match item.match_status {
aether_data::repository::shadow_results::ShadowResultMatchStatus::Pending => {
counts.pending += 1
}
aether_data::repository::shadow_results::ShadowResultMatchStatus::Match => {
counts.r#match += 1
}
aether_data::repository::shadow_results::ShadowResultMatchStatus::Mismatch => {
counts.mismatch += 1
}
aether_data::repository::shadow_results::ShadowResultMatchStatus::Error => {
counts.error += 1
}
}
}
Ok(Json(ListRecentShadowResultsResponse {
items,
limit_applied: limit,
counts,
}))
}
#[derive(Debug, Deserialize)]
pub(crate) struct GetRequestCandidateTraceQuery {
pub(crate) attempted_only: Option<bool>,

3
apps/aether-gateway/src/audit/mod.rs
View File

@@ -1,10 +1,7 @@
mod admin;
mod http;
mod shadow;
pub(crate) use admin::{attach_admin_audit_event, emit_admin_audit, AdminAuditEvent};
pub(crate) use http::get_auth_api_key_snapshot;
pub(crate) use http::get_decision_trace;
pub(crate) use http::get_request_candidate_trace;
pub(crate) use http::list_recent_shadow_results;
pub(crate) use shadow::record_shadow_result_non_blocking;

267
apps/aether-gateway/src/audit/shadow.rs
View File

@@ -1,267 +0,0 @@
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
use std::time::{SystemTime, UNIX_EPOCH};
use aether_data::repository::shadow_results::{RecordShadowResultSample, ShadowResultSampleOrigin};
use axum::body::Body;
use axum::http::header::CONTENT_TYPE;
use axum::http::Response;
use tracing::warn;
use crate::constants::{
CONTROL_CANDIDATE_ID_HEADER, CONTROL_REQUEST_ID_HEADER, EXECUTION_PATH_CONTROL_EXECUTE_STREAM,
EXECUTION_PATH_CONTROL_EXECUTE_SYNC,
};
use crate::control::GatewayControlDecision;
use crate::AppState;
pub(crate) fn record_shadow_result_non_blocking(
state: AppState,
trace_id: &str,
method: &http::Method,
path_and_query: &str,
control_decision: Option<&GatewayControlDecision>,
execution_path: &'static str,
response: &Response<Body>,
) {
let Some(decision) =
control_decision.filter(|decision| decision.route_class.as_deref() == Some("ai_public"))
else {
return;
};
if !state.has_shadow_result_data_writer() {
return;
}
let route_family = decision.route_family.clone();
let route_kind = decision.route_kind.clone();
let status_code = response.status().as_u16();
let content_type = response
.headers()
.get(CONTENT_TYPE)
.and_then(|value| value.to_str().ok())
.unwrap_or_default()
.to_string();
let candidate_id = response
.headers()
.get(CONTROL_CANDIDATE_ID_HEADER)
.and_then(|value| value.to_str().ok())
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned);
let request_id = response
.headers()
.get(CONTROL_REQUEST_ID_HEADER)
.and_then(|value| value.to_str().ok())
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned);
let now_unix_secs = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs();
let sample = RecordShadowResultSample {
trace_id: trace_id.to_string(),
request_fingerprint: build_request_fingerprint(
method,
path_and_query,
route_family.as_deref(),
route_kind.as_deref(),
),
request_id,
route_family,
route_kind,
candidate_id,
origin: sample_origin_for_execution_path(execution_path),
result_digest: build_result_digest(status_code, &content_type),
status_code: Some(status_code),
error_message: (status_code >= 400)
.then(|| format!("gateway response status {status_code}")),
recorded_at_unix_secs: now_unix_secs,
};
let trace_id = trace_id.to_string();
tokio::spawn(async move {
if let Err(err) = state.record_shadow_result_sample(sample).await {
warn!(trace_id = %trace_id, error = ?err, "gateway failed to record shadow result");
}
});
}
fn build_request_fingerprint(
method: &http::Method,
path_and_query: &str,
route_family: Option<&str>,
route_kind: Option<&str>,
) -> String {
let mut hasher = DefaultHasher::new();
method.as_str().hash(&mut hasher);
path_and_query.hash(&mut hasher);
route_family.unwrap_or_default().hash(&mut hasher);
route_kind.unwrap_or_default().hash(&mut hasher);
format!("{:x}", hasher.finish())
}
fn build_result_digest(status_code: u16, content_type: &str) -> String {
let mut hasher = DefaultHasher::new();
status_code.hash(&mut hasher);
content_type.hash(&mut hasher);
format!("{:x}", hasher.finish())
}
fn sample_origin_for_execution_path(execution_path: &str) -> ShadowResultSampleOrigin {
match execution_path {
EXECUTION_PATH_CONTROL_EXECUTE_SYNC | EXECUTION_PATH_CONTROL_EXECUTE_STREAM => {
ShadowResultSampleOrigin::Python
}
_ => ShadowResultSampleOrigin::Rust,
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use aether_data::repository::shadow_results::{
InMemoryShadowResultRepository, ShadowResultMatchStatus, ShadowResultReadRepository,
};
use axum::body::Body;
use axum::http::header::CONTENT_TYPE;
use axum::http::{Method, Response, StatusCode};
use super::record_shadow_result_non_blocking;
use crate::constants::{
CONTROL_REQUEST_ID_HEADER, EXECUTION_PATH_CONTROL_EXECUTE_SYNC,
EXECUTION_PATH_EXECUTION_RUNTIME_SYNC,
};
use crate::control::GatewayControlDecision;
use crate::AppState;
fn sample_decision() -> GatewayControlDecision {
GatewayControlDecision {
public_path: "/v1/chat/completions".to_string(),
public_query_string: Some("stream=true".to_string()),
route_class: Some("ai_public".to_string()),
route_family: Some("openai".to_string()),
route_kind: Some("chat".to_string()),
auth_endpoint_signature: Some("openai:chat".to_string()),
execution_runtime_candidate: true,
auth_context: None,
admin_principal: None,
local_auth_rejection: None,
}
}
#[tokio::test]
async fn records_shadow_result_for_ai_public_response() {
let repository = Arc::new(InMemoryShadowResultRepository::default());
let state = AppState::new()
.expect("app state should build")
.with_shadow_result_data_writer_for_tests(repository.clone());
let response = Response::builder()
.status(StatusCode::OK)
.header(CONTENT_TYPE, "application/json")
.header(CONTROL_REQUEST_ID_HEADER, "req-shadow-123")
.body(Body::from("{}"))
.expect("response should build");
record_shadow_result_non_blocking(
state,
"trace-shadow-123",
&Method::POST,
"/v1/chat/completions?stream=true",
Some(&sample_decision()),
EXECUTION_PATH_EXECUTION_RUNTIME_SYNC,
&response,
);
for _ in 0..30 {
if repository
.list_recent(1)
.await
.map(|rows| !rows.is_empty())
.unwrap_or(false)
{
break;
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
let stored = repository
.list_recent(1)
.await
.expect("list should succeed")
.into_iter()
.next()
.expect("stored result should exist");
assert_eq!(stored.trace_id, "trace-shadow-123");
assert_eq!(stored.request_id.as_deref(), Some("req-shadow-123"));
assert_eq!(stored.route_family.as_deref(), Some("openai"));
assert_eq!(stored.route_kind.as_deref(), Some("chat"));
assert_eq!(stored.match_status, ShadowResultMatchStatus::Pending);
assert_eq!(stored.status_code, Some(200));
assert!(stored.rust_result_digest.is_some());
}
#[tokio::test]
async fn merges_rust_and_python_shadow_samples_into_match() {
let repository = Arc::new(InMemoryShadowResultRepository::default());
let state = AppState::new()
.expect("app state should build")
.with_shadow_result_data_repository_for_tests(repository.clone());
let response = Response::builder()
.status(StatusCode::OK)
.header(CONTENT_TYPE, "application/json")
.header(CONTROL_REQUEST_ID_HEADER, "req-shadow-compare-123")
.body(Body::from("{}"))
.expect("response should build");
record_shadow_result_non_blocking(
state.clone(),
"trace-shadow-compare-123",
&Method::POST,
"/v1/chat/completions?stream=true",
Some(&sample_decision()),
EXECUTION_PATH_EXECUTION_RUNTIME_SYNC,
&response,
);
record_shadow_result_non_blocking(
state,
"trace-shadow-compare-123",
&Method::POST,
"/v1/chat/completions?stream=true",
Some(&sample_decision()),
EXECUTION_PATH_CONTROL_EXECUTE_SYNC,
&response,
);
for _ in 0..30 {
if repository
.list_recent(1)
.await
.map(|rows| {
rows.first()
.map(|row| row.match_status == ShadowResultMatchStatus::Match)
.unwrap_or(false)
})
.unwrap_or(false)
{
break;
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
let stored = repository
.list_recent(1)
.await
.expect("list should succeed")
.into_iter()
.next()
.expect("stored result should exist");
assert_eq!(stored.request_id.as_deref(), Some("req-shadow-compare-123"));
assert_eq!(stored.match_status, ShadowResultMatchStatus::Match);
assert!(stored.rust_result_digest.is_some());
assert!(stored.python_result_digest.is_some());
}
}

14
apps/aether-gateway/src/data/state/core.rs
View File

@@ -54,8 +54,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
});
}
@@ -94,8 +92,6 @@ impl GatewayDataState {
let wallet_reader = backends.read().wallets();
let wallet_writer = backends.write().wallets();
let settlement_writer = backends.write().settlement();
let shadow_result_reader = backends.read().shadow_results();
let shadow_result_writer = backends.write().shadow_results();
Ok(Self {
config,
@@ -134,8 +130,6 @@ impl GatewayDataState {
wallet_reader,
wallet_writer,
settlement_writer,
shadow_result_reader,
shadow_result_writer,
system_config_values: None,
})
}
@@ -297,14 +291,6 @@ impl GatewayDataState {
self.settlement_writer.is_some()
}
pub(crate) fn has_shadow_result_writer(&self) -> bool {
self.shadow_result_writer.is_some()
}
pub(crate) fn has_shadow_result_reader(&self) -> bool {
self.shadow_result_reader.is_some()
}
#[allow(dead_code)]
pub(crate) fn encryption_key(&self) -> Option<&str> {
self.config.encryption_key()

14
apps/aether-gateway/src/data/state/mod.rs
View File

@@ -44,10 +44,6 @@ use aether_data::repository::proxy_nodes::{
ProxyNodeRemoteConfigMutation, ProxyNodeTunnelStatusMutation, ProxyNodeWriteRepository,
StoredProxyNode, StoredProxyNodeEvent,
};
use aether_data::repository::shadow_results::{
merge_shadow_result_sample, RecordShadowResultSample, ShadowResultLookupKey,
ShadowResultReadRepository, ShadowResultWriteRepository, StoredShadowResult,
};
pub(crate) use aether_data::repository::system::{AdminSystemStats, StoredSystemConfigEntry};
use aether_data::repository::users::{
StoredUserAuthRecord, StoredUserExportRow, StoredUserSummary, UserReadRepository,
@@ -146,8 +142,6 @@ pub(crate) struct GatewayDataState {
wallet_reader: Option<Arc<dyn WalletReadRepository>>,
wallet_writer: Option<Arc<dyn WalletWriteRepository>>,
settlement_writer: Option<Arc<dyn SettlementWriteRepository>>,
shadow_result_reader: Option<Arc<dyn ShadowResultReadRepository>>,
shadow_result_writer: Option<Arc<dyn ShadowResultWriteRepository>>,
system_config_values: Option<Arc<RwLock<BTreeMap<String, StoredSystemConfigEntry>>>>,
}
@@ -249,14 +243,6 @@ impl fmt::Debug for GatewayDataState {
.field("has_wallet_reader", &self.wallet_reader.is_some())
.field("has_wallet_writer", &self.wallet_writer.is_some())
.field("has_settlement_writer", &self.settlement_writer.is_some())
.field(
"has_shadow_result_reader",
&self.shadow_result_reader.is_some(),
)
.field(
"has_shadow_result_writer",
&self.shadow_result_writer.is_some(),
)
.field(
"has_system_config_values",
&self.system_config_values.is_some(),

111
apps/aether-gateway/src/data/state/runtime.rs
View File

@@ -1,22 +1,21 @@
use super::{
merge_shadow_result_sample, read_decision_trace, read_provider_transport_snapshot,
read_request_candidate_trace, AdjustWalletBalanceInput, AdminPaymentOrderListQuery,
AdminWalletLedgerQuery, AdminWalletListQuery, AdminWalletRefundRequestListQuery,
AnnouncementListQuery, CompleteAdminWalletRefundInput, CreateAnnouncementRecord,
CreateManualWalletRechargeInput, CreateWalletRechargeOrderInput,
CreateWalletRechargeOrderOutcome, CreateWalletRefundRequestInput,
CreateWalletRefundRequestOutcome, CreditAdminPaymentOrderInput, DataLayerError, DecisionTrace,
FailAdminWalletRefundInput, GatewayDataState, GatewayProviderTransportSnapshot,
LocalVideoTaskReadResponse, ProcessAdminWalletRefundInput, ProcessPaymentCallbackInput,
ProcessPaymentCallbackOutcome, RecordShadowResultSample, RedisStreamRunner, RequestAuditBundle,
RequestCandidateTrace, ShadowResultLookupKey, StoredAdminPaymentCallbackPage,
read_decision_trace, read_provider_transport_snapshot, read_request_candidate_trace,
AdjustWalletBalanceInput, AdminPaymentOrderListQuery, AdminWalletLedgerQuery,
AdminWalletListQuery, AdminWalletRefundRequestListQuery, AnnouncementListQuery,
CompleteAdminWalletRefundInput, CreateAnnouncementRecord, CreateManualWalletRechargeInput,
CreateWalletRechargeOrderInput, CreateWalletRechargeOrderOutcome,
CreateWalletRefundRequestInput, CreateWalletRefundRequestOutcome, CreditAdminPaymentOrderInput,
DataLayerError, DecisionTrace, FailAdminWalletRefundInput, GatewayDataState,
GatewayProviderTransportSnapshot, LocalVideoTaskReadResponse, ProcessAdminWalletRefundInput,
ProcessPaymentCallbackInput, ProcessPaymentCallbackOutcome, RedisStreamRunner,
RequestAuditBundle, RequestCandidateTrace, StoredAdminPaymentCallbackPage,
StoredAdminPaymentOrder, StoredAdminPaymentOrderPage, StoredAdminWalletLedgerPage,
StoredAdminWalletListPage, StoredAdminWalletRefund, StoredAdminWalletRefundPage,
StoredAdminWalletRefundRequestPage, StoredAdminWalletTransaction,
StoredAdminWalletTransactionPage, StoredAnnouncement, StoredAnnouncementPage,
StoredBillingModelContext, StoredProviderQuotaSnapshot, StoredProviderUsageSummary,
StoredRequestUsageAudit, StoredShadowResult, StoredUsageSettlement, StoredUserAuthRecord,
StoredUserExportRow, StoredUserSummary, StoredVideoTask, StoredWalletDailyUsageLedger,
StoredRequestUsageAudit, StoredUsageSettlement, StoredUserAuthRecord, StoredUserExportRow,
StoredUserSummary, StoredVideoTask, StoredWalletDailyUsageLedger,
StoredWalletDailyUsageLedgerPage, StoredWalletSnapshot, UpdateAnnouncementRecord,
UpsertUsageRecord, UpsertVideoTask, UsageSettlementInput, VideoTaskLookupKey,
VideoTaskModelCount, VideoTaskQueryFilter, VideoTaskStatusCount, WalletLookupKey,
@@ -25,12 +24,6 @@ use super::{
use aether_data_contracts::repository::usage::UsageAuditListQuery;
use aether_video_tasks_core::read_data_backed_video_task_response;
fn is_missing_shadow_results_relation_error(error: &DataLayerError) -> bool {
error
.to_string()
.contains("relation \"gateway_shadow_results\" does not exist")
}
impl GatewayDataState {
pub(crate) async fn list_announcements(
&self,
@@ -873,68 +866,6 @@ impl GatewayDataState {
) -> Result<Option<LocalVideoTaskReadResponse>, DataLayerError> {
read_data_backed_video_task_response(self, route_family, request_path).await
}
#[cfg(test)]
pub(crate) async fn write_shadow_result(
&self,
result: aether_data::repository::shadow_results::UpsertShadowResult,
) -> Result<Option<StoredShadowResult>, DataLayerError> {
match &self.shadow_result_writer {
Some(repository) => match repository.upsert(result).await {
Ok(stored) => Ok(Some(stored)),
Err(err) if is_missing_shadow_results_relation_error(&err) => Ok(None),
Err(err) => Err(err),
},
None => Ok(None),
}
}
pub(crate) async fn record_shadow_result_sample(
&self,
sample: RecordShadowResultSample,
) -> Result<Option<StoredShadowResult>, DataLayerError> {
let Some(writer) = &self.shadow_result_writer else {
return Ok(None);
};
let existing = match &self.shadow_result_reader {
Some(reader) => {
match reader
.find(ShadowResultLookupKey::TraceFingerprint {
trace_id: &sample.trace_id,
request_fingerprint: &sample.request_fingerprint,
})
.await
{
Ok(existing) => existing,
Err(err) if is_missing_shadow_results_relation_error(&err) => return Ok(None),
Err(err) => return Err(err),
}
}
None => None,
};
let merged = merge_shadow_result_sample(existing.as_ref(), sample);
match writer.upsert(merged).await {
Ok(stored) => Ok(Some(stored)),
Err(err) if is_missing_shadow_results_relation_error(&err) => Ok(None),
Err(err) => Err(err),
}
}
pub(crate) async fn list_recent_shadow_results(
&self,
limit: usize,
) -> Result<Vec<StoredShadowResult>, DataLayerError> {
match &self.shadow_result_reader {
Some(repository) => match repository.list_recent(limit).await {
Ok(results) => Ok(results),
Err(err) if is_missing_shadow_results_relation_error(&err) => Ok(Vec::new()),
Err(err) => Err(err),
},
None => Ok(Vec::new()),
}
}
}
#[cfg(test)]
@@ -942,9 +873,8 @@ mod tests {
use std::sync::Arc;
use aether_data::repository::users::{InMemoryUserReadRepository, StoredUserExportRow};
use aether_data::DataLayerError;
use super::{is_missing_shadow_results_relation_error, GatewayDataState};
use super::GatewayDataState;
#[tokio::test]
async fn lists_non_admin_export_users_from_user_reader() {
@@ -983,19 +913,4 @@ mod tests {
Some(serde_json::json!({"gpt-4.1": {"cache_1h": true}}))
);
}
#[test]
fn detects_missing_shadow_results_relation_error_text() {
let missing_relation = DataLayerError::UnexpectedValue(
"postgres error: error returned from database: relation \"gateway_shadow_results\" does not exist"
.to_string(),
);
let other = DataLayerError::UnexpectedValue(
"postgres error: error returned from database: relation \"usage\" does not exist"
.to_string(),
);
assert!(is_missing_shadow_results_relation_error(&missing_relation));
assert!(!is_missing_shadow_results_relation_error(&other));
}
}

4
apps/aether-gateway/src/data/state/testing/announcements.rs
View File

@@ -46,8 +46,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -98,8 +96,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}

201
apps/aether-gateway/src/data/state/testing/mod.rs
View File

@@ -15,10 +15,9 @@ use super::{
OAuthProviderWriteRepository, ProviderCatalogReadRepository, ProviderCatalogWriteRepository,
ProviderQuotaReadRepository, ProviderQuotaWriteRepository, ProxyNodeReadRepository,
ProxyNodeWriteRepository, RequestCandidateReadRepository, RequestCandidateWriteRepository,
SettlementWriteRepository, ShadowResultReadRepository, ShadowResultWriteRepository,
StoredSystemConfigEntry, StoredUserPreferenceRecord, UsageReadRepository, UsageWriteRepository,
UserReadRepository, VideoTaskReadRepository, VideoTaskWriteRepository, WalletReadRepository,
WalletWriteRepository,
SettlementWriteRepository, StoredSystemConfigEntry, StoredUserPreferenceRecord,
UsageReadRepository, UsageWriteRepository, UserReadRepository, VideoTaskReadRepository,
VideoTaskWriteRepository, WalletReadRepository, WalletWriteRepository,
};
mod announcements;
@@ -80,8 +79,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -131,8 +128,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -178,8 +173,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -313,8 +306,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -376,8 +367,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -432,8 +421,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -497,8 +484,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -544,8 +529,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -592,8 +575,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -651,8 +632,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -712,8 +691,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -757,8 +734,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -817,8 +792,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -870,8 +843,6 @@ impl GatewayDataState {
wallet_reader: Some(wallet_reader),
wallet_writer: Some(wallet_writer),
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -928,8 +899,6 @@ impl GatewayDataState {
wallet_reader: Some(wallet_reader),
wallet_writer: Some(wallet_writer),
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -987,8 +956,6 @@ impl GatewayDataState {
wallet_reader: Some(wallet_reader),
wallet_writer: Some(wallet_writer),
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -1045,8 +1012,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -1092,8 +1057,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -1139,8 +1102,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -1198,8 +1159,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -1262,8 +1221,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -1309,8 +1266,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -1361,8 +1316,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -1430,8 +1383,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -1494,8 +1445,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -1542,8 +1491,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -1590,8 +1537,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -1640,106 +1585,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
#[cfg(test)]
pub(crate) fn with_shadow_result_writer_for_tests(
repository: Arc<dyn ShadowResultWriteRepository>,
) -> Self {
Self {
config: GatewayDataConfig::disabled(),
backends: None,
auth_api_key_reader: None,
auth_api_key_writer: None,
auth_module_reader: None,
auth_module_writer: None,
announcement_reader: None,
announcement_writer: None,
management_token_reader: None,
management_token_writer: None,
oauth_provider_reader: None,
oauth_provider_writer: None,
proxy_node_reader: None,
proxy_node_writer: None,
billing_reader: None,
gemini_file_mapping_reader: None,
gemini_file_mapping_writer: None,
global_model_reader: None,
global_model_writer: None,
minimal_candidate_selection_reader: None,
request_candidate_reader: None,
request_candidate_writer: None,
provider_catalog_reader: None,
provider_catalog_writer: None,
provider_quota_reader: None,
provider_quota_writer: None,
usage_reader: None,
usage_writer: None,
user_reader: None,
user_preferences: None,
usage_worker_runner: None,
video_task_reader: None,
video_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: Some(repository),
system_config_values: None,
}
}
#[cfg(test)]
pub(crate) fn with_shadow_result_repository_for_tests<T>(repository: Arc<T>) -> Self
where
T: aether_data::repository::shadow_results::ShadowResultRepository + 'static,
{
let shadow_result_reader: Arc<dyn ShadowResultReadRepository> = repository.clone();
let shadow_result_writer: Arc<dyn ShadowResultWriteRepository> = repository;
Self {
config: GatewayDataConfig::disabled(),
backends: None,
auth_api_key_reader: None,
auth_api_key_writer: None,
auth_module_reader: None,
auth_module_writer: None,
announcement_reader: None,
announcement_writer: None,
management_token_reader: None,
management_token_writer: None,
oauth_provider_reader: None,
oauth_provider_writer: None,
proxy_node_reader: None,
proxy_node_writer: None,
billing_reader: None,
gemini_file_mapping_reader: None,
gemini_file_mapping_writer: None,
global_model_reader: None,
global_model_writer: None,
minimal_candidate_selection_reader: None,
request_candidate_reader: None,
request_candidate_writer: None,
provider_catalog_reader: None,
provider_catalog_writer: None,
provider_quota_reader: None,
provider_quota_writer: None,
usage_reader: None,
usage_writer: None,
user_reader: None,
user_preferences: None,
usage_worker_runner: None,
video_task_reader: None,
video_task_writer: None,
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: Some(shadow_result_reader),
shadow_result_writer: Some(shadow_result_writer),
system_config_values: None,
}
}
@@ -1786,8 +1631,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -1834,8 +1677,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -1890,8 +1731,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -1947,8 +1786,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -2007,8 +1844,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -2073,29 +1908,24 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
#[cfg(test)]
pub(crate) fn with_auth_candidate_selection_provider_catalog_request_candidates_and_shadow_results_for_tests<
pub(crate) fn with_auth_candidate_selection_provider_catalog_request_candidates_for_tests<
T,
U,
V,
>(
auth_api_key_repository: Arc<dyn AuthApiKeyReadRepository>,
candidate_selection_repository: Arc<dyn MinimalCandidateSelectionReadRepository>,
provider_catalog_repository: Arc<U>,
request_candidate_repository: Arc<T>,
shadow_result_repository: Arc<V>,
encryption_key: impl Into<String>,
) -> Self
where
T: RequestCandidateRepository + 'static,
U: ProviderCatalogReadRepository + ProviderCatalogWriteRepository + 'static,
V: aether_data::repository::shadow_results::ShadowResultRepository + 'static,
{
let request_candidate_reader: Arc<dyn RequestCandidateReadRepository> =
request_candidate_repository.clone();
@@ -2105,9 +1935,6 @@ impl GatewayDataState {
provider_catalog_repository.clone();
let provider_catalog_writer: Arc<dyn ProviderCatalogWriteRepository> =
provider_catalog_repository;
let shadow_result_reader: Arc<dyn ShadowResultReadRepository> =
shadow_result_repository.clone();
let shadow_result_writer: Arc<dyn ShadowResultWriteRepository> = shadow_result_repository;
Self {
config: GatewayDataConfig::disabled().with_encryption_key(encryption_key),
@@ -2146,8 +1973,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: Some(shadow_result_reader),
shadow_result_writer: Some(shadow_result_writer),
system_config_values: None,
}
}
@@ -2217,8 +2042,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -2295,8 +2118,6 @@ impl GatewayDataState {
wallet_reader: Some(wallet_reader),
wallet_writer: Some(wallet_writer),
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -2355,8 +2176,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -2406,8 +2225,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -2453,8 +2270,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -2506,8 +2321,6 @@ impl GatewayDataState {
wallet_reader: Some(wallet_reader),
wallet_writer: Some(wallet_writer),
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -2563,8 +2376,6 @@ impl GatewayDataState {
wallet_reader: Some(wallet_reader),
wallet_writer: Some(wallet_writer),
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -2621,8 +2432,6 @@ impl GatewayDataState {
wallet_reader: Some(wallet_reader),
wallet_writer: Some(wallet_writer),
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -2672,8 +2481,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}

12
apps/aether-gateway/src/data/state/testing/video_tasks.rs
View File

@@ -50,8 +50,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -104,8 +102,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -155,8 +151,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -210,8 +204,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -269,8 +261,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}
@@ -337,8 +327,6 @@ impl GatewayDataState {
wallet_reader: None,
wallet_writer: None,
settlement_writer: None,
shadow_result_reader: None,
shadow_result_writer: None,
system_config_values: None,
}
}

202
apps/aether-gateway/src/data/tests.rs
View File

@@ -7,11 +7,6 @@ use aether_data::repository::auth::{
use aether_data::repository::candidate_selection::InMemoryMinimalCandidateSelectionReadRepository;
use aether_data::repository::candidates::InMemoryRequestCandidateRepository;
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
use aether_data::repository::shadow_results::{
InMemoryShadowResultRepository, RecordShadowResultSample, ShadowResultLookupKey,
ShadowResultMatchStatus, ShadowResultReadRepository, ShadowResultSampleOrigin,
ShadowResultWriteRepository, StoredShadowResult, UpsertShadowResult,
};
use aether_data::repository::usage::InMemoryUsageReadRepository;
use aether_data::repository::video_tasks::InMemoryVideoTaskRepository;
use aether_data::DataLayerError;
@@ -32,7 +27,6 @@ use aether_scheduler_core::{
build_minimal_candidate_selection, BuildMinimalCandidateSelectionInput,
SchedulerAuthConstraints, SchedulerPriorityMode,
};
use async_trait::async_trait;
use serde_json::json;
use super::{GatewayDataConfig, GatewayDataState};
@@ -52,8 +46,6 @@ fn disabled_gateway_data_state_has_no_backends() {
assert!(!state.has_proxy_node_writer());
assert!(!state.has_usage_reader());
assert!(!state.has_video_task_reader());
assert!(!state.has_shadow_result_reader());
assert!(!state.has_shadow_result_writer());
}
#[tokio::test]
@@ -73,8 +65,6 @@ async fn postgres_gateway_data_state_builds_video_task_reader() {
assert!(state.has_proxy_node_writer());
assert!(state.has_usage_reader());
assert!(state.has_video_task_reader());
assert!(state.has_shadow_result_reader());
assert!(state.has_shadow_result_writer());
}
#[tokio::test]
@@ -152,8 +142,6 @@ async fn app_state_wires_gateway_data_state_from_config() {
assert!(state.data.has_proxy_node_writer());
assert!(state.data.has_usage_reader());
assert!(state.data.has_video_task_reader());
assert!(state.data.has_shadow_result_reader());
assert!(state.data.has_shadow_result_writer());
}
fn sample_auth_snapshot(api_key_id: &str, user_id: &str) -> StoredAuthApiKeySnapshot {
@@ -283,40 +271,6 @@ fn sample_request_usage(request_id: &str) -> StoredRequestUsageAudit {
.expect("usage should build")
}
#[derive(Default)]
struct MissingShadowResultsRelationRepository;
fn missing_shadow_results_relation_error() -> DataLayerError {
DataLayerError::UnexpectedValue(
"postgres error: error returned from database: relation \"gateway_shadow_results\" does not exist"
.to_string(),
)
}
#[async_trait]
impl ShadowResultReadRepository for MissingShadowResultsRelationRepository {
async fn find(
&self,
_key: ShadowResultLookupKey<'_>,
) -> Result<Option<StoredShadowResult>, DataLayerError> {
Err(missing_shadow_results_relation_error())
}
async fn list_recent(&self, _limit: usize) -> Result<Vec<StoredShadowResult>, DataLayerError> {
Err(missing_shadow_results_relation_error())
}
}
#[async_trait]
impl ShadowResultWriteRepository for MissingShadowResultsRelationRepository {
async fn upsert(
&self,
_result: UpsertShadowResult,
) -> Result<StoredShadowResult, DataLayerError> {
Err(missing_shadow_results_relation_error())
}
}
fn sample_minimal_candidate_selection_row(
provider_id: &str,
provider_name: &str,
@@ -830,162 +784,6 @@ async fn maps_gemini_video_task_repository_row_into_read_response() {
assert_eq!(response.body_json["done"], true);
}
#[tokio::test]
async fn data_state_write_uses_configured_shadow_result_writer() {
let repository = Arc::new(InMemoryShadowResultRepository::default());
let state = GatewayDataState::with_shadow_result_writer_for_tests(repository.clone());
let written = state
.write_shadow_result(UpsertShadowResult {
trace_id: "trace-1".to_string(),
request_fingerprint: "fp-1".to_string(),
request_id: Some("req-1".to_string()),
route_family: Some("openai".to_string()),
route_kind: Some("chat".to_string()),
candidate_id: None,
rust_result_digest: Some("rust-digest".to_string()),
python_result_digest: None,
match_status: ShadowResultMatchStatus::Pending,
status_code: Some(200),
error_message: None,
created_at_unix_ms: 100,
updated_at_unix_secs: 100,
})
.await
.expect("write should succeed");
assert!(written.is_some());
let stored = repository
.find(ShadowResultLookupKey::TraceFingerprint {
trace_id: "trace-1",
request_fingerprint: "fp-1",
})
.await
.expect("find should succeed");
assert_eq!(
stored.expect("stored result should exist").match_status,
ShadowResultMatchStatus::Pending
);
}
#[tokio::test]
async fn data_state_records_shadow_result_samples_and_merges_match_status() {
let repository = Arc::new(InMemoryShadowResultRepository::default());
let state = GatewayDataState::with_shadow_result_repository_for_tests(repository);
let first = state
.record_shadow_result_sample(RecordShadowResultSample {
trace_id: "trace-1".to_string(),
request_fingerprint: "fp-1".to_string(),
request_id: Some("req-1".to_string()),
route_family: Some("openai".to_string()),
route_kind: Some("chat".to_string()),
candidate_id: None,
origin: ShadowResultSampleOrigin::Rust,
result_digest: "digest-1".to_string(),
status_code: Some(200),
error_message: None,
recorded_at_unix_secs: 100,
})
.await
.expect("first record should succeed")
.expect("first stored result should exist");
assert_eq!(first.match_status, ShadowResultMatchStatus::Pending);
let second = state
.record_shadow_result_sample(RecordShadowResultSample {
trace_id: "trace-1".to_string(),
request_fingerprint: "fp-1".to_string(),
request_id: Some("req-1".to_string()),
route_family: Some("openai".to_string()),
route_kind: Some("chat".to_string()),
candidate_id: None,
origin: ShadowResultSampleOrigin::Python,
result_digest: "digest-1".to_string(),
status_code: Some(200),
error_message: None,
recorded_at_unix_secs: 200,
})
.await
.expect("second record should succeed")
.expect("second stored result should exist");
assert_eq!(second.match_status, ShadowResultMatchStatus::Match);
assert_eq!(second.created_at_unix_ms, 100);
assert_eq!(second.updated_at_unix_secs, 200);
assert_eq!(second.request_id.as_deref(), Some("req-1"));
}
#[tokio::test]
async fn data_state_lists_recent_shadow_results_from_reader() {
let repository = Arc::new(InMemoryShadowResultRepository::default());
let state = GatewayDataState::with_shadow_result_repository_for_tests(repository.clone());
state
.record_shadow_result_sample(RecordShadowResultSample {
trace_id: "trace-1".to_string(),
request_fingerprint: "fp-1".to_string(),
request_id: Some("req-shadow-1".to_string()),
route_family: Some("openai".to_string()),
route_kind: Some("chat".to_string()),
candidate_id: None,
origin: ShadowResultSampleOrigin::Rust,
result_digest: "digest-1".to_string(),
status_code: Some(200),
error_message: None,
recorded_at_unix_secs: 100,
})
.await
.expect("record should succeed");
let recent = state
.list_recent_shadow_results(5)
.await
.expect("list recent should succeed");
assert_eq!(recent.len(), 1);
assert_eq!(recent[0].trace_id, "trace-1");
assert_eq!(recent[0].request_id.as_deref(), Some("req-shadow-1"));
}
#[tokio::test]
async fn data_state_ignores_missing_shadow_results_relation_when_recording_sample() {
let repository = Arc::new(MissingShadowResultsRelationRepository);
let state = GatewayDataState::with_shadow_result_repository_for_tests(repository);
let recorded = state
.record_shadow_result_sample(RecordShadowResultSample {
trace_id: "trace-missing-shadow".to_string(),
request_fingerprint: "fp-missing-shadow".to_string(),
request_id: Some("req-missing-shadow".to_string()),
route_family: Some("openai".to_string()),
route_kind: Some("chat".to_string()),
candidate_id: None,
origin: ShadowResultSampleOrigin::Rust,
result_digest: "digest-missing-shadow".to_string(),
status_code: Some(200),
error_message: None,
recorded_at_unix_secs: 100,
})
.await
.expect("missing shadow result table should be ignored");
assert!(recorded.is_none());
}
#[tokio::test]
async fn data_state_returns_empty_shadow_results_when_relation_missing() {
let repository = Arc::new(MissingShadowResultsRelationRepository);
let state = GatewayDataState::with_shadow_result_repository_for_tests(repository);
let recent = state
.list_recent_shadow_results(5)
.await
.expect("missing shadow result table should be ignored");
assert!(recent.is_empty());
}
fn sample_request_candidate(
id: &str,
request_id: &str,

77
apps/aether-gateway/src/handlers/admin/observability/monitoring/tests/trace.rs
View File

@@ -75,6 +75,83 @@ async fn admin_monitoring_trace_request_returns_local_payload() {
assert_eq!(payload["candidates"][0]["status_code"], json!(502));
}
#[tokio::test]
async fn admin_monitoring_trace_request_hides_format_conversion_disabled_candidates() {
let mut hidden_candidate = sample_candidate(
"cand-hidden",
"request-1",
0,
RequestCandidateStatus::Skipped,
None,
None,
None,
);
hidden_candidate.skip_reason = Some("format_conversion_disabled".to_string());
let mut visible_skipped_candidate = sample_candidate(
"cand-visible-skipped",
"request-1",
1,
RequestCandidateStatus::Skipped,
None,
None,
None,
);
visible_skipped_candidate.skip_reason = Some("transport_unsupported".to_string());
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![
hidden_candidate,
visible_skipped_candidate,
sample_candidate(
"cand-used",
"request-1",
2,
RequestCandidateStatus::Failed,
Some(101),
Some(33),
Some(502),
),
]));
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider()],
vec![sample_endpoint()],
vec![sample_key()],
));
let state = AppState::new()
.expect("state should build")
.with_decision_trace_data_readers_for_tests(request_candidates, provider_catalog);
let context = request_context(
http::Method::GET,
"/api/admin/monitoring/trace/request-1?attempted_only=false",
);
let response = local_monitoring_response(&state, &context)
.await
.expect("handler should not error")
.expect("route should be handled locally");
assert_eq!(response.status(), http::StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX)
.await
.expect("body should read");
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
assert_eq!(payload["total_candidates"], json!(2));
assert_eq!(
payload["candidates"]
.as_array()
.expect("candidates should be an array")
.iter()
.map(|item| item["id"].as_str().unwrap_or_default())
.collect::<Vec<_>>(),
vec!["cand-visible-skipped", "cand-used"]
);
assert_eq!(
payload["candidates"][0]["skip_reason"],
json!("transport_unsupported")
);
}
#[tokio::test]
async fn admin_monitoring_trace_provider_stats_returns_local_payload() {
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![

19
apps/aether-gateway/src/handlers/admin/observability/monitoring/trace.rs
View File

@@ -8,13 +8,29 @@ use aether_admin::observability::monitoring::{
build_admin_monitoring_trace_provider_stats_payload_response,
build_admin_monitoring_trace_request_payload_response, parse_admin_monitoring_attempted_only,
};
use aether_data_contracts::repository::candidates::RequestCandidateStatus;
use aether_data_contracts::repository::candidates::{DecisionTrace, RequestCandidateStatus};
use axum::{
body::Body,
response::{IntoResponse, Response},
};
use tracing::debug;
const HIDDEN_TRACE_SKIP_REASONS: &[&str] = &["format_conversion_disabled"];
fn filter_admin_monitoring_trace_candidates(mut trace: DecisionTrace) -> DecisionTrace {
trace.candidates.retain(|item| {
let skip_reason = item
.candidate
.skip_reason
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty());
!skip_reason.is_some_and(|reason| HIDDEN_TRACE_SKIP_REASONS.contains(&reason))
});
trace.total_candidates = trace.candidates.len();
trace
}
pub(super) async fn build_admin_monitoring_trace_request_response(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
@@ -51,6 +67,7 @@ pub(super) async fn build_admin_monitoring_trace_request_response(
attempted_only,
));
};
let trace = filter_admin_monitoring_trace_candidates(trace);
Ok(build_admin_monitoring_trace_request_payload_response(
&trace,

12
apps/aether-gateway/src/handlers/proxy/finalize.rs
View File

@@ -1,4 +1,4 @@
use crate::audit::{emit_admin_audit, record_shadow_result_non_blocking};
use crate::audit::emit_admin_audit;
use crate::constants::{
CONTROL_ENDPOINT_SIGNATURE_HEADER, CONTROL_EXECUTION_RUNTIME_HEADER, CONTROL_REQUEST_ID_HEADER,
CONTROL_ROUTE_CLASS_HEADER, CONTROL_ROUTE_FAMILY_HEADER, CONTROL_ROUTE_KIND_HEADER,
@@ -165,16 +165,6 @@ pub(super) fn finalize_gateway_response(
}
response.extensions_mut().insert(RequestLogEmitted);
record_shadow_result_non_blocking(
state.clone(),
trace_id,
method,
path_and_query,
control_decision,
execution_path,
&response,
);
maybe_hold_axum_response_permit(response, request_permit)
}

1
apps/aether-gateway/src/hooks/mod.rs
View File

@@ -1,2 +1 @@
pub(crate) use crate::audit::record_shadow_result_non_blocking;
pub(crate) use crate::usage::http::{get_request_audit_bundle, get_request_usage_audit};

9
apps/aether-gateway/src/state/runtime/mod.rs
View File

@@ -15,7 +15,6 @@ mod candidate_queries;
mod gemini_files;
mod payments;
mod security;
mod shadow_results;
mod usage_queries;
mod user_preferences;
mod wallet;
@@ -100,14 +99,6 @@ impl AppState {
self.data.has_provider_quota_writer()
}
pub fn has_shadow_result_data_writer(&self) -> bool {
self.data.has_shadow_result_writer()
}
pub fn has_shadow_result_data_reader(&self) -> bool {
self.data.has_shadow_result_reader()
}
pub(crate) async fn count_active_admin_users(&self) -> Result<u64, GatewayError> {
#[cfg(test)]
if let Some(store) = self.auth_user_store.as_ref() {

25
apps/aether-gateway/src/state/runtime/shadow_results.rs
View File

@@ -1,25 +0,0 @@
use crate::{AppState, GatewayError};
impl AppState {
pub(crate) async fn record_shadow_result_sample(
&self,
sample: aether_data::repository::shadow_results::RecordShadowResultSample,
) -> Result<Option<aether_data::repository::shadow_results::StoredShadowResult>, GatewayError>
{
self.data
.record_shadow_result_sample(sample)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))
}
pub(crate) async fn list_recent_shadow_results(
&self,
limit: usize,
) -> Result<Vec<aether_data::repository::shadow_results::StoredShadowResult>, GatewayError>
{
self.data
.list_recent_shadow_results(limit)
.await
.map_err(|err| GatewayError::Internal(err.to_string()))
}
}

23
apps/aether-gateway/src/state/testing.rs
View File

@@ -578,27 +578,4 @@ impl AppState {
.insert(provider_type.trim().to_ascii_lowercase(), token_url.into());
self
}
pub(crate) fn with_shadow_result_data_writer_for_tests(
mut self,
repository: Arc<dyn aether_data::repository::shadow_results::ShadowResultWriteRepository>,
) -> Self {
self.replace_data_state(Arc::new(
GatewayDataState::with_shadow_result_writer_for_tests(repository),
));
self
}
pub(crate) fn with_shadow_result_data_repository_for_tests<T>(
mut self,
repository: Arc<T>,
) -> Self
where
T: aether_data::repository::shadow_results::ShadowResultRepository + 'static,
{
self.replace_data_state(Arc::new(
GatewayDataState::with_shadow_result_repository_for_tests(repository),
));
self
}
}

4
apps/aether-gateway/src/tests/architecture/runtime_and_security.rs
View File

@@ -28,10 +28,6 @@ fn gateway_small_runtime_shims_stay_deleted() {
}
let hooks_mod = read_workspace_file("apps/aether-gateway/src/hooks/mod.rs");
assert!(
hooks_mod.contains("pub(crate) use crate::audit::record_shadow_result_non_blocking;"),
"hooks/mod.rs should re-export shadow audit directly from crate::audit"
);
assert!(
hooks_mod.contains("pub(crate) use crate::usage::http::{get_request_audit_bundle, get_request_usage_audit};"),
"hooks/mod.rs should re-export request audit helpers directly from usage/http"

107
apps/aether-gateway/src/tests/audit.rs
View File

@@ -7,9 +7,6 @@ use aether_data::repository::auth::{
use aether_data::repository::candidate_selection::InMemoryMinimalCandidateSelectionReadRepository;
use aether_data::repository::candidates::InMemoryRequestCandidateRepository;
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
use aether_data::repository::shadow_results::{
InMemoryShadowResultRepository, ShadowResultMatchStatus, ShadowResultReadRepository,
};
use aether_data::repository::usage::InMemoryUsageReadRepository;
use aether_data_contracts::repository::candidate_selection::{
StoredMinimalCandidateSelectionRow, StoredProviderModelMapping,
@@ -160,110 +157,6 @@ fn sample_local_openai_key() -> StoredProviderCatalogKey {
.expect("key transport should build")
}
#[tokio::test]
async fn gateway_records_candidate_id_in_shadow_result_for_local_execution_response() {
let repository = Arc::new(InMemoryShadowResultRepository::default());
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
let auth_repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
Some(hash_api_key("sk-client-openai-audit-local")),
sample_local_openai_auth_snapshot(
"api-key-openai-audit-local-1",
"user-openai-audit-local-1",
),
)]));
let candidate_selection_repository =
Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(vec![
sample_local_openai_candidate_row(),
]));
let provider = Router::new().route(
"/v1/chat/completions",
any(|_request: Request| async move {
Json(json!({
"id": "chatcmpl-shadow-local-123",
"object": "chat.completion",
"model": "gpt-5-upstream",
"choices": [],
"usage": {
"prompt_tokens": 2,
"completion_tokens": 3,
"total_tokens": 5
}
}))
}),
);
let (provider_url, provider_handle) = start_server(provider).await;
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_local_openai_provider()],
vec![sample_local_openai_endpoint(provider_url)],
vec![sample_local_openai_key()],
));
let gateway_state = AppState::new()
.expect("gateway state should build")
.with_data_state_for_tests(
crate::data::GatewayDataState::with_auth_candidate_selection_provider_catalog_request_candidates_and_shadow_results_for_tests(
auth_repository,
candidate_selection_repository,
provider_catalog_repository,
Arc::clone(&request_candidate_repository),
Arc::clone(&repository),
DEVELOPMENT_ENCRYPTION_KEY,
),
);
let gateway = build_router_with_state(gateway_state);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.post(format!("{gateway_url}/v1/chat/completions"))
.header(http::header::CONTENT_TYPE, "application/json")
.header(
http::header::AUTHORIZATION,
"Bearer sk-client-openai-audit-local",
)
.header(TRACE_ID_HEADER, "req-shadow-direct-123")
.body("{\"model\":\"gpt-5\",\"messages\":[]}")
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let candidate_id = response
.headers()
.get(CONTROL_CANDIDATE_ID_HEADER)
.and_then(|value| value.to_str().ok())
.expect("candidate id header should exist")
.to_string();
for _ in 0..50 {
if repository
.list_recent(1)
.await
.map(|rows| !rows.is_empty())
.unwrap_or(false)
{
break;
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
let stored = repository
.list_recent(1)
.await
.expect("list should succeed")
.into_iter()
.next()
.expect("stored result should exist");
assert_eq!(stored.request_id.as_deref(), Some("req-shadow-direct-123"));
assert_eq!(stored.candidate_id.as_deref(), Some(candidate_id.as_str()));
assert_eq!(stored.route_family.as_deref(), Some("openai"));
assert_eq!(stored.route_kind.as_deref(), Some("chat"));
assert_eq!(stored.match_status, ShadowResultMatchStatus::Pending);
gateway_handle.abort();
provider_handle.abort();
}
#[tokio::test]
async fn gateway_exposes_request_id_header_for_local_execution_response() {
let auth_repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(