i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-01 17:00:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor(gateway): 重构格式转换候选资格检查并优化测试基础设施

Browse Source 
- 将 format_conversion_disabled 的过滤提前到候选遴选阶段，替代原来的 skip_reason 标记机制
- 为测试添加 execution_runtime_sync_override 直接注入支持，避免启动额外 HTTP 服务器
- 将 submit_terminal_event 改为 async record_terminal_event 确保失败用量事件可靠入库
- 新增 test_support 模块封装可重试的 loopback 端口绑定逻辑
- 前端：poolTrace 将 skipped 从隐藏状态移除，新增 buildPoolParticipatedCandidates 统一新旧链路逻辑，并将 skipped 状态色改为 foreground
This commit is contained in:
fawney19
2026-04-20 16:59:18 +08:00
parent 87afe4898e
commit 226a6e58d5
21 changed files with 380 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
apps/aether-gateway/src/ai_pipeline/planner/candidate_affinity.rs
View File

@@ -1194,9 +1194,7 @@ mod tests {
assert_eq!(ranked.len(), 1);
assert_eq!(ranked[0].candidate.endpoint_id, "endpoint-same");
assert_eq!(skipped.len(), 1);
assert_eq!(skipped[0].candidate.endpoint_id, "endpoint-cross");
assert_eq!(skipped[0].skip_reason, "format_conversion_disabled");
assert!(skipped.is_empty());
}
#[tokio::test]

74
apps/aether-gateway/src/ai_pipeline/planner/candidate_eligibility.rs
View File

@@ -1,7 +1,6 @@
use tracing::warn;
use aether_scheduler_core::SchedulerMinimalCandidateSelectionCandidate;
use std::collections::BTreeSet;
use crate::ai_pipeline::{GatewayProviderTransportSnapshot, PlannerAppState};
use crate::orchestration::LocalExecutionCandidateMetadata;
@@ -101,7 +100,6 @@ where
{
let mut selectable = Vec::with_capacity(candidates.len());
let mut skipped = Vec::new();
let normalized_client_api_format = client_api_format.trim().to_ascii_lowercase();
for candidate in candidates {
let Some(transport) = read_candidate_transport_snapshot(state, &candidate).await else {
@@ -113,6 +111,10 @@ where
});
continue;
};
if candidate_is_ineligible_due_to_disabled_format_conversion(&transport, client_api_format)
{
continue;
}
match runtime_skip_reason(&candidate, &transport) {
Some(skip_reason) => skipped.push(SkippedLocalExecutionCandidate {
candidate,
@@ -129,28 +131,6 @@ where
}
}
let exact_selectable_keys = selectable
.iter()
.filter(|candidate| {
candidate
.provider_api_format
.eq_ignore_ascii_case(normalized_client_api_format.as_str())
})
.map(|candidate| {
(
candidate.candidate.provider_id.clone(),
candidate.candidate.key_id.clone(),
)
})
.collect::<BTreeSet<_>>();
skipped.retain(|candidate| {
candidate.skip_reason != "format_conversion_disabled"
|| !exact_selectable_keys.contains(&(
candidate.candidate.provider_id.clone(),
candidate.candidate.key_id.clone(),
))
});
let ranked = rank_eligible_local_execution_candidates(
state,
selectable,
@@ -234,6 +214,32 @@ fn current_local_execution_candidate_common_skip_reason_with_transport(
None
}
fn candidate_is_ineligible_due_to_disabled_format_conversion(
transport: &GatewayProviderTransportSnapshot,
client_api_format: &str,
) -> bool {
let endpoint_api_format = transport.endpoint.api_format.trim().to_ascii_lowercase();
let client_api_format = client_api_format.trim().to_ascii_lowercase();
if client_api_format == endpoint_api_format {
return false;
}
crate::ai_pipeline::conversion::request_conversion_kind(
client_api_format.as_str(),
endpoint_api_format.as_str(),
)
.is_some()
&& crate::ai_pipeline::conversion::request_conversion_requires_enable_flag(
client_api_format.as_str(),
endpoint_api_format.as_str(),
)
&& !crate::ai_pipeline::conversion::request_conversion_enabled_for_transport(
transport,
client_api_format.as_str(),
endpoint_api_format.as_str(),
)
}
fn current_local_execution_candidate_skip_reason_with_transport(
candidate: &SchedulerMinimalCandidateSelectionCandidate,
transport: &GatewayProviderTransportSnapshot,
@@ -259,25 +265,7 @@ fn current_local_execution_candidate_skip_reason_with_transport(
client_api_format.as_str(),
endpoint_api_format.as_str(),
) {
let skip_reason = if crate::ai_pipeline::conversion::request_conversion_kind(
client_api_format.as_str(),
endpoint_api_format.as_str(),
)
.is_some()
&& crate::ai_pipeline::conversion::request_conversion_requires_enable_flag(
client_api_format.as_str(),
endpoint_api_format.as_str(),
)
&& !crate::ai_pipeline::conversion::request_conversion_enabled_for_transport(
transport,
client_api_format.as_str(),
endpoint_api_format.as_str(),
) {
"format_conversion_disabled"
} else {
"transport_unsupported"
};
return Some(skip_reason);
return Some("transport_unsupported");
}
None

2
apps/aether-gateway/src/execution_runtime/server.rs
View File

@@ -386,7 +386,7 @@ mod tests {
use std::sync::Arc;
async fn start_server(app: Router) -> (String, tokio::task::JoinHandle<()>) {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
let listener = crate::test_support::bind_loopback_listener()
.await
.expect("listener should bind");
let addr = listener.local_addr().expect("local addr should resolve");

4
apps/aether-gateway/src/execution_runtime/stream_pump.rs
View File

@@ -385,7 +385,7 @@ mod tests {
#[tokio::test]
async fn direct_execution_frame_stream_reports_ttfb_after_first_upstream_chunk() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
let listener = crate::test_support::bind_loopback_listener()
.await
.expect("listener should bind");
let addr = listener.local_addr().expect("local addr should resolve");
@@ -471,7 +471,7 @@ mod tests {
#[tokio::test]
async fn direct_execution_frame_stream_emits_telemetry_before_first_data_frame() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
let listener = crate::test_support::bind_loopback_listener()
.await
.expect("listener should bind");
let addr = listener.local_addr().expect("local addr should resolve");

48
apps/aether-gateway/src/execution_runtime/sync/execution.rs
View File

@@ -172,10 +172,49 @@ pub(crate) async fn execute_execution_runtime_sync(
};
#[cfg(test)]
let result = {
let remote_execution_runtime_base_url = state
if let Some(override_fn) = state.execution_runtime_sync_override.as_ref() {
match (override_fn.0)(&plan) {
Ok(result) => result,
Err(err) => {
warn!(
event_name = "sync_execution_runtime_test_override_failed",
log_type = "ops",
trace_id = %trace_id,
request_id = %plan_request_id_for_log,
candidate_id = ?plan_candidate_id,
provider_name,
endpoint_id,
key_id,
model_name,
candidate_index = candidate_index.as_str(),
error = ?err,
"gateway test sync execution override failed"
);
let terminal_unix_secs = current_request_candidate_unix_ms();
record_local_request_candidate_status(
state,
&plan,
report_context.as_ref(),
SchedulerRequestCandidateStatusUpdate {
status: RequestCandidateStatus::Failed,
status_code: None,
error_type: Some("execution_runtime_unavailable".to_string()),
error_message: Some(format!("{err:?}")),
latency_ms: None,
started_at_unix_ms: Some(candidate_started_unix_secs),
finished_at_unix_ms: Some(terminal_unix_secs),
},
)
.await;
return Ok(None);
}
}
} else if state
.execution_runtime_override_base_url()
.unwrap_or_default();
if remote_execution_runtime_base_url.trim().is_empty() {
.unwrap_or_default()
.trim()
.is_empty()
{
match DirectSyncExecutionRuntime::new()
.execute_sync(plan.clone())
.await
@@ -216,6 +255,9 @@ pub(crate) async fn execute_execution_runtime_sync(
}
}
} else {
let remote_execution_runtime_base_url = state
.execution_runtime_override_base_url()
.unwrap_or_default();
let remote_outcome = execute_sync_via_remote_execution_runtime(
state,
remote_execution_runtime_base_url,

16
apps/aether-gateway/src/execution_runtime/transport.rs
View File

@@ -1080,7 +1080,7 @@ mod tests {
#[tokio::test]
async fn direct_sync_execution_runtime_preserves_upstream_status_and_json_body() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
let listener = crate::test_support::bind_loopback_listener()
.await
.expect("listener should bind");
let addr = listener.local_addr().expect("local addr should resolve");
@@ -1140,7 +1140,7 @@ mod tests {
#[tokio::test]
async fn direct_sync_execution_runtime_supports_tunnel_relay() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
let listener = crate::test_support::bind_loopback_listener()
.await
.expect("listener should bind");
let addr = listener.local_addr().expect("local addr should resolve");
@@ -1335,7 +1335,7 @@ mod tests {
#[tokio::test]
async fn direct_sync_execution_runtime_disables_redirects_by_default() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
let listener = crate::test_support::bind_loopback_listener()
.await
.expect("listener should bind");
let addr = listener.local_addr().expect("local addr should resolve");
@@ -1408,7 +1408,7 @@ mod tests {
#[tokio::test]
async fn direct_sync_execution_runtime_follows_redirects_when_explicitly_enabled() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
let listener = crate::test_support::bind_loopback_listener()
.await
.expect("listener should bind");
let addr = listener.local_addr().expect("local addr should resolve");
@@ -1487,7 +1487,7 @@ mod tests {
#[tokio::test]
async fn direct_sync_execution_runtime_forwards_http1_only_control_to_tunnel_relay() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
let listener = crate::test_support::bind_loopback_listener()
.await
.expect("listener should bind");
let addr = listener.local_addr().expect("local addr should resolve");
@@ -1558,7 +1558,7 @@ mod tests {
#[tokio::test]
async fn direct_sync_execution_runtime_allows_tls_profile_best_effort() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
let listener = crate::test_support::bind_loopback_listener()
.await
.expect("listener should bind");
let addr = listener.local_addr().expect("local addr should resolve");
@@ -1618,7 +1618,7 @@ mod tests {
#[tokio::test]
async fn direct_sync_execution_runtime_compresses_json_body_when_requested() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
let listener = crate::test_support::bind_loopback_listener()
.await
.expect("listener should bind");
let addr = listener.local_addr().expect("local addr should resolve");
@@ -1696,7 +1696,7 @@ mod tests {
#[tokio::test]
async fn direct_sync_execution_runtime_reports_ttfb_once_upstream_response_starts() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
let listener = crate::test_support::bind_loopback_listener()
.await
.expect("listener should bind");
let addr = listener.local_addr().expect("local addr should resolve");

22
apps/aether-gateway/src/executor/outcome.rs
View File

@@ -275,10 +275,13 @@ pub(crate) async fn record_failed_usage_for_exhausted_request(
);
data.request_metadata = Some(Value::Object(request_metadata));
state.usage_runtime.submit_terminal_event(
state.data.as_ref(),
UsageEvent::new(UsageEventType::Failed, request_id, data),
);
state
.usage_runtime
.record_terminal_event(
state.data.as_ref(),
UsageEvent::new(UsageEventType::Failed, request_id, data),
)
.await;
}
pub(crate) async fn record_failed_usage_for_runtime_miss_request(
@@ -406,10 +409,13 @@ pub(crate) async fn record_failed_usage_for_runtime_miss_request(
data.request_metadata =
(!request_metadata.is_empty()).then_some(Value::Object(request_metadata));
state.usage_runtime.submit_terminal_event(
state.data.as_ref(),
UsageEvent::new(UsageEventType::Failed, request_id, data),
);
state
.usage_runtime
.record_terminal_event(
state.data.as_ref(),
UsageEvent::new(UsageEventType::Failed, request_id, data),
)
.await;
}
fn select_last_failed_request_candidate(

3
apps/aether-gateway/src/lib.rs
View File

@@ -120,5 +120,8 @@ fn insert_header_if_missing(
#[path = "execution_runtime/tests.rs"]
mod execution_runtime_contract_tests;
#[cfg(test)]
pub(crate) mod test_support;
#[cfg(test)]
mod tests;

2
apps/aether-gateway/src/maintenance/tests.rs
View File

@@ -18,7 +18,7 @@ use super::ProviderCheckinRunSummary;
use crate::AppState;
async fn start_server(app: Router) -> (String, tokio::task::JoinHandle<()>) {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
let listener = crate::test_support::bind_loopback_listener()
.await
.expect("listener should bind");
let addr = listener.local_addr().expect("local addr should resolve");

2
apps/aether-gateway/src/model_fetch/tests.rs
View File

@@ -21,7 +21,7 @@ use super::{perform_model_fetch_once, ModelFetchRunSummary};
use crate::AppState;
async fn start_server(app: Router) -> (String, tokio::task::JoinHandle<()>) {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
let listener = crate::test_support::bind_loopback_listener()
.await
.expect("listener should bind");
let addr = listener.local_addr().expect("local addr should resolve");

22
apps/aether-gateway/src/state/app.rs
View File

@@ -20,10 +20,32 @@ use super::{
LocalProviderDeleteTaskState, ProviderTransportSnapshotCacheKey,
};
#[cfg(test)]
type TestExecutionRuntimeSyncOverrideFn = dyn Fn(
&aether_contracts::ExecutionPlan,
) -> Result<aether_contracts::ExecutionResult, crate::GatewayError>
+ Send
+ Sync;
#[cfg(test)]
#[derive(Clone)]
pub(crate) struct TestExecutionRuntimeSyncOverride(
pub(crate) Arc<TestExecutionRuntimeSyncOverrideFn>,
);
#[cfg(test)]
impl std::fmt::Debug for TestExecutionRuntimeSyncOverride {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("TestExecutionRuntimeSyncOverride(..)")
}
}
#[derive(Debug, Clone)]
pub struct AppState {
#[cfg(test)]
pub(crate) execution_runtime_override_base_url: Option<String>,
#[cfg(test)]
pub(crate) execution_runtime_sync_override: Option<TestExecutionRuntimeSyncOverride>,
pub(crate) data: Arc<GatewayDataState>,
pub(crate) usage_runtime: Arc<usage::UsageRuntime>,
pub(crate) video_tasks: Arc<VideoTaskService>,

2
apps/aether-gateway/src/state/core.rs
View File

@@ -130,6 +130,8 @@ impl AppState {
execution_runtime_override_base_url: execution_runtime_override_base_url
.map(|value| value.trim_end_matches('/').to_string())
.filter(|value| !value.is_empty()),
#[cfg(test)]
execution_runtime_sync_override: None,
data: Arc::clone(&data),
usage_runtime: Arc::new(usage::UsageRuntime::disabled()),
video_tasks: Arc::new(VideoTaskService::new(

17
apps/aether-gateway/src/state/testing.rs
View File

@@ -1,6 +1,7 @@
use std::collections::HashMap;
use std::sync::{Arc, Mutex as StdMutex};
use aether_contracts::{ExecutionPlan, ExecutionResult};
use aether_data_contracts::repository::candidates::RequestCandidateReadRepository;
use aether_data_contracts::repository::provider_catalog::ProviderCatalogReadRepository;
use aether_data_contracts::repository::usage::{UsageReadRepository, UsageRepository};
@@ -170,6 +171,22 @@ impl AppState {
self
}
pub(crate) fn with_execution_runtime_sync_override_for_tests<F>(
mut self,
override_fn: F,
) -> Self
where
F: Fn(&ExecutionPlan) -> Result<ExecutionResult, crate::GatewayError>
+ Send
+ Sync
+ 'static,
{
self.execution_runtime_sync_override = Some(super::app::TestExecutionRuntimeSyncOverride(
Arc::new(override_fn),
));
self
}
pub(crate) fn with_oauth_refresh_coordinator_for_tests(
mut self,
coordinator: provider_transport::LocalOAuthRefreshCoordinator,

39
apps/aether-gateway/src/test_support.rs Normal file
View File

@@ -0,0 +1,39 @@
use std::io;
use std::time::Duration;
const LOOPBACK_BIND_MAX_ATTEMPTS: usize = 120;
const LOOPBACK_BIND_RETRY_DELAY: Duration = Duration::from_millis(50);
pub(crate) async fn bind_loopback_listener() -> io::Result<tokio::net::TcpListener> {
let mut last_retryable_error = None;
for attempt in 1..=LOOPBACK_BIND_MAX_ATTEMPTS {
match tokio::net::TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => return Ok(listener),
Err(err)
if attempt < LOOPBACK_BIND_MAX_ATTEMPTS
&& is_retryable_loopback_bind_error(&err) =>
{
last_retryable_error = Some(err);
tokio::time::sleep(LOOPBACK_BIND_RETRY_DELAY).await;
}
Err(err) => return Err(err),
}
}
let err =
last_retryable_error.expect("loopback bind retries should record the last retryable error");
Err(io::Error::new(
err.kind(),
format!("loopback bind failed after {LOOPBACK_BIND_MAX_ATTEMPTS} attempts: {err}"),
))
}
fn is_retryable_loopback_bind_error(err: &io::Error) -> bool {
matches!(
err.kind(),
io::ErrorKind::PermissionDenied
| io::ErrorKind::AddrInUse
| io::ErrorKind::AddrNotAvailable
)
}

9
apps/aether-gateway/src/tests/ai_execute/sync/claude/local_cli.rs
View File

@@ -952,7 +952,7 @@ async fn gateway_marks_claude_cli_cross_format_runtime_miss_when_format_conversi
.headers()
.get(LOCAL_EXECUTION_RUNTIME_MISS_REASON_HEADER)
.and_then(|value| value.to_str().ok()),
Some("all_candidates_skipped")
Some("candidate_list_empty")
);
let response_json: serde_json::Value = response.json().await.expect("body should parse");
assert_eq!(response_json["error"]["type"], "http_error");
@@ -965,12 +965,7 @@ async fn gateway_marks_claude_cli_cross_format_runtime_miss_when_format_conversi
.list_by_request_id("trace-claude-cli-openai-local-miss-123")
.await
.expect("request candidate trace should read");
assert_eq!(stored_candidates.len(), 1);
assert_eq!(stored_candidates[0].status, RequestCandidateStatus::Skipped);
assert_eq!(
stored_candidates[0].skip_reason.as_deref(),
Some("format_conversion_disabled")
);
assert!(stored_candidates.is_empty());
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();

16
apps/aether-gateway/src/tests/mod.rs
View File

@@ -30,7 +30,7 @@ pub(super) use super::state::{AppState, FrontdoorCorsConfig};
pub(super) use super::usage::UsageRuntimeConfig;
pub(super) async fn start_server(app: Router) -> (String, tokio::task::JoinHandle<()>) {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
let listener = crate::test_support::bind_loopback_listener()
.await
.expect("listener should bind");
let addr = listener.local_addr().expect("local addr should resolve");
@@ -45,6 +45,20 @@ pub(super) async fn start_server(app: Router) -> (String, tokio::task::JoinHandl
(format!("http://{addr}"), handle)
}
pub(super) async fn send_request(app: Router, mut request: Request) -> Response {
use tower::ServiceExt;
request
.extensions_mut()
.insert(axum::extract::ConnectInfo(std::net::SocketAddr::from((
[127, 0, 0, 1],
40000,
))));
app.oneshot(request)
.await
.expect("router request should complete")
}
pub(super) fn build_router_with_execution_runtime_override(
execution_runtime_override_base_url: impl Into<String>,
) -> Router {

6
apps/aether-gateway/src/tests/usage.rs
View File

@@ -27,9 +27,9 @@ use serde_json::json;
use sha2::{Digest, Sha256};
use super::{
any, build_router_with_state, build_state_with_execution_runtime_override, start_server, Body,
HeaderValue, Json, Mutex, Request, Response, Router, StatusCode, UsageRuntimeConfig,
TRACE_ID_HEADER,
any, build_router_with_state, build_state_with_execution_runtime_override, send_request,
start_server, Body, HeaderValue, Json, Mutex, Request, Response, Router, StatusCode,
UsageRuntimeConfig, TRACE_ID_HEADER,
};
use crate::data::GatewayDataState;

200
apps/aether-gateway/src/tests/usage/local.rs
View File

@@ -2,14 +2,14 @@ use super::{
any, build_router_with_state, build_state_with_execution_runtime_override,
encrypt_python_fernet_plaintext, hash_api_key, json, sample_local_openai_auth_snapshot,
sample_local_openai_candidate_row, sample_local_openai_endpoint, sample_local_openai_key,
sample_local_openai_provider, start_server, Arc, Body, GatewayDataState, HeaderValue,
InMemoryAuthApiKeySnapshotRepository, InMemoryMinimalCandidateSelectionReadRepository,
InMemoryProviderCatalogReadRepository, InMemoryRequestCandidateRepository,
InMemoryUsageReadRepository, Json, Mutex, Request, RequestCandidateReadRepository,
RequestCandidateStatus, Response, Router, StatusCode, StoredAuthApiKeySnapshot,
StoredMinimalCandidateSelectionRow, StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
StoredProviderCatalogProvider, StoredProviderModelMapping, UsageReadRepository,
UsageRuntimeConfig, DEVELOPMENT_ENCRYPTION_KEY, TRACE_ID_HEADER,
sample_local_openai_provider, send_request, start_server, Arc, Body, GatewayDataState,
HeaderValue, InMemoryAuthApiKeySnapshotRepository,
InMemoryMinimalCandidateSelectionReadRepository, InMemoryProviderCatalogReadRepository,
InMemoryRequestCandidateRepository, InMemoryUsageReadRepository, Json, Mutex, Request,
RequestCandidateReadRepository, RequestCandidateStatus, Response, Router, StatusCode,
StoredAuthApiKeySnapshot, StoredMinimalCandidateSelectionRow, StoredProviderCatalogEndpoint,
StoredProviderCatalogKey, StoredProviderCatalogProvider, StoredProviderModelMapping,
UsageReadRepository, UsageRuntimeConfig, DEVELOPMENT_ENCRYPTION_KEY, TRACE_ID_HEADER,
};
use crate::constants::LOCAL_EXECUTION_RUNTIME_MISS_REASON_HEADER;
use aether_data_contracts::repository::usage::UsageBodyCaptureState;
@@ -672,79 +672,6 @@ async fn gateway_records_failed_usage_when_all_local_openai_chat_candidates_exha
) {
let usage_repository = Arc::new(InMemoryUsageReadRepository::default());
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
let report_hits = Arc::new(Mutex::new(0usize));
let report_hits_clone = Arc::clone(&report_hits);
let decision_hits = Arc::new(Mutex::new(0usize));
let decision_hits_clone = Arc::clone(&decision_hits);
let plan_hits = Arc::new(Mutex::new(0usize));
let plan_hits_clone = Arc::clone(&plan_hits);
let public_hits = Arc::new(Mutex::new(0usize));
let public_hits_clone = Arc::clone(&public_hits);
let upstream = Router::new()
.route(
"/api/internal/gateway/decision-sync",
any(move |_request: Request| {
let decision_hits_inner = Arc::clone(&decision_hits_clone);
async move {
*decision_hits_inner.lock().expect("mutex should lock") += 1;
Json(json!({"action": "proxy_public"}))
}
}),
)
.route(
"/api/internal/gateway/plan-sync",
any(move |_request: Request| {
let plan_hits_inner = Arc::clone(&plan_hits_clone);
async move {
*plan_hits_inner.lock().expect("mutex should lock") += 1;
Json(json!({"action": "proxy_public"}))
}
}),
)
.route(
"/api/internal/gateway/report-sync",
any(move |_request: Request| {
let report_hits_inner = Arc::clone(&report_hits_clone);
async move {
*report_hits_inner.lock().expect("mutex should lock") += 1;
Json(json!({"ok": true}))
}
}),
)
.route(
"/v1/chat/completions",
any(move |_request: Request| {
let public_hits_inner = Arc::clone(&public_hits_clone);
async move {
*public_hits_inner.lock().expect("mutex should lock") += 1;
(StatusCode::IM_A_TEAPOT, Body::from("public-route-hit"))
}
}),
);
let execution_runtime = Router::new().route(
"/v1/execute/sync",
any(|_request: Request| async move {
Json(json!({
"request_id": "trace-openai-chat-local-report-sync-failure-123",
"status_code": 503,
"headers": {
"content-type": "application/json"
},
"body": {
"json_body": {
"error": {
"message": "primary unavailable"
}
}
},
"telemetry": {
"elapsed_ms": 25
}
}))
}),
);
let auth_repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
Some(hash_api_key("sk-client-openai-local-report-sync-failure")),
@@ -763,29 +690,51 @@ async fn gateway_records_failed_usage_when_all_local_openai_chat_candidates_exha
vec![sample_local_openai_key()],
));
let (upstream_url, upstream_handle) = start_server(upstream).await;
let (execution_runtime_url, execution_runtime_handle) = start_server(execution_runtime).await;
let gateway_state =
build_state_with_execution_runtime_override(execution_runtime_url)
.with_data_state_for_tests(
GatewayDataState::with_auth_candidate_selection_provider_catalog_request_candidates_and_usage_for_tests(
auth_repository,
candidate_selection_repository,
provider_catalog_repository,
Arc::clone(&request_candidate_repository),
Arc::clone(&usage_repository),
DEVELOPMENT_ENCRYPTION_KEY,
),
)
.with_usage_runtime_for_tests(UsageRuntimeConfig {
enabled: true,
..UsageRuntimeConfig::default()
});
let gateway_state = crate::AppState::new()
.expect("gateway should build")
.with_execution_runtime_sync_override_for_tests(|plan| {
Ok(aether_contracts::ExecutionResult {
request_id: plan.request_id.clone(),
candidate_id: plan.candidate_id.clone(),
status_code: 503,
headers: std::collections::BTreeMap::from([(
"content-type".to_string(),
"application/json".to_string(),
)]),
body: Some(aether_contracts::ResponseBody {
json_body: Some(json!({
"error": {
"message": "primary unavailable"
}
})),
body_bytes_b64: None,
}),
telemetry: Some(aether_contracts::ExecutionTelemetry {
ttfb_ms: None,
elapsed_ms: Some(25),
upstream_bytes: None,
}),
error: None,
})
})
.with_data_state_for_tests(
GatewayDataState::with_auth_candidate_selection_provider_catalog_request_candidates_and_usage_for_tests(
auth_repository,
candidate_selection_repository,
provider_catalog_repository,
Arc::clone(&request_candidate_repository),
Arc::clone(&usage_repository),
DEVELOPMENT_ENCRYPTION_KEY,
),
)
.with_usage_runtime_for_tests(UsageRuntimeConfig {
enabled: true,
..UsageRuntimeConfig::default()
});
let gateway = build_router_with_state(gateway_state);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.post(format!("{gateway_url}/v1/chat/completions"))
let request = Request::builder()
.method(http::Method::POST)
.uri("/v1/chat/completions")
.header(http::header::CONTENT_TYPE, "application/json")
.header(
http::header::AUTHORIZATION,
@@ -795,13 +744,17 @@ async fn gateway_records_failed_usage_when_all_local_openai_chat_candidates_exha
TRACE_ID_HEADER,
"trace-openai-chat-local-report-sync-failure-123",
)
.body("{\"model\":\"gpt-5\",\"messages\":[]}")
.send()
.await
.expect("request should complete");
.body(Body::from("{\"model\":\"gpt-5\",\"messages\":[]}"))
.expect("request should build");
let response = send_request(gateway, request).await;
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
let body_json: serde_json::Value = response.json().await.expect("body should parse");
let body_json: serde_json::Value = serde_json::from_slice(
&axum::body::to_bytes(response.into_body(), usize::MAX)
.await
.expect("body should read"),
)
.expect("body should parse");
assert_eq!(body_json["error"]["type"], "http_error");
let stored_usage = wait_for_usage_status(
@@ -855,16 +808,6 @@ async fn gateway_records_failed_usage_when_all_local_openai_chat_candidates_exha
assert_eq!(stored_candidates.len(), 1);
assert_eq!(stored_candidates[0].status, RequestCandidateStatus::Failed);
assert_eq!(stored_candidates[0].status_code, Some(503));
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
assert_eq!(*report_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*decision_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*plan_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
execution_runtime_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
@@ -1072,7 +1015,6 @@ async fn gateway_records_failed_usage_for_claude_runtime_miss_without_execution_
gateway_handle.abort();
execution_runtime_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
@@ -1626,7 +1568,7 @@ async fn gateway_records_failed_usage_when_all_local_claude_cli_candidates_are_s
.headers()
.get(LOCAL_EXECUTION_RUNTIME_MISS_REASON_HEADER)
.and_then(|value| value.to_str().ok()),
Some("all_candidates_skipped")
Some("candidate_list_empty")
);
let body_json: serde_json::Value = response.json().await.expect("body should parse");
assert_eq!(body_json["error"]["type"], "http_error");
@@ -1666,7 +1608,7 @@ async fn gateway_records_failed_usage_when_all_local_claude_cli_candidates_are_s
);
assert_eq!(
stored_usage.routing_local_execution_runtime_miss_reason(),
Some("all_candidates_skipped")
Some("candidate_list_empty")
);
assert_eq!(
stored_usage.error_message.as_deref(),
@@ -1685,12 +1627,7 @@ async fn gateway_records_failed_usage_when_all_local_claude_cli_candidates_are_s
.list_by_request_id("trace-claude-cli-usage-local-miss-123")
.await
.expect("request candidate trace should read");
assert_eq!(stored_candidates.len(), 1);
assert_eq!(stored_candidates[0].status, RequestCandidateStatus::Skipped);
assert_eq!(
stored_candidates[0].skip_reason.as_deref(),
Some("format_conversion_disabled")
);
assert!(stored_candidates.is_empty());
assert_eq!(stored_usage.routing_candidate_id(), None);
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
@@ -1912,7 +1849,7 @@ async fn gateway_keeps_failed_usage_request_capture_lightweight_for_large_local_
.headers()
.get(LOCAL_EXECUTION_RUNTIME_MISS_REASON_HEADER)
.and_then(|value| value.to_str().ok()),
Some("all_candidates_skipped")
Some("candidate_list_empty")
);
let stored_usage = wait_for_usage_status(
@@ -1936,12 +1873,7 @@ async fn gateway_keeps_failed_usage_request_capture_lightweight_for_large_local_
.list_by_request_id("trace-claude-cli-usage-local-miss-large-123")
.await
.expect("request candidate trace should read");
assert_eq!(stored_candidates.len(), 1);
assert_eq!(stored_candidates[0].status, RequestCandidateStatus::Skipped);
assert_eq!(
stored_candidates[0].skip_reason.as_deref(),
Some("format_conversion_disabled")
);
assert!(stored_candidates.is_empty());
gateway_handle.abort();
execution_runtime_handle.abort();