mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 01:10:23 +08:00
refactor(gateway): 重构格式转换候选资格检查并优化测试基础设施
- 将 format_conversion_disabled 的过滤提前到候选遴选阶段,替代原来的 skip_reason 标记机制 - 为测试添加 execution_runtime_sync_override 直接注入支持,避免启动额外 HTTP 服务器 - 将 submit_terminal_event 改为 async record_terminal_event 确保失败用量事件可靠入库 - 新增 test_support 模块封装可重试的 loopback 端口绑定逻辑 - 前端:poolTrace 将 skipped 从隐藏状态移除,新增 buildPoolParticipatedCandidates 统一新旧链路逻辑,并将 skipped 状态色改为 foreground
This commit is contained in:
@@ -1194,9 +1194,7 @@ mod tests {
|
|||||||
|
|
||||||
assert_eq!(ranked.len(), 1);
|
assert_eq!(ranked.len(), 1);
|
||||||
assert_eq!(ranked[0].candidate.endpoint_id, "endpoint-same");
|
assert_eq!(ranked[0].candidate.endpoint_id, "endpoint-same");
|
||||||
assert_eq!(skipped.len(), 1);
|
assert!(skipped.is_empty());
|
||||||
assert_eq!(skipped[0].candidate.endpoint_id, "endpoint-cross");
|
|
||||||
assert_eq!(skipped[0].skip_reason, "format_conversion_disabled");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
|
|
||||||
use aether_scheduler_core::SchedulerMinimalCandidateSelectionCandidate;
|
use aether_scheduler_core::SchedulerMinimalCandidateSelectionCandidate;
|
||||||
use std::collections::BTreeSet;
|
|
||||||
|
|
||||||
use crate::ai_pipeline::{GatewayProviderTransportSnapshot, PlannerAppState};
|
use crate::ai_pipeline::{GatewayProviderTransportSnapshot, PlannerAppState};
|
||||||
use crate::orchestration::LocalExecutionCandidateMetadata;
|
use crate::orchestration::LocalExecutionCandidateMetadata;
|
||||||
@@ -101,7 +100,6 @@ where
|
|||||||
{
|
{
|
||||||
let mut selectable = Vec::with_capacity(candidates.len());
|
let mut selectable = Vec::with_capacity(candidates.len());
|
||||||
let mut skipped = Vec::new();
|
let mut skipped = Vec::new();
|
||||||
let normalized_client_api_format = client_api_format.trim().to_ascii_lowercase();
|
|
||||||
|
|
||||||
for candidate in candidates {
|
for candidate in candidates {
|
||||||
let Some(transport) = read_candidate_transport_snapshot(state, &candidate).await else {
|
let Some(transport) = read_candidate_transport_snapshot(state, &candidate).await else {
|
||||||
@@ -113,6 +111,10 @@ where
|
|||||||
});
|
});
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
|
if candidate_is_ineligible_due_to_disabled_format_conversion(&transport, client_api_format)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
match runtime_skip_reason(&candidate, &transport) {
|
match runtime_skip_reason(&candidate, &transport) {
|
||||||
Some(skip_reason) => skipped.push(SkippedLocalExecutionCandidate {
|
Some(skip_reason) => skipped.push(SkippedLocalExecutionCandidate {
|
||||||
candidate,
|
candidate,
|
||||||
@@ -129,28 +131,6 @@ where
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let exact_selectable_keys = selectable
|
|
||||||
.iter()
|
|
||||||
.filter(|candidate| {
|
|
||||||
candidate
|
|
||||||
.provider_api_format
|
|
||||||
.eq_ignore_ascii_case(normalized_client_api_format.as_str())
|
|
||||||
})
|
|
||||||
.map(|candidate| {
|
|
||||||
(
|
|
||||||
candidate.candidate.provider_id.clone(),
|
|
||||||
candidate.candidate.key_id.clone(),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.collect::<BTreeSet<_>>();
|
|
||||||
skipped.retain(|candidate| {
|
|
||||||
candidate.skip_reason != "format_conversion_disabled"
|
|
||||||
|| !exact_selectable_keys.contains(&(
|
|
||||||
candidate.candidate.provider_id.clone(),
|
|
||||||
candidate.candidate.key_id.clone(),
|
|
||||||
))
|
|
||||||
});
|
|
||||||
|
|
||||||
let ranked = rank_eligible_local_execution_candidates(
|
let ranked = rank_eligible_local_execution_candidates(
|
||||||
state,
|
state,
|
||||||
selectable,
|
selectable,
|
||||||
@@ -234,6 +214,32 @@ fn current_local_execution_candidate_common_skip_reason_with_transport(
|
|||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn candidate_is_ineligible_due_to_disabled_format_conversion(
|
||||||
|
transport: &GatewayProviderTransportSnapshot,
|
||||||
|
client_api_format: &str,
|
||||||
|
) -> bool {
|
||||||
|
let endpoint_api_format = transport.endpoint.api_format.trim().to_ascii_lowercase();
|
||||||
|
let client_api_format = client_api_format.trim().to_ascii_lowercase();
|
||||||
|
if client_api_format == endpoint_api_format {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
crate::ai_pipeline::conversion::request_conversion_kind(
|
||||||
|
client_api_format.as_str(),
|
||||||
|
endpoint_api_format.as_str(),
|
||||||
|
)
|
||||||
|
.is_some()
|
||||||
|
&& crate::ai_pipeline::conversion::request_conversion_requires_enable_flag(
|
||||||
|
client_api_format.as_str(),
|
||||||
|
endpoint_api_format.as_str(),
|
||||||
|
)
|
||||||
|
&& !crate::ai_pipeline::conversion::request_conversion_enabled_for_transport(
|
||||||
|
transport,
|
||||||
|
client_api_format.as_str(),
|
||||||
|
endpoint_api_format.as_str(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
fn current_local_execution_candidate_skip_reason_with_transport(
|
fn current_local_execution_candidate_skip_reason_with_transport(
|
||||||
candidate: &SchedulerMinimalCandidateSelectionCandidate,
|
candidate: &SchedulerMinimalCandidateSelectionCandidate,
|
||||||
transport: &GatewayProviderTransportSnapshot,
|
transport: &GatewayProviderTransportSnapshot,
|
||||||
@@ -259,25 +265,7 @@ fn current_local_execution_candidate_skip_reason_with_transport(
|
|||||||
client_api_format.as_str(),
|
client_api_format.as_str(),
|
||||||
endpoint_api_format.as_str(),
|
endpoint_api_format.as_str(),
|
||||||
) {
|
) {
|
||||||
let skip_reason = if crate::ai_pipeline::conversion::request_conversion_kind(
|
return Some("transport_unsupported");
|
||||||
client_api_format.as_str(),
|
|
||||||
endpoint_api_format.as_str(),
|
|
||||||
)
|
|
||||||
.is_some()
|
|
||||||
&& crate::ai_pipeline::conversion::request_conversion_requires_enable_flag(
|
|
||||||
client_api_format.as_str(),
|
|
||||||
endpoint_api_format.as_str(),
|
|
||||||
)
|
|
||||||
&& !crate::ai_pipeline::conversion::request_conversion_enabled_for_transport(
|
|
||||||
transport,
|
|
||||||
client_api_format.as_str(),
|
|
||||||
endpoint_api_format.as_str(),
|
|
||||||
) {
|
|
||||||
"format_conversion_disabled"
|
|
||||||
} else {
|
|
||||||
"transport_unsupported"
|
|
||||||
};
|
|
||||||
return Some(skip_reason);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
None
|
None
|
||||||
|
|||||||
@@ -386,7 +386,7 @@ mod tests {
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
async fn start_server(app: Router) -> (String, tokio::task::JoinHandle<()>) {
|
async fn start_server(app: Router) -> (String, tokio::task::JoinHandle<()>) {
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
let listener = crate::test_support::bind_loopback_listener()
|
||||||
.await
|
.await
|
||||||
.expect("listener should bind");
|
.expect("listener should bind");
|
||||||
let addr = listener.local_addr().expect("local addr should resolve");
|
let addr = listener.local_addr().expect("local addr should resolve");
|
||||||
|
|||||||
@@ -385,7 +385,7 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn direct_execution_frame_stream_reports_ttfb_after_first_upstream_chunk() {
|
async fn direct_execution_frame_stream_reports_ttfb_after_first_upstream_chunk() {
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
let listener = crate::test_support::bind_loopback_listener()
|
||||||
.await
|
.await
|
||||||
.expect("listener should bind");
|
.expect("listener should bind");
|
||||||
let addr = listener.local_addr().expect("local addr should resolve");
|
let addr = listener.local_addr().expect("local addr should resolve");
|
||||||
@@ -471,7 +471,7 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn direct_execution_frame_stream_emits_telemetry_before_first_data_frame() {
|
async fn direct_execution_frame_stream_emits_telemetry_before_first_data_frame() {
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
let listener = crate::test_support::bind_loopback_listener()
|
||||||
.await
|
.await
|
||||||
.expect("listener should bind");
|
.expect("listener should bind");
|
||||||
let addr = listener.local_addr().expect("local addr should resolve");
|
let addr = listener.local_addr().expect("local addr should resolve");
|
||||||
|
|||||||
@@ -172,10 +172,49 @@ pub(crate) async fn execute_execution_runtime_sync(
|
|||||||
};
|
};
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
let result = {
|
let result = {
|
||||||
let remote_execution_runtime_base_url = state
|
if let Some(override_fn) = state.execution_runtime_sync_override.as_ref() {
|
||||||
|
match (override_fn.0)(&plan) {
|
||||||
|
Ok(result) => result,
|
||||||
|
Err(err) => {
|
||||||
|
warn!(
|
||||||
|
event_name = "sync_execution_runtime_test_override_failed",
|
||||||
|
log_type = "ops",
|
||||||
|
trace_id = %trace_id,
|
||||||
|
request_id = %plan_request_id_for_log,
|
||||||
|
candidate_id = ?plan_candidate_id,
|
||||||
|
provider_name,
|
||||||
|
endpoint_id,
|
||||||
|
key_id,
|
||||||
|
model_name,
|
||||||
|
candidate_index = candidate_index.as_str(),
|
||||||
|
error = ?err,
|
||||||
|
"gateway test sync execution override failed"
|
||||||
|
);
|
||||||
|
let terminal_unix_secs = current_request_candidate_unix_ms();
|
||||||
|
record_local_request_candidate_status(
|
||||||
|
state,
|
||||||
|
&plan,
|
||||||
|
report_context.as_ref(),
|
||||||
|
SchedulerRequestCandidateStatusUpdate {
|
||||||
|
status: RequestCandidateStatus::Failed,
|
||||||
|
status_code: None,
|
||||||
|
error_type: Some("execution_runtime_unavailable".to_string()),
|
||||||
|
error_message: Some(format!("{err:?}")),
|
||||||
|
latency_ms: None,
|
||||||
|
started_at_unix_ms: Some(candidate_started_unix_secs),
|
||||||
|
finished_at_unix_ms: Some(terminal_unix_secs),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if state
|
||||||
.execution_runtime_override_base_url()
|
.execution_runtime_override_base_url()
|
||||||
.unwrap_or_default();
|
.unwrap_or_default()
|
||||||
if remote_execution_runtime_base_url.trim().is_empty() {
|
.trim()
|
||||||
|
.is_empty()
|
||||||
|
{
|
||||||
match DirectSyncExecutionRuntime::new()
|
match DirectSyncExecutionRuntime::new()
|
||||||
.execute_sync(plan.clone())
|
.execute_sync(plan.clone())
|
||||||
.await
|
.await
|
||||||
@@ -216,6 +255,9 @@ pub(crate) async fn execute_execution_runtime_sync(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
let remote_execution_runtime_base_url = state
|
||||||
|
.execution_runtime_override_base_url()
|
||||||
|
.unwrap_or_default();
|
||||||
let remote_outcome = execute_sync_via_remote_execution_runtime(
|
let remote_outcome = execute_sync_via_remote_execution_runtime(
|
||||||
state,
|
state,
|
||||||
remote_execution_runtime_base_url,
|
remote_execution_runtime_base_url,
|
||||||
|
|||||||
@@ -1080,7 +1080,7 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn direct_sync_execution_runtime_preserves_upstream_status_and_json_body() {
|
async fn direct_sync_execution_runtime_preserves_upstream_status_and_json_body() {
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
let listener = crate::test_support::bind_loopback_listener()
|
||||||
.await
|
.await
|
||||||
.expect("listener should bind");
|
.expect("listener should bind");
|
||||||
let addr = listener.local_addr().expect("local addr should resolve");
|
let addr = listener.local_addr().expect("local addr should resolve");
|
||||||
@@ -1140,7 +1140,7 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn direct_sync_execution_runtime_supports_tunnel_relay() {
|
async fn direct_sync_execution_runtime_supports_tunnel_relay() {
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
let listener = crate::test_support::bind_loopback_listener()
|
||||||
.await
|
.await
|
||||||
.expect("listener should bind");
|
.expect("listener should bind");
|
||||||
let addr = listener.local_addr().expect("local addr should resolve");
|
let addr = listener.local_addr().expect("local addr should resolve");
|
||||||
@@ -1335,7 +1335,7 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn direct_sync_execution_runtime_disables_redirects_by_default() {
|
async fn direct_sync_execution_runtime_disables_redirects_by_default() {
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
let listener = crate::test_support::bind_loopback_listener()
|
||||||
.await
|
.await
|
||||||
.expect("listener should bind");
|
.expect("listener should bind");
|
||||||
let addr = listener.local_addr().expect("local addr should resolve");
|
let addr = listener.local_addr().expect("local addr should resolve");
|
||||||
@@ -1408,7 +1408,7 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn direct_sync_execution_runtime_follows_redirects_when_explicitly_enabled() {
|
async fn direct_sync_execution_runtime_follows_redirects_when_explicitly_enabled() {
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
let listener = crate::test_support::bind_loopback_listener()
|
||||||
.await
|
.await
|
||||||
.expect("listener should bind");
|
.expect("listener should bind");
|
||||||
let addr = listener.local_addr().expect("local addr should resolve");
|
let addr = listener.local_addr().expect("local addr should resolve");
|
||||||
@@ -1487,7 +1487,7 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn direct_sync_execution_runtime_forwards_http1_only_control_to_tunnel_relay() {
|
async fn direct_sync_execution_runtime_forwards_http1_only_control_to_tunnel_relay() {
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
let listener = crate::test_support::bind_loopback_listener()
|
||||||
.await
|
.await
|
||||||
.expect("listener should bind");
|
.expect("listener should bind");
|
||||||
let addr = listener.local_addr().expect("local addr should resolve");
|
let addr = listener.local_addr().expect("local addr should resolve");
|
||||||
@@ -1558,7 +1558,7 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn direct_sync_execution_runtime_allows_tls_profile_best_effort() {
|
async fn direct_sync_execution_runtime_allows_tls_profile_best_effort() {
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
let listener = crate::test_support::bind_loopback_listener()
|
||||||
.await
|
.await
|
||||||
.expect("listener should bind");
|
.expect("listener should bind");
|
||||||
let addr = listener.local_addr().expect("local addr should resolve");
|
let addr = listener.local_addr().expect("local addr should resolve");
|
||||||
@@ -1618,7 +1618,7 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn direct_sync_execution_runtime_compresses_json_body_when_requested() {
|
async fn direct_sync_execution_runtime_compresses_json_body_when_requested() {
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
let listener = crate::test_support::bind_loopback_listener()
|
||||||
.await
|
.await
|
||||||
.expect("listener should bind");
|
.expect("listener should bind");
|
||||||
let addr = listener.local_addr().expect("local addr should resolve");
|
let addr = listener.local_addr().expect("local addr should resolve");
|
||||||
@@ -1696,7 +1696,7 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn direct_sync_execution_runtime_reports_ttfb_once_upstream_response_starts() {
|
async fn direct_sync_execution_runtime_reports_ttfb_once_upstream_response_starts() {
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
let listener = crate::test_support::bind_loopback_listener()
|
||||||
.await
|
.await
|
||||||
.expect("listener should bind");
|
.expect("listener should bind");
|
||||||
let addr = listener.local_addr().expect("local addr should resolve");
|
let addr = listener.local_addr().expect("local addr should resolve");
|
||||||
|
|||||||
@@ -275,10 +275,13 @@ pub(crate) async fn record_failed_usage_for_exhausted_request(
|
|||||||
);
|
);
|
||||||
data.request_metadata = Some(Value::Object(request_metadata));
|
data.request_metadata = Some(Value::Object(request_metadata));
|
||||||
|
|
||||||
state.usage_runtime.submit_terminal_event(
|
state
|
||||||
state.data.as_ref(),
|
.usage_runtime
|
||||||
UsageEvent::new(UsageEventType::Failed, request_id, data),
|
.record_terminal_event(
|
||||||
);
|
state.data.as_ref(),
|
||||||
|
UsageEvent::new(UsageEventType::Failed, request_id, data),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn record_failed_usage_for_runtime_miss_request(
|
pub(crate) async fn record_failed_usage_for_runtime_miss_request(
|
||||||
@@ -406,10 +409,13 @@ pub(crate) async fn record_failed_usage_for_runtime_miss_request(
|
|||||||
data.request_metadata =
|
data.request_metadata =
|
||||||
(!request_metadata.is_empty()).then_some(Value::Object(request_metadata));
|
(!request_metadata.is_empty()).then_some(Value::Object(request_metadata));
|
||||||
|
|
||||||
state.usage_runtime.submit_terminal_event(
|
state
|
||||||
state.data.as_ref(),
|
.usage_runtime
|
||||||
UsageEvent::new(UsageEventType::Failed, request_id, data),
|
.record_terminal_event(
|
||||||
);
|
state.data.as_ref(),
|
||||||
|
UsageEvent::new(UsageEventType::Failed, request_id, data),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
fn select_last_failed_request_candidate(
|
fn select_last_failed_request_candidate(
|
||||||
|
|||||||
@@ -120,5 +120,8 @@ fn insert_header_if_missing(
|
|||||||
#[path = "execution_runtime/tests.rs"]
|
#[path = "execution_runtime/tests.rs"]
|
||||||
mod execution_runtime_contract_tests;
|
mod execution_runtime_contract_tests;
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) mod test_support;
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests;
|
mod tests;
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ use super::ProviderCheckinRunSummary;
|
|||||||
use crate::AppState;
|
use crate::AppState;
|
||||||
|
|
||||||
async fn start_server(app: Router) -> (String, tokio::task::JoinHandle<()>) {
|
async fn start_server(app: Router) -> (String, tokio::task::JoinHandle<()>) {
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
let listener = crate::test_support::bind_loopback_listener()
|
||||||
.await
|
.await
|
||||||
.expect("listener should bind");
|
.expect("listener should bind");
|
||||||
let addr = listener.local_addr().expect("local addr should resolve");
|
let addr = listener.local_addr().expect("local addr should resolve");
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ use super::{perform_model_fetch_once, ModelFetchRunSummary};
|
|||||||
use crate::AppState;
|
use crate::AppState;
|
||||||
|
|
||||||
async fn start_server(app: Router) -> (String, tokio::task::JoinHandle<()>) {
|
async fn start_server(app: Router) -> (String, tokio::task::JoinHandle<()>) {
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
let listener = crate::test_support::bind_loopback_listener()
|
||||||
.await
|
.await
|
||||||
.expect("listener should bind");
|
.expect("listener should bind");
|
||||||
let addr = listener.local_addr().expect("local addr should resolve");
|
let addr = listener.local_addr().expect("local addr should resolve");
|
||||||
|
|||||||
@@ -20,10 +20,32 @@ use super::{
|
|||||||
LocalProviderDeleteTaskState, ProviderTransportSnapshotCacheKey,
|
LocalProviderDeleteTaskState, ProviderTransportSnapshotCacheKey,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
type TestExecutionRuntimeSyncOverrideFn = dyn Fn(
|
||||||
|
&aether_contracts::ExecutionPlan,
|
||||||
|
) -> Result<aether_contracts::ExecutionResult, crate::GatewayError>
|
||||||
|
+ Send
|
||||||
|
+ Sync;
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub(crate) struct TestExecutionRuntimeSyncOverride(
|
||||||
|
pub(crate) Arc<TestExecutionRuntimeSyncOverrideFn>,
|
||||||
|
);
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
impl std::fmt::Debug for TestExecutionRuntimeSyncOverride {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
f.write_str("TestExecutionRuntimeSyncOverride(..)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct AppState {
|
pub struct AppState {
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) execution_runtime_override_base_url: Option<String>,
|
pub(crate) execution_runtime_override_base_url: Option<String>,
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) execution_runtime_sync_override: Option<TestExecutionRuntimeSyncOverride>,
|
||||||
pub(crate) data: Arc<GatewayDataState>,
|
pub(crate) data: Arc<GatewayDataState>,
|
||||||
pub(crate) usage_runtime: Arc<usage::UsageRuntime>,
|
pub(crate) usage_runtime: Arc<usage::UsageRuntime>,
|
||||||
pub(crate) video_tasks: Arc<VideoTaskService>,
|
pub(crate) video_tasks: Arc<VideoTaskService>,
|
||||||
|
|||||||
@@ -130,6 +130,8 @@ impl AppState {
|
|||||||
execution_runtime_override_base_url: execution_runtime_override_base_url
|
execution_runtime_override_base_url: execution_runtime_override_base_url
|
||||||
.map(|value| value.trim_end_matches('/').to_string())
|
.map(|value| value.trim_end_matches('/').to_string())
|
||||||
.filter(|value| !value.is_empty()),
|
.filter(|value| !value.is_empty()),
|
||||||
|
#[cfg(test)]
|
||||||
|
execution_runtime_sync_override: None,
|
||||||
data: Arc::clone(&data),
|
data: Arc::clone(&data),
|
||||||
usage_runtime: Arc::new(usage::UsageRuntime::disabled()),
|
usage_runtime: Arc::new(usage::UsageRuntime::disabled()),
|
||||||
video_tasks: Arc::new(VideoTaskService::new(
|
video_tasks: Arc::new(VideoTaskService::new(
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::sync::{Arc, Mutex as StdMutex};
|
use std::sync::{Arc, Mutex as StdMutex};
|
||||||
|
|
||||||
|
use aether_contracts::{ExecutionPlan, ExecutionResult};
|
||||||
use aether_data_contracts::repository::candidates::RequestCandidateReadRepository;
|
use aether_data_contracts::repository::candidates::RequestCandidateReadRepository;
|
||||||
use aether_data_contracts::repository::provider_catalog::ProviderCatalogReadRepository;
|
use aether_data_contracts::repository::provider_catalog::ProviderCatalogReadRepository;
|
||||||
use aether_data_contracts::repository::usage::{UsageReadRepository, UsageRepository};
|
use aether_data_contracts::repository::usage::{UsageReadRepository, UsageRepository};
|
||||||
@@ -170,6 +171,22 @@ impl AppState {
|
|||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn with_execution_runtime_sync_override_for_tests<F>(
|
||||||
|
mut self,
|
||||||
|
override_fn: F,
|
||||||
|
) -> Self
|
||||||
|
where
|
||||||
|
F: Fn(&ExecutionPlan) -> Result<ExecutionResult, crate::GatewayError>
|
||||||
|
+ Send
|
||||||
|
+ Sync
|
||||||
|
+ 'static,
|
||||||
|
{
|
||||||
|
self.execution_runtime_sync_override = Some(super::app::TestExecutionRuntimeSyncOverride(
|
||||||
|
Arc::new(override_fn),
|
||||||
|
));
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn with_oauth_refresh_coordinator_for_tests(
|
pub(crate) fn with_oauth_refresh_coordinator_for_tests(
|
||||||
mut self,
|
mut self,
|
||||||
coordinator: provider_transport::LocalOAuthRefreshCoordinator,
|
coordinator: provider_transport::LocalOAuthRefreshCoordinator,
|
||||||
|
|||||||
39
apps/aether-gateway/src/test_support.rs
Normal file
39
apps/aether-gateway/src/test_support.rs
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
use std::io;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
const LOOPBACK_BIND_MAX_ATTEMPTS: usize = 120;
|
||||||
|
const LOOPBACK_BIND_RETRY_DELAY: Duration = Duration::from_millis(50);
|
||||||
|
|
||||||
|
pub(crate) async fn bind_loopback_listener() -> io::Result<tokio::net::TcpListener> {
|
||||||
|
let mut last_retryable_error = None;
|
||||||
|
|
||||||
|
for attempt in 1..=LOOPBACK_BIND_MAX_ATTEMPTS {
|
||||||
|
match tokio::net::TcpListener::bind("127.0.0.1:0").await {
|
||||||
|
Ok(listener) => return Ok(listener),
|
||||||
|
Err(err)
|
||||||
|
if attempt < LOOPBACK_BIND_MAX_ATTEMPTS
|
||||||
|
&& is_retryable_loopback_bind_error(&err) =>
|
||||||
|
{
|
||||||
|
last_retryable_error = Some(err);
|
||||||
|
tokio::time::sleep(LOOPBACK_BIND_RETRY_DELAY).await;
|
||||||
|
}
|
||||||
|
Err(err) => return Err(err),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let err =
|
||||||
|
last_retryable_error.expect("loopback bind retries should record the last retryable error");
|
||||||
|
Err(io::Error::new(
|
||||||
|
err.kind(),
|
||||||
|
format!("loopback bind failed after {LOOPBACK_BIND_MAX_ATTEMPTS} attempts: {err}"),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_retryable_loopback_bind_error(err: &io::Error) -> bool {
|
||||||
|
matches!(
|
||||||
|
err.kind(),
|
||||||
|
io::ErrorKind::PermissionDenied
|
||||||
|
| io::ErrorKind::AddrInUse
|
||||||
|
| io::ErrorKind::AddrNotAvailable
|
||||||
|
)
|
||||||
|
}
|
||||||
@@ -952,7 +952,7 @@ async fn gateway_marks_claude_cli_cross_format_runtime_miss_when_format_conversi
|
|||||||
.headers()
|
.headers()
|
||||||
.get(LOCAL_EXECUTION_RUNTIME_MISS_REASON_HEADER)
|
.get(LOCAL_EXECUTION_RUNTIME_MISS_REASON_HEADER)
|
||||||
.and_then(|value| value.to_str().ok()),
|
.and_then(|value| value.to_str().ok()),
|
||||||
Some("all_candidates_skipped")
|
Some("candidate_list_empty")
|
||||||
);
|
);
|
||||||
let response_json: serde_json::Value = response.json().await.expect("body should parse");
|
let response_json: serde_json::Value = response.json().await.expect("body should parse");
|
||||||
assert_eq!(response_json["error"]["type"], "http_error");
|
assert_eq!(response_json["error"]["type"], "http_error");
|
||||||
@@ -965,12 +965,7 @@ async fn gateway_marks_claude_cli_cross_format_runtime_miss_when_format_conversi
|
|||||||
.list_by_request_id("trace-claude-cli-openai-local-miss-123")
|
.list_by_request_id("trace-claude-cli-openai-local-miss-123")
|
||||||
.await
|
.await
|
||||||
.expect("request candidate trace should read");
|
.expect("request candidate trace should read");
|
||||||
assert_eq!(stored_candidates.len(), 1);
|
assert!(stored_candidates.is_empty());
|
||||||
assert_eq!(stored_candidates[0].status, RequestCandidateStatus::Skipped);
|
|
||||||
assert_eq!(
|
|
||||||
stored_candidates[0].skip_reason.as_deref(),
|
|
||||||
Some("format_conversion_disabled")
|
|
||||||
);
|
|
||||||
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
|
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
|
||||||
|
|
||||||
gateway_handle.abort();
|
gateway_handle.abort();
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ pub(super) use super::state::{AppState, FrontdoorCorsConfig};
|
|||||||
pub(super) use super::usage::UsageRuntimeConfig;
|
pub(super) use super::usage::UsageRuntimeConfig;
|
||||||
|
|
||||||
pub(super) async fn start_server(app: Router) -> (String, tokio::task::JoinHandle<()>) {
|
pub(super) async fn start_server(app: Router) -> (String, tokio::task::JoinHandle<()>) {
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
let listener = crate::test_support::bind_loopback_listener()
|
||||||
.await
|
.await
|
||||||
.expect("listener should bind");
|
.expect("listener should bind");
|
||||||
let addr = listener.local_addr().expect("local addr should resolve");
|
let addr = listener.local_addr().expect("local addr should resolve");
|
||||||
@@ -45,6 +45,20 @@ pub(super) async fn start_server(app: Router) -> (String, tokio::task::JoinHandl
|
|||||||
(format!("http://{addr}"), handle)
|
(format!("http://{addr}"), handle)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(super) async fn send_request(app: Router, mut request: Request) -> Response {
|
||||||
|
use tower::ServiceExt;
|
||||||
|
|
||||||
|
request
|
||||||
|
.extensions_mut()
|
||||||
|
.insert(axum::extract::ConnectInfo(std::net::SocketAddr::from((
|
||||||
|
[127, 0, 0, 1],
|
||||||
|
40000,
|
||||||
|
))));
|
||||||
|
app.oneshot(request)
|
||||||
|
.await
|
||||||
|
.expect("router request should complete")
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) fn build_router_with_execution_runtime_override(
|
pub(super) fn build_router_with_execution_runtime_override(
|
||||||
execution_runtime_override_base_url: impl Into<String>,
|
execution_runtime_override_base_url: impl Into<String>,
|
||||||
) -> Router {
|
) -> Router {
|
||||||
|
|||||||
@@ -27,9 +27,9 @@ use serde_json::json;
|
|||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
any, build_router_with_state, build_state_with_execution_runtime_override, start_server, Body,
|
any, build_router_with_state, build_state_with_execution_runtime_override, send_request,
|
||||||
HeaderValue, Json, Mutex, Request, Response, Router, StatusCode, UsageRuntimeConfig,
|
start_server, Body, HeaderValue, Json, Mutex, Request, Response, Router, StatusCode,
|
||||||
TRACE_ID_HEADER,
|
UsageRuntimeConfig, TRACE_ID_HEADER,
|
||||||
};
|
};
|
||||||
use crate::data::GatewayDataState;
|
use crate::data::GatewayDataState;
|
||||||
|
|
||||||
|
|||||||
@@ -2,14 +2,14 @@ use super::{
|
|||||||
any, build_router_with_state, build_state_with_execution_runtime_override,
|
any, build_router_with_state, build_state_with_execution_runtime_override,
|
||||||
encrypt_python_fernet_plaintext, hash_api_key, json, sample_local_openai_auth_snapshot,
|
encrypt_python_fernet_plaintext, hash_api_key, json, sample_local_openai_auth_snapshot,
|
||||||
sample_local_openai_candidate_row, sample_local_openai_endpoint, sample_local_openai_key,
|
sample_local_openai_candidate_row, sample_local_openai_endpoint, sample_local_openai_key,
|
||||||
sample_local_openai_provider, start_server, Arc, Body, GatewayDataState, HeaderValue,
|
sample_local_openai_provider, send_request, start_server, Arc, Body, GatewayDataState,
|
||||||
InMemoryAuthApiKeySnapshotRepository, InMemoryMinimalCandidateSelectionReadRepository,
|
HeaderValue, InMemoryAuthApiKeySnapshotRepository,
|
||||||
InMemoryProviderCatalogReadRepository, InMemoryRequestCandidateRepository,
|
InMemoryMinimalCandidateSelectionReadRepository, InMemoryProviderCatalogReadRepository,
|
||||||
InMemoryUsageReadRepository, Json, Mutex, Request, RequestCandidateReadRepository,
|
InMemoryRequestCandidateRepository, InMemoryUsageReadRepository, Json, Mutex, Request,
|
||||||
RequestCandidateStatus, Response, Router, StatusCode, StoredAuthApiKeySnapshot,
|
RequestCandidateReadRepository, RequestCandidateStatus, Response, Router, StatusCode,
|
||||||
StoredMinimalCandidateSelectionRow, StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
|
StoredAuthApiKeySnapshot, StoredMinimalCandidateSelectionRow, StoredProviderCatalogEndpoint,
|
||||||
StoredProviderCatalogProvider, StoredProviderModelMapping, UsageReadRepository,
|
StoredProviderCatalogKey, StoredProviderCatalogProvider, StoredProviderModelMapping,
|
||||||
UsageRuntimeConfig, DEVELOPMENT_ENCRYPTION_KEY, TRACE_ID_HEADER,
|
UsageReadRepository, UsageRuntimeConfig, DEVELOPMENT_ENCRYPTION_KEY, TRACE_ID_HEADER,
|
||||||
};
|
};
|
||||||
use crate::constants::LOCAL_EXECUTION_RUNTIME_MISS_REASON_HEADER;
|
use crate::constants::LOCAL_EXECUTION_RUNTIME_MISS_REASON_HEADER;
|
||||||
use aether_data_contracts::repository::usage::UsageBodyCaptureState;
|
use aether_data_contracts::repository::usage::UsageBodyCaptureState;
|
||||||
@@ -672,79 +672,6 @@ async fn gateway_records_failed_usage_when_all_local_openai_chat_candidates_exha
|
|||||||
) {
|
) {
|
||||||
let usage_repository = Arc::new(InMemoryUsageReadRepository::default());
|
let usage_repository = Arc::new(InMemoryUsageReadRepository::default());
|
||||||
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
|
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
|
||||||
let report_hits = Arc::new(Mutex::new(0usize));
|
|
||||||
let report_hits_clone = Arc::clone(&report_hits);
|
|
||||||
let decision_hits = Arc::new(Mutex::new(0usize));
|
|
||||||
let decision_hits_clone = Arc::clone(&decision_hits);
|
|
||||||
let plan_hits = Arc::new(Mutex::new(0usize));
|
|
||||||
let plan_hits_clone = Arc::clone(&plan_hits);
|
|
||||||
let public_hits = Arc::new(Mutex::new(0usize));
|
|
||||||
let public_hits_clone = Arc::clone(&public_hits);
|
|
||||||
|
|
||||||
let upstream = Router::new()
|
|
||||||
.route(
|
|
||||||
"/api/internal/gateway/decision-sync",
|
|
||||||
any(move |_request: Request| {
|
|
||||||
let decision_hits_inner = Arc::clone(&decision_hits_clone);
|
|
||||||
async move {
|
|
||||||
*decision_hits_inner.lock().expect("mutex should lock") += 1;
|
|
||||||
Json(json!({"action": "proxy_public"}))
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
.route(
|
|
||||||
"/api/internal/gateway/plan-sync",
|
|
||||||
any(move |_request: Request| {
|
|
||||||
let plan_hits_inner = Arc::clone(&plan_hits_clone);
|
|
||||||
async move {
|
|
||||||
*plan_hits_inner.lock().expect("mutex should lock") += 1;
|
|
||||||
Json(json!({"action": "proxy_public"}))
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
.route(
|
|
||||||
"/api/internal/gateway/report-sync",
|
|
||||||
any(move |_request: Request| {
|
|
||||||
let report_hits_inner = Arc::clone(&report_hits_clone);
|
|
||||||
async move {
|
|
||||||
*report_hits_inner.lock().expect("mutex should lock") += 1;
|
|
||||||
Json(json!({"ok": true}))
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
.route(
|
|
||||||
"/v1/chat/completions",
|
|
||||||
any(move |_request: Request| {
|
|
||||||
let public_hits_inner = Arc::clone(&public_hits_clone);
|
|
||||||
async move {
|
|
||||||
*public_hits_inner.lock().expect("mutex should lock") += 1;
|
|
||||||
(StatusCode::IM_A_TEAPOT, Body::from("public-route-hit"))
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
|
|
||||||
let execution_runtime = Router::new().route(
|
|
||||||
"/v1/execute/sync",
|
|
||||||
any(|_request: Request| async move {
|
|
||||||
Json(json!({
|
|
||||||
"request_id": "trace-openai-chat-local-report-sync-failure-123",
|
|
||||||
"status_code": 503,
|
|
||||||
"headers": {
|
|
||||||
"content-type": "application/json"
|
|
||||||
},
|
|
||||||
"body": {
|
|
||||||
"json_body": {
|
|
||||||
"error": {
|
|
||||||
"message": "primary unavailable"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"telemetry": {
|
|
||||||
"elapsed_ms": 25
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
|
|
||||||
let auth_repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
|
let auth_repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
|
||||||
Some(hash_api_key("sk-client-openai-local-report-sync-failure")),
|
Some(hash_api_key("sk-client-openai-local-report-sync-failure")),
|
||||||
@@ -763,29 +690,51 @@ async fn gateway_records_failed_usage_when_all_local_openai_chat_candidates_exha
|
|||||||
vec![sample_local_openai_key()],
|
vec![sample_local_openai_key()],
|
||||||
));
|
));
|
||||||
|
|
||||||
let (upstream_url, upstream_handle) = start_server(upstream).await;
|
let gateway_state = crate::AppState::new()
|
||||||
let (execution_runtime_url, execution_runtime_handle) = start_server(execution_runtime).await;
|
.expect("gateway should build")
|
||||||
let gateway_state =
|
.with_execution_runtime_sync_override_for_tests(|plan| {
|
||||||
build_state_with_execution_runtime_override(execution_runtime_url)
|
Ok(aether_contracts::ExecutionResult {
|
||||||
.with_data_state_for_tests(
|
request_id: plan.request_id.clone(),
|
||||||
GatewayDataState::with_auth_candidate_selection_provider_catalog_request_candidates_and_usage_for_tests(
|
candidate_id: plan.candidate_id.clone(),
|
||||||
auth_repository,
|
status_code: 503,
|
||||||
candidate_selection_repository,
|
headers: std::collections::BTreeMap::from([(
|
||||||
provider_catalog_repository,
|
"content-type".to_string(),
|
||||||
Arc::clone(&request_candidate_repository),
|
"application/json".to_string(),
|
||||||
Arc::clone(&usage_repository),
|
)]),
|
||||||
DEVELOPMENT_ENCRYPTION_KEY,
|
body: Some(aether_contracts::ResponseBody {
|
||||||
),
|
json_body: Some(json!({
|
||||||
)
|
"error": {
|
||||||
.with_usage_runtime_for_tests(UsageRuntimeConfig {
|
"message": "primary unavailable"
|
||||||
enabled: true,
|
}
|
||||||
..UsageRuntimeConfig::default()
|
})),
|
||||||
});
|
body_bytes_b64: None,
|
||||||
|
}),
|
||||||
|
telemetry: Some(aether_contracts::ExecutionTelemetry {
|
||||||
|
ttfb_ms: None,
|
||||||
|
elapsed_ms: Some(25),
|
||||||
|
upstream_bytes: None,
|
||||||
|
}),
|
||||||
|
error: None,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.with_data_state_for_tests(
|
||||||
|
GatewayDataState::with_auth_candidate_selection_provider_catalog_request_candidates_and_usage_for_tests(
|
||||||
|
auth_repository,
|
||||||
|
candidate_selection_repository,
|
||||||
|
provider_catalog_repository,
|
||||||
|
Arc::clone(&request_candidate_repository),
|
||||||
|
Arc::clone(&usage_repository),
|
||||||
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.with_usage_runtime_for_tests(UsageRuntimeConfig {
|
||||||
|
enabled: true,
|
||||||
|
..UsageRuntimeConfig::default()
|
||||||
|
});
|
||||||
let gateway = build_router_with_state(gateway_state);
|
let gateway = build_router_with_state(gateway_state);
|
||||||
let (gateway_url, gateway_handle) = start_server(gateway).await;
|
let request = Request::builder()
|
||||||
|
.method(http::Method::POST)
|
||||||
let response = reqwest::Client::new()
|
.uri("/v1/chat/completions")
|
||||||
.post(format!("{gateway_url}/v1/chat/completions"))
|
|
||||||
.header(http::header::CONTENT_TYPE, "application/json")
|
.header(http::header::CONTENT_TYPE, "application/json")
|
||||||
.header(
|
.header(
|
||||||
http::header::AUTHORIZATION,
|
http::header::AUTHORIZATION,
|
||||||
@@ -795,13 +744,17 @@ async fn gateway_records_failed_usage_when_all_local_openai_chat_candidates_exha
|
|||||||
TRACE_ID_HEADER,
|
TRACE_ID_HEADER,
|
||||||
"trace-openai-chat-local-report-sync-failure-123",
|
"trace-openai-chat-local-report-sync-failure-123",
|
||||||
)
|
)
|
||||||
.body("{\"model\":\"gpt-5\",\"messages\":[]}")
|
.body(Body::from("{\"model\":\"gpt-5\",\"messages\":[]}"))
|
||||||
.send()
|
.expect("request should build");
|
||||||
.await
|
let response = send_request(gateway, request).await;
|
||||||
.expect("request should complete");
|
|
||||||
|
|
||||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||||
let body_json: serde_json::Value = response.json().await.expect("body should parse");
|
let body_json: serde_json::Value = serde_json::from_slice(
|
||||||
|
&axum::body::to_bytes(response.into_body(), usize::MAX)
|
||||||
|
.await
|
||||||
|
.expect("body should read"),
|
||||||
|
)
|
||||||
|
.expect("body should parse");
|
||||||
assert_eq!(body_json["error"]["type"], "http_error");
|
assert_eq!(body_json["error"]["type"], "http_error");
|
||||||
|
|
||||||
let stored_usage = wait_for_usage_status(
|
let stored_usage = wait_for_usage_status(
|
||||||
@@ -855,16 +808,6 @@ async fn gateway_records_failed_usage_when_all_local_openai_chat_candidates_exha
|
|||||||
assert_eq!(stored_candidates.len(), 1);
|
assert_eq!(stored_candidates.len(), 1);
|
||||||
assert_eq!(stored_candidates[0].status, RequestCandidateStatus::Failed);
|
assert_eq!(stored_candidates[0].status, RequestCandidateStatus::Failed);
|
||||||
assert_eq!(stored_candidates[0].status_code, Some(503));
|
assert_eq!(stored_candidates[0].status_code, Some(503));
|
||||||
|
|
||||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
|
||||||
assert_eq!(*report_hits.lock().expect("mutex should lock"), 0);
|
|
||||||
assert_eq!(*decision_hits.lock().expect("mutex should lock"), 0);
|
|
||||||
assert_eq!(*plan_hits.lock().expect("mutex should lock"), 0);
|
|
||||||
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
|
|
||||||
|
|
||||||
gateway_handle.abort();
|
|
||||||
execution_runtime_handle.abort();
|
|
||||||
upstream_handle.abort();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -1072,7 +1015,6 @@ async fn gateway_records_failed_usage_for_claude_runtime_miss_without_execution_
|
|||||||
|
|
||||||
gateway_handle.abort();
|
gateway_handle.abort();
|
||||||
execution_runtime_handle.abort();
|
execution_runtime_handle.abort();
|
||||||
upstream_handle.abort();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -1626,7 +1568,7 @@ async fn gateway_records_failed_usage_when_all_local_claude_cli_candidates_are_s
|
|||||||
.headers()
|
.headers()
|
||||||
.get(LOCAL_EXECUTION_RUNTIME_MISS_REASON_HEADER)
|
.get(LOCAL_EXECUTION_RUNTIME_MISS_REASON_HEADER)
|
||||||
.and_then(|value| value.to_str().ok()),
|
.and_then(|value| value.to_str().ok()),
|
||||||
Some("all_candidates_skipped")
|
Some("candidate_list_empty")
|
||||||
);
|
);
|
||||||
let body_json: serde_json::Value = response.json().await.expect("body should parse");
|
let body_json: serde_json::Value = response.json().await.expect("body should parse");
|
||||||
assert_eq!(body_json["error"]["type"], "http_error");
|
assert_eq!(body_json["error"]["type"], "http_error");
|
||||||
@@ -1666,7 +1608,7 @@ async fn gateway_records_failed_usage_when_all_local_claude_cli_candidates_are_s
|
|||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
stored_usage.routing_local_execution_runtime_miss_reason(),
|
stored_usage.routing_local_execution_runtime_miss_reason(),
|
||||||
Some("all_candidates_skipped")
|
Some("candidate_list_empty")
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
stored_usage.error_message.as_deref(),
|
stored_usage.error_message.as_deref(),
|
||||||
@@ -1685,12 +1627,7 @@ async fn gateway_records_failed_usage_when_all_local_claude_cli_candidates_are_s
|
|||||||
.list_by_request_id("trace-claude-cli-usage-local-miss-123")
|
.list_by_request_id("trace-claude-cli-usage-local-miss-123")
|
||||||
.await
|
.await
|
||||||
.expect("request candidate trace should read");
|
.expect("request candidate trace should read");
|
||||||
assert_eq!(stored_candidates.len(), 1);
|
assert!(stored_candidates.is_empty());
|
||||||
assert_eq!(stored_candidates[0].status, RequestCandidateStatus::Skipped);
|
|
||||||
assert_eq!(
|
|
||||||
stored_candidates[0].skip_reason.as_deref(),
|
|
||||||
Some("format_conversion_disabled")
|
|
||||||
);
|
|
||||||
assert_eq!(stored_usage.routing_candidate_id(), None);
|
assert_eq!(stored_usage.routing_candidate_id(), None);
|
||||||
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
|
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
|
||||||
|
|
||||||
@@ -1912,7 +1849,7 @@ async fn gateway_keeps_failed_usage_request_capture_lightweight_for_large_local_
|
|||||||
.headers()
|
.headers()
|
||||||
.get(LOCAL_EXECUTION_RUNTIME_MISS_REASON_HEADER)
|
.get(LOCAL_EXECUTION_RUNTIME_MISS_REASON_HEADER)
|
||||||
.and_then(|value| value.to_str().ok()),
|
.and_then(|value| value.to_str().ok()),
|
||||||
Some("all_candidates_skipped")
|
Some("candidate_list_empty")
|
||||||
);
|
);
|
||||||
|
|
||||||
let stored_usage = wait_for_usage_status(
|
let stored_usage = wait_for_usage_status(
|
||||||
@@ -1936,12 +1873,7 @@ async fn gateway_keeps_failed_usage_request_capture_lightweight_for_large_local_
|
|||||||
.list_by_request_id("trace-claude-cli-usage-local-miss-large-123")
|
.list_by_request_id("trace-claude-cli-usage-local-miss-large-123")
|
||||||
.await
|
.await
|
||||||
.expect("request candidate trace should read");
|
.expect("request candidate trace should read");
|
||||||
assert_eq!(stored_candidates.len(), 1);
|
assert!(stored_candidates.is_empty());
|
||||||
assert_eq!(stored_candidates[0].status, RequestCandidateStatus::Skipped);
|
|
||||||
assert_eq!(
|
|
||||||
stored_candidates[0].skip_reason.as_deref(),
|
|
||||||
Some("format_conversion_disabled")
|
|
||||||
);
|
|
||||||
|
|
||||||
gateway_handle.abort();
|
gateway_handle.abort();
|
||||||
execution_runtime_handle.abort();
|
execution_runtime_handle.abort();
|
||||||
|
|||||||
@@ -33,9 +33,6 @@
|
|||||||
<h4 class="text-sm font-semibold">
|
<h4 class="text-sm font-semibold">
|
||||||
请求链路追踪
|
请求链路追踪
|
||||||
</h4>
|
</h4>
|
||||||
<span class="text-xs text-muted-foreground">
|
|
||||||
按实际调度顺序
|
|
||||||
</span>
|
|
||||||
<Badge :variant="getFinalStatusBadgeVariant(computedFinalStatus)">
|
<Badge :variant="getFinalStatusBadgeVariant(computedFinalStatus)">
|
||||||
{{ getFinalStatusLabel(computedFinalStatus) }}
|
{{ getFinalStatusLabel(computedFinalStatus) }}
|
||||||
</Badge>
|
</Badge>
|
||||||
@@ -499,9 +496,8 @@ import { parseApiError } from '@/utils/errorParser'
|
|||||||
import { formatApiFormat } from '@/api/endpoints/types/api-format'
|
import { formatApiFormat } from '@/api/endpoints/types/api-format'
|
||||||
import { resolveTimelineFinalStatus } from '../utils/status'
|
import { resolveTimelineFinalStatus } from '../utils/status'
|
||||||
import {
|
import {
|
||||||
buildPoolAttemptCandidatesFromAudit,
|
buildPoolParticipatedCandidates,
|
||||||
extractPoolGroupId,
|
extractPoolGroupId,
|
||||||
isPoolAttemptedCandidate,
|
|
||||||
makeAttemptKey,
|
makeAttemptKey,
|
||||||
TIMELINE_STATUS,
|
TIMELINE_STATUS,
|
||||||
} from '../utils/poolTrace'
|
} from '../utils/poolTrace'
|
||||||
@@ -736,21 +732,10 @@ const schedulingAudit = computed<Record<string, unknown> | null>(() => {
|
|||||||
})
|
})
|
||||||
|
|
||||||
const poolAttemptCandidates = computed<CandidateRecord[]>(() => {
|
const poolAttemptCandidates = computed<CandidateRecord[]>(() => {
|
||||||
// 新链路:优先使用后端写入的 extra_data.pool_group_id,
|
const auditAttempts = schedulingAudit.value?.attempts
|
||||||
// 但仅展示实际进入号池执行的 key(排除 available/unused/skipped)。
|
return buildPoolParticipatedCandidates(
|
||||||
const fromTrace = rawTimeline.value.filter(
|
|
||||||
(candidate) => extractPoolGroupId(candidate) !== null && isPoolAttemptedCandidate(candidate),
|
|
||||||
)
|
|
||||||
if (fromTrace.length > 0) {
|
|
||||||
return fromTrace
|
|
||||||
}
|
|
||||||
|
|
||||||
// 兼容旧链路:回退到 request_metadata.scheduling_audit.attempts。
|
|
||||||
const audit = schedulingAudit.value
|
|
||||||
if (!audit) return []
|
|
||||||
return buildPoolAttemptCandidatesFromAudit(
|
|
||||||
rawTimeline.value,
|
rawTimeline.value,
|
||||||
audit.attempts,
|
auditAttempts,
|
||||||
props.requestId,
|
props.requestId,
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
@@ -1732,7 +1717,7 @@ const getDisplayStatus = (attempt: CandidateRecord | null | undefined): string =
|
|||||||
.sub-dot.status-failed { background: #ef4444; color: #ef4444; }
|
.sub-dot.status-failed { background: #ef4444; color: #ef4444; }
|
||||||
.sub-dot.status-cancelled { background: #f59e0b; color: #f59e0b; }
|
.sub-dot.status-cancelled { background: #f59e0b; color: #f59e0b; }
|
||||||
.sub-dot.status-pending { background: #3b82f6; color: #3b82f6; }
|
.sub-dot.status-pending { background: #3b82f6; color: #3b82f6; }
|
||||||
.sub-dot.status-skipped { background: hsl(var(--primary)); color: hsl(var(--primary)); }
|
.sub-dot.status-skipped { background: hsl(var(--foreground)); color: hsl(var(--foreground)); }
|
||||||
.sub-dot.status-available { background: #d1d5db; color: #d1d5db; }
|
.sub-dot.status-available { background: #d1d5db; color: #d1d5db; }
|
||||||
|
|
||||||
/* 选中状态:呼吸动画 + 涟漪效果 */
|
/* 选中状态:呼吸动画 + 涟漪效果 */
|
||||||
@@ -1795,7 +1780,7 @@ const getDisplayStatus = (attempt: CandidateRecord | null | undefined): string =
|
|||||||
.node-dot.status-failed { color: #ef4444; }
|
.node-dot.status-failed { color: #ef4444; }
|
||||||
.node-dot.status-cancelled { color: #f59e0b; }
|
.node-dot.status-cancelled { color: #f59e0b; }
|
||||||
.node-dot.status-pending { color: #3b82f6; }
|
.node-dot.status-pending { color: #3b82f6; }
|
||||||
.node-dot.status-skipped { color: hsl(var(--primary)); }
|
.node-dot.status-skipped { color: hsl(var(--foreground)); }
|
||||||
.node-dot.status-available { color: #d1d5db; }
|
.node-dot.status-available { color: #d1d5db; }
|
||||||
|
|
||||||
/* 连接线容器 */
|
/* 连接线容器 */
|
||||||
@@ -1858,7 +1843,7 @@ const getDisplayStatus = (attempt: CandidateRecord | null | undefined): string =
|
|||||||
.title-dot.status-failed { background: #ef4444; }
|
.title-dot.status-failed { background: #ef4444; }
|
||||||
.title-dot.status-cancelled { background: #f59e0b; }
|
.title-dot.status-cancelled { background: #f59e0b; }
|
||||||
.title-dot.status-pending { background: #3b82f6; }
|
.title-dot.status-pending { background: #3b82f6; }
|
||||||
.title-dot.status-skipped { background: hsl(var(--primary)); }
|
.title-dot.status-skipped { background: hsl(var(--foreground)); }
|
||||||
.title-dot.status-available { background: #d1d5db; }
|
.title-dot.status-available { background: #d1d5db; }
|
||||||
|
|
||||||
.title-text {
|
.title-text {
|
||||||
@@ -1951,8 +1936,8 @@ const getDisplayStatus = (attempt: CandidateRecord | null | undefined): string =
|
|||||||
}
|
}
|
||||||
|
|
||||||
.status-tag.status-skipped {
|
.status-tag.status-skipped {
|
||||||
background: hsl(var(--primary) / 0.15);
|
background: hsl(var(--foreground) / 0.08);
|
||||||
color: hsl(var(--primary));
|
color: hsl(var(--foreground));
|
||||||
}
|
}
|
||||||
|
|
||||||
.status-tag.status-available {
|
.status-tag.status-available {
|
||||||
|
|||||||
@@ -1,7 +1,10 @@
|
|||||||
import { describe, expect, it } from 'vitest'
|
import { describe, expect, it } from 'vitest'
|
||||||
|
|
||||||
import type { CandidateRecord } from '@/api/requestTrace'
|
import type { CandidateRecord } from '@/api/requestTrace'
|
||||||
import { buildPoolAttemptCandidatesFromAudit } from '@/features/usage/utils/poolTrace'
|
import {
|
||||||
|
buildPoolAttemptCandidatesFromAudit,
|
||||||
|
buildPoolParticipatedCandidates,
|
||||||
|
} from '@/features/usage/utils/poolTrace'
|
||||||
|
|
||||||
function buildCandidate(
|
function buildCandidate(
|
||||||
overrides: Partial<CandidateRecord> = {},
|
overrides: Partial<CandidateRecord> = {},
|
||||||
@@ -19,7 +22,7 @@ function buildCandidate(
|
|||||||
}
|
}
|
||||||
|
|
||||||
describe('poolTrace', () => {
|
describe('poolTrace', () => {
|
||||||
it('keeps only actually attempted pool nodes from scheduling audit fallback', () => {
|
it('keeps only pool nodes that actually participated in scheduling audit fallback', () => {
|
||||||
const attempts = buildPoolAttemptCandidatesFromAudit([], [
|
const attempts = buildPoolAttemptCandidatesFromAudit([], [
|
||||||
{
|
{
|
||||||
candidate_index: 0,
|
candidate_index: 0,
|
||||||
@@ -63,9 +66,11 @@ describe('poolTrace', () => {
|
|||||||
},
|
},
|
||||||
], 'req-1')
|
], 'req-1')
|
||||||
|
|
||||||
expect(attempts).toHaveLength(1)
|
expect(attempts).toHaveLength(2)
|
||||||
expect(attempts[0].key_id).toBe('key-success')
|
expect(attempts[0].key_id).toBe('key-success')
|
||||||
expect(attempts[0].status).toBe('success')
|
expect(attempts[0].status).toBe('success')
|
||||||
|
expect(attempts[1].key_id).toBe('key-skipped')
|
||||||
|
expect(attempts[1].status).toBe('skipped')
|
||||||
})
|
})
|
||||||
|
|
||||||
it('preserves real trace attempts even when audit status is non-standard', () => {
|
it('preserves real trace attempts even when audit status is non-standard', () => {
|
||||||
@@ -111,4 +116,58 @@ describe('poolTrace', () => {
|
|||||||
expect(attempts[0].status).toBe('failed')
|
expect(attempts[0].status).toBe('failed')
|
||||||
expect(attempts[0].provider_name).toBe('Codex反代')
|
expect(attempts[0].provider_name).toBe('Codex反代')
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('merges audit-only skipped pool nodes when trace only carries partial pool metadata', () => {
|
||||||
|
const rawTimeline = [
|
||||||
|
buildCandidate({
|
||||||
|
id: 'cand-success',
|
||||||
|
candidate_index: 1,
|
||||||
|
retry_index: 0,
|
||||||
|
provider_id: 'provider-1',
|
||||||
|
provider_name: 'Codex反代',
|
||||||
|
key_id: 'key-success',
|
||||||
|
key_name: 'Success Key',
|
||||||
|
status: 'success',
|
||||||
|
extra_data: { pool_group_id: 'provider-1' },
|
||||||
|
started_at: '2026-04-19T12:00:00.000Z',
|
||||||
|
}),
|
||||||
|
buildCandidate({
|
||||||
|
id: 'cand-skipped',
|
||||||
|
candidate_index: 2,
|
||||||
|
retry_index: 0,
|
||||||
|
provider_id: 'provider-1',
|
||||||
|
provider_name: 'Codex反代',
|
||||||
|
key_id: 'key-skipped',
|
||||||
|
key_name: 'Skipped Key',
|
||||||
|
status: 'skipped',
|
||||||
|
}),
|
||||||
|
]
|
||||||
|
|
||||||
|
const attempts = buildPoolParticipatedCandidates(rawTimeline, [
|
||||||
|
{
|
||||||
|
candidate_index: 1,
|
||||||
|
retry_index: 0,
|
||||||
|
provider_id: 'provider-1',
|
||||||
|
provider_name: 'Codex反代',
|
||||||
|
key_id: 'key-success',
|
||||||
|
key_name: 'Success Key',
|
||||||
|
status: 'success',
|
||||||
|
pool_group_id: 'provider-1',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
candidate_index: 2,
|
||||||
|
retry_index: 0,
|
||||||
|
provider_id: 'provider-1',
|
||||||
|
provider_name: 'Codex反代',
|
||||||
|
key_id: 'key-skipped',
|
||||||
|
key_name: 'Skipped Key',
|
||||||
|
status: 'skipped',
|
||||||
|
pool_group_id: 'provider-1',
|
||||||
|
},
|
||||||
|
], 'req-1')
|
||||||
|
|
||||||
|
expect(attempts).toHaveLength(2)
|
||||||
|
expect(attempts.map(item => item.key_id)).toEqual(['key-success', 'key-skipped'])
|
||||||
|
expect(attempts[1].extra_data?.pool_group_id).toBe('provider-1')
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -12,10 +12,9 @@ export const TIMELINE_STATUS: CandidateRecord['status'][] = [
|
|||||||
'stream_interrupted',
|
'stream_interrupted',
|
||||||
]
|
]
|
||||||
|
|
||||||
const POOL_UNATTEMPTED_STATUS = new Set<CandidateRecord['status']>([
|
const POOL_HIDDEN_STATUS = new Set<CandidateRecord['status']>([
|
||||||
'available',
|
'available',
|
||||||
'unused',
|
'unused',
|
||||||
'skipped',
|
|
||||||
])
|
])
|
||||||
|
|
||||||
const PROVIDER_TYPE_LIKE_NAMES = new Set<string>([
|
const PROVIDER_TYPE_LIKE_NAMES = new Set<string>([
|
||||||
@@ -40,8 +39,8 @@ export const makeAttemptKey = (candidateIndex: number, retryIndex: number): stri
|
|||||||
return `${candidateIndex}:${retryIndex}`
|
return `${candidateIndex}:${retryIndex}`
|
||||||
}
|
}
|
||||||
|
|
||||||
export const isPoolAttemptedCandidate = (candidate: CandidateRecord): boolean => {
|
export const isPoolParticipatedCandidate = (candidate: CandidateRecord): boolean => {
|
||||||
if (POOL_UNATTEMPTED_STATUS.has(candidate.status)) return false
|
if (POOL_HIDDEN_STATUS.has(candidate.status)) return false
|
||||||
if (candidate.status === 'pending' && !candidate.started_at) return false
|
if (candidate.status === 'pending' && !candidate.started_at) return false
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
@@ -66,6 +65,38 @@ export const extractPoolGroupId = (
|
|||||||
return text || null
|
return text || null
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function buildPoolParticipatedCandidates(
|
||||||
|
rawTimeline: CandidateRecord[],
|
||||||
|
attempts: unknown,
|
||||||
|
requestId?: string | null,
|
||||||
|
): CandidateRecord[] {
|
||||||
|
const fromTrace = rawTimeline.filter(
|
||||||
|
candidate => extractPoolGroupId(candidate) !== null && isPoolParticipatedCandidate(candidate),
|
||||||
|
)
|
||||||
|
const fromAudit = buildPoolAttemptCandidatesFromAudit(rawTimeline, attempts, requestId)
|
||||||
|
|
||||||
|
if (fromTrace.length === 0) return fromAudit
|
||||||
|
if (fromAudit.length === 0) return fromTrace
|
||||||
|
|
||||||
|
const traceKeys = new Set(
|
||||||
|
fromTrace.map(candidate => makeAttemptKey(candidate.candidate_index, candidate.retry_index)),
|
||||||
|
)
|
||||||
|
const merged = [...fromTrace]
|
||||||
|
for (const candidate of fromAudit) {
|
||||||
|
const key = makeAttemptKey(candidate.candidate_index, candidate.retry_index)
|
||||||
|
if (!traceKeys.has(key)) {
|
||||||
|
merged.push(candidate)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return merged.sort((a, b) => {
|
||||||
|
if (a.candidate_index !== b.candidate_index) {
|
||||||
|
return a.candidate_index - b.candidate_index
|
||||||
|
}
|
||||||
|
return a.retry_index - b.retry_index
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
export function buildPoolAttemptCandidatesFromAudit(
|
export function buildPoolAttemptCandidatesFromAudit(
|
||||||
rawTimeline: CandidateRecord[],
|
rawTimeline: CandidateRecord[],
|
||||||
attempts: unknown,
|
attempts: unknown,
|
||||||
@@ -154,7 +185,7 @@ export function buildPoolAttemptCandidatesFromAudit(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return isPoolAttemptedCandidate(merged) ? merged : null
|
return isPoolParticipatedCandidate(merged) ? merged : null
|
||||||
})
|
})
|
||||||
.filter((item): item is CandidateRecord => item !== null)
|
.filter((item): item is CandidateRecord => item !== null)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user