mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-03 01:40:21 +08:00
Add Codex image progress heartbeat
This commit is contained in:
@@ -30,6 +30,7 @@ pub(crate) use crate::ai_serving::{
|
||||
maybe_build_stream_decision_payload, maybe_build_stream_plan_payload,
|
||||
maybe_build_sync_decision_payload, maybe_build_sync_plan_payload,
|
||||
set_local_openai_chat_execution_exhausted_diagnostic,
|
||||
set_local_openai_image_execution_exhausted_diagnostic,
|
||||
};
|
||||
pub(crate) use crate::ai_serving::{
|
||||
maybe_bridge_standard_sync_json_to_stream, maybe_build_provider_private_stream_normalizer,
|
||||
|
||||
@@ -49,7 +49,8 @@ pub(crate) use self::planner::{
|
||||
extract_pool_sticky_session_token, maybe_build_stream_decision_payload,
|
||||
maybe_build_stream_plan_payload, maybe_build_sync_decision_payload,
|
||||
maybe_build_sync_plan_payload, planner_is_matching_stream_request,
|
||||
set_local_openai_chat_execution_exhausted_diagnostic, CandidateFailureDiagnostic,
|
||||
set_local_openai_chat_execution_exhausted_diagnostic,
|
||||
set_local_openai_image_execution_exhausted_diagnostic, CandidateFailureDiagnostic,
|
||||
CandidateFailureDiagnosticKind, GatewayAuthApiKeySnapshot, GatewayProviderTransportSnapshot,
|
||||
LocalExecutionAttemptSource, LocalResolvedOAuthRequestAuth, PlannerAppState,
|
||||
};
|
||||
|
||||
@@ -48,6 +48,7 @@ pub(crate) use self::specialized::{
|
||||
build_local_image_sync_plan_and_reports_for_kind,
|
||||
build_local_video_sync_attempt_source_for_kind,
|
||||
build_local_video_sync_plan_and_reports_for_kind,
|
||||
set_local_openai_image_execution_exhausted_diagnostic,
|
||||
};
|
||||
pub(crate) use self::standard::{
|
||||
build_local_openai_chat_stream_attempt_source_for_kind,
|
||||
|
||||
@@ -10,6 +10,7 @@ use crate::ai_serving::planner::plan_builders::{
|
||||
build_passthrough_sync_plan_from_decision, build_standard_stream_plan_from_decision,
|
||||
AiStreamAttempt, AiSyncAttempt,
|
||||
};
|
||||
use crate::ai_serving::planner::runtime_miss::set_local_runtime_execution_exhausted_diagnostic;
|
||||
use crate::ai_serving::planner::spec_metadata::local_openai_image_spec_metadata;
|
||||
use crate::ai_serving::GatewayControlDecision;
|
||||
use crate::ai_serving::{
|
||||
@@ -49,6 +50,35 @@ pub(crate) struct LocalOpenAiImageStreamAttemptSource<'a> {
|
||||
candidates: LocalOpenAiImageCandidateAttemptSource<'a>,
|
||||
}
|
||||
|
||||
pub(crate) fn set_local_openai_image_execution_exhausted_diagnostic(
|
||||
state: &AppState,
|
||||
trace_id: &str,
|
||||
decision: &GatewayControlDecision,
|
||||
plan_kind: &str,
|
||||
body_json: &serde_json::Value,
|
||||
candidate_count: usize,
|
||||
) {
|
||||
warn!(
|
||||
event_name = "local_openai_image_candidates_exhausted",
|
||||
log_type = "event",
|
||||
trace_id = %trace_id,
|
||||
plan_kind,
|
||||
route_class = decision.route_class.as_deref().unwrap_or("passthrough"),
|
||||
route_family = decision.route_family.as_deref().unwrap_or("unknown"),
|
||||
candidate_count,
|
||||
model = body_json.get("model").and_then(|value| value.as_str()).unwrap_or(""),
|
||||
"gateway local openai image execution exhausted all candidates"
|
||||
);
|
||||
set_local_runtime_execution_exhausted_diagnostic(
|
||||
state,
|
||||
trace_id,
|
||||
decision,
|
||||
plan_kind,
|
||||
body_json.get("model").and_then(|value| value.as_str()),
|
||||
candidate_count,
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) async fn build_local_image_sync_plan_and_reports_for_kind(
|
||||
state: &AppState,
|
||||
parts: &http::request::Parts,
|
||||
|
||||
@@ -18,6 +18,7 @@ pub(crate) use self::image::{
|
||||
build_local_image_sync_attempt_source_for_kind,
|
||||
build_local_image_sync_plan_and_reports_for_kind,
|
||||
maybe_build_stream_local_image_decision_payload, maybe_build_sync_local_image_decision_payload,
|
||||
set_local_openai_image_execution_exhausted_diagnostic,
|
||||
};
|
||||
pub(crate) use self::video::{
|
||||
build_local_video_sync_attempt_source_for_kind,
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
use std::collections::{BTreeMap, VecDeque};
|
||||
use std::io::Error as IoError;
|
||||
use std::time::Instant;
|
||||
use std::sync::{
|
||||
atomic::{AtomicBool, AtomicU64, Ordering},
|
||||
Arc,
|
||||
};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use aether_contracts::{
|
||||
ExecutionPlan, ExecutionStreamTerminalSummary, ExecutionTelemetry, StreamFrame,
|
||||
@@ -25,6 +29,7 @@ use futures_util::stream::BoxStream;
|
||||
use futures_util::{StreamExt, TryStreamExt};
|
||||
use serde_json::Value;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::time::MissedTickBehavior;
|
||||
use tokio_util::codec::{FramedRead, LinesCodec};
|
||||
use tokio_util::io::StreamReader;
|
||||
use tracing::{debug, info, warn};
|
||||
@@ -88,6 +93,14 @@ use crate::usage::submit_stream_report;
|
||||
use crate::usage::{GatewayStreamReportRequest, GatewaySyncReportRequest};
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
const OPENAI_IMAGE_STREAM_PLAN_KIND: &str = "openai_image_stream";
|
||||
const SSE_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(15);
|
||||
const SSE_KEEPALIVE_BYTES: &[u8] = b": aether-keepalive\n\n";
|
||||
const STREAM_IDLE_LOG_INTERVAL: Duration = Duration::from_secs(60);
|
||||
const STREAM_IDLE_LOG_INTERVAL_MS: u64 = 60_000;
|
||||
const REWRITTEN_STREAM_PREFETCH_TIMEOUT: Duration = Duration::from_millis(750);
|
||||
const OPENAI_IMAGE_STREAM_DEFAULT_TOTAL_TIMEOUT_MS: u64 = 900_000;
|
||||
|
||||
fn record_sync_terminal_usage(
|
||||
state: &AppState,
|
||||
plan: &ExecutionPlan,
|
||||
@@ -750,6 +763,114 @@ fn encode_terminal_sse_error_event(failure: &StreamFailureReport) -> Result<Byte
|
||||
Ok(Bytes::from(event))
|
||||
}
|
||||
|
||||
fn image_stream_failed_event_name(report_context: Option<&Value>) -> &'static str {
|
||||
let operation = report_context
|
||||
.and_then(|value| value.get("image_request"))
|
||||
.and_then(|value| value.get("operation"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.unwrap_or_default();
|
||||
if operation == "edit" {
|
||||
"image_edit.failed"
|
||||
} else {
|
||||
"image_generation.failed"
|
||||
}
|
||||
}
|
||||
|
||||
fn encode_openai_image_failed_event(
|
||||
report_context: Option<&Value>,
|
||||
failure: &StreamFailureReport,
|
||||
) -> Result<Bytes, std::io::Error> {
|
||||
let event_name = image_stream_failed_event_name(report_context);
|
||||
let failure_body = failure
|
||||
.to_json_string()
|
||||
.map_err(|err| IoError::other(err.to_string()))?;
|
||||
let failure_json: Value =
|
||||
serde_json::from_str(&failure_body).map_err(|err| IoError::other(err.to_string()))?;
|
||||
let error = failure_json.get("error").cloned().unwrap_or_else(|| {
|
||||
serde_json::json!({
|
||||
"type": failure.error_type.as_str(),
|
||||
"message": failure.error_message.as_str(),
|
||||
"code": failure.status_code,
|
||||
})
|
||||
});
|
||||
let payload = serde_json::json!({
|
||||
"type": event_name,
|
||||
"error": error,
|
||||
});
|
||||
let payload = serde_json::to_string(&payload).map_err(|err| IoError::other(err.to_string()))?;
|
||||
let mut event = format!("event: {event_name}\n");
|
||||
for line in payload.lines() {
|
||||
event.push_str("data: ");
|
||||
event.push_str(line);
|
||||
event.push('\n');
|
||||
}
|
||||
event.push('\n');
|
||||
Ok(Bytes::from(event))
|
||||
}
|
||||
|
||||
fn resolve_openai_image_stream_total_timeout_ms(
|
||||
plan_kind: &str,
|
||||
plan: &ExecutionPlan,
|
||||
) -> Option<u64> {
|
||||
if plan_kind != OPENAI_IMAGE_STREAM_PLAN_KIND {
|
||||
return None;
|
||||
}
|
||||
Some(
|
||||
plan.timeouts
|
||||
.as_ref()
|
||||
.and_then(|timeouts| timeouts.total_ms)
|
||||
.unwrap_or(OPENAI_IMAGE_STREAM_DEFAULT_TOTAL_TIMEOUT_MS)
|
||||
.max(1),
|
||||
)
|
||||
}
|
||||
|
||||
fn should_limit_direct_finalize_prefetch(plan_kind: &str, has_local_stream_rewriter: bool) -> bool {
|
||||
plan_kind == OPENAI_IMAGE_STREAM_PLAN_KIND || has_local_stream_rewriter
|
||||
}
|
||||
|
||||
fn build_sse_body_stream(
|
||||
prefetched_chunks_for_body: Vec<Bytes>,
|
||||
mut rx: mpsc::Receiver<Result<Bytes, IoError>>,
|
||||
emit_keepalive: bool,
|
||||
keepalive_interval: Duration,
|
||||
) -> impl futures_util::Stream<Item = Result<Bytes, IoError>> + Send + 'static {
|
||||
stream! {
|
||||
let mut sent_prefetched_chunk = false;
|
||||
for chunk in prefetched_chunks_for_body {
|
||||
sent_prefetched_chunk = true;
|
||||
yield Ok(chunk);
|
||||
}
|
||||
|
||||
if emit_keepalive {
|
||||
if !sent_prefetched_chunk {
|
||||
yield Ok(Bytes::from_static(SSE_KEEPALIVE_BYTES));
|
||||
}
|
||||
let mut keepalive = tokio::time::interval(keepalive_interval);
|
||||
keepalive.set_missed_tick_behavior(MissedTickBehavior::Delay);
|
||||
keepalive.tick().await;
|
||||
loop {
|
||||
tokio::select! {
|
||||
biased;
|
||||
item = rx.recv() => {
|
||||
let Some(item) = item else {
|
||||
break;
|
||||
};
|
||||
yield item;
|
||||
}
|
||||
_ = keepalive.tick() => {
|
||||
yield Ok(Bytes::from_static(SSE_KEEPALIVE_BYTES));
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
while let Some(item) = rx.recv().await {
|
||||
yield item;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn next_stream_frame<R>(
|
||||
buffered_frames: &mut VecDeque<StreamFrame>,
|
||||
lines: &mut FramedRead<R, LinesCodec>,
|
||||
@@ -1217,6 +1338,8 @@ async fn execute_stream_from_frame_stream(
|
||||
private_stream_normalizer.is_some(),
|
||||
local_stream_rewriter.is_some(),
|
||||
);
|
||||
let limit_direct_finalize_prefetch =
|
||||
should_limit_direct_finalize_prefetch(plan_kind, local_stream_rewriter.is_some());
|
||||
let mut prefetched_chunks: Vec<Bytes> = Vec::new();
|
||||
let mut provider_prefetched_body = Vec::new();
|
||||
let mut prefetched_body = Vec::new();
|
||||
@@ -1250,7 +1373,38 @@ async fn execute_stream_from_frame_stream(
|
||||
while prefetched_chunks.len() < MAX_STREAM_PREFETCH_FRAMES
|
||||
&& prefetched_inspection_body.len() < MAX_STREAM_PREFETCH_BYTES
|
||||
{
|
||||
let Some(frame) = (match next_stream_frame(&mut buffered_frames, &mut lines).await {
|
||||
let next_frame_result = if limit_direct_finalize_prefetch {
|
||||
match tokio::time::timeout(
|
||||
REWRITTEN_STREAM_PREFETCH_TIMEOUT,
|
||||
next_stream_frame(&mut buffered_frames, &mut lines),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(_) => {
|
||||
debug!(
|
||||
event_name = "execution_runtime_stream_prefetch_limited",
|
||||
log_type = "debug",
|
||||
trace_id = %trace_id,
|
||||
request_id = %request_id_for_log,
|
||||
candidate_id = ?candidate_id,
|
||||
plan_kind,
|
||||
report_kind,
|
||||
provider_name,
|
||||
endpoint_id = %plan.endpoint_id,
|
||||
key_id = %plan.key_id,
|
||||
model_name,
|
||||
candidate_index = candidate_index.as_str(),
|
||||
timeout_ms = REWRITTEN_STREAM_PREFETCH_TIMEOUT.as_millis() as u64,
|
||||
"gateway stopped rewritten stream prefetch before client-visible body"
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
next_stream_frame(&mut buffered_frames, &mut lines).await
|
||||
};
|
||||
let Some(frame) = (match next_frame_result {
|
||||
Ok(frame) => frame,
|
||||
Err(err) => {
|
||||
let failure = build_stream_failure_report(
|
||||
@@ -1575,7 +1729,6 @@ async fn execute_stream_from_frame_stream(
|
||||
let candidate_id = candidate_id.map(ToOwned::to_owned);
|
||||
let (tx, mut rx) = mpsc::channel::<Result<Bytes, IoError>>(16);
|
||||
let state_for_report = state.clone();
|
||||
let plan_for_report = plan;
|
||||
let trace_id_owned = trace_id.to_string();
|
||||
let headers_for_report = headers.clone();
|
||||
let report_kind_owned = report_kind;
|
||||
@@ -1593,8 +1746,14 @@ async fn execute_stream_from_frame_stream(
|
||||
let request_id_for_report = request_id.clone();
|
||||
let request_id_for_report_log = short_request_id(&request_id);
|
||||
let candidate_id_for_report = candidate_id.clone();
|
||||
let emit_passthrough_sse_terminal_error =
|
||||
skip_direct_finalize_prefetch && response_headers_indicate_sse(&upstream_headers);
|
||||
let candidate_index_for_report = candidate_index.clone();
|
||||
let is_openai_image_stream_for_report = plan_kind == OPENAI_IMAGE_STREAM_PLAN_KIND;
|
||||
let openai_image_stream_total_timeout_ms =
|
||||
resolve_openai_image_stream_total_timeout_ms(plan_kind, &plan);
|
||||
let plan_for_report = plan;
|
||||
let emit_passthrough_sse_terminal_error = skip_direct_finalize_prefetch
|
||||
&& response_headers_indicate_sse(&upstream_headers)
|
||||
&& !is_openai_image_stream_for_report;
|
||||
let body_capture_policy = match UsageRuntimeAccess::body_capture_policy(state.data.as_ref())
|
||||
.await
|
||||
{
|
||||
@@ -1623,6 +1782,8 @@ async fn execute_stream_from_frame_stream(
|
||||
.max_response_body_bytes
|
||||
.unwrap_or(DEFAULT_USAGE_RESPONSE_BODY_CAPTURE_LIMIT_BYTES)
|
||||
};
|
||||
let plan_kind_for_report = plan_kind.to_string();
|
||||
let stream_started_at_for_report = stream_started_at;
|
||||
tokio::spawn(async move {
|
||||
let mut provider_buffered_body = Vec::new();
|
||||
let mut buffered_body = Vec::new();
|
||||
@@ -1667,6 +1828,115 @@ async fn execute_stream_from_frame_stream(
|
||||
let reached_eof = initial_reached_eof;
|
||||
let mut downstream_dropped = false;
|
||||
let mut terminal_failure: Option<StreamFailureReport> = None;
|
||||
let initial_elapsed_ms = stream_started_at_for_report
|
||||
.elapsed()
|
||||
.as_millis()
|
||||
.min(u128::from(u64::MAX)) as u64;
|
||||
let last_upstream_frame_elapsed_ms = Arc::new(AtomicU64::new(initial_elapsed_ms));
|
||||
let last_client_chunk_elapsed_ms =
|
||||
Arc::new(AtomicU64::new(if prefetched_body_for_report.is_empty() {
|
||||
0
|
||||
} else {
|
||||
initial_elapsed_ms
|
||||
}));
|
||||
let provider_stream_bytes = Arc::new(AtomicU64::new(
|
||||
u64::try_from(provider_prefetched_body_for_report.len()).unwrap_or(u64::MAX),
|
||||
));
|
||||
let client_stream_bytes = Arc::new(AtomicU64::new(
|
||||
u64::try_from(prefetched_body_for_report.len()).unwrap_or(u64::MAX),
|
||||
));
|
||||
let idle_monitor_done = Arc::new(AtomicBool::new(false));
|
||||
let idle_monitor_handle = {
|
||||
let done = Arc::clone(&idle_monitor_done);
|
||||
let last_upstream = Arc::clone(&last_upstream_frame_elapsed_ms);
|
||||
let last_client = Arc::clone(&last_client_chunk_elapsed_ms);
|
||||
let provider_bytes = Arc::clone(&provider_stream_bytes);
|
||||
let client_bytes = Arc::clone(&client_stream_bytes);
|
||||
let trace_id_for_idle = trace_id_owned.clone();
|
||||
let request_id_for_idle = request_id_for_report_log.clone();
|
||||
let candidate_id_for_idle = candidate_id_for_report.clone();
|
||||
let candidate_index_for_idle = candidate_index_for_report.clone();
|
||||
let plan_kind_for_idle = plan_kind_for_report.clone();
|
||||
let provider_name_for_idle = plan_for_report
|
||||
.provider_name
|
||||
.clone()
|
||||
.unwrap_or_else(|| "-".to_string());
|
||||
let endpoint_id_for_idle = plan_for_report.endpoint_id.clone();
|
||||
let key_id_for_idle = plan_for_report.key_id.clone();
|
||||
let model_name_for_idle = plan_for_report
|
||||
.model_name
|
||||
.clone()
|
||||
.unwrap_or_else(|| "-".to_string());
|
||||
let has_local_stream_rewriter_for_idle = local_stream_rewriter.is_some();
|
||||
tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(STREAM_IDLE_LOG_INTERVAL);
|
||||
interval.set_missed_tick_behavior(MissedTickBehavior::Delay);
|
||||
interval.tick().await;
|
||||
loop {
|
||||
interval.tick().await;
|
||||
if done.load(Ordering::Relaxed) {
|
||||
break;
|
||||
}
|
||||
let elapsed_ms = stream_started_at_for_report
|
||||
.elapsed()
|
||||
.as_millis()
|
||||
.min(u128::from(u64::MAX)) as u64;
|
||||
let last_upstream_frame_elapsed_ms = last_upstream.load(Ordering::Relaxed);
|
||||
let last_client_chunk_elapsed_ms = last_client.load(Ordering::Relaxed);
|
||||
let upstream_idle_ms =
|
||||
elapsed_ms.saturating_sub(last_upstream_frame_elapsed_ms);
|
||||
let client_idle_ms = if last_client_chunk_elapsed_ms == 0 {
|
||||
elapsed_ms
|
||||
} else {
|
||||
elapsed_ms.saturating_sub(last_client_chunk_elapsed_ms)
|
||||
};
|
||||
if upstream_idle_ms >= STREAM_IDLE_LOG_INTERVAL_MS {
|
||||
warn!(
|
||||
event_name = "stream_execution_upstream_idle",
|
||||
log_type = "ops",
|
||||
trace_id = %trace_id_for_idle,
|
||||
request_id = %request_id_for_idle,
|
||||
candidate_id = ?candidate_id_for_idle.as_deref(),
|
||||
candidate_index = candidate_index_for_idle.as_str(),
|
||||
plan_kind = plan_kind_for_idle.as_str(),
|
||||
provider_name = provider_name_for_idle.as_str(),
|
||||
endpoint_id = %endpoint_id_for_idle,
|
||||
key_id = %key_id_for_idle,
|
||||
model_name = model_name_for_idle.as_str(),
|
||||
elapsed_ms,
|
||||
provider_bytes = provider_bytes.load(Ordering::Relaxed),
|
||||
client_bytes = client_bytes.load(Ordering::Relaxed),
|
||||
last_upstream_frame_elapsed_ms,
|
||||
last_client_chunk_elapsed_ms,
|
||||
"gateway stream has not received an upstream frame within the idle window"
|
||||
);
|
||||
} else if client_idle_ms >= STREAM_IDLE_LOG_INTERVAL_MS
|
||||
&& last_upstream_frame_elapsed_ms >= last_client_chunk_elapsed_ms
|
||||
{
|
||||
warn!(
|
||||
event_name = "stream_execution_client_visible_idle",
|
||||
log_type = "ops",
|
||||
trace_id = %trace_id_for_idle,
|
||||
request_id = %request_id_for_idle,
|
||||
candidate_id = ?candidate_id_for_idle.as_deref(),
|
||||
candidate_index = candidate_index_for_idle.as_str(),
|
||||
plan_kind = plan_kind_for_idle.as_str(),
|
||||
provider_name = provider_name_for_idle.as_str(),
|
||||
endpoint_id = %endpoint_id_for_idle,
|
||||
key_id = %key_id_for_idle,
|
||||
model_name = model_name_for_idle.as_str(),
|
||||
elapsed_ms,
|
||||
provider_bytes = provider_bytes.load(Ordering::Relaxed),
|
||||
client_bytes = client_bytes.load(Ordering::Relaxed),
|
||||
last_upstream_frame_elapsed_ms,
|
||||
last_client_chunk_elapsed_ms,
|
||||
local_stream_rewriter = has_local_stream_rewriter_for_idle,
|
||||
"gateway stream received upstream frames but has no recent client-visible chunk"
|
||||
);
|
||||
}
|
||||
}
|
||||
})
|
||||
};
|
||||
if !provider_prefetched_body_for_report.is_empty() {
|
||||
let normalized_prefetched_chunk = if let Some(normalizer) =
|
||||
private_stream_normalizer.as_mut()
|
||||
@@ -1735,8 +2005,67 @@ async fn execute_stream_from_frame_stream(
|
||||
}
|
||||
|
||||
if terminal_failure.is_none() && !reached_eof {
|
||||
let mut image_stream_total_timeout = openai_image_stream_total_timeout_ms
|
||||
.map(|timeout_ms| Box::pin(tokio::time::sleep(Duration::from_millis(timeout_ms))));
|
||||
loop {
|
||||
let next_frame = match next_stream_frame(&mut buffered_frames, &mut lines).await {
|
||||
let next_frame_result = if let Some(timeout_sleep) =
|
||||
image_stream_total_timeout.as_mut()
|
||||
{
|
||||
tokio::select! {
|
||||
result = next_stream_frame(&mut buffered_frames, &mut lines) => result,
|
||||
_ = timeout_sleep.as_mut() => {
|
||||
let timeout_ms = openai_image_stream_total_timeout_ms
|
||||
.unwrap_or(OPENAI_IMAGE_STREAM_DEFAULT_TOTAL_TIMEOUT_MS);
|
||||
let elapsed_ms = stream_started_at_for_report
|
||||
.elapsed()
|
||||
.as_millis()
|
||||
.min(u128::from(u64::MAX)) as u64;
|
||||
warn!(
|
||||
event_name = "openai_image_stream_total_timeout",
|
||||
log_type = "ops",
|
||||
trace_id = %trace_id_owned,
|
||||
request_id = %request_id_for_report_log,
|
||||
candidate_id = ?candidate_id_for_report.as_deref(),
|
||||
candidate_index = candidate_index_for_report.as_str(),
|
||||
plan_kind = plan_kind_for_report.as_str(),
|
||||
provider_name = plan_for_report.provider_name.as_deref().unwrap_or("-"),
|
||||
endpoint_id = %plan_for_report.endpoint_id,
|
||||
key_id = %plan_for_report.key_id,
|
||||
model_name = plan_for_report.model_name.as_deref().unwrap_or("-"),
|
||||
elapsed_ms,
|
||||
timeout_ms,
|
||||
provider_bytes = provider_stream_bytes.load(Ordering::Relaxed),
|
||||
client_bytes = client_stream_bytes.load(Ordering::Relaxed),
|
||||
last_upstream_frame_elapsed_ms = last_upstream_frame_elapsed_ms.load(Ordering::Relaxed),
|
||||
last_client_chunk_elapsed_ms = last_client_chunk_elapsed_ms.load(Ordering::Relaxed),
|
||||
"gateway OpenAI image stream exceeded total timeout"
|
||||
);
|
||||
telemetry = Some(ExecutionTelemetry {
|
||||
ttfb_ms: telemetry
|
||||
.as_ref()
|
||||
.and_then(|telemetry| telemetry.ttfb_ms)
|
||||
.or_else(|| {
|
||||
usage_stream_telemetry
|
||||
.as_ref()
|
||||
.and_then(|telemetry| telemetry.ttfb_ms)
|
||||
}),
|
||||
elapsed_ms: Some(elapsed_ms),
|
||||
upstream_bytes: Some(provider_stream_bytes.load(Ordering::Relaxed)),
|
||||
});
|
||||
terminal_failure = Some(build_stream_failure_report(
|
||||
"image_stream_total_timeout",
|
||||
format!(
|
||||
"OpenAI image stream exceeded total timeout of {timeout_ms}ms"
|
||||
),
|
||||
504,
|
||||
));
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
next_stream_frame(&mut buffered_frames, &mut lines).await
|
||||
};
|
||||
let next_frame = match next_frame_result {
|
||||
Ok(frame) => frame,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
@@ -1759,6 +2088,11 @@ async fn execute_stream_from_frame_stream(
|
||||
let Some(frame) = next_frame else {
|
||||
break;
|
||||
};
|
||||
let frame_elapsed_ms = stream_started_at_for_report
|
||||
.elapsed()
|
||||
.as_millis()
|
||||
.min(u128::from(u64::MAX)) as u64;
|
||||
last_upstream_frame_elapsed_ms.store(frame_elapsed_ms, Ordering::Relaxed);
|
||||
match frame.payload {
|
||||
StreamFramePayload::Data { chunk_b64, text } => {
|
||||
if sync_json_stream_bridge_active_for_report {
|
||||
@@ -1792,6 +2126,10 @@ async fn execute_stream_from_frame_stream(
|
||||
continue;
|
||||
}
|
||||
|
||||
provider_stream_bytes.fetch_add(
|
||||
u64::try_from(chunk.len()).unwrap_or(u64::MAX),
|
||||
Ordering::Relaxed,
|
||||
);
|
||||
append_stream_capture_bytes(
|
||||
&mut provider_buffered_body,
|
||||
&chunk,
|
||||
@@ -1870,7 +2208,7 @@ async fn execute_stream_from_frame_stream(
|
||||
.and_then(|telemetry| telemetry.ttfb_ms)
|
||||
.is_none()
|
||||
{
|
||||
let first_data_elapsed_ms = stream_started_at
|
||||
let first_data_elapsed_ms = stream_started_at_for_report
|
||||
.elapsed()
|
||||
.as_millis()
|
||||
.min(u128::from(u64::MAX))
|
||||
@@ -1897,6 +2235,8 @@ async fn execute_stream_from_frame_stream(
|
||||
max_stream_body_buffer_bytes,
|
||||
&mut client_body_truncated,
|
||||
);
|
||||
let rewritten_chunk_len =
|
||||
u64::try_from(rewritten_chunk.len()).unwrap_or(u64::MAX);
|
||||
if tx.send(Ok(Bytes::from(rewritten_chunk))).await.is_err() {
|
||||
warn!(
|
||||
event_name = "stream_execution_downstream_disconnected",
|
||||
@@ -1908,6 +2248,16 @@ async fn execute_stream_from_frame_stream(
|
||||
);
|
||||
downstream_dropped = true;
|
||||
break;
|
||||
} else {
|
||||
client_stream_bytes.fetch_add(rewritten_chunk_len, Ordering::Relaxed);
|
||||
last_client_chunk_elapsed_ms.store(
|
||||
stream_started_at_for_report
|
||||
.elapsed()
|
||||
.as_millis()
|
||||
.min(u128::from(u64::MAX))
|
||||
as u64,
|
||||
Ordering::Relaxed,
|
||||
);
|
||||
}
|
||||
}
|
||||
StreamFramePayload::Telemetry {
|
||||
@@ -2010,16 +2360,29 @@ async fn execute_stream_from_frame_stream(
|
||||
max_stream_body_buffer_bytes,
|
||||
&mut client_body_truncated,
|
||||
);
|
||||
let rewritten_chunk_len =
|
||||
u64::try_from(rewritten_chunk.len()).unwrap_or(u64::MAX);
|
||||
if tx.send(Ok(Bytes::from(rewritten_chunk))).await.is_err() {
|
||||
warn!(
|
||||
event_name = "stream_execution_downstream_flush_disconnected",
|
||||
log_type = "ops",
|
||||
trace_id = %trace_id_owned,
|
||||
request_id = %request_id_for_report_log,
|
||||
candidate_id = ?candidate_id_for_report.as_deref(),
|
||||
"gateway stream downstream dropped while flushing private stream normalization"
|
||||
candidate_id = ?candidate_id_for_report.as_deref(),
|
||||
"gateway stream downstream dropped while flushing private stream normalization"
|
||||
);
|
||||
downstream_dropped = true;
|
||||
} else {
|
||||
client_stream_bytes
|
||||
.fetch_add(rewritten_chunk_len, Ordering::Relaxed);
|
||||
last_client_chunk_elapsed_ms.store(
|
||||
stream_started_at_for_report
|
||||
.elapsed()
|
||||
.as_millis()
|
||||
.min(u128::from(u64::MAX))
|
||||
as u64,
|
||||
Ordering::Relaxed,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2054,16 +2417,28 @@ async fn execute_stream_from_frame_stream(
|
||||
max_stream_body_buffer_bytes,
|
||||
&mut client_body_truncated,
|
||||
);
|
||||
let flushed_chunk_len =
|
||||
u64::try_from(flushed_chunk.len()).unwrap_or(u64::MAX);
|
||||
if tx.send(Ok(Bytes::from(flushed_chunk))).await.is_err() {
|
||||
warn!(
|
||||
event_name = "stream_execution_downstream_rewrite_flush_disconnected",
|
||||
log_type = "ops",
|
||||
trace_id = %trace_id_owned,
|
||||
request_id = %request_id_for_report_log,
|
||||
candidate_id = ?candidate_id_for_report.as_deref(),
|
||||
"gateway stream downstream dropped while flushing local stream rewrite"
|
||||
candidate_id = ?candidate_id_for_report.as_deref(),
|
||||
"gateway stream downstream dropped while flushing local stream rewrite"
|
||||
);
|
||||
downstream_dropped = true;
|
||||
} else {
|
||||
client_stream_bytes.fetch_add(flushed_chunk_len, Ordering::Relaxed);
|
||||
last_client_chunk_elapsed_ms.store(
|
||||
stream_started_at_for_report
|
||||
.elapsed()
|
||||
.as_millis()
|
||||
.min(u128::from(u64::MAX))
|
||||
as u64,
|
||||
Ordering::Relaxed,
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(_) => {}
|
||||
@@ -2090,44 +2465,70 @@ async fn execute_stream_from_frame_stream(
|
||||
}
|
||||
}
|
||||
|
||||
if !downstream_dropped && emit_passthrough_sse_terminal_error {
|
||||
if !downstream_dropped {
|
||||
if let Some(failure) = terminal_failure.as_ref() {
|
||||
match encode_terminal_sse_error_event(failure) {
|
||||
Ok(error_event) => {
|
||||
append_stream_capture_bytes(
|
||||
&mut buffered_body,
|
||||
error_event.as_ref(),
|
||||
max_stream_body_buffer_bytes,
|
||||
&mut client_body_truncated,
|
||||
);
|
||||
if tx.send(Ok(error_event)).await.is_err() {
|
||||
warn!(
|
||||
let terminal_event = if is_openai_image_stream_for_report {
|
||||
Some(encode_openai_image_failed_event(
|
||||
report_context_owned.as_ref(),
|
||||
failure,
|
||||
))
|
||||
} else if emit_passthrough_sse_terminal_error {
|
||||
Some(encode_terminal_sse_error_event(failure))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(terminal_event) = terminal_event {
|
||||
match terminal_event {
|
||||
Ok(error_event) => {
|
||||
let error_event_len =
|
||||
u64::try_from(error_event.len()).unwrap_or(u64::MAX);
|
||||
append_stream_capture_bytes(
|
||||
&mut buffered_body,
|
||||
error_event.as_ref(),
|
||||
max_stream_body_buffer_bytes,
|
||||
&mut client_body_truncated,
|
||||
);
|
||||
if tx.send(Ok(error_event)).await.is_err() {
|
||||
warn!(
|
||||
event_name = "stream_execution_downstream_terminal_error_disconnected",
|
||||
log_type = "ops",
|
||||
trace_id = %trace_id_owned,
|
||||
request_id = %request_id_for_report_log,
|
||||
candidate_id = ?candidate_id_for_report.as_deref(),
|
||||
"gateway stream downstream dropped while sending terminal SSE error event"
|
||||
);
|
||||
downstream_dropped = true;
|
||||
);
|
||||
downstream_dropped = true;
|
||||
} else {
|
||||
client_stream_bytes.fetch_add(error_event_len, Ordering::Relaxed);
|
||||
last_client_chunk_elapsed_ms.store(
|
||||
stream_started_at_for_report
|
||||
.elapsed()
|
||||
.as_millis()
|
||||
.min(u128::from(u64::MAX))
|
||||
as u64,
|
||||
Ordering::Relaxed,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event_name = "stream_execution_terminal_error_event_encode_failed",
|
||||
log_type = "ops",
|
||||
trace_id = %trace_id_owned,
|
||||
request_id = %request_id_for_report_log,
|
||||
candidate_id = ?candidate_id_for_report.as_deref(),
|
||||
error = ?err,
|
||||
"gateway failed to encode terminal SSE error event"
|
||||
);
|
||||
"gateway failed to encode terminal SSE error event"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
drop(tx);
|
||||
idle_monitor_done.store(true, Ordering::Relaxed);
|
||||
idle_monitor_handle.abort();
|
||||
|
||||
stream_terminal_summary = merge_stream_terminal_summary(
|
||||
stream_terminal_summary,
|
||||
@@ -2292,15 +2693,6 @@ async fn execute_stream_from_frame_stream(
|
||||
}
|
||||
});
|
||||
|
||||
let body_stream = stream! {
|
||||
for chunk in prefetched_chunks_for_body {
|
||||
yield Ok(chunk);
|
||||
}
|
||||
while let Some(item) = rx.recv().await {
|
||||
yield item;
|
||||
}
|
||||
};
|
||||
|
||||
headers.insert(CONTROL_REQUEST_ID_HEADER.to_string(), request_id.clone());
|
||||
|
||||
if let Some(candidate_id) = candidate_id
|
||||
@@ -2314,6 +2706,17 @@ async fn execute_stream_from_frame_stream(
|
||||
);
|
||||
}
|
||||
|
||||
let emit_sse_keepalive = response_headers_indicate_sse(&headers);
|
||||
if emit_sse_keepalive {
|
||||
headers.remove("content-length");
|
||||
}
|
||||
let body_stream = build_sse_body_stream(
|
||||
prefetched_chunks_for_body,
|
||||
rx,
|
||||
emit_sse_keepalive,
|
||||
SSE_KEEPALIVE_INTERVAL,
|
||||
);
|
||||
|
||||
Ok(Some(build_client_response_from_parts(
|
||||
status_code,
|
||||
&headers,
|
||||
@@ -2337,7 +2740,7 @@ mod tests {
|
||||
use std::collections::BTreeMap;
|
||||
use std::convert::Infallible;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use aether_contracts::{
|
||||
ExecutionPlan, ExecutionStreamTerminalSummary, ExecutionTimeouts, RequestBody,
|
||||
@@ -2352,11 +2755,13 @@ mod tests {
|
||||
use axum::extract::Request;
|
||||
use axum::routing::any;
|
||||
use axum::{http::header, http::HeaderValue, Router};
|
||||
use futures_util::StreamExt as _;
|
||||
use serde_json::{json, Value};
|
||||
use tokio::sync::{watch, Notify};
|
||||
use tokio::sync::{mpsc, watch, Notify};
|
||||
|
||||
use super::{
|
||||
execute_execution_runtime_stream, merge_stream_terminal_summary,
|
||||
build_sse_body_stream, execute_execution_runtime_stream, execute_stream_from_frame_stream,
|
||||
merge_stream_terminal_summary, should_limit_direct_finalize_prefetch,
|
||||
should_probe_success_failover_before_stream, should_skip_direct_finalize_prefetch,
|
||||
};
|
||||
use crate::control::GatewayControlDecision;
|
||||
@@ -2482,6 +2887,132 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn limits_prefetch_for_openai_image_and_rewritten_streams() {
|
||||
assert!(should_limit_direct_finalize_prefetch(
|
||||
"openai_image_stream",
|
||||
false
|
||||
));
|
||||
assert!(should_limit_direct_finalize_prefetch(
|
||||
"openai_chat_stream",
|
||||
true
|
||||
));
|
||||
assert!(!should_limit_direct_finalize_prefetch(
|
||||
"openai_chat_stream",
|
||||
false
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sse_body_stream_emits_initial_and_periodic_keepalive_without_business_chunks() {
|
||||
let (_tx, rx) = mpsc::channel::<Result<Bytes, std::io::Error>>(1);
|
||||
let mut body_stream = Box::pin(build_sse_body_stream(
|
||||
Vec::new(),
|
||||
rx,
|
||||
true,
|
||||
Duration::from_millis(10),
|
||||
));
|
||||
|
||||
let first = tokio::time::timeout(Duration::from_millis(50), body_stream.next())
|
||||
.await
|
||||
.expect("initial keepalive should be immediate")
|
||||
.expect("stream should yield initial keepalive")
|
||||
.expect("initial keepalive should be ok");
|
||||
assert_eq!(first.as_ref(), b": aether-keepalive\n\n");
|
||||
|
||||
let second = tokio::time::timeout(Duration::from_millis(100), body_stream.next())
|
||||
.await
|
||||
.expect("periodic keepalive should arrive")
|
||||
.expect("stream should yield periodic keepalive")
|
||||
.expect("periodic keepalive should be ok");
|
||||
assert_eq!(second.as_ref(), b": aether-keepalive\n\n");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn openai_image_stream_total_timeout_emits_image_failed_event() {
|
||||
let state = AppState::new().expect("app state should build");
|
||||
let plan = ExecutionPlan {
|
||||
request_id: "req-image-stream-timeout".into(),
|
||||
candidate_id: Some("cand-image-stream-timeout".into()),
|
||||
provider_name: Some("codex".into()),
|
||||
provider_id: "prov-1".into(),
|
||||
endpoint_id: "ep-1".into(),
|
||||
key_id: "key-1".into(),
|
||||
method: "POST".into(),
|
||||
url: "https://chatgpt.com/backend-api/codex/responses".into(),
|
||||
headers: BTreeMap::from([
|
||||
("content-type".into(), "application/json".into()),
|
||||
("accept".into(), "text/event-stream".into()),
|
||||
]),
|
||||
content_type: Some("application/json".into()),
|
||||
content_encoding: None,
|
||||
body: RequestBody::from_json(json!({
|
||||
"model": "gpt-image-1",
|
||||
"prompt": "hello",
|
||||
"stream": true
|
||||
})),
|
||||
stream: true,
|
||||
client_api_format: "openai:image".into(),
|
||||
provider_api_format: "openai:image".into(),
|
||||
model_name: Some("gpt-image-1".into()),
|
||||
proxy: None,
|
||||
transport_profile: None,
|
||||
timeouts: Some(ExecutionTimeouts {
|
||||
total_ms: Some(25),
|
||||
..ExecutionTimeouts::default()
|
||||
}),
|
||||
};
|
||||
let decision = GatewayControlDecision::synthetic(
|
||||
"/v1/images/generations",
|
||||
Some("ai_public".to_string()),
|
||||
Some("openai".to_string()),
|
||||
Some("image".to_string()),
|
||||
Some("openai:image".to_string()),
|
||||
)
|
||||
.with_execution_runtime_candidate(true);
|
||||
let frame_stream = stream! {
|
||||
yield Ok::<Bytes, std::io::Error>(Bytes::from_static(
|
||||
b"{\"type\":\"headers\",\"payload\":{\"kind\":\"headers\",\"status_code\":200,\"headers\":{\"content-type\":\"text/event-stream\"}}}\n",
|
||||
));
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
.boxed();
|
||||
|
||||
let response = execute_stream_from_frame_stream(
|
||||
&state,
|
||||
plan,
|
||||
"trace-image-stream-timeout",
|
||||
&decision,
|
||||
"openai_image_stream",
|
||||
None,
|
||||
Some(json!({
|
||||
"provider_api_format": "openai:image",
|
||||
"client_api_format": "openai:image",
|
||||
"image_request": {
|
||||
"operation": "generate"
|
||||
}
|
||||
})),
|
||||
crate::clock::current_unix_ms(),
|
||||
Instant::now(),
|
||||
frame_stream,
|
||||
)
|
||||
.await
|
||||
.expect("execution should succeed")
|
||||
.expect("execution should return a client response");
|
||||
|
||||
let body = tokio::time::timeout(
|
||||
Duration::from_secs(2),
|
||||
to_bytes(response.into_body(), usize::MAX),
|
||||
)
|
||||
.await
|
||||
.expect("timeout failure should close the response body")
|
||||
.expect("response body should read");
|
||||
let text = String::from_utf8(body.to_vec()).expect("response body should be utf8");
|
||||
assert!(text.contains(": aether-keepalive\n\n"));
|
||||
assert!(text.contains("event: image_generation.failed"));
|
||||
assert!(text.contains("\"type\":\"image_stream_total_timeout\""));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_execution_runtime_stream_records_first_data_as_streaming_before_terminal_telemetry(
|
||||
) {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -551,7 +551,7 @@ fn build_direct_tunnel_request_meta(
|
||||
}
|
||||
}
|
||||
|
||||
async fn send_request(
|
||||
pub(crate) async fn send_request(
|
||||
plan: &ExecutionPlan,
|
||||
body_bytes: Vec<u8>,
|
||||
) -> Result<reqwest::Response, ExecutionRuntimeTransportError> {
|
||||
@@ -728,7 +728,9 @@ async fn send_via_tunnel_relay(
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
fn build_request_body(plan: &ExecutionPlan) -> Result<Vec<u8>, ExecutionRuntimeTransportError> {
|
||||
pub(crate) fn build_request_body(
|
||||
plan: &ExecutionPlan,
|
||||
) -> Result<Vec<u8>, ExecutionRuntimeTransportError> {
|
||||
let mut body_bytes = if let Some(json_body) = plan.body.json_body.clone() {
|
||||
serde_json::to_vec(&json_body).map_err(ExecutionRuntimeTransportError::BodyEncode)?
|
||||
} else if let Some(body_b64) = plan.body.body_bytes_b64.as_deref() {
|
||||
@@ -1102,7 +1104,7 @@ fn is_hop_by_hop_header(name: &str) -> bool {
|
||||
)
|
||||
}
|
||||
|
||||
fn collect_response_headers(headers: &HeaderMap) -> BTreeMap<String, String> {
|
||||
pub(crate) fn collect_response_headers(headers: &HeaderMap) -> BTreeMap<String, String> {
|
||||
header_map_to_string_map(headers)
|
||||
}
|
||||
|
||||
@@ -1130,7 +1132,7 @@ fn execution_log_url_host(url: &str) -> String {
|
||||
.unwrap_or_else(|| "-".to_string())
|
||||
}
|
||||
|
||||
fn decode_response_body_bytes(
|
||||
pub(crate) fn decode_response_body_bytes(
|
||||
headers: &BTreeMap<String, String>,
|
||||
body_bytes: &[u8],
|
||||
) -> Option<Vec<u8>> {
|
||||
@@ -1157,7 +1159,7 @@ fn decode_response_body_bytes(
|
||||
}
|
||||
}
|
||||
|
||||
fn response_body_is_json(headers: &BTreeMap<String, String>, body_bytes: &[u8]) -> bool {
|
||||
pub(crate) fn response_body_is_json(headers: &BTreeMap<String, String>, body_bytes: &[u8]) -> bool {
|
||||
if headers
|
||||
.get("content-type")
|
||||
.map(|value| value.to_ascii_lowercase())
|
||||
|
||||
@@ -158,6 +158,19 @@ where
|
||||
last_plan: aether_contracts::ExecutionPlan,
|
||||
last_report_context: Option<serde_json::Value>,
|
||||
) -> Result<Self::Exhaustion, Self::Error> {
|
||||
warn!(
|
||||
event_name = "candidate_loop_exhausted",
|
||||
log_type = "ops",
|
||||
trace_id = %self.trace_id,
|
||||
plan_kind = self.plan_kind,
|
||||
request_id = %short_request_id(last_plan.request_id.as_str()),
|
||||
candidate_id = ?last_plan.candidate_id,
|
||||
provider_name = last_plan.provider_name.as_deref().unwrap_or("-"),
|
||||
endpoint_id = %last_plan.endpoint_id,
|
||||
key_id = %last_plan.key_id,
|
||||
model_name = last_plan.model_name.as_deref().unwrap_or("-"),
|
||||
"candidate loop exhausted local sync candidates"
|
||||
);
|
||||
Ok(
|
||||
build_local_execution_exhaustion(self.state, &last_plan, last_report_context.as_ref())
|
||||
.await,
|
||||
|
||||
@@ -18,7 +18,8 @@ use crate::ai_serving::api::{
|
||||
resolve_claude_stream_spec, resolve_claude_sync_spec, resolve_gemini_stream_spec,
|
||||
resolve_gemini_sync_spec, resolve_local_same_format_stream_spec,
|
||||
resolve_local_same_format_sync_spec, set_local_openai_chat_execution_exhausted_diagnostic,
|
||||
AiStreamAttempt, AiSyncAttempt, LocalStandardSpec, EXECUTION_RUNTIME_STREAM_DECISION_ACTION,
|
||||
set_local_openai_image_execution_exhausted_diagnostic, AiStreamAttempt, AiSyncAttempt,
|
||||
LocalStandardSpec, EXECUTION_RUNTIME_STREAM_DECISION_ACTION,
|
||||
EXECUTION_RUNTIME_SYNC_DECISION_ACTION,
|
||||
};
|
||||
use crate::control::GatewayControlDecision;
|
||||
@@ -457,7 +458,7 @@ pub(crate) async fn maybe_execute_sync_via_local_image_decision(
|
||||
decision: &GatewayControlDecision,
|
||||
plan_kind: &str,
|
||||
) -> Result<LocalExecutionRequestOutcome, GatewayError> {
|
||||
let Some((attempt_source, _candidate_count)) = build_local_image_sync_attempt_source_for_kind(
|
||||
let Some((attempt_source, candidate_count)) = build_local_image_sync_attempt_source_for_kind(
|
||||
state,
|
||||
parts,
|
||||
body_json,
|
||||
@@ -471,7 +472,7 @@ pub(crate) async fn maybe_execute_sync_via_local_image_decision(
|
||||
return Ok(LocalExecutionRequestOutcome::NoPath);
|
||||
};
|
||||
|
||||
execute_sync_attempt_source::<AiSyncAttempt, _>(
|
||||
let outcome = execute_sync_attempt_source::<AiSyncAttempt, _>(
|
||||
state,
|
||||
parts,
|
||||
trace_id,
|
||||
@@ -479,7 +480,20 @@ pub(crate) async fn maybe_execute_sync_via_local_image_decision(
|
||||
plan_kind,
|
||||
attempt_source,
|
||||
)
|
||||
.await
|
||||
.await?;
|
||||
|
||||
if let LocalExecutionRequestOutcome::Exhausted(_) = &outcome {
|
||||
set_local_openai_image_execution_exhausted_diagnostic(
|
||||
state,
|
||||
trace_id,
|
||||
decision,
|
||||
plan_kind,
|
||||
body_json,
|
||||
candidate_count,
|
||||
);
|
||||
}
|
||||
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
pub(crate) async fn maybe_execute_stream_via_local_gemini_files_decision(
|
||||
@@ -517,29 +531,41 @@ pub(crate) async fn maybe_execute_stream_via_local_image_decision(
|
||||
decision: &GatewayControlDecision,
|
||||
plan_kind: &str,
|
||||
) -> Result<LocalExecutionRequestOutcome, GatewayError> {
|
||||
let Some((attempt_source, _candidate_count)) =
|
||||
build_local_image_stream_attempt_source_for_kind(
|
||||
state,
|
||||
parts,
|
||||
body_json,
|
||||
body_base64,
|
||||
trace_id,
|
||||
decision,
|
||||
plan_kind,
|
||||
)
|
||||
.await?
|
||||
let Some((attempt_source, candidate_count)) = build_local_image_stream_attempt_source_for_kind(
|
||||
state,
|
||||
parts,
|
||||
body_json,
|
||||
body_base64,
|
||||
trace_id,
|
||||
decision,
|
||||
plan_kind,
|
||||
)
|
||||
.await?
|
||||
else {
|
||||
return Ok(LocalExecutionRequestOutcome::NoPath);
|
||||
};
|
||||
|
||||
execute_stream_attempt_source::<AiStreamAttempt, _>(
|
||||
let outcome = execute_stream_attempt_source::<AiStreamAttempt, _>(
|
||||
state,
|
||||
trace_id,
|
||||
decision,
|
||||
plan_kind,
|
||||
attempt_source,
|
||||
)
|
||||
.await
|
||||
.await?;
|
||||
|
||||
if let LocalExecutionRequestOutcome::Exhausted(_) = &outcome {
|
||||
set_local_openai_image_execution_exhausted_diagnostic(
|
||||
state,
|
||||
trace_id,
|
||||
decision,
|
||||
plan_kind,
|
||||
body_json,
|
||||
candidate_count,
|
||||
);
|
||||
}
|
||||
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
pub(crate) async fn maybe_execute_sync_via_local_video_decision(
|
||||
|
||||
@@ -194,6 +194,57 @@ async fn resolve_admin_usage_attempt_flags_by_usage_id(
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn resolve_admin_usage_image_progress_by_request_id(
|
||||
state: &AdminAppState<'_>,
|
||||
items: &[StoredRequestUsageAudit],
|
||||
) -> Result<BTreeMap<String, serde_json::Value>, GatewayError> {
|
||||
if !state.has_request_candidate_data_reader() || items.is_empty() {
|
||||
return Ok(BTreeMap::new());
|
||||
}
|
||||
|
||||
let request_ids = items
|
||||
.iter()
|
||||
.map(|item| item.request_id.clone())
|
||||
.collect::<BTreeSet<_>>();
|
||||
let mut progress_by_request_id = BTreeMap::new();
|
||||
for request_id in request_ids {
|
||||
let candidates = state
|
||||
.app()
|
||||
.read_request_candidates_by_request_id(&request_id)
|
||||
.await?;
|
||||
if let Some(progress) = latest_admin_usage_image_progress(&candidates) {
|
||||
progress_by_request_id.insert(request_id, progress);
|
||||
}
|
||||
}
|
||||
Ok(progress_by_request_id)
|
||||
}
|
||||
|
||||
fn latest_admin_usage_image_progress(
|
||||
candidates: &[StoredRequestCandidate],
|
||||
) -> Option<serde_json::Value> {
|
||||
candidates
|
||||
.iter()
|
||||
.filter_map(|candidate| {
|
||||
let progress = candidate
|
||||
.extra_data
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("image_progress"))?
|
||||
.clone();
|
||||
Some((
|
||||
candidate
|
||||
.started_at_unix_ms
|
||||
.unwrap_or(candidate.created_at_unix_ms),
|
||||
candidate.candidate_index,
|
||||
candidate.retry_index,
|
||||
progress,
|
||||
))
|
||||
})
|
||||
.max_by_key(|(started_at, candidate_index, retry_index, _)| {
|
||||
(*started_at, *candidate_index, *retry_index)
|
||||
})
|
||||
.map(|(_, _, _, progress)| progress)
|
||||
}
|
||||
|
||||
fn admin_usage_matches_attempt_status(
|
||||
item: &StoredRequestUsageAudit,
|
||||
status: &str,
|
||||
@@ -490,6 +541,7 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
|
||||
&BTreeMap::new(),
|
||||
state.has_auth_api_key_data_reader(),
|
||||
&BTreeMap::new(),
|
||||
&BTreeMap::new(),
|
||||
)));
|
||||
};
|
||||
state
|
||||
@@ -513,12 +565,15 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
|
||||
};
|
||||
let api_key_names = admin_usage_api_key_names(state, &items).await?;
|
||||
let provider_key_names = admin_usage_provider_key_names(state, &items).await?;
|
||||
let image_progress_by_request_id =
|
||||
resolve_admin_usage_image_progress_by_request_id(state, &items).await?;
|
||||
|
||||
return Ok(Some(build_admin_usage_active_requests_response(
|
||||
&items,
|
||||
&api_key_names,
|
||||
state.has_auth_api_key_data_reader(),
|
||||
&provider_key_names,
|
||||
&image_progress_by_request_id,
|
||||
)));
|
||||
}
|
||||
Some("records")
|
||||
|
||||
@@ -274,6 +274,47 @@ pub(crate) async fn record_local_request_candidate_status(
|
||||
persist_local_request_candidate_status_record(state, record).await;
|
||||
}
|
||||
|
||||
pub(crate) async fn record_local_request_candidate_extra_data(
|
||||
state: &(impl RequestCandidateRuntimeWriter + ?Sized),
|
||||
plan: &ExecutionPlan,
|
||||
report_context: Option<&Value>,
|
||||
status: RequestCandidateStatus,
|
||||
status_code: Option<u16>,
|
||||
latency_ms: Option<u64>,
|
||||
extra_data: Value,
|
||||
) {
|
||||
let Some(snapshot) = snapshot_local_request_candidate_status(plan, report_context) else {
|
||||
return;
|
||||
};
|
||||
let record = UpsertRequestCandidateRecord {
|
||||
id: snapshot.candidate_id.clone(),
|
||||
request_id: snapshot.request_id.clone(),
|
||||
user_id: snapshot.user_id.clone(),
|
||||
api_key_id: snapshot.api_key_id.clone(),
|
||||
username: None,
|
||||
api_key_name: None,
|
||||
candidate_index: snapshot.candidate_index,
|
||||
retry_index: snapshot.retry_index,
|
||||
provider_id: Some(snapshot.provider_id.clone()),
|
||||
endpoint_id: Some(snapshot.endpoint_id.clone()),
|
||||
key_id: Some(snapshot.key_id.clone()),
|
||||
status,
|
||||
skip_reason: None,
|
||||
is_cached: None,
|
||||
status_code,
|
||||
error_type: None,
|
||||
error_message: None,
|
||||
latency_ms,
|
||||
concurrent_requests: None,
|
||||
extra_data: Some(extra_data),
|
||||
required_capabilities: None,
|
||||
created_at_unix_ms: None,
|
||||
started_at_unix_ms: None,
|
||||
finished_at_unix_ms: None,
|
||||
};
|
||||
persist_local_request_candidate_status_record(state, record).await;
|
||||
}
|
||||
|
||||
pub(crate) async fn record_local_request_candidate_status_snapshot(
|
||||
state: &(impl RequestCandidateRuntimeWriter + ?Sized),
|
||||
snapshot: &LocalRequestCandidateStatusSnapshot,
|
||||
|
||||
@@ -20,6 +20,7 @@ use aether_data_contracts::repository::provider_catalog::{
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::data::GatewayDataState;
|
||||
use crate::tests::next_non_keepalive_chunk;
|
||||
|
||||
fn hash_api_key(value: &str) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
@@ -370,11 +371,7 @@ async fn gateway_stops_execution_runtime_stream_when_client_disconnects() {
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let mut response = response;
|
||||
let first_chunk = response
|
||||
.chunk()
|
||||
.await
|
||||
.expect("first chunk should read")
|
||||
.expect("first chunk should exist");
|
||||
let first_chunk = next_non_keepalive_chunk(&mut response).await;
|
||||
assert_eq!(
|
||||
first_chunk,
|
||||
Bytes::from_static(b"data: {\"id\":\"chatcmpl-first\"}\n\n")
|
||||
|
||||
@@ -18,9 +18,10 @@ use crate::constants::{
|
||||
|
||||
use super::{
|
||||
build_router, build_router_with_execution_runtime_override, build_router_with_state,
|
||||
build_state_with_execution_runtime_override, start_server, wait_until, AppState,
|
||||
FrontdoorCorsConfig, FrontdoorUserRpmConfig, GatewayFallbackMetricKind, GatewayFallbackReason,
|
||||
UsageRuntimeConfig, VideoTaskTruthSourceMode,
|
||||
build_state_with_execution_runtime_override, next_non_keepalive_chunk, start_server,
|
||||
strip_sse_keepalive_comments, wait_until, AppState, FrontdoorCorsConfig,
|
||||
FrontdoorUserRpmConfig, GatewayFallbackMetricKind, GatewayFallbackReason, UsageRuntimeConfig,
|
||||
VideoTaskTruthSourceMode,
|
||||
};
|
||||
|
||||
mod control_execute;
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
use super::{
|
||||
any, build_router_with_state, build_state_with_execution_runtime_override, json, start_server,
|
||||
to_bytes, AppState, Arc, Body, Bytes, HeaderName, HeaderValue, Json, Mutex, Request, Response,
|
||||
Router, StatusCode, EXECUTION_PATH_EXECUTION_RUNTIME_STREAM, EXECUTION_PATH_HEADER,
|
||||
TRACE_ID_HEADER,
|
||||
strip_sse_keepalive_comments, to_bytes, AppState, Arc, Body, Bytes, HeaderName, HeaderValue,
|
||||
Json, Mutex, Request, Response, Router, StatusCode, EXECUTION_PATH_EXECUTION_RUNTIME_STREAM,
|
||||
EXECUTION_PATH_HEADER, TRACE_ID_HEADER,
|
||||
};
|
||||
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
|
||||
use aether_data::repository::auth::{
|
||||
@@ -391,7 +391,7 @@ async fn gateway_executes_openai_chat_stream_via_local_decision_gate_without_exe
|
||||
Some(EXECUTION_PATH_EXECUTION_RUNTIME_STREAM)
|
||||
);
|
||||
assert_eq!(
|
||||
response.text().await.expect("body should read"),
|
||||
strip_sse_keepalive_comments(&response.text().await.expect("body should read")),
|
||||
"data: {\"id\":\"chatcmpl-local-123\"}\n\ndata: [DONE]\n\n"
|
||||
);
|
||||
|
||||
@@ -1352,7 +1352,7 @@ async fn gateway_executes_openai_chat_stream_with_custom_path_via_local_decision
|
||||
Some(EXECUTION_PATH_EXECUTION_RUNTIME_STREAM)
|
||||
);
|
||||
assert_eq!(
|
||||
response.text().await.expect("body should read"),
|
||||
strip_sse_keepalive_comments(&response.text().await.expect("body should read")),
|
||||
"data: {\"id\":\"chatcmpl-local-custom-path-123\"}\n\ndata: [DONE]\n\n"
|
||||
);
|
||||
|
||||
@@ -1863,7 +1863,7 @@ async fn gateway_retries_next_local_openai_chat_stream_candidate_after_retryable
|
||||
Some(EXECUTION_PATH_EXECUTION_RUNTIME_STREAM)
|
||||
);
|
||||
assert_eq!(
|
||||
response.text().await.expect("body should read"),
|
||||
strip_sse_keepalive_comments(&response.text().await.expect("body should read")),
|
||||
"data: {\"id\":\"chatcmpl-local-stream-failover-123\"}\n\ndata: [DONE]\n\n"
|
||||
);
|
||||
|
||||
|
||||
@@ -18,9 +18,10 @@ use crate::constants::{
|
||||
|
||||
use super::{
|
||||
build_router, build_router_with_execution_runtime_override, build_router_with_state,
|
||||
build_state_with_execution_runtime_override, start_server, wait_until, AppState,
|
||||
FrontdoorCorsConfig, FrontdoorUserRpmConfig, GatewayFallbackMetricKind, GatewayFallbackReason,
|
||||
UsageRuntimeConfig, VideoTaskTruthSourceMode,
|
||||
build_state_with_execution_runtime_override, next_non_keepalive_chunk, start_server,
|
||||
strip_sse_keepalive_comments, wait_until, AppState, FrontdoorCorsConfig,
|
||||
FrontdoorUserRpmConfig, GatewayFallbackMetricKind, GatewayFallbackReason, UsageRuntimeConfig,
|
||||
VideoTaskTruthSourceMode,
|
||||
};
|
||||
|
||||
mod decision;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use super::{
|
||||
any, build_router_with_state, build_state_with_execution_runtime_override, json, start_server,
|
||||
to_bytes, Arc, Body, Bytes, HeaderName, HeaderValue, Infallible, Json, Mutex, Request,
|
||||
Response, Router, StatusCode, UsageRuntimeConfig, TRACE_ID_HEADER,
|
||||
strip_sse_keepalive_comments, to_bytes, Arc, Body, Bytes, HeaderName, HeaderValue, Infallible,
|
||||
Json, Mutex, Request, Response, Router, StatusCode, UsageRuntimeConfig, TRACE_ID_HEADER,
|
||||
};
|
||||
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
|
||||
use aether_data::repository::auth::{
|
||||
@@ -469,7 +469,7 @@ async fn gateway_executes_codex_cli_stream_via_local_decision_gate_after_oauth_r
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
response.text().await.expect("body should read"),
|
||||
strip_sse_keepalive_comments(&response.text().await.expect("body should read")),
|
||||
"event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_codex_cli_stream_local_123\",\"object\":\"response\",\"model\":\"gpt-5.4\",\"status\":\"completed\",\"usage\":{\"input_tokens\":1,\"output_tokens\":2,\"total_tokens\":3}}}\n\n"
|
||||
);
|
||||
|
||||
|
||||
@@ -18,9 +18,9 @@ use crate::constants::{
|
||||
|
||||
use super::{
|
||||
build_router, build_router_with_execution_runtime_override, build_router_with_state,
|
||||
build_state_with_execution_runtime_override, start_server, wait_until, AppState,
|
||||
FrontdoorCorsConfig, FrontdoorUserRpmConfig, GatewayFallbackMetricKind, GatewayFallbackReason,
|
||||
UsageRuntimeConfig, VideoTaskTruthSourceMode,
|
||||
build_state_with_execution_runtime_override, start_server, strip_sse_keepalive_comments,
|
||||
wait_until, AppState, FrontdoorCorsConfig, FrontdoorUserRpmConfig, GatewayFallbackMetricKind,
|
||||
GatewayFallbackReason, UsageRuntimeConfig, VideoTaskTruthSourceMode,
|
||||
};
|
||||
|
||||
mod compact;
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
use super::{
|
||||
any, build_router_with_state, build_state_with_execution_runtime_override, json, start_server,
|
||||
to_bytes, Arc, Body, Bytes, HeaderName, HeaderValue, Json, Mutex, Request, Response, Router,
|
||||
StatusCode, TRACE_ID_HEADER,
|
||||
any, build_router_with_state, build_state_with_execution_runtime_override, json,
|
||||
next_non_keepalive_chunk, start_server, strip_sse_keepalive_comments, to_bytes, Arc, Body,
|
||||
Bytes, HeaderName, HeaderValue, Json, Mutex, Request, Response, Router, StatusCode,
|
||||
TRACE_ID_HEADER,
|
||||
};
|
||||
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
|
||||
use aether_data::repository::auth::{
|
||||
@@ -952,11 +953,12 @@ async fn gateway_executes_claude_cli_stream_via_local_decision_gate_without_wait
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
tokio::time::timeout(std::time::Duration::from_millis(100), response.chunk())
|
||||
.await
|
||||
.expect("same-format passthrough should yield first chunk before eof")
|
||||
.expect("first chunk should read")
|
||||
.expect("first chunk should exist"),
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_millis(100),
|
||||
next_non_keepalive_chunk(&mut response),
|
||||
)
|
||||
.await
|
||||
.expect("same-format passthrough should yield first chunk before eof"),
|
||||
Bytes::from_static(b"event: message_start\ndata: {\"type\":\"message_start\"}\n\n")
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -1470,7 +1472,7 @@ async fn gateway_executes_claude_code_cli_stream_via_local_decision_gate_with_lo
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
response.text().await.expect("body should read"),
|
||||
strip_sse_keepalive_comments(&response.text().await.expect("body should read")),
|
||||
"event: message_start\ndata: {\"type\":\"message_start\"}\n\n"
|
||||
);
|
||||
|
||||
@@ -1924,7 +1926,7 @@ async fn gateway_executes_claude_chat_stream_via_local_decision_gate_with_local_
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
response.text().await.expect("body should read"),
|
||||
strip_sse_keepalive_comments(&response.text().await.expect("body should read")),
|
||||
"event: message_start\ndata: {\"type\":\"message_start\"}\n\n"
|
||||
);
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use super::{
|
||||
any, build_router_with_state, build_state_with_execution_runtime_override,
|
||||
encrypt_python_fernet_plaintext, json, start_server, to_bytes, Arc, Body, Bytes, Digest,
|
||||
HeaderName, HeaderValue, InMemoryAuthApiKeySnapshotRepository,
|
||||
encrypt_python_fernet_plaintext, json, start_server, strip_sse_keepalive_comments, to_bytes,
|
||||
Arc, Body, Bytes, Digest, HeaderName, HeaderValue, InMemoryAuthApiKeySnapshotRepository,
|
||||
InMemoryMinimalCandidateSelectionReadRepository, InMemoryProviderCatalogReadRepository,
|
||||
InMemoryRequestCandidateRepository, Json, Mutex, Request, RequestCandidateReadRepository,
|
||||
RequestCandidateStatus, Response, Router, Sha256, StatusCode, StoredAuthApiKeySnapshot,
|
||||
@@ -404,7 +404,7 @@ async fn gateway_executes_gemini_chat_stream_via_local_decision_gate_with_local_
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
response.text().await.expect("body should read"),
|
||||
strip_sse_keepalive_comments(&response.text().await.expect("body should read")),
|
||||
"data: {\"candidates\":[]}\n\n"
|
||||
);
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use super::{
|
||||
any, build_router_with_state, build_state_with_execution_runtime_override,
|
||||
encrypt_python_fernet_plaintext, json, start_server, to_bytes, Arc, Body, Bytes, Digest,
|
||||
HeaderName, HeaderValue, InMemoryAuthApiKeySnapshotRepository,
|
||||
encrypt_python_fernet_plaintext, json, start_server, strip_sse_keepalive_comments, to_bytes,
|
||||
Arc, Body, Bytes, Digest, HeaderName, HeaderValue, InMemoryAuthApiKeySnapshotRepository,
|
||||
InMemoryMinimalCandidateSelectionReadRepository, InMemoryProviderCatalogReadRepository,
|
||||
InMemoryRequestCandidateRepository, Json, Mutex, Request, RequestCandidateReadRepository,
|
||||
RequestCandidateStatus, Response, Router, Sha256, StatusCode, StoredAuthApiKeySnapshot,
|
||||
@@ -381,7 +381,7 @@ async fn gateway_executes_gemini_cli_stream_via_local_decision_gate_with_local_s
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
response.text().await.expect("body should read"),
|
||||
strip_sse_keepalive_comments(&response.text().await.expect("body should read")),
|
||||
"data: {\"candidates\":[]}\n\n"
|
||||
);
|
||||
|
||||
@@ -872,7 +872,7 @@ async fn gateway_executes_gemini_cli_stream_via_local_decision_gate_after_oauth_
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
response.text().await.expect("body should read"),
|
||||
strip_sse_keepalive_comments(&response.text().await.expect("body should read")),
|
||||
"data: {\"candidates\":[]}\n\n"
|
||||
);
|
||||
|
||||
@@ -1339,7 +1339,7 @@ async fn gateway_executes_vertex_ai_gemini_cli_stream_via_local_decision_gate_wi
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
response.text().await.expect("body should read"),
|
||||
strip_sse_keepalive_comments(&response.text().await.expect("body should read")),
|
||||
"data: {\"candidates\":[]}\n\n"
|
||||
);
|
||||
|
||||
@@ -1849,7 +1849,8 @@ async fn gateway_executes_antigravity_gemini_cli_stream_via_local_decision_gate_
|
||||
.expect("request should succeed");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let response_text = response.text().await.expect("body should read");
|
||||
let response_text =
|
||||
strip_sse_keepalive_comments(&response.text().await.expect("body should read"));
|
||||
let payload = response_text
|
||||
.trim()
|
||||
.strip_prefix("data: ")
|
||||
|
||||
@@ -18,9 +18,9 @@ use crate::constants::{
|
||||
|
||||
use super::{
|
||||
build_router, build_router_with_execution_runtime_override, build_router_with_state,
|
||||
build_state_with_execution_runtime_override, start_server, wait_until, AppState,
|
||||
FrontdoorCorsConfig, FrontdoorUserRpmConfig, GatewayFallbackMetricKind, GatewayFallbackReason,
|
||||
UsageRuntimeConfig, VideoTaskTruthSourceMode,
|
||||
build_state_with_execution_runtime_override, start_server, strip_sse_keepalive_comments,
|
||||
wait_until, AppState, FrontdoorCorsConfig, FrontdoorUserRpmConfig, GatewayFallbackMetricKind,
|
||||
GatewayFallbackReason, UsageRuntimeConfig, VideoTaskTruthSourceMode,
|
||||
};
|
||||
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
|
||||
use aether_data::repository::auth::{
|
||||
|
||||
@@ -6,8 +6,8 @@ use super::{
|
||||
};
|
||||
use crate::tests::{
|
||||
any, build_router, build_router_with_state, build_state_with_execution_runtime_override, json,
|
||||
start_server, AppState, Arc, Body, HeaderValue, Json, Mutex, Request, Response, Router,
|
||||
StatusCode, CONTROL_ACTION_PROXY_PUBLIC, CONTROL_EXECUTED_HEADER,
|
||||
start_server, strip_sse_keepalive_comments, AppState, Arc, Body, HeaderValue, Json, Mutex,
|
||||
Request, Response, Router, StatusCode, CONTROL_ACTION_PROXY_PUBLIC, CONTROL_EXECUTED_HEADER,
|
||||
EXECUTION_PATH_EXECUTION_RUNTIME_STREAM, EXECUTION_PATH_EXECUTION_RUNTIME_SYNC,
|
||||
EXECUTION_PATH_HEADER,
|
||||
};
|
||||
@@ -458,7 +458,7 @@ async fn gateway_handles_internal_gateway_execute_stream_locally() {
|
||||
Some(EXECUTION_PATH_EXECUTION_RUNTIME_STREAM)
|
||||
);
|
||||
assert_eq!(
|
||||
response.text().await.expect("body should read"),
|
||||
strip_sse_keepalive_comments(&response.text().await.expect("body should read")),
|
||||
"data: one\n\ndata: [DONE]\n\n"
|
||||
);
|
||||
assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0);
|
||||
|
||||
@@ -88,3 +88,20 @@ pub(super) async fn wait_until(timeout_ms: u64, mut predicate: impl FnMut() -> b
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn strip_sse_keepalive_comments(body: &str) -> String {
|
||||
body.replace(": aether-keepalive\n\n", "")
|
||||
}
|
||||
|
||||
pub(crate) async fn next_non_keepalive_chunk(response: &mut reqwest::Response) -> Bytes {
|
||||
loop {
|
||||
let chunk = response
|
||||
.chunk()
|
||||
.await
|
||||
.expect("chunk should read")
|
||||
.expect("chunk should exist");
|
||||
if chunk.as_ref() != b": aether-keepalive\n\n" {
|
||||
return chunk;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,8 +28,8 @@ use sha2::{Digest, Sha256};
|
||||
|
||||
use super::{
|
||||
any, build_router_with_state, build_state_with_execution_runtime_override, send_request,
|
||||
start_server, Body, HeaderValue, Json, Mutex, Request, Response, Router, StatusCode,
|
||||
UsageRuntimeConfig, TRACE_ID_HEADER,
|
||||
start_server, strip_sse_keepalive_comments, Body, HeaderValue, Json, Mutex, Request, Response,
|
||||
Router, StatusCode, UsageRuntimeConfig, TRACE_ID_HEADER,
|
||||
};
|
||||
use crate::data::GatewayDataState;
|
||||
|
||||
|
||||
@@ -2,8 +2,8 @@ use super::{
|
||||
any, build_router_with_state, build_state_with_execution_runtime_override,
|
||||
encrypt_python_fernet_plaintext, hash_api_key, json, sample_local_openai_auth_snapshot,
|
||||
sample_local_openai_candidate_row, sample_local_openai_endpoint, sample_local_openai_key,
|
||||
sample_local_openai_provider, send_request, start_server, Arc, Body, GatewayDataState,
|
||||
HeaderValue, InMemoryAuthApiKeySnapshotRepository,
|
||||
sample_local_openai_provider, send_request, start_server, strip_sse_keepalive_comments, Arc,
|
||||
Body, GatewayDataState, HeaderValue, InMemoryAuthApiKeySnapshotRepository,
|
||||
InMemoryMinimalCandidateSelectionReadRepository, InMemoryProviderCatalogReadRepository,
|
||||
InMemoryRequestCandidateRepository, InMemoryUsageReadRepository, Json, Mutex, Request,
|
||||
RequestCandidateReadRepository, RequestCandidateStatus, Response, Router, StatusCode,
|
||||
@@ -840,6 +840,104 @@ async fn gateway_records_failed_usage_when_all_local_openai_chat_candidates_exha
|
||||
assert_eq!(stored_candidates[0].status_code, Some(503));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn gateway_records_failed_usage_when_sync_runtime_transport_is_unavailable_without_plan_fallback(
|
||||
) {
|
||||
let usage_repository = Arc::new(InMemoryUsageReadRepository::default());
|
||||
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
|
||||
let execution_hits = Arc::new(Mutex::new(0usize));
|
||||
let execution_hits_clone = Arc::clone(&execution_hits);
|
||||
|
||||
let auth_repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
|
||||
Some(hash_api_key("sk-client-openai-local-transport-unavailable")),
|
||||
sample_local_openai_auth_snapshot(
|
||||
"api-key-openai-usage-local-transport-unavailable-1",
|
||||
"user-openai-usage-local-transport-unavailable-1",
|
||||
),
|
||||
)]));
|
||||
let candidate_selection_repository =
|
||||
Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(vec![
|
||||
sample_local_openai_candidate_row(),
|
||||
]));
|
||||
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||
vec![sample_local_openai_provider()],
|
||||
vec![sample_local_openai_endpoint()],
|
||||
vec![sample_local_openai_key()],
|
||||
));
|
||||
|
||||
let gateway_state = crate::AppState::new()
|
||||
.expect("gateway should build")
|
||||
.with_execution_runtime_sync_override_for_tests(move |_plan| {
|
||||
*execution_hits_clone.lock().expect("mutex should lock") += 1;
|
||||
Err(crate::GatewayError::Internal(
|
||||
"simulated transport unavailable".to_string(),
|
||||
))
|
||||
})
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::with_auth_candidate_selection_provider_catalog_request_candidates_and_usage_for_tests(
|
||||
auth_repository,
|
||||
candidate_selection_repository,
|
||||
provider_catalog_repository,
|
||||
Arc::clone(&request_candidate_repository),
|
||||
Arc::clone(&usage_repository),
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
),
|
||||
)
|
||||
.with_usage_runtime_for_tests(UsageRuntimeConfig {
|
||||
enabled: true,
|
||||
..UsageRuntimeConfig::default()
|
||||
});
|
||||
let gateway = build_router_with_state(gateway_state);
|
||||
let request = Request::builder()
|
||||
.method(http::Method::POST)
|
||||
.uri("/v1/chat/completions")
|
||||
.header(http::header::CONTENT_TYPE, "application/json")
|
||||
.header(
|
||||
http::header::AUTHORIZATION,
|
||||
"Bearer sk-client-openai-local-transport-unavailable",
|
||||
)
|
||||
.header(
|
||||
TRACE_ID_HEADER,
|
||||
"trace-openai-chat-local-transport-unavailable-123",
|
||||
)
|
||||
.body(Body::from("{\"model\":\"gpt-5\",\"messages\":[]}"))
|
||||
.expect("request should build");
|
||||
let response = send_request(gateway, request).await;
|
||||
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(*execution_hits.lock().expect("mutex should lock"), 1);
|
||||
|
||||
let stored_usage = wait_for_usage_status(
|
||||
usage_repository.as_ref(),
|
||||
"trace-openai-chat-local-transport-unavailable-123",
|
||||
"failed",
|
||||
)
|
||||
.await;
|
||||
assert_eq!(stored_usage.status, "failed");
|
||||
assert_eq!(stored_usage.billing_status, "void");
|
||||
assert_eq!(stored_usage.status_code, Some(503));
|
||||
assert_eq!(
|
||||
stored_usage
|
||||
.response_body
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("error"))
|
||||
.and_then(|value| value.get("type"))
|
||||
.and_then(|value| value.as_str()),
|
||||
Some("execution_runtime_unavailable")
|
||||
);
|
||||
|
||||
let stored_candidates = request_candidate_repository
|
||||
.list_by_request_id("trace-openai-chat-local-transport-unavailable-123")
|
||||
.await
|
||||
.expect("request candidate trace should read");
|
||||
assert_eq!(stored_candidates.len(), 1);
|
||||
assert_eq!(stored_candidates[0].status, RequestCandidateStatus::Failed);
|
||||
assert_eq!(
|
||||
stored_candidates[0].error_type.as_deref(),
|
||||
Some("execution_runtime_unavailable")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gateway_records_failed_usage_for_claude_runtime_miss_without_execution_exhaustion() {
|
||||
run_async_test_on_large_stack(
|
||||
@@ -1184,7 +1282,8 @@ async fn gateway_handles_local_openai_chat_stream_report_with_local_reporting_wh
|
||||
.expect("request should succeed");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body_text = response.text().await.expect("stream body should read");
|
||||
let body_text =
|
||||
strip_sse_keepalive_comments(&response.text().await.expect("stream body should read"));
|
||||
assert_eq!(
|
||||
body_text,
|
||||
"data: {\"id\":\"chatcmpl-local-report-stream-123\",\"usage\":{\"input_tokens\":2,\"output_tokens\":4,\"total_tokens\":6}}\n\ndata: [DONE]\n\n"
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
use super::{
|
||||
any, build_router_with_state, build_state_with_execution_runtime_override,
|
||||
encrypt_python_fernet_plaintext, hash_api_key, json, start_server, Arc, Body, GatewayDataState,
|
||||
HeaderValue, InMemoryAuthApiKeySnapshotRepository,
|
||||
InMemoryMinimalCandidateSelectionReadRepository, InMemoryProviderCatalogReadRepository,
|
||||
InMemoryRequestCandidateRepository, InMemoryUsageReadRepository, Json, Request,
|
||||
RequestCandidateReadRepository, RequestCandidateStatus, Response, Router, StatusCode,
|
||||
StoredAuthApiKeySnapshot, StoredMinimalCandidateSelectionRow, StoredProviderCatalogEndpoint,
|
||||
StoredProviderCatalogKey, StoredProviderCatalogProvider, StoredProviderModelMapping,
|
||||
UsageReadRepository, UsageRuntimeConfig, DEVELOPMENT_ENCRYPTION_KEY, TRACE_ID_HEADER,
|
||||
encrypt_python_fernet_plaintext, hash_api_key, json, start_server,
|
||||
strip_sse_keepalive_comments, Arc, Body, GatewayDataState, HeaderValue,
|
||||
InMemoryAuthApiKeySnapshotRepository, InMemoryMinimalCandidateSelectionReadRepository,
|
||||
InMemoryProviderCatalogReadRepository, InMemoryRequestCandidateRepository,
|
||||
InMemoryUsageReadRepository, Json, Request, RequestCandidateReadRepository,
|
||||
RequestCandidateStatus, Response, Router, StatusCode, StoredAuthApiKeySnapshot,
|
||||
StoredMinimalCandidateSelectionRow, StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
|
||||
StoredProviderCatalogProvider, StoredProviderModelMapping, UsageReadRepository,
|
||||
UsageRuntimeConfig, DEVELOPMENT_ENCRYPTION_KEY, TRACE_ID_HEADER,
|
||||
};
|
||||
use aether_data::repository::billing::InMemoryBillingReadRepository;
|
||||
use aether_data::repository::wallet::{InMemoryWalletRepository, StoredWalletSnapshot};
|
||||
@@ -939,7 +940,7 @@ async fn gateway_records_openai_stream_usage_and_pricing_with_cache_tokens_impl(
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
response.text().await.expect("stream body should read"),
|
||||
strip_sse_keepalive_comments(&response.text().await.expect("stream body should read")),
|
||||
stream_body.concat()
|
||||
);
|
||||
|
||||
@@ -1132,7 +1133,7 @@ async fn gateway_records_claude_stream_usage_and_pricing_with_cache_breakdown_im
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
response.text().await.expect("stream body should read"),
|
||||
strip_sse_keepalive_comments(&response.text().await.expect("stream body should read")),
|
||||
stream_body.concat()
|
||||
);
|
||||
|
||||
@@ -1310,7 +1311,7 @@ async fn gateway_records_gemini_stream_usage_and_pricing_with_cache_read_tokens_
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
response.text().await.expect("stream body should read"),
|
||||
strip_sse_keepalive_comments(&response.text().await.expect("stream body should read")),
|
||||
stream_body.concat()
|
||||
);
|
||||
|
||||
|
||||
@@ -366,6 +366,10 @@ pub fn build_admin_monitoring_trace_request_candidate_payload_with_key_accounts(
|
||||
"latency_ms": candidate.latency_ms,
|
||||
"concurrent_requests": candidate.concurrent_requests,
|
||||
"ranking": build_admin_monitoring_trace_candidate_ranking(candidate.extra_data.as_ref()),
|
||||
"image_progress": candidate.extra_data.as_ref()
|
||||
.and_then(|value| value.get("image_progress"))
|
||||
.cloned()
|
||||
.unwrap_or(Value::Null),
|
||||
"extra_data": build_admin_monitoring_trace_candidate_extra_data(candidate.extra_data.as_ref(), usage),
|
||||
"created_at": unix_ms_to_rfc3339(candidate.created_at_unix_ms),
|
||||
"started_at": candidate.started_at_unix_ms.and_then(unix_ms_to_rfc3339),
|
||||
|
||||
@@ -983,6 +983,7 @@ fn admin_usage_active_request_json(
|
||||
item: &StoredRequestUsageAudit,
|
||||
api_key_name: Option<String>,
|
||||
provider_key_name: Option<String>,
|
||||
image_progress: Option<&Value>,
|
||||
) -> Value {
|
||||
let cache_creation_input_tokens = admin_usage_cache_creation_tokens(item);
|
||||
let client_is_stream = admin_usage_client_is_stream(item);
|
||||
@@ -1022,6 +1023,9 @@ fn admin_usage_active_request_json(
|
||||
if let Some(target_model) = item.target_model.as_ref() {
|
||||
value["target_model"] = json!(target_model);
|
||||
}
|
||||
if let Some(image_progress) = image_progress {
|
||||
value["image_progress"] = image_progress.clone();
|
||||
}
|
||||
value
|
||||
}
|
||||
|
||||
@@ -2017,6 +2021,7 @@ pub fn build_admin_usage_active_requests_response(
|
||||
api_key_names: &BTreeMap<String, String>,
|
||||
auth_api_key_reader_available: bool,
|
||||
provider_key_names: &BTreeMap<String, String>,
|
||||
image_progress_by_request_id: &BTreeMap<String, Value>,
|
||||
) -> Response<Body> {
|
||||
let payload: Vec<_> = items
|
||||
.iter()
|
||||
@@ -2024,7 +2029,12 @@ pub fn build_admin_usage_active_requests_response(
|
||||
let provider_key_name = admin_usage_provider_key_name(item, provider_key_names);
|
||||
let api_key_name =
|
||||
admin_usage_api_key_name(item, api_key_names, auth_api_key_reader_available);
|
||||
admin_usage_active_request_json(item, api_key_name, provider_key_name)
|
||||
admin_usage_active_request_json(
|
||||
item,
|
||||
api_key_name,
|
||||
provider_key_name,
|
||||
image_progress_by_request_id.get(&item.request_id),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
|
||||
@@ -2362,7 +2372,7 @@ mod tests {
|
||||
|
||||
assert!(!admin_usage_client_is_stream(&item));
|
||||
|
||||
let active = admin_usage_active_request_json(&item, None, None);
|
||||
let active = admin_usage_active_request_json(&item, None, None, None);
|
||||
assert_eq!(active["is_stream"], true);
|
||||
assert_eq!(active["upstream_is_stream"], true);
|
||||
assert_eq!(active["client_requested_stream"], false);
|
||||
|
||||
@@ -24,10 +24,10 @@ pub use crate::contracts::{
|
||||
OPENAI_CHAT_SYNC_ERROR_REPORT_KIND, OPENAI_CHAT_SYNC_FINALIZE_REPORT_KIND,
|
||||
OPENAI_CHAT_SYNC_PLAN_KIND, OPENAI_CHAT_SYNC_SUCCESS_REPORT_KIND,
|
||||
OPENAI_EMBEDDING_SYNC_PLAN_KIND, OPENAI_IMAGE_STREAM_PLAN_KIND,
|
||||
OPENAI_IMAGE_STREAM_SUCCESS_REPORT_KIND, OPENAI_IMAGE_SYNC_FINALIZE_REPORT_KIND,
|
||||
OPENAI_IMAGE_SYNC_PLAN_KIND, OPENAI_IMAGE_SYNC_SUCCESS_REPORT_KIND,
|
||||
OPENAI_RERANK_SYNC_PLAN_KIND, OPENAI_RESPONSES_COMPACT_STREAM_PLAN_KIND,
|
||||
OPENAI_RESPONSES_COMPACT_STREAM_SUCCESS_REPORT_KIND,
|
||||
OPENAI_IMAGE_STREAM_SUCCESS_REPORT_KIND, OPENAI_IMAGE_SYNC_ERROR_REPORT_KIND,
|
||||
OPENAI_IMAGE_SYNC_FINALIZE_REPORT_KIND, OPENAI_IMAGE_SYNC_PLAN_KIND,
|
||||
OPENAI_IMAGE_SYNC_SUCCESS_REPORT_KIND, OPENAI_RERANK_SYNC_PLAN_KIND,
|
||||
OPENAI_RESPONSES_COMPACT_STREAM_PLAN_KIND, OPENAI_RESPONSES_COMPACT_STREAM_SUCCESS_REPORT_KIND,
|
||||
OPENAI_RESPONSES_COMPACT_SYNC_ERROR_REPORT_KIND,
|
||||
OPENAI_RESPONSES_COMPACT_SYNC_FINALIZE_REPORT_KIND, OPENAI_RESPONSES_COMPACT_SYNC_PLAN_KIND,
|
||||
OPENAI_RESPONSES_COMPACT_SYNC_SUCCESS_REPORT_KIND, OPENAI_RESPONSES_STREAM_PLAN_KIND,
|
||||
|
||||
@@ -39,8 +39,9 @@ pub use report_kinds::{
|
||||
GEMINI_CLI_SYNC_SUCCESS_REPORT_KIND, GEMINI_VIDEO_CREATE_SYNC_FINALIZE_REPORT_KIND,
|
||||
OPENAI_CHAT_STREAM_SUCCESS_REPORT_KIND, OPENAI_CHAT_SYNC_ERROR_REPORT_KIND,
|
||||
OPENAI_CHAT_SYNC_FINALIZE_REPORT_KIND, OPENAI_CHAT_SYNC_SUCCESS_REPORT_KIND,
|
||||
OPENAI_IMAGE_STREAM_SUCCESS_REPORT_KIND, OPENAI_IMAGE_SYNC_FINALIZE_REPORT_KIND,
|
||||
OPENAI_IMAGE_SYNC_SUCCESS_REPORT_KIND, OPENAI_RESPONSES_COMPACT_STREAM_SUCCESS_REPORT_KIND,
|
||||
OPENAI_IMAGE_STREAM_SUCCESS_REPORT_KIND, OPENAI_IMAGE_SYNC_ERROR_REPORT_KIND,
|
||||
OPENAI_IMAGE_SYNC_FINALIZE_REPORT_KIND, OPENAI_IMAGE_SYNC_SUCCESS_REPORT_KIND,
|
||||
OPENAI_RESPONSES_COMPACT_STREAM_SUCCESS_REPORT_KIND,
|
||||
OPENAI_RESPONSES_COMPACT_SYNC_ERROR_REPORT_KIND,
|
||||
OPENAI_RESPONSES_COMPACT_SYNC_FINALIZE_REPORT_KIND,
|
||||
OPENAI_RESPONSES_COMPACT_SYNC_SUCCESS_REPORT_KIND, OPENAI_RESPONSES_STREAM_SUCCESS_REPORT_KIND,
|
||||
|
||||
@@ -45,6 +45,7 @@ pub const GEMINI_CHAT_SYNC_ERROR_REPORT_KIND: &str = "gemini_chat_sync_error";
|
||||
pub const OPENAI_RESPONSES_SYNC_ERROR_REPORT_KIND: &str = "openai_responses_sync_error";
|
||||
pub const OPENAI_RESPONSES_COMPACT_SYNC_ERROR_REPORT_KIND: &str =
|
||||
"openai_responses_compact_sync_error";
|
||||
pub const OPENAI_IMAGE_SYNC_ERROR_REPORT_KIND: &str = "openai_image_sync_error";
|
||||
pub const CLAUDE_CLI_SYNC_ERROR_REPORT_KIND: &str = "claude_cli_sync_error";
|
||||
pub const GEMINI_CLI_SYNC_ERROR_REPORT_KIND: &str = "gemini_cli_sync_error";
|
||||
|
||||
@@ -95,6 +96,7 @@ pub fn core_error_background_report_kind(report_kind: &str) -> Option<&'static s
|
||||
LEGACY_OPENAI_COMPACT_SYNC_FINALIZE_REPORT_KIND => {
|
||||
Some(OPENAI_RESPONSES_COMPACT_SYNC_ERROR_REPORT_KIND)
|
||||
}
|
||||
OPENAI_IMAGE_SYNC_FINALIZE_REPORT_KIND => Some(OPENAI_IMAGE_SYNC_ERROR_REPORT_KIND),
|
||||
CLAUDE_CLI_SYNC_FINALIZE_REPORT_KIND => Some(CLAUDE_CLI_SYNC_ERROR_REPORT_KIND),
|
||||
GEMINI_CLI_SYNC_FINALIZE_REPORT_KIND => Some(GEMINI_CLI_SYNC_ERROR_REPORT_KIND),
|
||||
_ => None,
|
||||
|
||||
@@ -107,6 +107,10 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(outcome) = exhausted {
|
||||
return Ok(AiServingExecutionOutcome::Exhausted(outcome));
|
||||
}
|
||||
|
||||
let fallback_reason = if port.scheduler_decision_supported() {
|
||||
AiPlanFallbackReason::RemoteDecisionMiss
|
||||
} else {
|
||||
@@ -119,9 +123,7 @@ where
|
||||
AiServingExecutionOutcome::Exhausted(outcome) => {
|
||||
Ok(AiServingExecutionOutcome::Exhausted(outcome))
|
||||
}
|
||||
AiServingExecutionOutcome::NoPath => Ok(exhausted
|
||||
.map(AiServingExecutionOutcome::Exhausted)
|
||||
.unwrap_or(AiServingExecutionOutcome::NoPath)),
|
||||
AiServingExecutionOutcome::NoPath => Ok(AiServingExecutionOutcome::NoPath),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -159,6 +161,10 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(outcome) = exhausted {
|
||||
return Ok(AiServingExecutionOutcome::Exhausted(outcome));
|
||||
}
|
||||
|
||||
let fallback_reason = if port.scheduler_decision_supported() {
|
||||
AiPlanFallbackReason::RemoteDecisionMiss
|
||||
} else {
|
||||
@@ -171,9 +177,7 @@ where
|
||||
AiServingExecutionOutcome::Exhausted(outcome) => {
|
||||
Ok(AiServingExecutionOutcome::Exhausted(outcome))
|
||||
}
|
||||
AiServingExecutionOutcome::NoPath => Ok(exhausted
|
||||
.map(AiServingExecutionOutcome::Exhausted)
|
||||
.unwrap_or(AiServingExecutionOutcome::NoPath)),
|
||||
AiServingExecutionOutcome::NoPath => Ok(AiServingExecutionOutcome::NoPath),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -365,6 +369,20 @@ mod tests {
|
||||
outcome,
|
||||
AiServingExecutionOutcome::Exhausted("local_video_exhausted")
|
||||
));
|
||||
assert_eq!(
|
||||
port.calls.lock().unwrap().as_slice(),
|
||||
[
|
||||
"VideoTaskFollowUp",
|
||||
"LocalVideo",
|
||||
"LocalImage",
|
||||
"LocalOpenAiChat",
|
||||
"LocalOpenAiResponses",
|
||||
"LocalStandardFamily",
|
||||
"LocalSameFormatProvider",
|
||||
"LocalGeminiFiles",
|
||||
"RemoteDecision",
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -405,4 +423,36 @@ mod tests {
|
||||
["LocalVideoContent", "LocalImage"]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stream_path_returns_last_exhaustion_without_plan_fallback() {
|
||||
let port = TestStreamPort {
|
||||
scheduler_supported: true,
|
||||
outcomes: Mutex::new(VecDeque::from([
|
||||
AiServingExecutionOutcome::NoPath,
|
||||
AiServingExecutionOutcome::Exhausted("local_image_exhausted"),
|
||||
])),
|
||||
calls: Mutex::default(),
|
||||
};
|
||||
|
||||
let outcome = run_ai_stream_execution_path(&port).await.unwrap();
|
||||
|
||||
assert!(matches!(
|
||||
outcome,
|
||||
AiServingExecutionOutcome::Exhausted("local_image_exhausted")
|
||||
));
|
||||
assert_eq!(
|
||||
port.calls.lock().unwrap().as_slice(),
|
||||
[
|
||||
"LocalVideoContent",
|
||||
"LocalImage",
|
||||
"LocalOpenAiChat",
|
||||
"LocalOpenAiResponses",
|
||||
"LocalStandardFamily",
|
||||
"LocalSameFormatProvider",
|
||||
"LocalGeminiFiles",
|
||||
"RemoteDecision",
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -233,6 +233,7 @@ pub fn is_local_ai_sync_report_kind(report_kind: &str) -> bool {
|
||||
| "openai_responses_compact_sync_error"
|
||||
| "openai_cli_sync_success"
|
||||
| "openai_image_sync_success"
|
||||
| "openai_image_sync_error"
|
||||
| "claude_cli_sync_success"
|
||||
| "gemini_cli_sync_success"
|
||||
| "openai_cli_sync_error"
|
||||
@@ -424,6 +425,7 @@ mod tests {
|
||||
"openai_responses_compact_sync_error"
|
||||
));
|
||||
assert!(is_local_ai_sync_report_kind("openai_image_sync_success"));
|
||||
assert!(is_local_ai_sync_report_kind("openai_image_sync_error"));
|
||||
assert!(is_local_ai_sync_report_kind("gemini_files_delete_mapping"));
|
||||
assert!(!is_local_ai_sync_report_kind("unknown_sync_kind"));
|
||||
}
|
||||
|
||||
@@ -9,6 +9,19 @@ export interface CandidateRankingMetadata {
|
||||
demoted_by?: string
|
||||
}
|
||||
|
||||
export interface ImageProgress {
|
||||
phase?: 'upstream_connecting' | 'upstream_streaming' | 'upstream_completed' | 'failed' | string
|
||||
upstream_ttfb_ms?: number | null
|
||||
upstream_sse_frame_count?: number | null
|
||||
last_upstream_event?: string | null
|
||||
last_upstream_frame_at_unix_ms?: number | null
|
||||
partial_image_count?: number | null
|
||||
last_client_visible_event?: string | null
|
||||
downstream_heartbeat_count?: number | null
|
||||
last_downstream_heartbeat_at_unix_ms?: number | null
|
||||
downstream_heartbeat_interval_ms?: number | null
|
||||
}
|
||||
|
||||
export interface CandidateRecord {
|
||||
id: string
|
||||
request_id: string
|
||||
@@ -46,6 +59,7 @@ export interface CandidateRecord {
|
||||
latency_ms?: number
|
||||
concurrent_requests?: number
|
||||
ranking?: CandidateRankingMetadata | null
|
||||
image_progress?: ImageProgress | null
|
||||
extra_data?: Record<string, unknown>
|
||||
created_at: string
|
||||
started_at?: string
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import apiClient from './client'
|
||||
import { cachedRequest, dedupedRequest, buildCacheKey } from '@/utils/cache'
|
||||
import type { ActivityHeatmap } from '@/types/activity'
|
||||
import type { ImageProgress } from './requestTrace'
|
||||
|
||||
const ACTIVITY_HEATMAP_CACHE_TTL_MS = 30 * 60 * 1000
|
||||
|
||||
@@ -321,6 +322,7 @@ export const usageApi = {
|
||||
has_format_conversion?: boolean | null
|
||||
has_fallback?: boolean | null
|
||||
target_model?: string | null
|
||||
image_progress?: ImageProgress | null
|
||||
}>
|
||||
}> {
|
||||
const params: Record<string, string | number> = {}
|
||||
|
||||
@@ -404,6 +404,67 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div
|
||||
v-if="currentImageProgress"
|
||||
class="image-progress-block"
|
||||
>
|
||||
<div class="image-progress-header">
|
||||
<span class="image-progress-title">图片生成进度</span>
|
||||
<span
|
||||
class="image-progress-phase"
|
||||
:class="imageProgressPhaseClass(currentImageProgress.phase)"
|
||||
>
|
||||
{{ formatImageProgressPhase(currentImageProgress.phase) }}
|
||||
</span>
|
||||
</div>
|
||||
<div class="image-progress-grid">
|
||||
<div class="image-progress-item">
|
||||
<span class="image-progress-label">上游 TTFB</span>
|
||||
<span class="image-progress-value mono">{{ formatLatency(currentImageProgress.upstream_ttfb_ms) }}</span>
|
||||
</div>
|
||||
<div class="image-progress-item">
|
||||
<span class="image-progress-label">SSE 帧数</span>
|
||||
<span class="image-progress-value mono">{{ formatProgressCount(currentImageProgress.upstream_sse_frame_count) }}</span>
|
||||
</div>
|
||||
<div class="image-progress-item">
|
||||
<span class="image-progress-label">Partial 图片</span>
|
||||
<span class="image-progress-value mono">{{ formatProgressCount(currentImageProgress.partial_image_count) }}</span>
|
||||
</div>
|
||||
<div class="image-progress-item">
|
||||
<span class="image-progress-label">最后帧</span>
|
||||
<span class="image-progress-value mono">{{ formatProgressFrameTime(currentImageProgress.last_upstream_frame_at_unix_ms) }}</span>
|
||||
</div>
|
||||
<template v-if="hasDownstreamHeartbeatProgress">
|
||||
<div class="image-progress-item">
|
||||
<span class="image-progress-label">下游心跳</span>
|
||||
<span class="image-progress-value mono">{{ formatProgressCount(currentImageProgress.downstream_heartbeat_count) }}</span>
|
||||
</div>
|
||||
<div class="image-progress-item">
|
||||
<span class="image-progress-label">心跳间隔</span>
|
||||
<span class="image-progress-value mono">{{ formatLatency(currentImageProgress.downstream_heartbeat_interval_ms) }}</span>
|
||||
</div>
|
||||
<div class="image-progress-item">
|
||||
<span class="image-progress-label">最后心跳</span>
|
||||
<span class="image-progress-value mono">{{ formatProgressFrameTime(currentImageProgress.last_downstream_heartbeat_at_unix_ms) }}</span>
|
||||
</div>
|
||||
</template>
|
||||
<div
|
||||
v-if="currentImageProgress.last_upstream_event"
|
||||
class="image-progress-item full-width"
|
||||
>
|
||||
<span class="image-progress-label">上游事件</span>
|
||||
<code class="image-progress-code">{{ currentImageProgress.last_upstream_event }}</code>
|
||||
</div>
|
||||
<div
|
||||
v-if="currentImageProgress.last_client_visible_event"
|
||||
class="image-progress-item full-width"
|
||||
>
|
||||
<span class="image-progress-label">客户端可见事件</span>
|
||||
<code class="image-progress-code">{{ currentImageProgress.last_client_visible_event }}</code>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 用量与费用(仅成功节点显示) -->
|
||||
<div
|
||||
v-if="currentAttempt.status === 'success' && usageData"
|
||||
@@ -528,14 +589,14 @@
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { ref, watch, computed } from 'vue'
|
||||
import { ref, watch, computed, onBeforeUnmount } from 'vue'
|
||||
import { isAxiosError } from 'axios'
|
||||
import Card from '@/components/ui/card.vue'
|
||||
import Badge from '@/components/ui/badge.vue'
|
||||
import Skeleton from '@/components/ui/skeleton.vue'
|
||||
import JsonContentPanel from './JsonContentPanel.vue'
|
||||
import { ChevronLeft, ChevronRight, ExternalLink } from 'lucide-vue-next'
|
||||
import { requestTraceApi, type RequestTrace, type CandidateRecord } from '@/api/requestTrace'
|
||||
import { requestTraceApi, type RequestTrace, type CandidateRecord, type ImageProgress } from '@/api/requestTrace'
|
||||
import { log } from '@/utils/logger'
|
||||
import { parseApiError } from '@/utils/errorParser'
|
||||
import { formatApiFormat } from '@/api/endpoints/types/api-format'
|
||||
@@ -619,7 +680,15 @@ const props = defineProps<{
|
||||
|
||||
const emit = defineEmits<{
|
||||
selectAttempt: [attempt: CandidateRecord | null]
|
||||
traceState: [state: { loaded: boolean, hasTrace: boolean }]
|
||||
traceState: [state: {
|
||||
loaded: boolean
|
||||
hasTrace: boolean
|
||||
finalStatus?: RequestTrace['final_status'] | null
|
||||
statusCode?: number | null
|
||||
latencyMs?: number | null
|
||||
imageProgress?: ImageProgress | null
|
||||
errorMessage?: string | null
|
||||
}]
|
||||
}>()
|
||||
|
||||
// 用量数据(从 props 获取)
|
||||
@@ -678,18 +747,9 @@ const selectedGroupIndex = ref(0)
|
||||
const selectedAttemptIndex = ref(0)
|
||||
const hoveredGroupIndex = ref<number | null>(null)
|
||||
const traceLoadStarted = ref(false)
|
||||
|
||||
watch(
|
||||
[trace, loading],
|
||||
([value, isLoading]) => {
|
||||
const waitingForInternalTrace = Boolean(props.requestId && !props.traceData && !traceLoadStarted.value && !value)
|
||||
emit('traceState', {
|
||||
loaded: !isLoading && !waitingForInternalTrace,
|
||||
hasTrace: Boolean(value?.candidates?.length),
|
||||
})
|
||||
},
|
||||
{ immediate: true },
|
||||
)
|
||||
let tracePollTimer: ReturnType<typeof setTimeout> | null = null
|
||||
let traceLoadInFlight: Promise<void> | null = null
|
||||
const TRACE_POLL_INTERVAL_MS = 1000
|
||||
|
||||
// 格式化延迟(自动调整单位)
|
||||
const formatLatency = (ms: number | undefined | null): string => {
|
||||
@@ -1109,6 +1169,112 @@ const readBooleanField = (obj: Record<string, unknown>, key: string): boolean |
|
||||
return typeof value === 'boolean' ? value : undefined
|
||||
}
|
||||
|
||||
const normalizeImageProgress = (value: unknown): ImageProgress | null => {
|
||||
const raw = extractObject(value)
|
||||
if (!raw) return null
|
||||
|
||||
const progress: ImageProgress = {
|
||||
phase: readStringField(raw, 'phase'),
|
||||
upstream_ttfb_ms: readNumberField(raw, 'upstream_ttfb_ms') ?? null,
|
||||
upstream_sse_frame_count: readNumberField(raw, 'upstream_sse_frame_count') ?? null,
|
||||
last_upstream_event: readStringField(raw, 'last_upstream_event') ?? null,
|
||||
last_upstream_frame_at_unix_ms: readNumberField(raw, 'last_upstream_frame_at_unix_ms') ?? null,
|
||||
partial_image_count: readNumberField(raw, 'partial_image_count') ?? null,
|
||||
last_client_visible_event: readStringField(raw, 'last_client_visible_event') ?? null,
|
||||
downstream_heartbeat_count: readNumberField(raw, 'downstream_heartbeat_count') ?? null,
|
||||
last_downstream_heartbeat_at_unix_ms: readNumberField(raw, 'last_downstream_heartbeat_at_unix_ms') ?? null,
|
||||
downstream_heartbeat_interval_ms: readNumberField(raw, 'downstream_heartbeat_interval_ms') ?? null,
|
||||
}
|
||||
|
||||
return Object.values(progress).some(value => value !== undefined && value !== null && value !== '') ? progress : null
|
||||
}
|
||||
|
||||
const currentImageProgress = computed<ImageProgress | null>(() => {
|
||||
const attempt = currentAttempt.value
|
||||
if (!attempt) return null
|
||||
return normalizeImageProgress(attempt.image_progress)
|
||||
?? normalizeImageProgress(extractObject(attempt.extra_data)?.image_progress)
|
||||
})
|
||||
|
||||
const formatImageProgressPhase = (phase?: string | null): string => {
|
||||
const labels: Record<string, string> = {
|
||||
upstream_connecting: '连接上游',
|
||||
upstream_streaming: '上游生成中',
|
||||
upstream_completed: '上游已完成',
|
||||
failed: '失败',
|
||||
}
|
||||
if (!phase) return '未知'
|
||||
return labels[phase] || phase
|
||||
}
|
||||
|
||||
const imageProgressPhaseClass = (phase?: string | null): string => {
|
||||
if (phase === 'upstream_completed') return 'phase-completed'
|
||||
if (phase === 'failed') return 'phase-failed'
|
||||
if (phase === 'upstream_streaming') return 'phase-streaming'
|
||||
return 'phase-connecting'
|
||||
}
|
||||
|
||||
const formatProgressCount = (value?: number | null): string => {
|
||||
return typeof value === 'number' && Number.isFinite(value) ? String(value) : '-'
|
||||
}
|
||||
|
||||
const formatProgressFrameTime = (value?: number | null): string => {
|
||||
if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) return '-'
|
||||
const date = new Date(value)
|
||||
const time = formatTime(date.toISOString())
|
||||
const ageMs = Date.now() - value
|
||||
if (ageMs >= 0 && ageMs < 60_000) {
|
||||
return `${Math.max(0, Math.round(ageMs / 1000))}s 前 (${time})`
|
||||
}
|
||||
return time
|
||||
}
|
||||
|
||||
const hasDownstreamHeartbeatProgress = computed(() => {
|
||||
const progress = currentImageProgress.value
|
||||
return typeof progress?.downstream_heartbeat_count === 'number' ||
|
||||
typeof progress?.last_downstream_heartbeat_at_unix_ms === 'number' ||
|
||||
typeof progress?.downstream_heartbeat_interval_ms === 'number'
|
||||
})
|
||||
|
||||
const latestTraceAttemptForState = computed<CandidateRecord | null>(() => {
|
||||
const candidates = rawTimeline.value
|
||||
for (let index = candidates.length - 1; index >= 0; index -= 1) {
|
||||
const candidate = candidates[index]
|
||||
if (candidate.status !== 'available' && candidate.status !== 'unused') {
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
return null
|
||||
})
|
||||
|
||||
const latestTraceImageProgress = computed<ImageProgress | null>(() => {
|
||||
const candidates = rawTimeline.value
|
||||
for (let index = candidates.length - 1; index >= 0; index -= 1) {
|
||||
const candidate = candidates[index]
|
||||
const progress = normalizeImageProgress(candidate.image_progress)
|
||||
?? normalizeImageProgress(extractObject(candidate.extra_data)?.image_progress)
|
||||
if (progress) return progress
|
||||
}
|
||||
return null
|
||||
})
|
||||
|
||||
watch(
|
||||
[trace, loading, latestTraceImageProgress, latestTraceAttemptForState, computedFinalStatus],
|
||||
([value, isLoading, imageProgress, attempt, finalStatus]) => {
|
||||
const waitingForInternalTrace = Boolean(props.requestId && !props.traceData && !traceLoadStarted.value && !value)
|
||||
emit('traceState', {
|
||||
loaded: !isLoading && !waitingForInternalTrace,
|
||||
hasTrace: Boolean(value?.candidates?.length),
|
||||
finalStatus: finalStatus ?? value?.final_status ?? null,
|
||||
statusCode: attempt?.status_code ?? null,
|
||||
latencyMs: attempt?.latency_ms ?? value?.total_latency_ms ?? null,
|
||||
imageProgress,
|
||||
errorMessage: attempt?.error_message ?? null,
|
||||
})
|
||||
},
|
||||
{ immediate: true },
|
||||
)
|
||||
|
||||
const normalizeAttemptErrorFlow = (value: unknown): AttemptErrorFlow | null => {
|
||||
const raw = extractObject(value)
|
||||
if (!raw) return null
|
||||
@@ -1556,6 +1722,15 @@ const keyCapabilities = computed(() => {
|
||||
.map(([key]) => key)
|
||||
})
|
||||
|
||||
const hasActiveImageProgress = computed(() => {
|
||||
return rawTimeline.value.some((candidate) => {
|
||||
const progress = normalizeImageProgress(candidate.image_progress)
|
||||
?? normalizeImageProgress(extractObject(candidate.extra_data)?.image_progress)
|
||||
if (!progress?.phase) return false
|
||||
return progress.phase !== 'upstream_completed' && progress.phase !== 'failed'
|
||||
})
|
||||
})
|
||||
|
||||
// 判断是否为 OAuth 类型(provider_type 为具体值时也算 OAuth)
|
||||
const isOAuthType = (authType?: string): boolean => {
|
||||
if (!authType) return false
|
||||
@@ -1694,34 +1869,80 @@ const navigateAttempt = (direction: number) => {
|
||||
const isSilentRefresh = ref(false)
|
||||
const loadTrace = async (silent = false) => {
|
||||
if (!props.requestId || props.traceData) return
|
||||
if (traceLoadInFlight) return traceLoadInFlight
|
||||
|
||||
isSilentRefresh.value = silent
|
||||
traceLoadStarted.value = true
|
||||
traceLoadInFlight = (async () => {
|
||||
isSilentRefresh.value = silent
|
||||
traceLoadStarted.value = true
|
||||
|
||||
if (!silent) {
|
||||
loading.value = true
|
||||
}
|
||||
error.value = null
|
||||
|
||||
try {
|
||||
internalTrace.value = await requestTraceApi.getRequestTrace(props.requestId)
|
||||
} catch (err: unknown) {
|
||||
if (isAxiosError(err) && err.response?.status === 404) {
|
||||
internalTrace.value = null
|
||||
error.value = null
|
||||
return
|
||||
}
|
||||
if (!silent) {
|
||||
error.value = parseApiError(err, '加载失败')
|
||||
loading.value = true
|
||||
}
|
||||
log.error('加载请求追踪失败:', err)
|
||||
} finally {
|
||||
if (!silent) {
|
||||
loading.value = false
|
||||
error.value = null
|
||||
|
||||
try {
|
||||
internalTrace.value = await requestTraceApi.getRequestTrace(props.requestId)
|
||||
} catch (err: unknown) {
|
||||
if (isAxiosError(err) && err.response?.status === 404) {
|
||||
internalTrace.value = null
|
||||
error.value = null
|
||||
return
|
||||
}
|
||||
if (!silent) {
|
||||
error.value = parseApiError(err, '加载失败')
|
||||
}
|
||||
log.error('加载请求追踪失败:', err)
|
||||
} finally {
|
||||
if (!silent) {
|
||||
loading.value = false
|
||||
}
|
||||
traceLoadInFlight = null
|
||||
}
|
||||
})()
|
||||
|
||||
return traceLoadInFlight
|
||||
}
|
||||
|
||||
const propsRequestIsActive = computed(() => {
|
||||
const status = props.requestStatus ?? usageData.value?.status
|
||||
return status === 'pending' || status === 'streaming'
|
||||
})
|
||||
|
||||
const traceHasActiveCandidate = computed(() => {
|
||||
return rawTimeline.value.some((candidate) => {
|
||||
const status = getDisplayStatus(candidate)
|
||||
return status === 'pending' || status === 'streaming'
|
||||
})
|
||||
})
|
||||
|
||||
const traceFinalIsTerminal = computed(() => {
|
||||
const status = trace.value?.final_status
|
||||
return status === 'success' || status === 'failed' || status === 'cancelled'
|
||||
})
|
||||
|
||||
const shouldPollTrace = computed(() => {
|
||||
if (!props.requestId || props.traceData) return false
|
||||
if (traceHasActiveCandidate.value || hasActiveImageProgress.value) return true
|
||||
return propsRequestIsActive.value && !traceFinalIsTerminal.value
|
||||
})
|
||||
|
||||
const stopTracePolling = () => {
|
||||
if (tracePollTimer) {
|
||||
clearTimeout(tracePollTimer)
|
||||
tracePollTimer = null
|
||||
}
|
||||
}
|
||||
|
||||
const scheduleTracePolling = () => {
|
||||
stopTracePolling()
|
||||
if (!shouldPollTrace.value) return
|
||||
|
||||
tracePollTimer = setTimeout(async () => {
|
||||
await loadTrace(true)
|
||||
scheduleTracePolling()
|
||||
}, TRACE_POLL_INTERVAL_MS)
|
||||
}
|
||||
|
||||
// 监听 groupedTimeline 变化,自动选择最有意义的组
|
||||
watch(groupedTimeline, (newGroups) => {
|
||||
if (!newGroups || newGroups.length === 0) return
|
||||
@@ -1809,6 +2030,14 @@ watch(
|
||||
{ immediate: true },
|
||||
)
|
||||
|
||||
watch(shouldPollTrace, () => {
|
||||
scheduleTracePolling()
|
||||
}, { immediate: true })
|
||||
|
||||
onBeforeUnmount(() => {
|
||||
stopTracePolling()
|
||||
})
|
||||
|
||||
defineExpose({ refresh: () => loadTrace(true) })
|
||||
|
||||
// 格式化时间(详细)
|
||||
@@ -2514,6 +2743,112 @@ function getDisplayStatus(attempt: CandidateRecord | null | undefined): string {
|
||||
background: hsl(var(--primary) / 0.08);
|
||||
}
|
||||
|
||||
.image-progress-block {
|
||||
margin-top: 0.875rem;
|
||||
padding: 0.75rem;
|
||||
border: 1px solid hsl(var(--border) / 0.7);
|
||||
border-radius: 8px;
|
||||
background: hsl(var(--background) / 0.72);
|
||||
}
|
||||
|
||||
.image-progress-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 0.75rem;
|
||||
margin-bottom: 0.625rem;
|
||||
}
|
||||
|
||||
.image-progress-title {
|
||||
font-size: 0.82rem;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.image-progress-phase {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
padding: 0.15rem 0.5rem;
|
||||
border-radius: 999px;
|
||||
font-size: 0.7rem;
|
||||
font-weight: 600;
|
||||
white-space: nowrap;
|
||||
border: 1px solid hsl(var(--border));
|
||||
}
|
||||
|
||||
.image-progress-phase.phase-connecting,
|
||||
.image-progress-phase.phase-streaming {
|
||||
color: #2563eb;
|
||||
background: #3b82f614;
|
||||
border-color: #3b82f633;
|
||||
}
|
||||
|
||||
.image-progress-phase.phase-completed {
|
||||
color: #16a34a;
|
||||
background: #22c55e14;
|
||||
border-color: #22c55e33;
|
||||
}
|
||||
|
||||
.image-progress-phase.phase-failed {
|
||||
color: #dc2626;
|
||||
background: #ef444414;
|
||||
border-color: #ef444433;
|
||||
}
|
||||
|
||||
.image-progress-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||
gap: 0.625rem 0.875rem;
|
||||
}
|
||||
|
||||
.image-progress-item {
|
||||
min-width: 0;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.2rem;
|
||||
}
|
||||
|
||||
.image-progress-item.full-width {
|
||||
grid-column: span 2;
|
||||
}
|
||||
|
||||
.image-progress-label {
|
||||
font-size: 0.68rem;
|
||||
color: hsl(var(--muted-foreground));
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.image-progress-value {
|
||||
min-width: 0;
|
||||
font-size: 0.82rem;
|
||||
font-weight: 600;
|
||||
color: hsl(var(--foreground));
|
||||
}
|
||||
|
||||
.image-progress-code {
|
||||
min-width: 0;
|
||||
width: fit-content;
|
||||
max-width: 100%;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
padding: 0.12rem 0.35rem;
|
||||
border-radius: 4px;
|
||||
background: hsl(var(--muted));
|
||||
color: hsl(var(--muted-foreground));
|
||||
font-size: 0.72rem;
|
||||
font-family: ui-monospace, monospace;
|
||||
}
|
||||
|
||||
@media (max-width: 768px) {
|
||||
.image-progress-grid {
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
}
|
||||
|
||||
.image-progress-item.full-width {
|
||||
grid-column: 1 / -1;
|
||||
}
|
||||
}
|
||||
|
||||
/* Provider 官网链接 */
|
||||
.provider-link {
|
||||
display: inline-flex;
|
||||
|
||||
@@ -751,6 +751,7 @@ import Tabs from '@/components/ui/tabs.vue'
|
||||
import TabsContent from '@/components/ui/tabs-content.vue'
|
||||
import { Check, Columns2, RefreshCw, X, Monitor, Server, MessageSquareText, Code2, Terminal, Play } from 'lucide-vue-next'
|
||||
import { dashboardApi, type RequestDetail, type RequestErrorDomain } from '@/api/dashboard'
|
||||
import type { ImageProgress, RequestTrace } from '@/api/requestTrace'
|
||||
import { formatApiFormat } from '@/api/endpoints/types/api-format'
|
||||
import { formatShortRequestId } from '@/utils/format'
|
||||
import { log } from '@/utils/logger'
|
||||
@@ -792,6 +793,15 @@ const props = defineProps<{
|
||||
|
||||
const emit = defineEmits<{
|
||||
close: []
|
||||
requestState: [state: {
|
||||
id: string
|
||||
requestId?: string | null
|
||||
status?: 'pending' | 'streaming' | 'completed' | 'failed' | 'cancelled'
|
||||
statusCode?: number | null
|
||||
responseTimeMs?: number | null
|
||||
imageProgress?: ImageProgress | null
|
||||
errorMessage?: string | null
|
||||
}]
|
||||
}>()
|
||||
|
||||
const loading = ref(false)
|
||||
@@ -868,9 +878,52 @@ function formatErrorDomainMeta(domain: NormalizedErrorDomain): string {
|
||||
return parts.join(' · ')
|
||||
}
|
||||
|
||||
function handleTraceState(state: { loaded: boolean, hasTrace: boolean }) {
|
||||
function mapTraceFinalStatusToRequestStatus(
|
||||
status?: RequestTrace['final_status'] | null
|
||||
): 'pending' | 'streaming' | 'completed' | 'failed' | 'cancelled' | undefined {
|
||||
switch (status) {
|
||||
case 'success':
|
||||
return 'completed'
|
||||
case 'failed':
|
||||
return 'failed'
|
||||
case 'cancelled':
|
||||
return 'cancelled'
|
||||
case 'streaming':
|
||||
return 'streaming'
|
||||
case 'pending':
|
||||
return 'pending'
|
||||
default:
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
function handleTraceState(state: {
|
||||
loaded: boolean
|
||||
hasTrace: boolean
|
||||
finalStatus?: RequestTrace['final_status'] | null
|
||||
statusCode?: number | null
|
||||
latencyMs?: number | null
|
||||
imageProgress?: ImageProgress | null
|
||||
errorMessage?: string | null
|
||||
}) {
|
||||
timelineLoaded.value = state.loaded
|
||||
timelineHasTrace.value = state.hasTrace
|
||||
const id = props.requestId
|
||||
if (!id) return
|
||||
|
||||
const status = mapTraceFinalStatusToRequestStatus(state.finalStatus)
|
||||
const imageFailed = state.imageProgress?.phase === 'failed'
|
||||
if (!status && !state.imageProgress && state.statusCode == null && state.latencyMs == null) return
|
||||
|
||||
emit('requestState', {
|
||||
id,
|
||||
requestId: detail.value?.request_id || detail.value?.id || null,
|
||||
status: imageFailed ? 'failed' : status,
|
||||
statusCode: state.statusCode ?? undefined,
|
||||
responseTimeMs: state.latencyMs ?? undefined,
|
||||
imageProgress: state.imageProgress ?? null,
|
||||
errorMessage: state.errorMessage ?? undefined,
|
||||
})
|
||||
}
|
||||
|
||||
function toNumber(value: unknown): number | null {
|
||||
|
||||
@@ -269,4 +269,42 @@ describe('HorizontalRequestTimeline', () => {
|
||||
expect(nodeDots[0].classList.contains('status-success')).toBe(false)
|
||||
expect(nodeDots[1].classList.contains('status-success')).toBe(true)
|
||||
})
|
||||
|
||||
it('renders Codex image progress from candidate image_progress', async () => {
|
||||
const trace = buildTrace([
|
||||
buildCandidate({
|
||||
id: 'cand-image-progress',
|
||||
provider_id: 'provider-image',
|
||||
provider_name: 'Codex Image',
|
||||
key_id: 'key-image',
|
||||
key_name: 'Image Key',
|
||||
candidate_index: 0,
|
||||
status: 'streaming',
|
||||
finished_at: undefined,
|
||||
image_progress: {
|
||||
phase: 'upstream_streaming',
|
||||
upstream_ttfb_ms: 3807,
|
||||
upstream_sse_frame_count: 12,
|
||||
partial_image_count: 1,
|
||||
last_upstream_event: 'response.output_item.added',
|
||||
last_upstream_frame_at_unix_ms: Date.now(),
|
||||
last_client_visible_event: 'image_generation.partial_image',
|
||||
downstream_heartbeat_count: 3,
|
||||
downstream_heartbeat_interval_ms: 15000,
|
||||
last_downstream_heartbeat_at_unix_ms: Date.now(),
|
||||
},
|
||||
}),
|
||||
])
|
||||
|
||||
const root = mountTimeline(trace)
|
||||
await nextTick()
|
||||
|
||||
expect(root.textContent).toContain('图片生成进度')
|
||||
expect(root.textContent).toContain('上游生成中')
|
||||
expect(root.textContent).toContain('3.81s')
|
||||
expect(root.textContent).toContain('下游心跳')
|
||||
expect(root.textContent).toContain('15.00s')
|
||||
expect(root.textContent).toContain('response.output_item.added')
|
||||
expect(root.textContent).toContain('image_generation.partial_image')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -233,6 +233,20 @@ describe('UsageRecordsTable', () => {
|
||||
expect(root.querySelector('[data-active-latency-state="waiting-first-byte"]')).toBeNull()
|
||||
})
|
||||
|
||||
it('shows failed when Codex image progress fails before the usage record finalizes', () => {
|
||||
const root = mountUsageRecordsTable([buildRecord({
|
||||
status: 'pending',
|
||||
response_time_ms: null,
|
||||
first_byte_time_ms: null,
|
||||
image_progress: {
|
||||
phase: 'failed',
|
||||
},
|
||||
})])
|
||||
|
||||
expect(root.textContent).toContain('失败')
|
||||
expect(root.textContent).not.toContain('等待中')
|
||||
})
|
||||
|
||||
it('renders output TPS in the non-admin usage table', () => {
|
||||
const root = mountUsageRecordsTable([buildRecord()], { isAdmin: false })
|
||||
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import type { ImageProgress } from '@/api/requestTrace'
|
||||
|
||||
// 统计数据状态
|
||||
export interface UsageStatsState {
|
||||
total_requests: number
|
||||
@@ -116,6 +118,7 @@ export interface UsageRecord {
|
||||
created_at: string
|
||||
has_fallback?: boolean
|
||||
has_retry?: boolean
|
||||
image_progress?: ImageProgress | null
|
||||
}
|
||||
|
||||
// 日期范围参数
|
||||
|
||||
@@ -91,6 +91,20 @@ describe('usage status helpers', () => {
|
||||
expect(isUsageRecordFailed(record)).toBe(true)
|
||||
})
|
||||
|
||||
it('treats failed image progress as failed before the usage record finalizes', () => {
|
||||
const record = buildUsageRecord({
|
||||
status: 'streaming',
|
||||
status_code: undefined,
|
||||
error_message: undefined,
|
||||
image_progress: {
|
||||
phase: 'failed',
|
||||
},
|
||||
})
|
||||
|
||||
expect(resolveDisplayRequestStatus(record)).toBe('failed')
|
||||
expect(isUsageRecordFailed(record)).toBe(true)
|
||||
})
|
||||
|
||||
it('prefers terminal request lifecycle status over status code for the timeline', () => {
|
||||
expect(resolveTimelineFinalStatus({
|
||||
traceFinalStatus: 'success',
|
||||
|
||||
@@ -7,6 +7,9 @@ type RequestStatusLike = RequestStatus | string | null | undefined
|
||||
type UsageFailureSignal = {
|
||||
status_code?: number | null
|
||||
error_message?: string | null
|
||||
image_progress?: {
|
||||
phase?: string | null
|
||||
} | null
|
||||
}
|
||||
|
||||
type UsageDisplayStatusRecord = UsageFailureSignal & {
|
||||
@@ -21,6 +24,19 @@ function hasLegacyFailureSignal(
|
||||
(typeof record.error_message === 'string' && record.error_message.trim().length > 0)
|
||||
}
|
||||
|
||||
function hasImageProgressFailureSignal(
|
||||
record: UsageFailureSignal
|
||||
): boolean {
|
||||
return typeof record.image_progress?.phase === 'string' &&
|
||||
record.image_progress.phase.trim().toLowerCase() === 'failed'
|
||||
}
|
||||
|
||||
function hasAnyFailureSignal(
|
||||
record: UsageFailureSignal
|
||||
): boolean {
|
||||
return hasLegacyFailureSignal(record) || hasImageProgressFailureSignal(record)
|
||||
}
|
||||
|
||||
export function hasUsageFallback(
|
||||
record: Pick<UsageRecord, 'has_fallback'>
|
||||
): boolean {
|
||||
@@ -160,13 +176,11 @@ function hasTerminalSuccessStatusCode(
|
||||
record.status_code < 400
|
||||
}
|
||||
|
||||
export function isUsageRecordFailed(
|
||||
record: Pick<UsageRecord, 'status' | 'status_code' | 'error_message'>
|
||||
): boolean {
|
||||
export function isUsageRecordFailed(record: UsageFailureSignal & Pick<UsageRecord, 'status'>): boolean {
|
||||
const status = typeof record.status === 'string' ? record.status.trim().toLowerCase() : ''
|
||||
if (status) {
|
||||
if (status === 'pending' || status === 'streaming') {
|
||||
return !hasTerminalSuccessStatusCode(record) && hasLegacyFailureSignal(record)
|
||||
return !hasTerminalSuccessStatusCode(record) && hasAnyFailureSignal(record)
|
||||
}
|
||||
if (status === 'cancelled') {
|
||||
return false
|
||||
@@ -184,12 +198,10 @@ export function isUsageRecordFailed(
|
||||
if (status) {
|
||||
return status === 'failed'
|
||||
}
|
||||
return hasLegacyFailureSignal(record)
|
||||
return hasAnyFailureSignal(record)
|
||||
}
|
||||
|
||||
export function isUsageRecordSuccessful(
|
||||
record: Pick<UsageRecord, 'status' | 'status_code' | 'error_message'>
|
||||
): boolean {
|
||||
export function isUsageRecordSuccessful(record: UsageFailureSignal & Pick<UsageRecord, 'status'>): boolean {
|
||||
const status = typeof record.status === 'string' ? record.status.trim().toLowerCase() : ''
|
||||
if (status) {
|
||||
if (status === 'completed') {
|
||||
@@ -203,7 +215,7 @@ export function isUsageRecordSuccessful(
|
||||
if (hasTerminalSuccessStatusCode(record)) {
|
||||
return true
|
||||
}
|
||||
return !hasLegacyFailureSignal(record)
|
||||
return !hasAnyFailureSignal(record)
|
||||
}
|
||||
|
||||
export function normalizeRequestStatus(status: RequestStatusLike): RequestStatus | undefined {
|
||||
@@ -224,13 +236,13 @@ export function resolveDisplayRequestStatus(record: UsageDisplayStatusRecord): R
|
||||
const status = normalizeRequestStatus(record.status)
|
||||
if ((status === 'pending' || status === 'streaming') &&
|
||||
!hasTerminalSuccessStatusCode(record) &&
|
||||
hasLegacyFailureSignal(record)) {
|
||||
hasAnyFailureSignal(record)) {
|
||||
return 'failed'
|
||||
}
|
||||
if (status === 'streaming' && record.first_byte_time_ms == null) {
|
||||
return 'pending'
|
||||
}
|
||||
return status
|
||||
return status ?? (hasAnyFailureSignal(record) ? 'failed' : undefined)
|
||||
}
|
||||
|
||||
export function mapRequestStatusToTimelineStatus(
|
||||
|
||||
@@ -119,6 +119,7 @@
|
||||
:is-open="detailModalOpen"
|
||||
:request-id="selectedRequestId"
|
||||
@close="detailModalOpen = false"
|
||||
@request-state="handleDetailRequestState"
|
||||
/>
|
||||
</div>
|
||||
</template>
|
||||
@@ -129,6 +130,7 @@ import { useRoute } from 'vue-router'
|
||||
import { useLocalStorage } from '@vueuse/core'
|
||||
import { useAuthStore } from '@/stores/auth'
|
||||
import { usageApi } from '@/api/usage'
|
||||
import type { ImageProgress } from '@/api/requestTrace'
|
||||
import { usersApi } from '@/api/users'
|
||||
import { meApi } from '@/api/me'
|
||||
import { dashboardApi } from '@/api/dashboard'
|
||||
@@ -153,7 +155,7 @@ import {
|
||||
isUsageUpstreamStream,
|
||||
resolveDisplayRequestStatus,
|
||||
} from '@/features/usage/utils/status'
|
||||
import type { DateRangeParams, FilterStatusValue } from '@/features/usage/types'
|
||||
import type { DateRangeParams, FilterStatusValue, RequestStatus } from '@/features/usage/types'
|
||||
import type { UserOption } from '@/features/usage/components/UsageRecordsTable.vue'
|
||||
import { log } from '@/utils/logger'
|
||||
import type { ActivityHeatmap } from '@/types/activity'
|
||||
@@ -445,12 +447,16 @@ async function pollActiveRequests() {
|
||||
const shouldApply = newRank >= currentRank
|
||||
const updateHasFailureSignal =
|
||||
(typeof update.status_code === 'number' && update.status_code >= 400) ||
|
||||
(typeof update.error_message === 'string' && update.error_message.trim().length > 0)
|
||||
(typeof update.error_message === 'string' && update.error_message.trim().length > 0) ||
|
||||
update.image_progress?.phase === 'failed'
|
||||
const shouldApplyData = shouldApply || updateHasFailureSignal
|
||||
|
||||
if (shouldApply && record.status !== update.status) {
|
||||
record.status = update.status
|
||||
}
|
||||
if ('image_progress' in update) {
|
||||
record.image_progress = update.image_progress ?? null
|
||||
}
|
||||
if (shouldApplyData) {
|
||||
// 进行中状态也需要持续更新(provider/key/TTFB 可能在 streaming 后才落库)
|
||||
record.input_tokens = update.input_tokens
|
||||
@@ -877,6 +883,63 @@ function showRequestDetail(id: string) {
|
||||
detailModalOpen.value = true
|
||||
}
|
||||
|
||||
function sameImageProgress(left?: ImageProgress | null, right?: ImageProgress | null): boolean {
|
||||
if (!left && !right) return true
|
||||
if (!left || !right) return false
|
||||
return left.phase === right.phase &&
|
||||
left.upstream_ttfb_ms === right.upstream_ttfb_ms &&
|
||||
left.upstream_sse_frame_count === right.upstream_sse_frame_count &&
|
||||
left.last_upstream_event === right.last_upstream_event &&
|
||||
left.last_upstream_frame_at_unix_ms === right.last_upstream_frame_at_unix_ms &&
|
||||
left.partial_image_count === right.partial_image_count &&
|
||||
left.last_client_visible_event === right.last_client_visible_event &&
|
||||
left.downstream_heartbeat_count === right.downstream_heartbeat_count &&
|
||||
left.last_downstream_heartbeat_at_unix_ms === right.last_downstream_heartbeat_at_unix_ms &&
|
||||
left.downstream_heartbeat_interval_ms === right.downstream_heartbeat_interval_ms
|
||||
}
|
||||
|
||||
function handleDetailRequestState(update: {
|
||||
id: string
|
||||
status?: RequestStatus
|
||||
statusCode?: number | null
|
||||
responseTimeMs?: number | null
|
||||
imageProgress?: ImageProgress | null
|
||||
errorMessage?: string | null
|
||||
}) {
|
||||
const record = currentRecords.value.find(record => record.id === update.id)
|
||||
if (!record) return
|
||||
|
||||
const statusPriority: Record<RequestStatus, number> = {
|
||||
pending: 0,
|
||||
streaming: 1,
|
||||
completed: 2,
|
||||
failed: 2,
|
||||
cancelled: 2,
|
||||
}
|
||||
if (update.status) {
|
||||
const currentRank = record.status ? statusPriority[record.status] : 0
|
||||
const nextRank = statusPriority[update.status]
|
||||
if (nextRank >= currentRank) {
|
||||
record.status = update.status
|
||||
}
|
||||
}
|
||||
if ('statusCode' in update) {
|
||||
record.status_code = update.statusCode ?? undefined
|
||||
}
|
||||
if ('responseTimeMs' in update && update.responseTimeMs != null) {
|
||||
record.response_time_ms = update.responseTimeMs
|
||||
}
|
||||
if ('imageProgress' in update) {
|
||||
const nextProgress = update.imageProgress ?? null
|
||||
if (!sameImageProgress(record.image_progress, nextProgress)) {
|
||||
record.image_progress = nextProgress
|
||||
}
|
||||
}
|
||||
if ('errorMessage' in update) {
|
||||
record.error_message = update.errorMessage ?? undefined
|
||||
}
|
||||
}
|
||||
|
||||
function prefetchRequestDetail(id: string) {
|
||||
if (!isAdminPage.value) return
|
||||
void dashboardApi.prefetchRequestDetail(id).catch(error => {
|
||||
|
||||
Reference in New Issue
Block a user