mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 01:10:23 +08:00
fix(usage): 修复含失败信号的 pending 请求被误判为活跃的问题
- pending/streaming 状态含 status_code >= 400 或 error_message 时视为 failed - 候选状态派生改为 failed 优先于 stale pending/streaming - 活跃请求轮询时保留失败信号字段的更新 - 图片流支持 partial_image 事件转发及从 response.completed 读取最终图片
This commit is contained in:
@@ -106,6 +106,7 @@ struct OpenAiImageStreamState {
|
|||||||
buffered: Vec<u8>,
|
buffered: Vec<u8>,
|
||||||
latest_image: Option<OpenAiImageFrame>,
|
latest_image: Option<OpenAiImageFrame>,
|
||||||
emitted_partial_count: u64,
|
emitted_partial_count: u64,
|
||||||
|
saw_upstream_partial: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
@@ -166,12 +167,55 @@ impl OpenAiImageStreamState {
|
|||||||
.or(event_name.as_deref())
|
.or(event_name.as_deref())
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
match event_type {
|
match event_type {
|
||||||
|
"response.image_generation_call.partial_image" => {
|
||||||
|
self.handle_image_generation_partial(report_context, &event)
|
||||||
|
}
|
||||||
"response.output_item.done" => self.handle_output_item_done(report_context, &event),
|
"response.output_item.done" => self.handle_output_item_done(report_context, &event),
|
||||||
"response.completed" => self.handle_completed(report_context, &event),
|
"response.completed" => self.handle_completed(report_context, &event),
|
||||||
_ => Ok(Vec::new()),
|
_ => Ok(Vec::new()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn handle_image_generation_partial(
|
||||||
|
&mut self,
|
||||||
|
report_context: &Value,
|
||||||
|
event: &Value,
|
||||||
|
) -> Result<Vec<u8>, GatewayError> {
|
||||||
|
if requested_partial_images(report_context) == 0 {
|
||||||
|
return Ok(Vec::new());
|
||||||
|
}
|
||||||
|
let Some(result) = event
|
||||||
|
.get("partial_image_b64")
|
||||||
|
.or_else(|| event.get("b64_json"))
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
else {
|
||||||
|
return Ok(Vec::new());
|
||||||
|
};
|
||||||
|
let partial_image_index = event
|
||||||
|
.get("partial_image_index")
|
||||||
|
.or_else(|| event.get("output_index"))
|
||||||
|
.and_then(Value::as_u64)
|
||||||
|
.unwrap_or(self.emitted_partial_count);
|
||||||
|
self.emitted_partial_count = self
|
||||||
|
.emitted_partial_count
|
||||||
|
.max(partial_image_index.saturating_add(1));
|
||||||
|
self.saw_upstream_partial = true;
|
||||||
|
self.latest_image = Some(OpenAiImageFrame {
|
||||||
|
b64_json: result.to_string(),
|
||||||
|
});
|
||||||
|
|
||||||
|
encode_json_sse(
|
||||||
|
Some(image_partial_event_name(report_context)),
|
||||||
|
&serde_json::json!({
|
||||||
|
"type": image_partial_event_name(report_context),
|
||||||
|
"b64_json": result,
|
||||||
|
"partial_image_index": partial_image_index,
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
fn handle_output_item_done(
|
fn handle_output_item_done(
|
||||||
&mut self,
|
&mut self,
|
||||||
report_context: &Value,
|
report_context: &Value,
|
||||||
@@ -193,7 +237,7 @@ impl OpenAiImageStreamState {
|
|||||||
b64_json: result.to_string(),
|
b64_json: result.to_string(),
|
||||||
});
|
});
|
||||||
|
|
||||||
if requested_partial_images(report_context) == 0 {
|
if requested_partial_images(report_context) == 0 || self.saw_upstream_partial {
|
||||||
return Ok(Vec::new());
|
return Ok(Vec::new());
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -218,6 +262,13 @@ impl OpenAiImageStreamState {
|
|||||||
report_context: &Value,
|
report_context: &Value,
|
||||||
event: &Value,
|
event: &Value,
|
||||||
) -> Result<Vec<u8>, GatewayError> {
|
) -> Result<Vec<u8>, GatewayError> {
|
||||||
|
if self.latest_image.is_none() {
|
||||||
|
if let Some(result) = completed_response_image_result(event) {
|
||||||
|
self.latest_image = Some(OpenAiImageFrame {
|
||||||
|
b64_json: result.to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
let Some(latest_image) = self.latest_image.clone() else {
|
let Some(latest_image) = self.latest_image.clone() else {
|
||||||
return Ok(Vec::new());
|
return Ok(Vec::new());
|
||||||
};
|
};
|
||||||
@@ -244,6 +295,19 @@ impl OpenAiImageStreamState {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn completed_response_image_result(event: &Value) -> Option<&str> {
|
||||||
|
event
|
||||||
|
.get("response")
|
||||||
|
.and_then(|value| value.get("output"))
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.into_iter()
|
||||||
|
.flatten()
|
||||||
|
.filter(|item| item.get("type").and_then(Value::as_str) == Some("image_generation_call"))
|
||||||
|
.filter_map(|item| item.get("result").and_then(Value::as_str))
|
||||||
|
.map(str::trim)
|
||||||
|
.find(|value| !value.is_empty())
|
||||||
|
}
|
||||||
|
|
||||||
fn requested_partial_images(report_context: &Value) -> u64 {
|
fn requested_partial_images(report_context: &Value) -> u64 {
|
||||||
report_context
|
report_context
|
||||||
.get("image_request")
|
.get("image_request")
|
||||||
|
|||||||
@@ -172,6 +172,91 @@ fn openai_image_stream_rewriter_emits_completed_event_for_generate() {
|
|||||||
assert!(rewriter.finish().expect("finish should succeed").is_empty());
|
assert!(rewriter.finish().expect("finish should succeed").is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn openai_image_stream_rewriter_maps_responses_partial_image_events() {
|
||||||
|
let report_context = json!({
|
||||||
|
"provider_api_format": "openai:image",
|
||||||
|
"client_api_format": "openai:image",
|
||||||
|
"needs_conversion": false,
|
||||||
|
"image_request": {
|
||||||
|
"operation": "generate",
|
||||||
|
"partial_images": 1
|
||||||
|
}
|
||||||
|
});
|
||||||
|
let mut rewriter =
|
||||||
|
maybe_build_local_stream_rewriter(Some(&report_context)).expect("rewriter should exist");
|
||||||
|
|
||||||
|
let partial = rewriter
|
||||||
|
.push_chunk(
|
||||||
|
concat!(
|
||||||
|
"event: response.image_generation_call.partial_image\n",
|
||||||
|
"data: {\"type\":\"response.image_generation_call.partial_image\",\"partial_image_index\":0,\"partial_image_b64\":\"cGFydGlhbA==\"}\n\n"
|
||||||
|
)
|
||||||
|
.as_bytes(),
|
||||||
|
)
|
||||||
|
.expect("rewrite should succeed");
|
||||||
|
let partial_text = utf8(partial);
|
||||||
|
assert!(partial_text.contains("event: image_generation.partial_image"));
|
||||||
|
assert!(partial_text.contains("\"type\":\"image_generation.partial_image\""));
|
||||||
|
assert!(partial_text.contains("\"b64_json\":\"cGFydGlhbA==\""));
|
||||||
|
assert!(partial_text.contains("\"partial_image_index\":0"));
|
||||||
|
assert!(!partial_text.contains("response.image_generation_call.partial_image"));
|
||||||
|
|
||||||
|
let done = rewriter
|
||||||
|
.push_chunk(
|
||||||
|
concat!(
|
||||||
|
"event: response.output_item.done\n",
|
||||||
|
"data: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"ig_123\",\"type\":\"image_generation_call\",\"result\":\"ZmluYWw=\"}}\n\n"
|
||||||
|
)
|
||||||
|
.as_bytes(),
|
||||||
|
)
|
||||||
|
.expect("rewrite should succeed");
|
||||||
|
assert!(done.is_empty());
|
||||||
|
|
||||||
|
let completed = rewriter
|
||||||
|
.push_chunk(
|
||||||
|
concat!(
|
||||||
|
"event: response.completed\n",
|
||||||
|
"data: {\"type\":\"response.completed\",\"response\":{\"usage\":{\"input_tokens\":4,\"output_tokens\":5,\"total_tokens\":9}}}\n\n"
|
||||||
|
)
|
||||||
|
.as_bytes(),
|
||||||
|
)
|
||||||
|
.expect("rewrite should succeed");
|
||||||
|
let completed_text = utf8(completed);
|
||||||
|
assert!(completed_text.contains("event: image_generation.completed"));
|
||||||
|
assert!(completed_text.contains("\"type\":\"image_generation.completed\""));
|
||||||
|
assert!(completed_text.contains("\"b64_json\":\"ZmluYWw=\""));
|
||||||
|
assert!(completed_text.contains("\"total_tokens\":9"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn openai_image_stream_rewriter_reads_final_image_from_completed_response_output() {
|
||||||
|
let report_context = json!({
|
||||||
|
"provider_api_format": "openai:image",
|
||||||
|
"client_api_format": "openai:image",
|
||||||
|
"needs_conversion": false,
|
||||||
|
"image_request": {
|
||||||
|
"operation": "generate"
|
||||||
|
}
|
||||||
|
});
|
||||||
|
let mut rewriter =
|
||||||
|
maybe_build_local_stream_rewriter(Some(&report_context)).expect("rewriter should exist");
|
||||||
|
|
||||||
|
let completed = rewriter
|
||||||
|
.push_chunk(
|
||||||
|
concat!(
|
||||||
|
"event: response.completed\n",
|
||||||
|
"data: {\"type\":\"response.completed\",\"response\":{\"output\":[{\"type\":\"message\"},{\"type\":\"image_generation_call\",\"result\":\"ZnJvbV9vdXRwdXQ=\"}],\"usage\":{\"input_tokens\":1,\"output_tokens\":2,\"total_tokens\":3}}}\n\n"
|
||||||
|
)
|
||||||
|
.as_bytes(),
|
||||||
|
)
|
||||||
|
.expect("rewrite should succeed");
|
||||||
|
let completed_text = utf8(completed);
|
||||||
|
assert!(completed_text.contains("event: image_generation.completed"));
|
||||||
|
assert!(completed_text.contains("\"b64_json\":\"ZnJvbV9vdXRwdXQ=\""));
|
||||||
|
assert!(completed_text.contains("\"total_tokens\":3"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn openai_image_stream_rewriter_emits_partial_and_completed_events_for_edit() {
|
fn openai_image_stream_rewriter_emits_partial_and_completed_events_for_edit() {
|
||||||
let report_context = json!({
|
let report_context = json!({
|
||||||
|
|||||||
@@ -76,6 +76,51 @@ async fn admin_monitoring_trace_request_returns_local_payload() {
|
|||||||
assert_eq!(payload["candidates"][0]["status_code"], json!(502));
|
assert_eq!(payload["candidates"][0]["status_code"], json!(502));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn admin_monitoring_trace_final_status_prefers_failed_over_stale_pending() {
|
||||||
|
let request_candidates = Arc::new(InMemoryRequestCandidateRepository::seed(vec![
|
||||||
|
sample_candidate(
|
||||||
|
"cand-stale-pending",
|
||||||
|
"request-1",
|
||||||
|
0,
|
||||||
|
RequestCandidateStatus::Pending,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
),
|
||||||
|
sample_candidate(
|
||||||
|
"cand-failed",
|
||||||
|
"request-1",
|
||||||
|
1,
|
||||||
|
RequestCandidateStatus::Failed,
|
||||||
|
Some(101),
|
||||||
|
Some(33),
|
||||||
|
Some(502),
|
||||||
|
),
|
||||||
|
]));
|
||||||
|
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||||
|
vec![sample_provider()],
|
||||||
|
vec![sample_endpoint()],
|
||||||
|
vec![sample_key()],
|
||||||
|
));
|
||||||
|
let state = AppState::new()
|
||||||
|
.expect("state should build")
|
||||||
|
.with_decision_trace_data_readers_for_tests(request_candidates, provider_catalog);
|
||||||
|
let context = request_context(http::Method::GET, "/api/admin/monitoring/trace/request-1");
|
||||||
|
|
||||||
|
let response = local_monitoring_response(&state, &context)
|
||||||
|
.await
|
||||||
|
.expect("handler should not error")
|
||||||
|
.expect("route should be handled locally");
|
||||||
|
|
||||||
|
assert_eq!(response.status(), http::StatusCode::OK);
|
||||||
|
let body = to_bytes(response.into_body(), usize::MAX)
|
||||||
|
.await
|
||||||
|
.expect("body should read");
|
||||||
|
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
|
||||||
|
assert_eq!(payload["final_status"], json!("failed"));
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn admin_monitoring_trace_request_keeps_format_conversion_disabled_candidates_visible() {
|
async fn admin_monitoring_trace_request_keeps_format_conversion_disabled_candidates_visible() {
|
||||||
let mut format_disabled_candidate = sample_candidate(
|
let mut format_disabled_candidate = sample_candidate(
|
||||||
|
|||||||
@@ -6,10 +6,11 @@ use crate::handlers::admin::shared::query_param_value;
|
|||||||
use crate::GatewayError;
|
use crate::GatewayError;
|
||||||
use aether_admin::observability::usage::{
|
use aether_admin::observability::usage::{
|
||||||
admin_usage_bad_request_response, admin_usage_data_unavailable_response,
|
admin_usage_bad_request_response, admin_usage_data_unavailable_response,
|
||||||
admin_usage_has_fallback, admin_usage_matches_search, admin_usage_matches_username,
|
admin_usage_has_fallback, admin_usage_is_failed, admin_usage_matches_search,
|
||||||
admin_usage_parse_ids, admin_usage_parse_limit, admin_usage_parse_offset,
|
admin_usage_matches_username, admin_usage_parse_ids, admin_usage_parse_limit,
|
||||||
build_admin_usage_active_requests_response, build_admin_usage_records_response,
|
admin_usage_parse_offset, build_admin_usage_active_requests_response,
|
||||||
build_admin_usage_summary_stats_response_from_summary, ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
|
build_admin_usage_records_response, build_admin_usage_summary_stats_response_from_summary,
|
||||||
|
ADMIN_USAGE_DATA_UNAVAILABLE_DETAIL,
|
||||||
};
|
};
|
||||||
use aether_data_contracts::repository::usage::{
|
use aether_data_contracts::repository::usage::{
|
||||||
StoredRequestUsageAudit, UsageAuditKeywordSearchQuery, UsageAuditListQuery,
|
StoredRequestUsageAudit, UsageAuditKeywordSearchQuery, UsageAuditListQuery,
|
||||||
@@ -305,6 +306,14 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
|
|||||||
})
|
})
|
||||||
.await?
|
.await?
|
||||||
};
|
};
|
||||||
|
let items = if requested_ids.is_some() {
|
||||||
|
items
|
||||||
|
} else {
|
||||||
|
items
|
||||||
|
.into_iter()
|
||||||
|
.filter(|item| !admin_usage_is_failed(item))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
};
|
||||||
let api_key_names = admin_usage_api_key_names(state, &items).await?;
|
let api_key_names = admin_usage_api_key_names(state, &items).await?;
|
||||||
let provider_key_names = admin_usage_provider_key_names(state, &items).await?;
|
let provider_key_names = admin_usage_provider_key_names(state, &items).await?;
|
||||||
|
|
||||||
|
|||||||
@@ -404,6 +404,8 @@ fn build_users_me_usage_active_payload(item: &StoredRequestUsageAudit) -> serde_
|
|||||||
"rate_multiplier": item.settlement_rate_multiplier(),
|
"rate_multiplier": item.settlement_rate_multiplier(),
|
||||||
"response_time_ms": item.response_time_ms,
|
"response_time_ms": item.response_time_ms,
|
||||||
"first_byte_time_ms": item.first_byte_time_ms,
|
"first_byte_time_ms": item.first_byte_time_ms,
|
||||||
|
"status_code": item.status_code,
|
||||||
|
"error_message": item.error_message,
|
||||||
"api_format": item.api_format,
|
"api_format": item.api_format,
|
||||||
"endpoint_api_format": item.endpoint_api_format,
|
"endpoint_api_format": item.endpoint_api_format,
|
||||||
"is_stream": item.is_stream,
|
"is_stream": item.is_stream,
|
||||||
@@ -435,6 +437,24 @@ fn build_users_me_usage_active_payload(item: &StoredRequestUsageAudit) -> serde_
|
|||||||
payload
|
payload
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn users_me_usage_is_failed(item: &StoredRequestUsageAudit) -> bool {
|
||||||
|
let has_failure_signal = item.status_code.is_some_and(|value| value >= 400)
|
||||||
|
|| item
|
||||||
|
.error_message
|
||||||
|
.as_deref()
|
||||||
|
.is_some_and(|value| !value.trim().is_empty());
|
||||||
|
let status = item.status.trim().to_ascii_lowercase();
|
||||||
|
if status.is_empty() {
|
||||||
|
return has_failure_signal;
|
||||||
|
}
|
||||||
|
match status.as_str() {
|
||||||
|
"completed" | "cancelled" => false,
|
||||||
|
"pending" | "streaming" => has_failure_signal,
|
||||||
|
"failed" => true,
|
||||||
|
_ => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn build_users_me_usage_summary_by_model(
|
fn build_users_me_usage_summary_by_model(
|
||||||
rows: &[StoredUsageBreakdownSummaryRow],
|
rows: &[StoredUsageBreakdownSummaryRow],
|
||||||
include_actual_cost: bool,
|
include_actual_cost: bool,
|
||||||
@@ -997,6 +1017,15 @@ pub(super) async fn handle_users_me_usage_active_get(
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let items = if ids.is_some() {
|
||||||
|
items
|
||||||
|
} else {
|
||||||
|
items
|
||||||
|
.into_iter()
|
||||||
|
.filter(|item| !users_me_usage_is_failed(item))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
};
|
||||||
|
|
||||||
Json(json!({
|
Json(json!({
|
||||||
"requests": items
|
"requests": items
|
||||||
.iter()
|
.iter()
|
||||||
@@ -1233,7 +1262,8 @@ mod tests {
|
|||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
build_users_me_usage_active_payload, build_users_me_usage_record_payload,
|
build_users_me_usage_active_payload, build_users_me_usage_record_payload,
|
||||||
users_me_usage_client_is_stream, users_me_usage_upstream_is_stream,
|
users_me_usage_client_is_stream, users_me_usage_is_failed,
|
||||||
|
users_me_usage_upstream_is_stream,
|
||||||
};
|
};
|
||||||
|
|
||||||
fn sample_usage(status: &str) -> StoredRequestUsageAudit {
|
fn sample_usage(status: &str) -> StoredRequestUsageAudit {
|
||||||
@@ -1310,6 +1340,17 @@ mod tests {
|
|||||||
assert_eq!(payload["cache_creation_ephemeral_1h_input_tokens"], 6);
|
assert_eq!(payload["cache_creation_ephemeral_1h_input_tokens"], 6);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn user_usage_active_pending_with_failure_signal_is_not_active() {
|
||||||
|
let item = StoredRequestUsageAudit {
|
||||||
|
status_code: Some(503),
|
||||||
|
error_message: Some("upstream failed".to_string()),
|
||||||
|
..sample_usage("pending")
|
||||||
|
};
|
||||||
|
|
||||||
|
assert!(users_me_usage_is_failed(&item));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn user_usage_payloads_include_symmetric_stream_fields() {
|
fn user_usage_payloads_include_symmetric_stream_fields() {
|
||||||
let item = StoredRequestUsageAudit {
|
let item = StoredRequestUsageAudit {
|
||||||
|
|||||||
@@ -764,6 +764,26 @@ async fn gateway_handles_admin_usage_active_locally_with_trusted_admin_principal
|
|||||||
row.candidate_index = Some(1);
|
row.candidate_index = Some(1);
|
||||||
row
|
row
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
let mut row = sample_usage_row(
|
||||||
|
"usage-failed-pending",
|
||||||
|
"req-failed-pending",
|
||||||
|
Some("user-1"),
|
||||||
|
Some("key-1"),
|
||||||
|
Some("primary"),
|
||||||
|
"OpenAI",
|
||||||
|
"gpt-5",
|
||||||
|
"pending",
|
||||||
|
10,
|
||||||
|
0,
|
||||||
|
0.0,
|
||||||
|
0.0,
|
||||||
|
DAY_2_UNIX_SECS + 1,
|
||||||
|
);
|
||||||
|
row.status_code = Some(503);
|
||||||
|
row.error_message = Some("upstream failed".to_string());
|
||||||
|
row
|
||||||
|
},
|
||||||
sample_usage_row(
|
sample_usage_row(
|
||||||
"usage-done",
|
"usage-done",
|
||||||
"req-done",
|
"req-done",
|
||||||
|
|||||||
@@ -227,15 +227,21 @@ pub fn admin_usage_matches_api_format(
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn admin_usage_is_failed(item: &StoredRequestUsageAudit) -> bool {
|
pub fn admin_usage_is_failed(item: &StoredRequestUsageAudit) -> bool {
|
||||||
let status = item.status.trim();
|
let has_failure_signal = item.status_code.is_some_and(|value| value >= 400)
|
||||||
if !status.is_empty() {
|
|
||||||
return status.eq_ignore_ascii_case("failed");
|
|
||||||
}
|
|
||||||
item.status_code.is_some_and(|value| value >= 400)
|
|
||||||
|| item
|
|| item
|
||||||
.error_message
|
.error_message
|
||||||
.as_deref()
|
.as_deref()
|
||||||
.is_some_and(|value| !value.trim().is_empty())
|
.is_some_and(|value| !value.trim().is_empty());
|
||||||
|
let status = item.status.trim().to_ascii_lowercase();
|
||||||
|
if !status.is_empty() {
|
||||||
|
return match status.as_str() {
|
||||||
|
"completed" | "cancelled" => false,
|
||||||
|
"pending" | "streaming" => has_failure_signal,
|
||||||
|
"failed" => true,
|
||||||
|
_ => false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
has_failure_signal
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn admin_usage_has_fallback(item: &StoredRequestUsageAudit) -> bool {
|
pub fn admin_usage_has_fallback(item: &StoredRequestUsageAudit) -> bool {
|
||||||
@@ -588,6 +594,8 @@ fn admin_usage_active_request_json(
|
|||||||
"actual_cost": round_to(item.actual_total_cost_usd, 6),
|
"actual_cost": round_to(item.actual_total_cost_usd, 6),
|
||||||
"response_time_ms": item.response_time_ms,
|
"response_time_ms": item.response_time_ms,
|
||||||
"first_byte_time_ms": item.first_byte_time_ms,
|
"first_byte_time_ms": item.first_byte_time_ms,
|
||||||
|
"status_code": item.status_code,
|
||||||
|
"error_message": item.error_message,
|
||||||
"provider": item.provider_name,
|
"provider": item.provider_name,
|
||||||
"api_key_name": api_key_name,
|
"api_key_name": api_key_name,
|
||||||
"provider_key_name": provider_key_name,
|
"provider_key_name": provider_key_name,
|
||||||
@@ -2093,6 +2101,14 @@ mod tests {
|
|||||||
assert!(admin_usage_matches_status(&item, Some("failed")));
|
assert!(admin_usage_matches_status(&item, Some("failed")));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn active_status_with_failure_signal_counts_as_failed() {
|
||||||
|
let item = sample_usage("pending", Some(503), Some("upstream failed"));
|
||||||
|
|
||||||
|
assert!(admin_usage_is_failed(&item));
|
||||||
|
assert!(admin_usage_matches_status(&item, Some("failed")));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn admin_usage_fallback_flag_uses_routing_candidate_index() {
|
fn admin_usage_fallback_flag_uses_routing_candidate_index() {
|
||||||
let mut item = sample_usage("completed", Some(200), None);
|
let mut item = sample_usage("completed", Some(200), None);
|
||||||
|
|||||||
@@ -269,6 +269,20 @@ pub fn derive_request_candidate_final_status(
|
|||||||
return RequestCandidateFinalStatus::Success;
|
return RequestCandidateFinalStatus::Success;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let has_failed = candidates
|
||||||
|
.iter()
|
||||||
|
.any(|candidate| candidate.status == RequestCandidateStatus::Failed);
|
||||||
|
if has_failed {
|
||||||
|
return RequestCandidateFinalStatus::Failed;
|
||||||
|
}
|
||||||
|
|
||||||
|
let has_cancelled = candidates
|
||||||
|
.iter()
|
||||||
|
.any(|candidate| candidate.status == RequestCandidateStatus::Cancelled);
|
||||||
|
if has_cancelled {
|
||||||
|
return RequestCandidateFinalStatus::Cancelled;
|
||||||
|
}
|
||||||
|
|
||||||
if candidates
|
if candidates
|
||||||
.iter()
|
.iter()
|
||||||
.any(|candidate| candidate.status == RequestCandidateStatus::Streaming)
|
.any(|candidate| candidate.status == RequestCandidateStatus::Streaming)
|
||||||
@@ -283,16 +297,6 @@ pub fn derive_request_candidate_final_status(
|
|||||||
return RequestCandidateFinalStatus::Pending;
|
return RequestCandidateFinalStatus::Pending;
|
||||||
}
|
}
|
||||||
|
|
||||||
let has_cancelled = candidates
|
|
||||||
.iter()
|
|
||||||
.any(|candidate| candidate.status == RequestCandidateStatus::Cancelled);
|
|
||||||
let has_failed = candidates
|
|
||||||
.iter()
|
|
||||||
.any(|candidate| candidate.status == RequestCandidateStatus::Failed);
|
|
||||||
if has_cancelled && !has_failed {
|
|
||||||
return RequestCandidateFinalStatus::Cancelled;
|
|
||||||
}
|
|
||||||
|
|
||||||
RequestCandidateFinalStatus::Failed
|
RequestCandidateFinalStatus::Failed
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -302,6 +302,8 @@ export const meApi = {
|
|||||||
rate_multiplier?: number | null
|
rate_multiplier?: number | null
|
||||||
response_time_ms: number | null
|
response_time_ms: number | null
|
||||||
first_byte_time_ms: number | null
|
first_byte_time_ms: number | null
|
||||||
|
status_code?: number | null
|
||||||
|
error_message?: string | null
|
||||||
api_format?: string | null
|
api_format?: string | null
|
||||||
endpoint_api_format?: string | null
|
endpoint_api_format?: string | null
|
||||||
is_stream?: boolean | null
|
is_stream?: boolean | null
|
||||||
|
|||||||
@@ -307,6 +307,8 @@ export const usageApi = {
|
|||||||
rate_multiplier?: number | null
|
rate_multiplier?: number | null
|
||||||
response_time_ms: number | null
|
response_time_ms: number | null
|
||||||
first_byte_time_ms: number | null
|
first_byte_time_ms: number | null
|
||||||
|
status_code?: number | null
|
||||||
|
error_message?: string | null
|
||||||
provider?: string | null
|
provider?: string | null
|
||||||
api_key_name?: string | null
|
api_key_name?: string | null
|
||||||
provider_key_name?: string | null
|
provider_key_name?: string | null
|
||||||
|
|||||||
@@ -520,6 +520,7 @@ interface NodeGroup {
|
|||||||
|
|
||||||
// 用量数据类型
|
// 用量数据类型
|
||||||
interface UsageData {
|
interface UsageData {
|
||||||
|
status?: string | null
|
||||||
tokens: {
|
tokens: {
|
||||||
input: number
|
input: number
|
||||||
output: number
|
output: number
|
||||||
@@ -547,6 +548,8 @@ const props = defineProps<{
|
|||||||
requestId?: string | null
|
requestId?: string | null
|
||||||
/** 外部传入的状态码,用于覆盖 trace.final_status 的判断 */
|
/** 外部传入的状态码,用于覆盖 trace.final_status 的判断 */
|
||||||
overrideStatusCode?: number
|
overrideStatusCode?: number
|
||||||
|
/** 外部传入的请求状态,用于识别已失败/取消的终态请求 */
|
||||||
|
requestStatus?: string | null
|
||||||
/** 请求侧 API 格式(客户端入口格式) */
|
/** 请求侧 API 格式(客户端入口格式) */
|
||||||
requestApiFormat?: string | null
|
requestApiFormat?: string | null
|
||||||
/** 用量和费用数据 */
|
/** 用量和费用数据 */
|
||||||
@@ -577,6 +580,7 @@ const computedFinalStatus = computed(() => {
|
|||||||
return resolveTimelineFinalStatus({
|
return resolveTimelineFinalStatus({
|
||||||
hasPendingCandidates: hasPending,
|
hasPendingCandidates: hasPending,
|
||||||
statusCode: props.overrideStatusCode,
|
statusCode: props.overrideStatusCode,
|
||||||
|
requestStatus: props.requestStatus ?? usageData.value?.status,
|
||||||
traceFinalStatus: trace.value?.final_status,
|
traceFinalStatus: trace.value?.final_status,
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -411,6 +411,7 @@
|
|||||||
ref="timelineRef"
|
ref="timelineRef"
|
||||||
:request-id="detail.request_id || detail.id"
|
:request-id="detail.request_id || detail.id"
|
||||||
:override-status-code="detail.status_code"
|
:override-status-code="detail.status_code"
|
||||||
|
:request-status="detail.status"
|
||||||
:request-api-format="detail.api_format || null"
|
:request-api-format="detail.api_format || null"
|
||||||
:request-metadata="traceRequestMetadata"
|
:request-metadata="traceRequestMetadata"
|
||||||
/>
|
/>
|
||||||
@@ -711,6 +712,7 @@ import { getEffectiveInputTokens } from '../token-normalization'
|
|||||||
import {
|
import {
|
||||||
formatUsageStreamLabel,
|
formatUsageStreamLabel,
|
||||||
isUsageUpstreamStream,
|
isUsageUpstreamStream,
|
||||||
|
resolveDisplayRequestStatus,
|
||||||
resolveUsageStreamLabelSegments,
|
resolveUsageStreamLabelSegments,
|
||||||
} from '../utils/status'
|
} from '../utils/status'
|
||||||
|
|
||||||
@@ -1766,7 +1768,8 @@ function handleClose() {
|
|||||||
|
|
||||||
function isRequestCompleted(): boolean {
|
function isRequestCompleted(): boolean {
|
||||||
if (!detail.value?.status) return true
|
if (!detail.value?.status) return true
|
||||||
return !['pending', 'streaming'].includes(detail.value.status)
|
const displayStatus = resolveDisplayRequestStatus(detail.value)
|
||||||
|
return displayStatus !== 'pending' && displayStatus !== 'streaming'
|
||||||
}
|
}
|
||||||
|
|
||||||
function stopAutoRefresh() {
|
function stopAutoRefresh() {
|
||||||
|
|||||||
@@ -1,6 +1,11 @@
|
|||||||
import { ref, computed, type Ref } from 'vue'
|
import { ref, computed, type Ref } from 'vue'
|
||||||
import type { UsageRecord, FilterStatusValue } from '../types'
|
import type { UsageRecord, FilterStatusValue } from '../types'
|
||||||
import { hasUsageFallback, isUsageRecordFailed, isUsageUpstreamStream } from '../utils/status'
|
import {
|
||||||
|
hasUsageFallback,
|
||||||
|
isUsageRecordFailed,
|
||||||
|
isUsageUpstreamStream,
|
||||||
|
resolveDisplayRequestStatus,
|
||||||
|
} from '../utils/status'
|
||||||
|
|
||||||
export interface UseUsageFiltersOptions {
|
export interface UseUsageFiltersOptions {
|
||||||
/** 所有记录的响应式引用 */
|
/** 所有记录的响应式引用 */
|
||||||
@@ -73,7 +78,8 @@ export function useUsageFilters(options: UseUsageFiltersOptions) {
|
|||||||
)
|
)
|
||||||
} else if (filterStatus.value === 'active') {
|
} else if (filterStatus.value === 'active') {
|
||||||
records = records.filter(record =>
|
records = records.filter(record =>
|
||||||
record.status === 'pending' || record.status === 'streaming'
|
resolveDisplayRequestStatus(record) === 'pending' ||
|
||||||
|
resolveDisplayRequestStatus(record) === 'streaming'
|
||||||
)
|
)
|
||||||
} else if (filterStatus.value === 'failed') {
|
} else if (filterStatus.value === 'failed') {
|
||||||
records = records.filter(record => isUsageRecordFailed(record))
|
records = records.filter(record => isUsageRecordFailed(record))
|
||||||
|
|||||||
@@ -79,6 +79,17 @@ describe('usage status helpers', () => {
|
|||||||
}))).toBe('streaming')
|
}))).toBe('streaming')
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('treats active lifecycle records with failure signals as failed for display', () => {
|
||||||
|
const record = buildUsageRecord({
|
||||||
|
status: 'pending',
|
||||||
|
status_code: 503,
|
||||||
|
error_message: 'upstream failed',
|
||||||
|
})
|
||||||
|
|
||||||
|
expect(resolveDisplayRequestStatus(record)).toBe('failed')
|
||||||
|
expect(isUsageRecordFailed(record)).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
it('treats explicit success status code as authoritative for the timeline', () => {
|
it('treats explicit success status code as authoritative for the timeline', () => {
|
||||||
expect(resolveTimelineFinalStatus({
|
expect(resolveTimelineFinalStatus({
|
||||||
traceFinalStatus: 'success',
|
traceFinalStatus: 'success',
|
||||||
@@ -93,6 +104,13 @@ describe('usage status helpers', () => {
|
|||||||
})).toBe('failed')
|
})).toBe('failed')
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('does not let stale pending candidates override terminal request status', () => {
|
||||||
|
expect(resolveTimelineFinalStatus({
|
||||||
|
hasPendingCandidates: true,
|
||||||
|
requestStatus: 'failed',
|
||||||
|
})).toBe('failed')
|
||||||
|
})
|
||||||
|
|
||||||
it('uses explicit has_fallback flag for transfer filtering', () => {
|
it('uses explicit has_fallback flag for transfer filtering', () => {
|
||||||
expect(hasUsageFallback(buildUsageRecord({ has_fallback: true }))).toBe(true)
|
expect(hasUsageFallback(buildUsageRecord({ has_fallback: true }))).toBe(true)
|
||||||
expect(hasUsageFallback(buildUsageRecord({ has_fallback: false }))).toBe(false)
|
expect(hasUsageFallback(buildUsageRecord({ has_fallback: false }))).toBe(false)
|
||||||
|
|||||||
@@ -4,8 +4,18 @@ export type TimelineFinalStatus = 'success' | 'failed' | 'streaming' | 'pending'
|
|||||||
|
|
||||||
type RequestStatusLike = RequestStatus | string | null | undefined
|
type RequestStatusLike = RequestStatus | string | null | undefined
|
||||||
|
|
||||||
|
type UsageFailureSignal = {
|
||||||
|
status_code?: number | null
|
||||||
|
error_message?: string | null
|
||||||
|
}
|
||||||
|
|
||||||
|
type UsageDisplayStatusRecord = UsageFailureSignal & {
|
||||||
|
status?: RequestStatusLike
|
||||||
|
first_byte_time_ms?: number | null
|
||||||
|
}
|
||||||
|
|
||||||
function hasLegacyFailureSignal(
|
function hasLegacyFailureSignal(
|
||||||
record: Pick<UsageRecord, 'status_code' | 'error_message'>
|
record: UsageFailureSignal
|
||||||
): boolean {
|
): boolean {
|
||||||
return (typeof record.status_code === 'number' && record.status_code >= 400) ||
|
return (typeof record.status_code === 'number' && record.status_code >= 400) ||
|
||||||
(typeof record.error_message === 'string' && record.error_message.trim().length > 0)
|
(typeof record.error_message === 'string' && record.error_message.trim().length > 0)
|
||||||
@@ -138,7 +148,7 @@ function usageApiFormatDefaultsToNonStream(apiFormat: string): boolean {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function hasTerminalSuccessStatusCode(
|
function hasTerminalSuccessStatusCode(
|
||||||
record: Pick<UsageRecord, 'status_code'>
|
record: UsageFailureSignal
|
||||||
): boolean {
|
): boolean {
|
||||||
return typeof record.status_code === 'number' &&
|
return typeof record.status_code === 'number' &&
|
||||||
record.status_code >= 200 &&
|
record.status_code >= 200 &&
|
||||||
@@ -150,7 +160,10 @@ export function isUsageRecordFailed(
|
|||||||
): boolean {
|
): boolean {
|
||||||
const status = typeof record.status === 'string' ? record.status.trim().toLowerCase() : ''
|
const status = typeof record.status === 'string' ? record.status.trim().toLowerCase() : ''
|
||||||
if (status) {
|
if (status) {
|
||||||
if (status === 'pending' || status === 'streaming' || status === 'cancelled') {
|
if (status === 'pending' || status === 'streaming') {
|
||||||
|
return !hasTerminalSuccessStatusCode(record) && hasLegacyFailureSignal(record)
|
||||||
|
}
|
||||||
|
if (status === 'cancelled') {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
if (status === 'completed') {
|
if (status === 'completed') {
|
||||||
@@ -202,10 +215,13 @@ export function normalizeRequestStatus(status: RequestStatusLike): RequestStatus
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export function resolveDisplayRequestStatus(
|
export function resolveDisplayRequestStatus(record: UsageDisplayStatusRecord): RequestStatus | undefined {
|
||||||
record: Pick<UsageRecord, 'status' | 'first_byte_time_ms'>
|
|
||||||
): RequestStatus | undefined {
|
|
||||||
const status = normalizeRequestStatus(record.status)
|
const status = normalizeRequestStatus(record.status)
|
||||||
|
if ((status === 'pending' || status === 'streaming') &&
|
||||||
|
!hasTerminalSuccessStatusCode(record) &&
|
||||||
|
hasLegacyFailureSignal(record)) {
|
||||||
|
return 'failed'
|
||||||
|
}
|
||||||
if (status === 'streaming' && record.first_byte_time_ms == null) {
|
if (status === 'streaming' && record.first_byte_time_ms == null) {
|
||||||
return 'pending'
|
return 'pending'
|
||||||
}
|
}
|
||||||
@@ -251,20 +267,28 @@ export function resolveTimelineFinalStatus(params: {
|
|||||||
requestStatus?: RequestStatusLike
|
requestStatus?: RequestStatusLike
|
||||||
statusCode?: number
|
statusCode?: number
|
||||||
}): TimelineFinalStatus {
|
}): TimelineFinalStatus {
|
||||||
if (params.hasPendingCandidates) {
|
|
||||||
return 'pending'
|
|
||||||
}
|
|
||||||
|
|
||||||
if (typeof params.statusCode === 'number') {
|
if (typeof params.statusCode === 'number') {
|
||||||
return params.statusCode >= 200 && params.statusCode < 400 ? 'success' : 'failed'
|
return params.statusCode >= 200 && params.statusCode < 400 ? 'success' : 'failed'
|
||||||
}
|
}
|
||||||
|
|
||||||
const traceStatus = normalizeTimelineFinalStatus(params.traceFinalStatus)
|
const traceStatus = normalizeTimelineFinalStatus(params.traceFinalStatus)
|
||||||
if (traceStatus) {
|
if (traceStatus === 'success' || traceStatus === 'failed' || traceStatus === 'cancelled') {
|
||||||
return traceStatus
|
return traceStatus
|
||||||
}
|
}
|
||||||
|
|
||||||
const requestStatus = mapRequestStatusToTimelineStatus(params.requestStatus)
|
const requestStatus = mapRequestStatusToTimelineStatus(params.requestStatus)
|
||||||
|
if (requestStatus === 'success' || requestStatus === 'failed' || requestStatus === 'cancelled') {
|
||||||
|
return requestStatus
|
||||||
|
}
|
||||||
|
|
||||||
|
if (params.hasPendingCandidates) {
|
||||||
|
return 'pending'
|
||||||
|
}
|
||||||
|
|
||||||
|
if (traceStatus) {
|
||||||
|
return traceStatus
|
||||||
|
}
|
||||||
|
|
||||||
if (requestStatus) {
|
if (requestStatus) {
|
||||||
return requestStatus
|
return requestStatus
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -150,7 +150,8 @@ import { reconcileActiveRequestDiscovery } from '@/features/usage/utils/activeRe
|
|||||||
import {
|
import {
|
||||||
hasUsageFallback,
|
hasUsageFallback,
|
||||||
isUsageRecordFailed,
|
isUsageRecordFailed,
|
||||||
isUsageUpstreamStream
|
isUsageUpstreamStream,
|
||||||
|
resolveDisplayRequestStatus,
|
||||||
} from '@/features/usage/utils/status'
|
} from '@/features/usage/utils/status'
|
||||||
import type { DateRangeParams, FilterStatusValue } from '@/features/usage/types'
|
import type { DateRangeParams, FilterStatusValue } from '@/features/usage/types'
|
||||||
import type { UserOption } from '@/features/usage/components/UsageRecordsTable.vue'
|
import type { UserOption } from '@/features/usage/components/UsageRecordsTable.vue'
|
||||||
@@ -346,7 +347,8 @@ const filteredRecords = computed(() => {
|
|||||||
)
|
)
|
||||||
} else if (filterStatus.value === 'active') {
|
} else if (filterStatus.value === 'active') {
|
||||||
records = records.filter(record =>
|
records = records.filter(record =>
|
||||||
record.status === 'pending' || record.status === 'streaming'
|
resolveDisplayRequestStatus(record) === 'pending' ||
|
||||||
|
resolveDisplayRequestStatus(record) === 'streaming'
|
||||||
)
|
)
|
||||||
} else if (filterStatus.value === 'failed') {
|
} else if (filterStatus.value === 'failed') {
|
||||||
records = records.filter(record => isUsageRecordFailed(record))
|
records = records.filter(record => isUsageRecordFailed(record))
|
||||||
@@ -365,7 +367,10 @@ const filteredRecords = computed(() => {
|
|||||||
// 获取活跃请求的 ID 列表
|
// 获取活跃请求的 ID 列表
|
||||||
const activeRequestIds = computed(() => {
|
const activeRequestIds = computed(() => {
|
||||||
return currentRecords.value
|
return currentRecords.value
|
||||||
.filter(record => record.status === 'pending' || record.status === 'streaming')
|
.filter((record) => {
|
||||||
|
const displayStatus = resolveDisplayRequestStatus(record)
|
||||||
|
return displayStatus === 'pending' || displayStatus === 'streaming'
|
||||||
|
})
|
||||||
.map(record => record.id)
|
.map(record => record.id)
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -424,11 +429,15 @@ async function pollActiveRequests() {
|
|||||||
const currentRank = record.status ? (statusPriority[record.status] ?? 0) : 0
|
const currentRank = record.status ? (statusPriority[record.status] ?? 0) : 0
|
||||||
const newRank = update.status ? (statusPriority[update.status] ?? 0) : 0
|
const newRank = update.status ? (statusPriority[update.status] ?? 0) : 0
|
||||||
const shouldApply = newRank >= currentRank
|
const shouldApply = newRank >= currentRank
|
||||||
|
const updateHasFailureSignal =
|
||||||
|
(typeof update.status_code === 'number' && update.status_code >= 400) ||
|
||||||
|
(typeof update.error_message === 'string' && update.error_message.trim().length > 0)
|
||||||
|
const shouldApplyData = shouldApply || updateHasFailureSignal
|
||||||
|
|
||||||
if (shouldApply && record.status !== update.status) {
|
if (shouldApply && record.status !== update.status) {
|
||||||
record.status = update.status
|
record.status = update.status
|
||||||
}
|
}
|
||||||
if (shouldApply) {
|
if (shouldApplyData) {
|
||||||
// 进行中状态也需要持续更新(provider/key/TTFB 可能在 streaming 后才落库)
|
// 进行中状态也需要持续更新(provider/key/TTFB 可能在 streaming 后才落库)
|
||||||
record.input_tokens = update.input_tokens
|
record.input_tokens = update.input_tokens
|
||||||
record.effective_input_tokens = update.effective_input_tokens ?? record.effective_input_tokens
|
record.effective_input_tokens = update.effective_input_tokens ?? record.effective_input_tokens
|
||||||
@@ -444,6 +453,8 @@ async function pollActiveRequests() {
|
|||||||
record.rate_multiplier = update.rate_multiplier ?? undefined
|
record.rate_multiplier = update.rate_multiplier ?? undefined
|
||||||
record.response_time_ms = update.response_time_ms ?? undefined
|
record.response_time_ms = update.response_time_ms ?? undefined
|
||||||
record.first_byte_time_ms = update.first_byte_time_ms ?? undefined
|
record.first_byte_time_ms = update.first_byte_time_ms ?? undefined
|
||||||
|
record.status_code = update.status_code ?? undefined
|
||||||
|
record.error_message = update.error_message ?? undefined
|
||||||
if (typeof update.upstream_is_stream === 'boolean') {
|
if (typeof update.upstream_is_stream === 'boolean') {
|
||||||
record.upstream_is_stream = update.upstream_is_stream
|
record.upstream_is_stream = update.upstream_is_stream
|
||||||
record.is_stream = update.upstream_is_stream
|
record.is_stream = update.upstream_is_stream
|
||||||
|
|||||||
Reference in New Issue
Block a user