Merge pull request #410 from Entropy-Xu/codex/codex-image-progress-heartbeat

Add Codex image progress heartbeat
This commit is contained in:
fawney19
2026-05-10 17:57:33 +08:00
committed by GitHub
48 changed files with 3232 additions and 221 deletions

View File

@@ -20,6 +20,7 @@ use aether_data_contracts::repository::provider_catalog::{
use sha2::{Digest, Sha256};
use crate::data::GatewayDataState;
use crate::tests::next_non_keepalive_chunk;
fn hash_api_key(value: &str) -> String {
let mut hasher = Sha256::new();
@@ -370,11 +371,7 @@ async fn gateway_stops_execution_runtime_stream_when_client_disconnects() {
assert_eq!(response.status(), StatusCode::OK);
let mut response = response;
let first_chunk = response
.chunk()
.await
.expect("first chunk should read")
.expect("first chunk should exist");
let first_chunk = next_non_keepalive_chunk(&mut response).await;
assert_eq!(
first_chunk,
Bytes::from_static(b"data: {\"id\":\"chatcmpl-first\"}\n\n")

View File

@@ -18,9 +18,10 @@ use crate::constants::{
use super::{
build_router, build_router_with_execution_runtime_override, build_router_with_state,
build_state_with_execution_runtime_override, start_server, wait_until, AppState,
FrontdoorCorsConfig, FrontdoorUserRpmConfig, GatewayFallbackMetricKind, GatewayFallbackReason,
UsageRuntimeConfig, VideoTaskTruthSourceMode,
build_state_with_execution_runtime_override, next_non_keepalive_chunk, start_server,
strip_sse_keepalive_comments, wait_until, AppState, FrontdoorCorsConfig,
FrontdoorUserRpmConfig, GatewayFallbackMetricKind, GatewayFallbackReason, UsageRuntimeConfig,
VideoTaskTruthSourceMode,
};
mod control_execute;

View File

@@ -1,8 +1,8 @@
use super::{
any, build_router_with_state, build_state_with_execution_runtime_override, json, start_server,
to_bytes, AppState, Arc, Body, Bytes, HeaderName, HeaderValue, Json, Mutex, Request, Response,
Router, StatusCode, EXECUTION_PATH_EXECUTION_RUNTIME_STREAM, EXECUTION_PATH_HEADER,
TRACE_ID_HEADER,
strip_sse_keepalive_comments, to_bytes, AppState, Arc, Body, Bytes, HeaderName, HeaderValue,
Json, Mutex, Request, Response, Router, StatusCode, EXECUTION_PATH_EXECUTION_RUNTIME_STREAM,
EXECUTION_PATH_HEADER, TRACE_ID_HEADER,
};
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
use aether_data::repository::auth::{
@@ -391,7 +391,7 @@ async fn gateway_executes_openai_chat_stream_via_local_decision_gate_without_exe
Some(EXECUTION_PATH_EXECUTION_RUNTIME_STREAM)
);
assert_eq!(
response.text().await.expect("body should read"),
strip_sse_keepalive_comments(&response.text().await.expect("body should read")),
"data: {\"id\":\"chatcmpl-local-123\"}\n\ndata: [DONE]\n\n"
);
@@ -1774,7 +1774,7 @@ async fn gateway_executes_openai_chat_stream_with_custom_path_via_local_decision
Some(EXECUTION_PATH_EXECUTION_RUNTIME_STREAM)
);
assert_eq!(
response.text().await.expect("body should read"),
strip_sse_keepalive_comments(&response.text().await.expect("body should read")),
"data: {\"id\":\"chatcmpl-local-custom-path-123\"}\n\ndata: [DONE]\n\n"
);
@@ -2285,7 +2285,7 @@ async fn gateway_retries_next_local_openai_chat_stream_candidate_after_retryable
Some(EXECUTION_PATH_EXECUTION_RUNTIME_STREAM)
);
assert_eq!(
response.text().await.expect("body should read"),
strip_sse_keepalive_comments(&response.text().await.expect("body should read")),
"data: {\"id\":\"chatcmpl-local-stream-failover-123\"}\n\ndata: [DONE]\n\n"
);

View File

@@ -18,9 +18,10 @@ use crate::constants::{
use super::{
build_router, build_router_with_execution_runtime_override, build_router_with_state,
build_state_with_execution_runtime_override, start_server, wait_until, AppState,
FrontdoorCorsConfig, FrontdoorUserRpmConfig, GatewayFallbackMetricKind, GatewayFallbackReason,
UsageRuntimeConfig, VideoTaskTruthSourceMode,
build_state_with_execution_runtime_override, next_non_keepalive_chunk, start_server,
strip_sse_keepalive_comments, wait_until, AppState, FrontdoorCorsConfig,
FrontdoorUserRpmConfig, GatewayFallbackMetricKind, GatewayFallbackReason, UsageRuntimeConfig,
VideoTaskTruthSourceMode,
};
mod decision;

View File

@@ -1,7 +1,7 @@
use super::{
any, build_router_with_state, build_state_with_execution_runtime_override, json, start_server,
to_bytes, Arc, Body, Bytes, HeaderName, HeaderValue, Infallible, Json, Mutex, Request,
Response, Router, StatusCode, UsageRuntimeConfig, TRACE_ID_HEADER,
strip_sse_keepalive_comments, to_bytes, Arc, Body, Bytes, HeaderName, HeaderValue, Infallible,
Json, Mutex, Request, Response, Router, StatusCode, UsageRuntimeConfig, TRACE_ID_HEADER,
};
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
use aether_data::repository::auth::{
@@ -469,7 +469,7 @@ async fn gateway_executes_codex_cli_stream_via_local_decision_gate_after_oauth_r
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response.text().await.expect("body should read"),
strip_sse_keepalive_comments(&response.text().await.expect("body should read")),
"event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_codex_cli_stream_local_123\",\"object\":\"response\",\"model\":\"gpt-5.4\",\"status\":\"completed\",\"usage\":{\"input_tokens\":1,\"output_tokens\":2,\"total_tokens\":3}}}\n\n"
);

View File

@@ -18,9 +18,9 @@ use crate::constants::{
use super::{
build_router, build_router_with_execution_runtime_override, build_router_with_state,
build_state_with_execution_runtime_override, start_server, wait_until, AppState,
FrontdoorCorsConfig, FrontdoorUserRpmConfig, GatewayFallbackMetricKind, GatewayFallbackReason,
UsageRuntimeConfig, VideoTaskTruthSourceMode,
build_state_with_execution_runtime_override, start_server, strip_sse_keepalive_comments,
wait_until, AppState, FrontdoorCorsConfig, FrontdoorUserRpmConfig, GatewayFallbackMetricKind,
GatewayFallbackReason, UsageRuntimeConfig, VideoTaskTruthSourceMode,
};
mod compact;

View File

@@ -1,7 +1,8 @@
use super::{
any, build_router_with_state, build_state_with_execution_runtime_override, json, start_server,
to_bytes, Arc, Body, Bytes, HeaderName, HeaderValue, Json, Mutex, Request, Response, Router,
StatusCode, TRACE_ID_HEADER,
any, build_router_with_state, build_state_with_execution_runtime_override, json,
next_non_keepalive_chunk, start_server, strip_sse_keepalive_comments, to_bytes, Arc, Body,
Bytes, HeaderName, HeaderValue, Json, Mutex, Request, Response, Router, StatusCode,
TRACE_ID_HEADER,
};
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
use aether_data::repository::auth::{
@@ -952,11 +953,12 @@ async fn gateway_executes_claude_cli_stream_via_local_decision_gate_without_wait
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
tokio::time::timeout(std::time::Duration::from_millis(100), response.chunk())
.await
.expect("same-format passthrough should yield first chunk before eof")
.expect("first chunk should read")
.expect("first chunk should exist"),
tokio::time::timeout(
std::time::Duration::from_millis(100),
next_non_keepalive_chunk(&mut response),
)
.await
.expect("same-format passthrough should yield first chunk before eof"),
Bytes::from_static(b"event: message_start\ndata: {\"type\":\"message_start\"}\n\n")
);
assert_eq!(
@@ -1470,7 +1472,7 @@ async fn gateway_executes_claude_code_cli_stream_via_local_decision_gate_with_lo
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response.text().await.expect("body should read"),
strip_sse_keepalive_comments(&response.text().await.expect("body should read")),
"event: message_start\ndata: {\"type\":\"message_start\"}\n\n"
);
@@ -1924,7 +1926,7 @@ async fn gateway_executes_claude_chat_stream_via_local_decision_gate_with_local_
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response.text().await.expect("body should read"),
strip_sse_keepalive_comments(&response.text().await.expect("body should read")),
"event: message_start\ndata: {\"type\":\"message_start\"}\n\n"
);

View File

@@ -1,7 +1,7 @@
use super::{
any, build_router_with_state, build_state_with_execution_runtime_override,
encrypt_python_fernet_plaintext, json, start_server, to_bytes, Arc, Body, Bytes, Digest,
HeaderName, HeaderValue, InMemoryAuthApiKeySnapshotRepository,
encrypt_python_fernet_plaintext, json, start_server, strip_sse_keepalive_comments, to_bytes,
Arc, Body, Bytes, Digest, HeaderName, HeaderValue, InMemoryAuthApiKeySnapshotRepository,
InMemoryMinimalCandidateSelectionReadRepository, InMemoryProviderCatalogReadRepository,
InMemoryRequestCandidateRepository, Json, Mutex, Request, RequestCandidateReadRepository,
RequestCandidateStatus, Response, Router, Sha256, StatusCode, StoredAuthApiKeySnapshot,
@@ -404,7 +404,7 @@ async fn gateway_executes_gemini_chat_stream_via_local_decision_gate_with_local_
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response.text().await.expect("body should read"),
strip_sse_keepalive_comments(&response.text().await.expect("body should read")),
"data: {\"candidates\":[]}\n\n"
);

View File

@@ -1,7 +1,7 @@
use super::{
any, build_router_with_state, build_state_with_execution_runtime_override,
encrypt_python_fernet_plaintext, json, start_server, to_bytes, Arc, Body, Bytes, Digest,
HeaderName, HeaderValue, InMemoryAuthApiKeySnapshotRepository,
encrypt_python_fernet_plaintext, json, start_server, strip_sse_keepalive_comments, to_bytes,
Arc, Body, Bytes, Digest, HeaderName, HeaderValue, InMemoryAuthApiKeySnapshotRepository,
InMemoryMinimalCandidateSelectionReadRepository, InMemoryProviderCatalogReadRepository,
InMemoryRequestCandidateRepository, Json, Mutex, Request, RequestCandidateReadRepository,
RequestCandidateStatus, Response, Router, Sha256, StatusCode, StoredAuthApiKeySnapshot,
@@ -381,7 +381,7 @@ async fn gateway_executes_gemini_cli_stream_via_local_decision_gate_with_local_s
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response.text().await.expect("body should read"),
strip_sse_keepalive_comments(&response.text().await.expect("body should read")),
"data: {\"candidates\":[]}\n\n"
);
@@ -872,7 +872,7 @@ async fn gateway_executes_gemini_cli_stream_via_local_decision_gate_after_oauth_
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response.text().await.expect("body should read"),
strip_sse_keepalive_comments(&response.text().await.expect("body should read")),
"data: {\"candidates\":[]}\n\n"
);
@@ -1339,7 +1339,7 @@ async fn gateway_executes_vertex_ai_gemini_cli_stream_via_local_decision_gate_wi
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response.text().await.expect("body should read"),
strip_sse_keepalive_comments(&response.text().await.expect("body should read")),
"data: {\"candidates\":[]}\n\n"
);
@@ -1849,7 +1849,8 @@ async fn gateway_executes_antigravity_gemini_cli_stream_via_local_decision_gate_
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let response_text = response.text().await.expect("body should read");
let response_text =
strip_sse_keepalive_comments(&response.text().await.expect("body should read"));
let payload = response_text
.trim()
.strip_prefix("data: ")

View File

@@ -18,9 +18,9 @@ use crate::constants::{
use super::{
build_router, build_router_with_execution_runtime_override, build_router_with_state,
build_state_with_execution_runtime_override, start_server, wait_until, AppState,
FrontdoorCorsConfig, FrontdoorUserRpmConfig, GatewayFallbackMetricKind, GatewayFallbackReason,
UsageRuntimeConfig, VideoTaskTruthSourceMode,
build_state_with_execution_runtime_override, start_server, strip_sse_keepalive_comments,
wait_until, AppState, FrontdoorCorsConfig, FrontdoorUserRpmConfig, GatewayFallbackMetricKind,
GatewayFallbackReason, UsageRuntimeConfig, VideoTaskTruthSourceMode,
};
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
use aether_data::repository::auth::{

View File

@@ -6,8 +6,8 @@ use super::{
};
use crate::tests::{
any, build_router, build_router_with_state, build_state_with_execution_runtime_override, json,
start_server, AppState, Arc, Body, HeaderValue, Json, Mutex, Request, Response, Router,
StatusCode, CONTROL_ACTION_PROXY_PUBLIC, CONTROL_EXECUTED_HEADER,
start_server, strip_sse_keepalive_comments, AppState, Arc, Body, HeaderValue, Json, Mutex,
Request, Response, Router, StatusCode, CONTROL_ACTION_PROXY_PUBLIC, CONTROL_EXECUTED_HEADER,
EXECUTION_PATH_EXECUTION_RUNTIME_STREAM, EXECUTION_PATH_EXECUTION_RUNTIME_SYNC,
EXECUTION_PATH_HEADER,
};
@@ -458,7 +458,7 @@ async fn gateway_handles_internal_gateway_execute_stream_locally() {
Some(EXECUTION_PATH_EXECUTION_RUNTIME_STREAM)
);
assert_eq!(
response.text().await.expect("body should read"),
strip_sse_keepalive_comments(&response.text().await.expect("body should read")),
"data: one\n\ndata: [DONE]\n\n"
);
assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0);

View File

@@ -88,3 +88,20 @@ pub(super) async fn wait_until(timeout_ms: u64, mut predicate: impl FnMut() -> b
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
}
pub(crate) fn strip_sse_keepalive_comments(body: &str) -> String {
body.replace(": aether-keepalive\n\n", "")
}
pub(crate) async fn next_non_keepalive_chunk(response: &mut reqwest::Response) -> Bytes {
loop {
let chunk = response
.chunk()
.await
.expect("chunk should read")
.expect("chunk should exist");
if chunk.as_ref() != b": aether-keepalive\n\n" {
return chunk;
}
}
}

View File

@@ -28,8 +28,8 @@ use sha2::{Digest, Sha256};
use super::{
any, build_router_with_state, build_state_with_execution_runtime_override, send_request,
start_server, Body, HeaderValue, Json, Mutex, Request, Response, Router, StatusCode,
UsageRuntimeConfig, TRACE_ID_HEADER,
start_server, strip_sse_keepalive_comments, Body, HeaderValue, Json, Mutex, Request, Response,
Router, StatusCode, UsageRuntimeConfig, TRACE_ID_HEADER,
};
use crate::data::GatewayDataState;

View File

@@ -2,8 +2,8 @@ use super::{
any, build_router_with_state, build_state_with_execution_runtime_override,
encrypt_python_fernet_plaintext, hash_api_key, json, sample_local_openai_auth_snapshot,
sample_local_openai_candidate_row, sample_local_openai_endpoint, sample_local_openai_key,
sample_local_openai_provider, send_request, start_server, Arc, Body, GatewayDataState,
HeaderValue, InMemoryAuthApiKeySnapshotRepository,
sample_local_openai_provider, send_request, start_server, strip_sse_keepalive_comments, Arc,
Body, GatewayDataState, HeaderValue, InMemoryAuthApiKeySnapshotRepository,
InMemoryMinimalCandidateSelectionReadRepository, InMemoryProviderCatalogReadRepository,
InMemoryRequestCandidateRepository, InMemoryUsageReadRepository, Json, Mutex, Request,
RequestCandidateReadRepository, RequestCandidateStatus, Response, Router, StatusCode,
@@ -840,6 +840,104 @@ async fn gateway_records_failed_usage_when_all_local_openai_chat_candidates_exha
assert_eq!(stored_candidates[0].status_code, Some(503));
}
#[tokio::test]
async fn gateway_records_failed_usage_when_sync_runtime_transport_is_unavailable_without_plan_fallback(
) {
let usage_repository = Arc::new(InMemoryUsageReadRepository::default());
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
let execution_hits = Arc::new(Mutex::new(0usize));
let execution_hits_clone = Arc::clone(&execution_hits);
let auth_repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
Some(hash_api_key("sk-client-openai-local-transport-unavailable")),
sample_local_openai_auth_snapshot(
"api-key-openai-usage-local-transport-unavailable-1",
"user-openai-usage-local-transport-unavailable-1",
),
)]));
let candidate_selection_repository =
Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(vec![
sample_local_openai_candidate_row(),
]));
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_local_openai_provider()],
vec![sample_local_openai_endpoint()],
vec![sample_local_openai_key()],
));
let gateway_state = crate::AppState::new()
.expect("gateway should build")
.with_execution_runtime_sync_override_for_tests(move |_plan| {
*execution_hits_clone.lock().expect("mutex should lock") += 1;
Err(crate::GatewayError::Internal(
"simulated transport unavailable".to_string(),
))
})
.with_data_state_for_tests(
GatewayDataState::with_auth_candidate_selection_provider_catalog_request_candidates_and_usage_for_tests(
auth_repository,
candidate_selection_repository,
provider_catalog_repository,
Arc::clone(&request_candidate_repository),
Arc::clone(&usage_repository),
DEVELOPMENT_ENCRYPTION_KEY,
),
)
.with_usage_runtime_for_tests(UsageRuntimeConfig {
enabled: true,
..UsageRuntimeConfig::default()
});
let gateway = build_router_with_state(gateway_state);
let request = Request::builder()
.method(http::Method::POST)
.uri("/v1/chat/completions")
.header(http::header::CONTENT_TYPE, "application/json")
.header(
http::header::AUTHORIZATION,
"Bearer sk-client-openai-local-transport-unavailable",
)
.header(
TRACE_ID_HEADER,
"trace-openai-chat-local-transport-unavailable-123",
)
.body(Body::from("{\"model\":\"gpt-5\",\"messages\":[]}"))
.expect("request should build");
let response = send_request(gateway, request).await;
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(*execution_hits.lock().expect("mutex should lock"), 1);
let stored_usage = wait_for_usage_status(
usage_repository.as_ref(),
"trace-openai-chat-local-transport-unavailable-123",
"failed",
)
.await;
assert_eq!(stored_usage.status, "failed");
assert_eq!(stored_usage.billing_status, "void");
assert_eq!(stored_usage.status_code, Some(503));
assert_eq!(
stored_usage
.response_body
.as_ref()
.and_then(|value| value.get("error"))
.and_then(|value| value.get("type"))
.and_then(|value| value.as_str()),
Some("execution_runtime_unavailable")
);
let stored_candidates = request_candidate_repository
.list_by_request_id("trace-openai-chat-local-transport-unavailable-123")
.await
.expect("request candidate trace should read");
assert_eq!(stored_candidates.len(), 1);
assert_eq!(stored_candidates[0].status, RequestCandidateStatus::Failed);
assert_eq!(
stored_candidates[0].error_type.as_deref(),
Some("execution_runtime_unavailable")
);
}
#[test]
fn gateway_records_failed_usage_for_claude_runtime_miss_without_execution_exhaustion() {
run_async_test_on_large_stack(
@@ -1184,7 +1282,8 @@ async fn gateway_handles_local_openai_chat_stream_report_with_local_reporting_wh
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let body_text = response.text().await.expect("stream body should read");
let body_text =
strip_sse_keepalive_comments(&response.text().await.expect("stream body should read"));
assert_eq!(
body_text,
"data: {\"id\":\"chatcmpl-local-report-stream-123\",\"usage\":{\"input_tokens\":2,\"output_tokens\":4,\"total_tokens\":6}}\n\ndata: [DONE]\n\n"

View File

@@ -1,13 +1,14 @@
use super::{
any, build_router_with_state, build_state_with_execution_runtime_override,
encrypt_python_fernet_plaintext, hash_api_key, json, start_server, Arc, Body, GatewayDataState,
HeaderValue, InMemoryAuthApiKeySnapshotRepository,
InMemoryMinimalCandidateSelectionReadRepository, InMemoryProviderCatalogReadRepository,
InMemoryRequestCandidateRepository, InMemoryUsageReadRepository, Json, Request,
RequestCandidateReadRepository, RequestCandidateStatus, Response, Router, StatusCode,
StoredAuthApiKeySnapshot, StoredMinimalCandidateSelectionRow, StoredProviderCatalogEndpoint,
StoredProviderCatalogKey, StoredProviderCatalogProvider, StoredProviderModelMapping,
UsageReadRepository, UsageRuntimeConfig, DEVELOPMENT_ENCRYPTION_KEY, TRACE_ID_HEADER,
encrypt_python_fernet_plaintext, hash_api_key, json, start_server,
strip_sse_keepalive_comments, Arc, Body, GatewayDataState, HeaderValue,
InMemoryAuthApiKeySnapshotRepository, InMemoryMinimalCandidateSelectionReadRepository,
InMemoryProviderCatalogReadRepository, InMemoryRequestCandidateRepository,
InMemoryUsageReadRepository, Json, Request, RequestCandidateReadRepository,
RequestCandidateStatus, Response, Router, StatusCode, StoredAuthApiKeySnapshot,
StoredMinimalCandidateSelectionRow, StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
StoredProviderCatalogProvider, StoredProviderModelMapping, UsageReadRepository,
UsageRuntimeConfig, DEVELOPMENT_ENCRYPTION_KEY, TRACE_ID_HEADER,
};
use aether_data::repository::billing::InMemoryBillingReadRepository;
use aether_data::repository::wallet::{InMemoryWalletRepository, StoredWalletSnapshot};
@@ -939,7 +940,7 @@ async fn gateway_records_openai_stream_usage_and_pricing_with_cache_tokens_impl(
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response.text().await.expect("stream body should read"),
strip_sse_keepalive_comments(&response.text().await.expect("stream body should read")),
stream_body.concat()
);
@@ -1132,7 +1133,7 @@ async fn gateway_records_claude_stream_usage_and_pricing_with_cache_breakdown_im
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response.text().await.expect("stream body should read"),
strip_sse_keepalive_comments(&response.text().await.expect("stream body should read")),
stream_body.concat()
);
@@ -1310,7 +1311,7 @@ async fn gateway_records_gemini_stream_usage_and_pricing_with_cache_read_tokens_
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response.text().await.expect("stream body should read"),
strip_sse_keepalive_comments(&response.text().await.expect("stream body should read")),
stream_body.concat()
);