refactor: 拆分 gateway 单体为独立 crate,新增 systemd 部署方案

将 gateway 内部的 model-fetch、provider-transport、scheduler-core、
usage-runtime、video-tasks-core 模块提取为独立 crate;重构 gateway
内部模块结构(state/router/cache/data/query 等);移除大量遗留模块
文件;新增 systemd 二进制部署骨架及相关文档;更新前端 usage 相关
API 和组件。
This commit is contained in:
fawney19
2026-04-05 20:23:16 +08:00
parent cbc811f6ce
commit 763ff03a7b
777 changed files with 42659 additions and 21469 deletions

View File

@@ -0,0 +1,240 @@
use std::sync::{Arc, Mutex};
use aether_data::repository::video_tasks::{
InMemoryVideoTaskRepository, UpsertVideoTask, VideoTaskWriteRepository,
};
use axum::body::Body;
use axum::routing::any;
use axum::{extract::Request, Json, Router};
use http::StatusCode;
use serde_json::json;
use super::{build_router_with_state, build_state_with_execution_runtime_override, start_server};
#[tokio::test]
async fn gateway_reads_openai_video_task_via_data_read_side_without_hitting_public_route() {
let public_hits = Arc::new(Mutex::new(0usize));
let public_hits_clone = Arc::clone(&public_hits);
let upstream = Router::new()
.route(
"/api/internal/gateway/resolve",
any(|_request: Request| async move {
Json(json!({
"action": "proxy_public",
"route_class": "ai_public",
"route_family": "openai",
"route_kind": "video",
"auth_endpoint_signature": "openai:video",
"execution_runtime_candidate": true,
"auth_context": {
"user_id": "user-video-db-123",
"api_key_id": "key-video-db-123",
"access_allowed": true
},
"public_path": "/v1/videos/task-db-123"
}))
}),
)
.route(
"/v1/videos/task-db-123",
any(move |_request: Request| {
let public_hits_inner = Arc::clone(&public_hits_clone);
async move {
*public_hits_inner.lock().expect("mutex should lock") += 1;
(StatusCode::IM_A_TEAPOT, Body::from("public-route-hit"))
}
}),
);
let (upstream_url, upstream_handle) = start_server(upstream).await;
let repository = Arc::new(InMemoryVideoTaskRepository::default());
repository
.upsert(UpsertVideoTask {
id: "task-db-123".to_string(),
short_id: Some("short-task-db-123".to_string()),
request_id: "request-db-123".to_string(),
user_id: Some("user-video-db-123".to_string()),
api_key_id: Some("api-key-video-db-123".to_string()),
username: Some("video-user".to_string()),
api_key_name: Some("video-key".to_string()),
external_task_id: Some("ext-video-db-123".to_string()),
provider_id: Some("provider-video-db-123".to_string()),
endpoint_id: Some("endpoint-video-db-123".to_string()),
key_id: Some("provider-key-video-db-123".to_string()),
client_api_format: Some("openai:video".to_string()),
provider_api_format: Some("openai:video".to_string()),
format_converted: false,
model: Some("sora-2".to_string()),
prompt: Some("hello from db".to_string()),
original_request_body: Some(json!({"prompt": "hello from db"})),
duration_seconds: Some(4),
resolution: Some("720p".to_string()),
aspect_ratio: Some("16:9".to_string()),
size: Some("1280x720".to_string()),
status: aether_data::repository::video_tasks::VideoTaskStatus::Processing,
progress_percent: 45,
progress_message: Some("working".to_string()),
retry_count: 0,
poll_interval_seconds: 10,
next_poll_at_unix_secs: Some(124),
poll_count: 1,
max_poll_count: 360,
created_at_unix_secs: 123,
submitted_at_unix_secs: Some(123),
completed_at_unix_secs: None,
updated_at_unix_secs: 124,
error_code: None,
error_message: None,
video_url: None,
request_metadata: None,
})
.await
.expect("upsert should succeed");
let gateway = build_router_with_state(
build_state_with_execution_runtime_override(upstream_url.clone())
.with_video_task_data_reader_for_tests(repository),
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.get(format!("{gateway_url}/v1/videos/task-db-123"))
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let body: serde_json::Value = response.json().await.expect("body should parse");
assert_eq!(body["id"], "task-db-123");
assert_eq!(body["status"], "processing");
assert_eq!(body["progress"], 45);
assert_eq!(body["created_at"], 123);
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_reads_gemini_video_task_via_data_read_side_without_hitting_public_route() {
let public_hits = Arc::new(Mutex::new(0usize));
let public_hits_clone = Arc::clone(&public_hits);
let upstream = Router::new()
.route(
"/api/internal/gateway/resolve",
any(|_request: Request| async move {
Json(json!({
"action": "proxy_public",
"route_class": "ai_public",
"route_family": "gemini",
"route_kind": "video",
"auth_endpoint_signature": "gemini:video",
"execution_runtime_candidate": true,
"auth_context": {
"user_id": "user-video-db-123",
"api_key_id": "key-video-db-123",
"access_allowed": true
},
"public_path": "/v1beta/models/veo-3/operations/localshort123"
}))
}),
)
.route(
"/v1beta/models/veo-3/operations/localshort123",
any(move |_request: Request| {
let public_hits_inner = Arc::clone(&public_hits_clone);
async move {
*public_hits_inner.lock().expect("mutex should lock") += 1;
(StatusCode::IM_A_TEAPOT, Body::from("public-route-hit"))
}
}),
);
let (upstream_url, upstream_handle) = start_server(upstream).await;
let repository = Arc::new(InMemoryVideoTaskRepository::default());
repository
.upsert(UpsertVideoTask {
id: "task-db-456".to_string(),
short_id: Some("localshort123".to_string()),
request_id: "request-db-456".to_string(),
user_id: Some("user-video-db-123".to_string()),
api_key_id: Some("api-key-video-db-123".to_string()),
username: Some("video-user".to_string()),
api_key_name: Some("video-key".to_string()),
external_task_id: Some("operations/ext-video-db-123".to_string()),
provider_id: Some("provider-video-db-123".to_string()),
endpoint_id: Some("endpoint-video-db-123".to_string()),
key_id: Some("provider-key-video-db-123".to_string()),
client_api_format: Some("gemini:video".to_string()),
provider_api_format: Some("gemini:video".to_string()),
format_converted: false,
model: Some("veo-3".to_string()),
prompt: Some("hello from gemini db".to_string()),
original_request_body: Some(json!({"prompt": "hello from gemini db"})),
duration_seconds: Some(8),
resolution: Some("720p".to_string()),
aspect_ratio: Some("16:9".to_string()),
size: Some("720p".to_string()),
status: aether_data::repository::video_tasks::VideoTaskStatus::Completed,
progress_percent: 100,
progress_message: None,
retry_count: 0,
poll_interval_seconds: 10,
next_poll_at_unix_secs: None,
poll_count: 3,
max_poll_count: 360,
created_at_unix_secs: 223,
submitted_at_unix_secs: Some(223),
completed_at_unix_secs: Some(224),
updated_at_unix_secs: 224,
error_code: None,
error_message: None,
video_url: None,
request_metadata: Some(json!({
"rust_local_snapshot": {
"metadata": {
"generateVideoResponse": {
"generatedSamples": [
{
"video": {
"uri": "/v1beta/files/aev_localshort123:download?alt=media"
}
}
]
}
}
}
})),
})
.await
.expect("upsert should succeed");
let gateway = build_router_with_state(
build_state_with_execution_runtime_override(upstream_url.clone())
.with_video_task_data_reader_for_tests(repository),
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.get(format!(
"{gateway_url}/v1beta/models/veo-3/operations/localshort123"
))
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let body: serde_json::Value = response.json().await.expect("body should parse");
assert_eq!(body["name"], "models/veo-3/operations/localshort123");
assert_eq!(body["done"], true);
assert_eq!(
body["response"]["generateVideoResponse"]["generatedSamples"][0]["video"]["uri"],
"/v1beta/files/aev_localshort123:download?alt=media"
);
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
upstream_handle.abort();
}

View File

@@ -0,0 +1,485 @@
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
use aether_data::repository::auth::{
InMemoryAuthApiKeySnapshotRepository, StoredAuthApiKeySnapshot,
};
use aether_data::repository::candidate_selection::{
InMemoryMinimalCandidateSelectionReadRepository, StoredMinimalCandidateSelectionRow,
StoredProviderModelMapping,
};
use aether_data::repository::candidates::{
InMemoryRequestCandidateRepository, RequestCandidateReadRepository, RequestCandidateStatus,
};
use aether_data::repository::provider_catalog::{
InMemoryProviderCatalogReadRepository, StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
StoredProviderCatalogProvider,
};
use axum::body::{to_bytes, Body};
use axum::routing::any;
use axum::{extract::Request, Json, Router};
use http::StatusCode;
use serde_json::json;
use sha2::{Digest, Sha256};
use std::sync::{Arc, Mutex};
use crate::constants::TRACE_ID_HEADER;
use super::{build_router_with_state, build_state_with_execution_runtime_override, start_server};
#[tokio::test]
async fn gateway_executes_gemini_video_create_via_local_decision_gate_with_local_planning_only() {
#[derive(Debug, Clone)]
struct SeenExecutionRuntimeSyncRequest {
method: String,
url: String,
auth_header_value: String,
prompt: String,
endpoint_tag: String,
conditional_header: String,
renamed_header: String,
dropped_header_present: bool,
metadata_mode: String,
metadata_source: String,
store_present: bool,
proxy_node_id: String,
tls_profile: String,
}
fn hash_api_key(value: &str) -> String {
let mut hasher = Sha256::new();
hasher.update(value.as_bytes());
format!("{:x}", hasher.finalize())
}
fn sample_auth_snapshot(api_key_id: &str, user_id: &str) -> StoredAuthApiKeySnapshot {
StoredAuthApiKeySnapshot::new(
user_id.to_string(),
"video-user".to_string(),
Some("video@example.com".to_string()),
"user".to_string(),
"local".to_string(),
true,
false,
Some(json!(["gemini"])),
Some(json!(["gemini:video"])),
Some(json!(["veo-3"])),
api_key_id.to_string(),
Some("default".to_string()),
true,
false,
false,
Some(60),
Some(5),
Some(4_102_444_800),
Some(json!(["gemini"])),
Some(json!(["gemini:video"])),
Some(json!(["veo-3"])),
)
.expect("auth snapshot should build")
}
fn sample_candidate_row() -> StoredMinimalCandidateSelectionRow {
StoredMinimalCandidateSelectionRow {
provider_id: "provider-gemini-video-local-1".to_string(),
provider_name: "gemini".to_string(),
provider_type: "custom".to_string(),
provider_priority: 10,
provider_is_active: true,
endpoint_id: "endpoint-gemini-video-local-1".to_string(),
endpoint_api_format: "gemini:video".to_string(),
endpoint_api_family: Some("gemini".to_string()),
endpoint_kind: Some("video".to_string()),
endpoint_is_active: true,
key_id: "key-gemini-video-local-1".to_string(),
key_name: "prod".to_string(),
key_auth_type: "api_key".to_string(),
key_is_active: true,
key_api_formats: Some(vec!["gemini:video".to_string()]),
key_allowed_models: None,
key_capabilities: None,
key_internal_priority: 5,
key_global_priority_by_format: Some(json!({"gemini:video": 1})),
model_id: "model-gemini-video-local-1".to_string(),
global_model_id: "global-model-gemini-video-local-1".to_string(),
global_model_name: "veo-3".to_string(),
global_model_mappings: None,
global_model_supports_streaming: Some(false),
model_provider_model_name: "veo-3-upstream".to_string(),
model_provider_model_mappings: Some(vec![StoredProviderModelMapping {
name: "veo-3-upstream".to_string(),
priority: 1,
api_formats: Some(vec!["gemini:video".to_string()]),
}]),
model_supports_streaming: Some(false),
model_is_active: true,
model_is_available: true,
}
}
fn sample_provider() -> StoredProviderCatalogProvider {
StoredProviderCatalogProvider::new(
"provider-gemini-video-local-1".to_string(),
"gemini".to_string(),
Some("https://example.com".to_string()),
"custom".to_string(),
)
.expect("provider should build")
.with_transport_fields(
true,
false,
false,
None,
Some(2),
Some(serde_json::json!({"enabled": true, "node_id":"proxy-node-gemini-video-local"})),
Some(20.0),
None,
None,
)
}
fn sample_endpoint() -> StoredProviderCatalogEndpoint {
StoredProviderCatalogEndpoint::new(
"endpoint-gemini-video-local-1".to_string(),
"provider-gemini-video-local-1".to_string(),
"gemini:video".to_string(),
Some("gemini".to_string()),
Some("video".to_string()),
true,
)
.expect("endpoint should build")
.with_transport_fields(
"https://generativelanguage.googleapis.com".to_string(),
Some(json!([
{"action":"set","key":"x-endpoint-tag","value":"gemini-video-local"},
{"action":"set","key":"x-conditional-tag","value":"video-body-rule-applied","condition":{"path":"metadata.mode","op":"eq","value":"safe","source":"current"}},
{"action":"rename","from":"x-client-rename","to":"x-upstream-rename"},
{"action":"drop","key":"x-drop-me"}
])),
Some(json!([
{"action":"set","path":"metadata.mode","value":"safe","condition":{"path":"metadata.mode","op":"not_exists","source":"current"}},
{"action":"rename","from":"metadata.client","to":"metadata.source"},
{"action":"drop","path":"store"}
])),
Some(2),
Some("/custom/v1beta/models/veo-3-upstream:predictLongRunning".to_string()),
None,
None,
None,
)
.expect("endpoint transport should build")
}
fn sample_key() -> StoredProviderCatalogKey {
StoredProviderCatalogKey::new(
"key-gemini-video-local-1".to_string(),
"provider-gemini-video-local-1".to_string(),
"prod".to_string(),
"api_key".to_string(),
None,
true,
)
.expect("key should build")
.with_transport_fields(
Some(json!(["gemini:video"])),
encrypt_python_fernet_plaintext(DEVELOPMENT_ENCRYPTION_KEY, "sk-upstream-gemini-video")
.expect("api key should encrypt"),
None,
None,
Some(json!({"gemini:video": 1})),
None,
None,
None,
Some(json!({"tls_profile":"chrome_136"})),
)
.expect("key transport should build")
}
let decision_hits = Arc::new(Mutex::new(0usize));
let decision_hits_clone = Arc::clone(&decision_hits);
let plan_hits = Arc::new(Mutex::new(0usize));
let plan_hits_clone = Arc::clone(&plan_hits);
let seen_execution_runtime = Arc::new(Mutex::new(None::<SeenExecutionRuntimeSyncRequest>));
let seen_execution_runtime_clone = Arc::clone(&seen_execution_runtime);
let report_hits = Arc::new(Mutex::new(0usize));
let report_hits_clone = Arc::clone(&report_hits);
let public_hits = Arc::new(Mutex::new(0usize));
let public_hits_clone = Arc::clone(&public_hits);
let upstream = Router::new()
.route(
"/api/internal/gateway/resolve",
any(|_request: Request| async move {
Json(json!({
"action": "proxy_public",
"route_class": "ai_public",
"route_family": "gemini",
"route_kind": "video",
"auth_endpoint_signature": "gemini:video",
"execution_runtime_candidate": true,
"auth_context": {
"user_id": "user-gemini-video-local-123",
"api_key_id": "key-gemini-video-local-123",
"access_allowed": true
},
"public_path": "/v1beta/models/veo-3:predictLongRunning"
}))
}),
)
.route(
"/api/internal/gateway/decision-sync",
any(move |_request: Request| {
let decision_hits_inner = Arc::clone(&decision_hits_clone);
async move {
*decision_hits_inner.lock().expect("mutex should lock") += 1;
Json(json!({"action": "proxy_public"}))
}
}),
)
.route(
"/api/internal/gateway/plan-sync",
any(move |_request: Request| {
let plan_hits_inner = Arc::clone(&plan_hits_clone);
async move {
*plan_hits_inner.lock().expect("mutex should lock") += 1;
Json(json!({"action": "proxy_public"}))
}
}),
)
.route(
"/api/internal/gateway/report-sync",
any(move |_request: Request| {
let report_hits_inner = Arc::clone(&report_hits_clone);
async move {
*report_hits_inner.lock().expect("mutex should lock") += 1;
Json(json!({"ok": true}))
}
}),
)
.route(
"/v1beta/models/veo-3:predictLongRunning",
any(move |_request: Request| {
let public_hits_inner = Arc::clone(&public_hits_clone);
async move {
*public_hits_inner.lock().expect("mutex should lock") += 1;
(StatusCode::IM_A_TEAPOT, Body::from("public-route-hit"))
}
}),
);
let execution_runtime = Router::new().route(
"/v1/execute/sync",
any(move |request: Request| {
let seen_execution_runtime_inner = Arc::clone(&seen_execution_runtime_clone);
async move {
let (_parts, body) = request.into_parts();
let raw_body = to_bytes(body, usize::MAX).await.expect("body should read");
let payload: serde_json::Value = serde_json::from_slice(&raw_body)
.expect("execution runtime payload should parse");
*seen_execution_runtime_inner
.lock()
.expect("mutex should lock") = Some(SeenExecutionRuntimeSyncRequest {
method: payload
.get("method")
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
url: payload
.get("url")
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
auth_header_value: payload
.get("headers")
.and_then(|value| value.get("x-goog-api-key"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
prompt: payload
.get("body")
.and_then(|value| value.get("json_body"))
.and_then(|value| value.get("prompt"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
endpoint_tag: payload
.get("headers")
.and_then(|value| value.get("x-endpoint-tag"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
conditional_header: payload
.get("headers")
.and_then(|value| value.get("x-conditional-tag"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
renamed_header: payload
.get("headers")
.and_then(|value| value.get("x-upstream-rename"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
dropped_header_present: payload
.get("headers")
.and_then(|value| value.get("x-drop-me"))
.is_some(),
metadata_mode: payload
.get("body")
.and_then(|value| value.get("json_body"))
.and_then(|value| value.get("metadata"))
.and_then(|value| value.get("mode"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
metadata_source: payload
.get("body")
.and_then(|value| value.get("json_body"))
.and_then(|value| value.get("metadata"))
.and_then(|value| value.get("source"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
store_present: payload
.get("body")
.and_then(|value| value.get("json_body"))
.and_then(|value| value.get("store"))
.is_some(),
proxy_node_id: payload
.get("proxy")
.and_then(|value| value.get("node_id"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
tls_profile: payload
.get("tls_profile")
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
});
Json(json!({
"request_id": "trace-gemini-video-local-123",
"status_code": 200,
"headers": {
"content-type": "application/json"
},
"body": {
"json_body": {
"name": "operations/ext-video-123"
}
},
"telemetry": {
"elapsed_ms": 20
}
}))
}
}),
);
let auth_repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
Some(hash_api_key("client-gemini-video-local-key")),
sample_auth_snapshot("key-gemini-video-local-123", "user-gemini-video-local-123"),
)]));
let candidate_selection_repository =
Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(vec![
sample_candidate_row(),
]));
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider()],
vec![sample_endpoint()],
vec![sample_key()],
));
let (upstream_url, upstream_handle) = start_server(upstream).await;
let (execution_runtime_url, execution_runtime_handle) = start_server(execution_runtime).await;
let gateway_state =
build_state_with_execution_runtime_override(execution_runtime_url)
.with_data_state_for_tests(
crate::data::GatewayDataState::with_auth_candidate_selection_provider_catalog_and_request_candidate_repository_for_tests(
auth_repository,
candidate_selection_repository,
provider_catalog_repository,
Arc::clone(&request_candidate_repository),
DEVELOPMENT_ENCRYPTION_KEY,
),
);
let gateway = build_router_with_state(gateway_state);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.post(format!(
"{gateway_url}/v1beta/models/veo-3:predictLongRunning?key=client-gemini-video-local-key&view=full"
))
.header(http::header::CONTENT_TYPE, "application/json")
.header("x-client-rename", "rename-gemini-video")
.header("x-drop-me", "drop-gemini-video")
.header(TRACE_ID_HEADER, "trace-gemini-video-local-123")
.body("{\"prompt\":\"make a local video\",\"metadata\":{\"client\":\"desktop-gemini-video\"},\"store\":false}")
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let body: serde_json::Value = response.json().await.expect("body should parse");
assert_eq!(body.get("done"), Some(&json!(false)));
assert!(body
.get("name")
.and_then(|value| value.as_str())
.is_some_and(|value| value.starts_with("models/veo-3/operations/")));
let seen_execution_runtime_request = seen_execution_runtime
.lock()
.expect("mutex should lock")
.clone()
.expect("execution runtime sync should be captured");
assert_eq!(seen_execution_runtime_request.method, "POST");
assert_eq!(
seen_execution_runtime_request.url,
"https://generativelanguage.googleapis.com/custom/v1beta/models/veo-3-upstream:predictLongRunning?view=full"
);
assert_eq!(
seen_execution_runtime_request.auth_header_value,
"sk-upstream-gemini-video"
);
assert_eq!(seen_execution_runtime_request.prompt, "make a local video");
assert_eq!(
seen_execution_runtime_request.endpoint_tag,
"gemini-video-local"
);
assert_eq!(
seen_execution_runtime_request.conditional_header,
"video-body-rule-applied"
);
assert_eq!(
seen_execution_runtime_request.renamed_header,
"rename-gemini-video"
);
assert!(!seen_execution_runtime_request.dropped_header_present);
assert_eq!(seen_execution_runtime_request.metadata_mode, "safe");
assert_eq!(
seen_execution_runtime_request.metadata_source,
"desktop-gemini-video"
);
assert!(!seen_execution_runtime_request.store_present);
assert_eq!(
seen_execution_runtime_request.proxy_node_id,
"proxy-node-gemini-video-local"
);
assert_eq!(seen_execution_runtime_request.tls_profile, "chrome_136");
let stored_candidates = request_candidate_repository
.list_by_request_id("trace-gemini-video-local-123")
.await
.expect("request candidate trace should read");
assert_eq!(stored_candidates.len(), 1);
assert_eq!(stored_candidates[0].status, RequestCandidateStatus::Success);
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
assert_eq!(*report_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*decision_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*plan_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
execution_runtime_handle.abort();
upstream_handle.abort();
}

View File

@@ -0,0 +1,637 @@
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
use aether_data::repository::candidates::{
InMemoryRequestCandidateRepository, RequestCandidateReadRepository, RequestCandidateStatus,
};
use aether_data::repository::provider_catalog::{
InMemoryProviderCatalogReadRepository, StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
StoredProviderCatalogProvider,
};
use aether_data::repository::video_tasks::{
InMemoryVideoTaskRepository, UpsertVideoTask, VideoTaskWriteRepository,
};
use axum::body::{to_bytes, Body};
use axum::response::Response;
use axum::routing::any;
use axum::{extract::Request, Json, Router};
use http::header::{HeaderName, HeaderValue};
use http::StatusCode;
use serde_json::json;
use std::sync::{Arc, Mutex};
use crate::constants::{
CONTROL_EXECUTED_HEADER, CONTROL_EXECUTE_FALLBACK_HEADER, TRACE_ID_HEADER,
};
use super::{
build_router_with_state, build_state_with_execution_runtime_override, start_server,
VideoTaskTruthSourceMode,
};
#[tokio::test]
async fn gateway_executes_gemini_video_cancel_via_data_backed_local_follow_up_with_local_planning_only(
) {
#[derive(Debug, Clone)]
struct SeenExecutionRuntimeSyncRequest {
method: String,
url: String,
auth_header_value: String,
}
let decision_hits = Arc::new(Mutex::new(0usize));
let decision_hits_clone = Arc::clone(&decision_hits);
let plan_hits = Arc::new(Mutex::new(0usize));
let plan_hits_clone = Arc::clone(&plan_hits);
let seen_execution_runtime = Arc::new(Mutex::new(None::<SeenExecutionRuntimeSyncRequest>));
let seen_execution_runtime_clone = Arc::clone(&seen_execution_runtime);
let report_hits = Arc::new(Mutex::new(0usize));
let report_hits_clone = Arc::clone(&report_hits);
let public_hits = Arc::new(Mutex::new(0usize));
let public_hits_clone = Arc::clone(&public_hits);
let upstream = Router::new()
.route(
"/api/internal/gateway/resolve",
any(|_request: Request| async move {
Json(json!({
"action": "proxy_public",
"route_class": "ai_public",
"route_family": "gemini",
"route_kind": "video",
"auth_endpoint_signature": "gemini:video",
"execution_runtime_candidate": true,
"auth_context": {
"user_id": "user-gemini-video-cancel-local-123",
"api_key_id": "key-gemini-video-cancel-local-123",
"access_allowed": true
},
"public_path": "/v1beta/models/veo-3/operations/localshort123:cancel"
}))
}),
)
.route(
"/api/internal/gateway/decision-sync",
any(move |_request: Request| {
let decision_hits_inner = Arc::clone(&decision_hits_clone);
async move {
*decision_hits_inner.lock().expect("mutex should lock") += 1;
Json(json!({"action": "proxy_public"}))
}
}),
)
.route(
"/api/internal/gateway/plan-sync",
any(move |_request: Request| {
let plan_hits_inner = Arc::clone(&plan_hits_clone);
async move {
*plan_hits_inner.lock().expect("mutex should lock") += 1;
Json(json!({"action": "proxy_public"}))
}
}),
)
.route(
"/api/internal/gateway/report-sync",
any(move |_request: Request| {
let report_hits_inner = Arc::clone(&report_hits_clone);
async move {
*report_hits_inner.lock().expect("mutex should lock") += 1;
Json(json!({"ok": true}))
}
}),
)
.route(
"/v1beta/models/veo-3/operations/localshort123:cancel",
any(move |_request: Request| {
let public_hits_inner = Arc::clone(&public_hits_clone);
async move {
*public_hits_inner.lock().expect("mutex should lock") += 1;
(StatusCode::IM_A_TEAPOT, Body::from("public-route-hit"))
}
}),
);
let execution_runtime = Router::new().route(
"/v1/execute/sync",
any(move |request: Request| {
let seen_execution_runtime_inner = Arc::clone(&seen_execution_runtime_clone);
async move {
let (_parts, body) = request.into_parts();
let raw_body = to_bytes(body, usize::MAX).await.expect("body should read");
let payload: serde_json::Value = serde_json::from_slice(&raw_body)
.expect("execution runtime payload should parse");
*seen_execution_runtime_inner
.lock()
.expect("mutex should lock") = Some(SeenExecutionRuntimeSyncRequest {
method: payload
.get("method")
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
url: payload
.get("url")
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
auth_header_value: payload
.get("headers")
.and_then(|value| value.get("x-goog-api-key"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
});
Json(json!({
"request_id": "trace-gemini-video-cancel-local-123",
"status_code": 200,
"headers": {
"content-type": "application/json"
},
"body": {
"json_body": {}
},
"telemetry": {
"elapsed_ms": 17
}
}))
}
}),
);
let repository = Arc::new(InMemoryVideoTaskRepository::default());
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
repository
.upsert(UpsertVideoTask {
id: "task-gemini-cancel-local-123".to_string(),
short_id: Some("localshort123".to_string()),
request_id: "request-gemini-video-cancel-local-123".to_string(),
user_id: Some("user-gemini-video-cancel-local-123".to_string()),
api_key_id: Some("key-gemini-video-cancel-local-123".to_string()),
username: Some("video-user".to_string()),
api_key_name: Some("video-key".to_string()),
external_task_id: Some("operations/ext-123".to_string()),
provider_id: Some("provider-gemini-video-local-1".to_string()),
endpoint_id: Some("endpoint-gemini-video-local-1".to_string()),
key_id: Some("key-gemini-video-local-1".to_string()),
client_api_format: Some("gemini:video".to_string()),
provider_api_format: Some("gemini:video".to_string()),
format_converted: false,
model: Some("veo-3".to_string()),
prompt: Some("gemini prompt".to_string()),
original_request_body: Some(json!({"prompt": "gemini prompt"})),
duration_seconds: Some(8),
resolution: Some("720p".to_string()),
aspect_ratio: Some("16:9".to_string()),
size: Some("720p".to_string()),
status: aether_data::repository::video_tasks::VideoTaskStatus::Submitted,
progress_percent: 0,
progress_message: None,
retry_count: 0,
poll_interval_seconds: 10,
next_poll_at_unix_secs: Some(124),
poll_count: 0,
max_poll_count: 360,
created_at_unix_secs: 123,
submitted_at_unix_secs: Some(123),
completed_at_unix_secs: None,
updated_at_unix_secs: 123,
error_code: None,
error_message: None,
video_url: None,
request_metadata: Some(json!({
"rust_local_snapshot": {
"Gemini": {
"local_short_id": "localshort123",
"upstream_operation_name": "operations/ext-123",
"user_id": "user-gemini-video-cancel-local-123",
"api_key_id": "key-gemini-video-cancel-local-123",
"model": "veo-3",
"status": "Submitted",
"progress_percent": 0,
"error_code": null,
"error_message": null,
"metadata": {},
"persistence": {
"request_id": "request-gemini-video-cancel-local-123",
"username": "video-user",
"api_key_name": "video-key",
"client_api_format": "gemini:video",
"provider_api_format": "gemini:video",
"original_request_body": {
"prompt": "gemini prompt"
},
"format_converted": false
},
"transport": {
"upstream_base_url": "https://generativelanguage.googleapis.com",
"provider_name": "gemini-video",
"provider_id": "provider-gemini-video-local-1",
"endpoint_id": "endpoint-gemini-video-local-1",
"key_id": "key-gemini-video-local-1",
"headers": {
"x-goog-api-key": "sk-upstream-gemini-video",
"content-type": "application/json"
},
"content_type": "application/json",
"model_name": "veo-3-upstream",
"proxy": null,
"tls_profile": null,
"timeouts": null
}
}
}
})),
})
.await
.expect("upsert should succeed");
let (upstream_url, upstream_handle) = start_server(upstream).await;
let (execution_runtime_url, execution_runtime_handle) = start_server(execution_runtime).await;
let gateway_state =
build_state_with_execution_runtime_override(execution_runtime_url)
.with_data_state_for_tests(
crate::data::GatewayDataState::with_video_task_and_request_candidate_repository_for_tests(
repository,
Arc::clone(&request_candidate_repository),
),
);
let gateway = build_router_with_state(gateway_state);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.post(format!(
"{gateway_url}/v1beta/models/veo-3/operations/localshort123:cancel"
))
.header("x-goog-api-key", "client-key")
.header(http::header::CONTENT_TYPE, "application/json")
.header(TRACE_ID_HEADER, "trace-gemini-video-cancel-local-123")
.body("{}")
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response
.json::<serde_json::Value>()
.await
.expect("body should parse"),
json!({})
);
let seen_execution_runtime_request = seen_execution_runtime
.lock()
.expect("mutex should lock")
.clone()
.expect("execution runtime sync should be captured");
assert_eq!(seen_execution_runtime_request.method, "POST");
assert_eq!(
seen_execution_runtime_request.url,
"https://generativelanguage.googleapis.com/v1beta/models/veo-3/operations/ext-123:cancel"
);
assert_eq!(
seen_execution_runtime_request.auth_header_value,
"sk-upstream-gemini-video"
);
let stored_candidates = request_candidate_repository
.list_by_request_id("request-gemini-video-cancel-local-123")
.await
.expect("request candidate trace should read");
assert_eq!(stored_candidates.len(), 1);
assert_eq!(stored_candidates[0].status, RequestCandidateStatus::Success);
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
assert_eq!(*report_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*decision_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*plan_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
execution_runtime_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_executes_gemini_video_cancel_via_reconstructed_data_backed_local_follow_up_with_local_follow_up_routing(
) {
#[derive(Debug, Clone, PartialEq, Eq)]
struct SeenExecutionRuntimeSyncRequest {
method: String,
url: String,
api_key: String,
}
fn sample_provider() -> StoredProviderCatalogProvider {
StoredProviderCatalogProvider::new(
"provider-gemini-video-followup-1".to_string(),
"gemini".to_string(),
Some("https://example.com".to_string()),
"custom".to_string(),
)
.expect("provider should build")
.with_transport_fields(
true,
false,
false,
None,
Some(2),
None,
Some(20.0),
None,
None,
)
}
fn sample_endpoint() -> StoredProviderCatalogEndpoint {
StoredProviderCatalogEndpoint::new(
"endpoint-gemini-video-followup-1".to_string(),
"provider-gemini-video-followup-1".to_string(),
"gemini:video".to_string(),
Some("gemini".to_string()),
Some("video".to_string()),
true,
)
.expect("endpoint should build")
.with_transport_fields(
"https://generativelanguage.googleapis.com".to_string(),
None,
None,
Some(2),
None,
None,
None,
None,
)
.expect("endpoint transport should build")
}
fn sample_key() -> StoredProviderCatalogKey {
StoredProviderCatalogKey::new(
"key-gemini-video-followup-1".to_string(),
"provider-gemini-video-followup-1".to_string(),
"prod".to_string(),
"api_key".to_string(),
None,
true,
)
.expect("key should build")
.with_transport_fields(
Some(json!(["gemini:video"])),
encrypt_python_fernet_plaintext(DEVELOPMENT_ENCRYPTION_KEY, "sk-upstream-gemini-video")
.expect("api key should encrypt"),
None,
None,
Some(json!({"gemini:video": 1})),
None,
None,
None,
None,
)
.expect("key transport should build")
}
let decision_hits = Arc::new(Mutex::new(0usize));
let decision_hits_clone = Arc::clone(&decision_hits);
let execute_hits = Arc::new(Mutex::new(0usize));
let execute_hits_clone = Arc::clone(&execute_hits);
let public_hits = Arc::new(Mutex::new(0usize));
let public_hits_clone = Arc::clone(&public_hits);
let report_hits = Arc::new(Mutex::new(0usize));
let report_hits_clone = Arc::clone(&report_hits);
let seen_execution_runtime = Arc::new(Mutex::new(None::<SeenExecutionRuntimeSyncRequest>));
let seen_execution_runtime_clone = Arc::clone(&seen_execution_runtime);
let upstream = Router::new()
.route(
"/api/internal/gateway/resolve",
any(|_request: Request| async move {
Json(json!({
"action": "proxy_public",
"route_class": "ai_public",
"route_family": "gemini",
"route_kind": "video",
"auth_endpoint_signature": "gemini:video",
"execution_runtime_candidate": true,
"auth_context": {
"user_id": "user-gemini-video-cancel-op-123",
"api_key_id": "key-gemini-video-cancel-op-123",
"access_allowed": true
},
"public_path": "/v1beta/models/veo-3/operations/opshort123:cancel"
}))
}),
)
.route(
"/api/internal/gateway/decision-sync",
any(move |_request: Request| {
let decision_hits_inner = Arc::clone(&decision_hits_clone);
async move {
*decision_hits_inner.lock().expect("mutex should lock") += 1;
Json(json!({
"action": "execution_runtime_sync_decision",
"decision_kind": "gemini_video_cancel_sync",
"request_id": "unexpected-decision-hit"
}))
}
}),
)
.route(
"/api/internal/gateway/report-sync",
any(move |_request: Request| {
let report_hits_inner = Arc::clone(&report_hits_clone);
async move {
*report_hits_inner.lock().expect("mutex should lock") += 1;
Json(json!({"ok": true}))
}
}),
)
.route(
"/api/internal/gateway/execute-sync",
any(move |_request: Request| {
let execute_hits_inner = Arc::clone(&execute_hits_clone);
async move {
*execute_hits_inner.lock().expect("mutex should lock") += 1;
let mut response = Response::builder()
.status(StatusCode::OK)
.body(Body::from("{\"fallback\":true}"))
.expect("response should build");
response.headers_mut().insert(
HeaderName::from_static(CONTROL_EXECUTED_HEADER),
HeaderValue::from_static("true"),
);
response
}
}),
)
.route(
"/v1beta/models/veo-3/operations/opshort123:cancel",
any(move |_request: Request| {
let public_hits_inner = Arc::clone(&public_hits_clone);
async move {
*public_hits_inner.lock().expect("mutex should lock") += 1;
(StatusCode::IM_A_TEAPOT, Body::from("public-route-hit"))
}
}),
);
let execution_runtime = Router::new().route(
"/v1/execute/sync",
any(move |request: Request| {
let seen_execution_runtime_inner = Arc::clone(&seen_execution_runtime_clone);
async move {
let (_parts, body) = request.into_parts();
let raw_body = to_bytes(body, usize::MAX).await.expect("body should read");
let payload: serde_json::Value = serde_json::from_slice(&raw_body)
.expect("execution runtime payload should parse");
*seen_execution_runtime_inner
.lock()
.expect("mutex should lock") = Some(SeenExecutionRuntimeSyncRequest {
method: payload
.get("method")
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
url: payload
.get("url")
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
api_key: payload
.get("headers")
.and_then(|value| value.get("x-goog-api-key"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
});
Json(json!({
"request_id": "trace-gemini-video-cancel-op-123",
"status_code": 200,
"headers": {
"content-type": "application/json"
},
"body": {
"json_body": {}
}
}))
}
}),
);
let (upstream_url, upstream_handle) = start_server(upstream).await;
let (execution_runtime_url, execution_runtime_handle) = start_server(execution_runtime).await;
let repository = Arc::new(InMemoryVideoTaskRepository::default());
repository
.upsert(UpsertVideoTask {
id: "task-gemini-cancel-op-123".to_string(),
short_id: Some("opshort123".to_string()),
request_id: "request-gemini-video-cancel-op-123".to_string(),
user_id: Some("user-gemini-video-cancel-op-123".to_string()),
api_key_id: Some("key-gemini-video-cancel-op-123".to_string()),
username: Some("video-user".to_string()),
api_key_name: Some("video-key".to_string()),
external_task_id: Some("operations/ext-op-123".to_string()),
provider_id: Some("provider-gemini-video-followup-1".to_string()),
endpoint_id: Some("endpoint-gemini-video-followup-1".to_string()),
key_id: Some("key-gemini-video-followup-1".to_string()),
client_api_format: Some("gemini:video".to_string()),
provider_api_format: Some("gemini:video".to_string()),
format_converted: false,
model: Some("veo-3".to_string()),
prompt: Some("operation cancel".to_string()),
original_request_body: Some(json!({
"prompt": "operation cancel"
})),
duration_seconds: Some(4),
resolution: Some("720p".to_string()),
aspect_ratio: Some("16:9".to_string()),
size: None,
status: aether_data::repository::video_tasks::VideoTaskStatus::Processing,
progress_percent: 50,
progress_message: None,
retry_count: 0,
poll_interval_seconds: 10,
next_poll_at_unix_secs: Some(123),
poll_count: 0,
max_poll_count: 360,
created_at_unix_secs: 123,
submitted_at_unix_secs: Some(123),
completed_at_unix_secs: None,
updated_at_unix_secs: 123,
error_code: None,
error_message: None,
video_url: None,
request_metadata: None,
})
.await
.expect("upsert should succeed");
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider()],
vec![sample_endpoint()],
vec![sample_key()],
));
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
let gateway = build_router_with_state(
build_state_with_execution_runtime_override(execution_runtime_url)
.with_video_task_truth_source_mode(VideoTaskTruthSourceMode::RustAuthoritative)
.with_data_state_for_tests(
crate::data::GatewayDataState::with_video_task_provider_transport_and_request_candidate_repository_for_tests(
repository,
provider_catalog_repository,
Arc::clone(&request_candidate_repository),
DEVELOPMENT_ENCRYPTION_KEY,
),
),
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.post(format!(
"{gateway_url}/v1beta/models/veo-3/operations/opshort123:cancel"
))
.header(CONTROL_EXECUTE_FALLBACK_HEADER, "true")
.header(TRACE_ID_HEADER, "trace-gemini-video-cancel-op-123")
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response
.json::<serde_json::Value>()
.await
.expect("body should parse"),
json!({})
);
let seen_execution_runtime_request = seen_execution_runtime
.lock()
.expect("mutex should lock")
.clone()
.expect("execution runtime sync should be captured");
assert_eq!(seen_execution_runtime_request.method, "POST");
assert_eq!(
seen_execution_runtime_request.url,
"https://generativelanguage.googleapis.com/v1beta/models/veo-3/operations/ext-op-123:cancel"
);
assert_eq!(
seen_execution_runtime_request.api_key,
"sk-upstream-gemini-video"
);
let stored_candidates = request_candidate_repository
.list_by_request_id("request-gemini-video-cancel-op-123")
.await
.expect("request candidate trace should read");
assert_eq!(stored_candidates.len(), 1);
assert_eq!(stored_candidates[0].status, RequestCandidateStatus::Success);
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
assert_eq!(*decision_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*execute_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*report_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
execution_runtime_handle.abort();
upstream_handle.abort();
}

View File

@@ -0,0 +1,32 @@
use std::sync::{Arc, Mutex};
use aether_data::repository::video_tasks::{
InMemoryVideoTaskRepository, UpsertVideoTask, VideoTaskLookupKey, VideoTaskReadRepository,
VideoTaskWriteRepository,
};
use axum::body::{to_bytes, Body, Bytes};
use axum::response::Response;
use axum::routing::any;
use axum::{extract::Request, Json, Router};
use http::header::{HeaderName, HeaderValue};
use http::StatusCode;
use serde_json::json;
use crate::constants::{
CONTROL_EXECUTED_HEADER, CONTROL_EXECUTE_FALLBACK_HEADER, EXECUTION_PATH_HEADER,
TRACE_ID_HEADER,
};
use super::{
build_router, build_router_with_state, build_state_with_execution_runtime_override,
start_server, AppState, VideoTaskTruthSourceMode,
};
mod data_read;
mod gemini_sync_create;
mod gemini_sync_task;
mod openai_sync_create;
mod openai_sync_task;
mod registry_poller;
mod routing;
mod stream;

View File

@@ -0,0 +1,797 @@
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
use aether_data::repository::auth::{
InMemoryAuthApiKeySnapshotRepository, StoredAuthApiKeySnapshot,
};
use aether_data::repository::candidate_selection::{
InMemoryMinimalCandidateSelectionReadRepository, StoredMinimalCandidateSelectionRow,
StoredProviderModelMapping,
};
use aether_data::repository::candidates::{
InMemoryRequestCandidateRepository, RequestCandidateReadRepository, RequestCandidateStatus,
};
use aether_data::repository::provider_catalog::{
InMemoryProviderCatalogReadRepository, StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
StoredProviderCatalogProvider,
};
use aether_data::repository::video_tasks::{
InMemoryVideoTaskRepository, UpsertVideoTask, VideoTaskWriteRepository,
};
use axum::body::{to_bytes, Body};
use axum::routing::any;
use axum::{extract::Request, Json, Router};
use http::StatusCode;
use serde_json::json;
use sha2::{Digest, Sha256};
use std::sync::{Arc, Mutex};
use crate::constants::TRACE_ID_HEADER;
use super::{build_router_with_state, build_state_with_execution_runtime_override, start_server};
#[tokio::test]
async fn gateway_executes_openai_video_create_via_local_decision_gate_with_local_planning_only() {
#[derive(Debug, Clone)]
struct SeenExecutionRuntimeSyncRequest {
method: String,
url: String,
authorization: String,
model: String,
prompt: String,
endpoint_tag: String,
conditional_header: String,
renamed_header: String,
dropped_header_present: bool,
metadata_mode: String,
metadata_source: String,
store_present: bool,
proxy_node_id: String,
tls_profile: String,
}
fn hash_api_key(value: &str) -> String {
let mut hasher = Sha256::new();
hasher.update(value.as_bytes());
format!("{:x}", hasher.finalize())
}
fn sample_auth_snapshot(api_key_id: &str, user_id: &str) -> StoredAuthApiKeySnapshot {
StoredAuthApiKeySnapshot::new(
user_id.to_string(),
"video-user".to_string(),
Some("video@example.com".to_string()),
"user".to_string(),
"local".to_string(),
true,
false,
Some(json!(["openai"])),
Some(json!(["openai:video"])),
Some(json!(["sora-2"])),
api_key_id.to_string(),
Some("default".to_string()),
true,
false,
false,
Some(60),
Some(5),
Some(4_102_444_800),
Some(json!(["openai"])),
Some(json!(["openai:video"])),
Some(json!(["sora-2"])),
)
.expect("auth snapshot should build")
}
fn sample_candidate_row() -> StoredMinimalCandidateSelectionRow {
StoredMinimalCandidateSelectionRow {
provider_id: "provider-openai-video-local-1".to_string(),
provider_name: "openai".to_string(),
provider_type: "custom".to_string(),
provider_priority: 10,
provider_is_active: true,
endpoint_id: "endpoint-openai-video-local-1".to_string(),
endpoint_api_format: "openai:video".to_string(),
endpoint_api_family: Some("openai".to_string()),
endpoint_kind: Some("video".to_string()),
endpoint_is_active: true,
key_id: "key-openai-video-local-1".to_string(),
key_name: "prod".to_string(),
key_auth_type: "api_key".to_string(),
key_is_active: true,
key_api_formats: Some(vec!["openai:video".to_string()]),
key_allowed_models: None,
key_capabilities: None,
key_internal_priority: 5,
key_global_priority_by_format: Some(json!({"openai:video": 1})),
model_id: "model-openai-video-local-1".to_string(),
global_model_id: "global-model-openai-video-local-1".to_string(),
global_model_name: "sora-2".to_string(),
global_model_mappings: None,
global_model_supports_streaming: Some(false),
model_provider_model_name: "sora-2-upstream".to_string(),
model_provider_model_mappings: Some(vec![StoredProviderModelMapping {
name: "sora-2-upstream".to_string(),
priority: 1,
api_formats: Some(vec!["openai:video".to_string()]),
}]),
model_supports_streaming: Some(false),
model_is_active: true,
model_is_available: true,
}
}
fn sample_provider() -> StoredProviderCatalogProvider {
StoredProviderCatalogProvider::new(
"provider-openai-video-local-1".to_string(),
"openai".to_string(),
Some("https://example.com".to_string()),
"custom".to_string(),
)
.expect("provider should build")
.with_transport_fields(
true,
false,
false,
None,
Some(2),
Some(serde_json::json!({"enabled": true, "node_id":"proxy-node-openai-video-local"})),
Some(20.0),
None,
None,
)
}
fn sample_endpoint() -> StoredProviderCatalogEndpoint {
StoredProviderCatalogEndpoint::new(
"endpoint-openai-video-local-1".to_string(),
"provider-openai-video-local-1".to_string(),
"openai:video".to_string(),
Some("openai".to_string()),
Some("video".to_string()),
true,
)
.expect("endpoint should build")
.with_transport_fields(
"https://api.openai.example".to_string(),
Some(json!([
{"action":"set","key":"x-endpoint-tag","value":"openai-video-local"},
{"action":"set","key":"x-conditional-tag","value":"video-body-rule-applied","condition":{"path":"metadata.mode","op":"eq","value":"safe","source":"current"}},
{"action":"rename","from":"x-client-rename","to":"x-upstream-rename"},
{"action":"drop","key":"x-drop-me"}
])),
Some(json!([
{"action":"set","path":"metadata.mode","value":"safe","condition":{"path":"metadata.mode","op":"not_exists","source":"current"}},
{"action":"rename","from":"metadata.client","to":"metadata.source"},
{"action":"drop","path":"store"}
])),
Some(2),
Some("/custom/v1/videos".to_string()),
None,
None,
None,
)
.expect("endpoint transport should build")
}
fn sample_key() -> StoredProviderCatalogKey {
StoredProviderCatalogKey::new(
"key-openai-video-local-1".to_string(),
"provider-openai-video-local-1".to_string(),
"prod".to_string(),
"api_key".to_string(),
None,
true,
)
.expect("key should build")
.with_transport_fields(
Some(json!(["openai:video"])),
encrypt_python_fernet_plaintext(DEVELOPMENT_ENCRYPTION_KEY, "sk-upstream-openai-video")
.expect("api key should encrypt"),
None,
None,
Some(json!({"openai:video": 1})),
None,
None,
None,
Some(json!({"tls_profile":"chrome_136"})),
)
.expect("key transport should build")
}
let decision_hits = Arc::new(Mutex::new(0usize));
let decision_hits_clone = Arc::clone(&decision_hits);
let plan_hits = Arc::new(Mutex::new(0usize));
let plan_hits_clone = Arc::clone(&plan_hits);
let seen_execution_runtime = Arc::new(Mutex::new(None::<SeenExecutionRuntimeSyncRequest>));
let seen_execution_runtime_clone = Arc::clone(&seen_execution_runtime);
let report_hits = Arc::new(Mutex::new(0usize));
let report_hits_clone = Arc::clone(&report_hits);
let public_hits = Arc::new(Mutex::new(0usize));
let public_hits_clone = Arc::clone(&public_hits);
let upstream = Router::new()
.route(
"/api/internal/gateway/resolve",
any(|_request: Request| async move {
Json(json!({
"action": "proxy_public",
"route_class": "ai_public",
"route_family": "openai",
"route_kind": "video",
"auth_endpoint_signature": "openai:video",
"execution_runtime_candidate": true,
"auth_context": {
"user_id": "user-openai-video-local-123",
"api_key_id": "key-openai-video-local-123",
"access_allowed": true
},
"public_path": "/v1/videos"
}))
}),
)
.route(
"/api/internal/gateway/decision-sync",
any(move |_request: Request| {
let decision_hits_inner = Arc::clone(&decision_hits_clone);
async move {
*decision_hits_inner.lock().expect("mutex should lock") += 1;
Json(json!({"action": "proxy_public"}))
}
}),
)
.route(
"/api/internal/gateway/plan-sync",
any(move |_request: Request| {
let plan_hits_inner = Arc::clone(&plan_hits_clone);
async move {
*plan_hits_inner.lock().expect("mutex should lock") += 1;
Json(json!({"action": "proxy_public"}))
}
}),
)
.route(
"/api/internal/gateway/report-sync",
any(move |_request: Request| {
let report_hits_inner = Arc::clone(&report_hits_clone);
async move {
*report_hits_inner.lock().expect("mutex should lock") += 1;
Json(json!({"ok": true}))
}
}),
)
.route(
"/v1/videos",
any(move |_request: Request| {
let public_hits_inner = Arc::clone(&public_hits_clone);
async move {
*public_hits_inner.lock().expect("mutex should lock") += 1;
(StatusCode::IM_A_TEAPOT, Body::from("public-route-hit"))
}
}),
);
let execution_runtime = Router::new().route(
"/v1/execute/sync",
any(move |request: Request| {
let seen_execution_runtime_inner = Arc::clone(&seen_execution_runtime_clone);
async move {
let (_parts, body) = request.into_parts();
let raw_body = to_bytes(body, usize::MAX).await.expect("body should read");
let payload: serde_json::Value = serde_json::from_slice(&raw_body)
.expect("execution runtime payload should parse");
*seen_execution_runtime_inner
.lock()
.expect("mutex should lock") = Some(SeenExecutionRuntimeSyncRequest {
method: payload
.get("method")
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
url: payload
.get("url")
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
authorization: payload
.get("headers")
.and_then(|value| value.get("authorization"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
model: payload
.get("body")
.and_then(|value| value.get("json_body"))
.and_then(|value| value.get("model"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
prompt: payload
.get("body")
.and_then(|value| value.get("json_body"))
.and_then(|value| value.get("prompt"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
endpoint_tag: payload
.get("headers")
.and_then(|value| value.get("x-endpoint-tag"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
conditional_header: payload
.get("headers")
.and_then(|value| value.get("x-conditional-tag"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
renamed_header: payload
.get("headers")
.and_then(|value| value.get("x-upstream-rename"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
dropped_header_present: payload
.get("headers")
.and_then(|value| value.get("x-drop-me"))
.is_some(),
metadata_mode: payload
.get("body")
.and_then(|value| value.get("json_body"))
.and_then(|value| value.get("metadata"))
.and_then(|value| value.get("mode"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
metadata_source: payload
.get("body")
.and_then(|value| value.get("json_body"))
.and_then(|value| value.get("metadata"))
.and_then(|value| value.get("source"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
store_present: payload
.get("body")
.and_then(|value| value.get("json_body"))
.and_then(|value| value.get("store"))
.is_some(),
proxy_node_id: payload
.get("proxy")
.and_then(|value| value.get("node_id"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
tls_profile: payload
.get("tls_profile")
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
});
Json(json!({
"request_id": "trace-openai-video-local-123",
"status_code": 200,
"headers": {
"content-type": "application/json"
},
"body": {
"json_body": {
"id": "ext-video-task-123",
"status": "submitted"
}
},
"telemetry": {
"elapsed_ms": 18
}
}))
}
}),
);
let auth_repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
Some(hash_api_key("client-openai-video-local-key")),
sample_auth_snapshot("key-openai-video-local-123", "user-openai-video-local-123"),
)]));
let candidate_selection_repository =
Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(vec![
sample_candidate_row(),
]));
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider()],
vec![sample_endpoint()],
vec![sample_key()],
));
let (upstream_url, upstream_handle) = start_server(upstream).await;
let (execution_runtime_url, execution_runtime_handle) = start_server(execution_runtime).await;
let gateway_state =
build_state_with_execution_runtime_override(execution_runtime_url)
.with_data_state_for_tests(
crate::data::GatewayDataState::with_auth_candidate_selection_provider_catalog_and_request_candidate_repository_for_tests(
auth_repository,
candidate_selection_repository,
provider_catalog_repository,
Arc::clone(&request_candidate_repository),
DEVELOPMENT_ENCRYPTION_KEY,
),
);
let gateway = build_router_with_state(gateway_state);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.post(format!("{gateway_url}/v1/videos"))
.header(http::header::CONTENT_TYPE, "application/json")
.header(
http::header::AUTHORIZATION,
"Bearer client-openai-video-local-key",
)
.header("x-client-rename", "rename-openai-video")
.header("x-drop-me", "drop-openai-video")
.header(TRACE_ID_HEADER, "trace-openai-video-local-123")
.body("{\"model\":\"sora-2\",\"prompt\":\"hello local video\",\"metadata\":{\"client\":\"desktop-openai-video\"},\"store\":false}")
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let body: serde_json::Value = response.json().await.expect("body should parse");
assert_eq!(body.get("object"), Some(&json!("video")));
assert_eq!(body.get("status"), Some(&json!("queued")));
assert_eq!(body.get("prompt"), Some(&json!("hello local video")));
let seen_execution_runtime_request = seen_execution_runtime
.lock()
.expect("mutex should lock")
.clone()
.expect("execution runtime sync should be captured");
assert_eq!(seen_execution_runtime_request.method, "POST");
assert_eq!(
seen_execution_runtime_request.url,
"https://api.openai.example/custom/v1/videos"
);
assert_eq!(
seen_execution_runtime_request.authorization,
"Bearer sk-upstream-openai-video"
);
assert_eq!(seen_execution_runtime_request.model, "sora-2-upstream");
assert_eq!(seen_execution_runtime_request.prompt, "hello local video");
assert_eq!(
seen_execution_runtime_request.endpoint_tag,
"openai-video-local"
);
assert_eq!(
seen_execution_runtime_request.conditional_header,
"video-body-rule-applied"
);
assert_eq!(
seen_execution_runtime_request.renamed_header,
"rename-openai-video"
);
assert!(!seen_execution_runtime_request.dropped_header_present);
assert_eq!(seen_execution_runtime_request.metadata_mode, "safe");
assert_eq!(
seen_execution_runtime_request.metadata_source,
"desktop-openai-video"
);
assert!(!seen_execution_runtime_request.store_present);
assert_eq!(
seen_execution_runtime_request.proxy_node_id,
"proxy-node-openai-video-local"
);
assert_eq!(seen_execution_runtime_request.tls_profile, "chrome_136");
let stored_candidates = request_candidate_repository
.list_by_request_id("trace-openai-video-local-123")
.await
.expect("request candidate trace should read");
assert_eq!(stored_candidates.len(), 1);
assert_eq!(stored_candidates[0].status, RequestCandidateStatus::Success);
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
assert_eq!(*report_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*decision_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*plan_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
execution_runtime_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_executes_openai_video_remix_via_data_backed_local_follow_up_with_local_planning_only(
) {
#[derive(Debug, Clone)]
struct SeenExecutionRuntimeSyncRequest {
method: String,
url: String,
authorization: String,
prompt: String,
}
let decision_hits = Arc::new(Mutex::new(0usize));
let decision_hits_clone = Arc::clone(&decision_hits);
let plan_hits = Arc::new(Mutex::new(0usize));
let plan_hits_clone = Arc::clone(&plan_hits);
let seen_execution_runtime = Arc::new(Mutex::new(None::<SeenExecutionRuntimeSyncRequest>));
let seen_execution_runtime_clone = Arc::clone(&seen_execution_runtime);
let report_hits = Arc::new(Mutex::new(0usize));
let report_hits_clone = Arc::clone(&report_hits);
let public_hits = Arc::new(Mutex::new(0usize));
let public_hits_clone = Arc::clone(&public_hits);
let upstream = Router::new()
.route(
"/api/internal/gateway/resolve",
any(|_request: Request| async move {
Json(json!({
"action": "proxy_public",
"route_class": "ai_public",
"route_family": "openai",
"route_kind": "video",
"auth_endpoint_signature": "openai:video",
"execution_runtime_candidate": true,
"auth_context": {
"user_id": "user-openai-video-remix-local-123",
"api_key_id": "key-openai-video-remix-local-123",
"access_allowed": true
},
"public_path": "/v1/videos/task-local-123/remix"
}))
}),
)
.route(
"/api/internal/gateway/decision-sync",
any(move |_request: Request| {
let decision_hits_inner = Arc::clone(&decision_hits_clone);
async move {
*decision_hits_inner.lock().expect("mutex should lock") += 1;
Json(json!({"action": "proxy_public"}))
}
}),
)
.route(
"/api/internal/gateway/plan-sync",
any(move |_request: Request| {
let plan_hits_inner = Arc::clone(&plan_hits_clone);
async move {
*plan_hits_inner.lock().expect("mutex should lock") += 1;
Json(json!({"action": "proxy_public"}))
}
}),
)
.route(
"/api/internal/gateway/report-sync",
any(move |_request: Request| {
let report_hits_inner = Arc::clone(&report_hits_clone);
async move {
*report_hits_inner.lock().expect("mutex should lock") += 1;
Json(json!({"ok": true}))
}
}),
)
.route(
"/v1/videos/task-local-123/remix",
any(move |_request: Request| {
let public_hits_inner = Arc::clone(&public_hits_clone);
async move {
*public_hits_inner.lock().expect("mutex should lock") += 1;
(StatusCode::IM_A_TEAPOT, Body::from("public-route-hit"))
}
}),
);
let execution_runtime = Router::new().route(
"/v1/execute/sync",
any(move |request: Request| {
let seen_execution_runtime_inner = Arc::clone(&seen_execution_runtime_clone);
async move {
let (_parts, body) = request.into_parts();
let raw_body = to_bytes(body, usize::MAX).await.expect("body should read");
let payload: serde_json::Value = serde_json::from_slice(&raw_body)
.expect("execution runtime payload should parse");
*seen_execution_runtime_inner
.lock()
.expect("mutex should lock") = Some(SeenExecutionRuntimeSyncRequest {
method: payload
.get("method")
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
url: payload
.get("url")
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
authorization: payload
.get("headers")
.and_then(|value| value.get("authorization"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
prompt: payload
.get("body")
.and_then(|value| value.get("json_body"))
.and_then(|value| value.get("prompt"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
});
Json(json!({
"request_id": "trace-openai-video-remix-local-123",
"status_code": 200,
"headers": {
"content-type": "application/json"
},
"body": {
"json_body": {
"id": "ext-remix-task-123",
"status": "submitted"
}
},
"telemetry": {
"elapsed_ms": 23
}
}))
}
}),
);
let repository = Arc::new(InMemoryVideoTaskRepository::default());
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
repository
.upsert(UpsertVideoTask {
id: "task-local-123".to_string(),
short_id: Some("task-local-123".to_string()),
request_id: "request-openai-video-remix-local-123".to_string(),
user_id: Some("user-openai-video-remix-local-123".to_string()),
api_key_id: Some("key-openai-video-remix-local-123".to_string()),
username: Some("video-user".to_string()),
api_key_name: Some("video-key".to_string()),
external_task_id: Some("ext-video-task-123".to_string()),
provider_id: Some("provider-openai-video-local-1".to_string()),
endpoint_id: Some("endpoint-openai-video-local-1".to_string()),
key_id: Some("key-openai-video-local-1".to_string()),
client_api_format: Some("openai:video".to_string()),
provider_api_format: Some("openai:video".to_string()),
format_converted: false,
model: Some("sora-2".to_string()),
prompt: Some("original prompt".to_string()),
original_request_body: Some(json!({"prompt": "original prompt"})),
duration_seconds: Some(4),
resolution: Some("720p".to_string()),
aspect_ratio: Some("16:9".to_string()),
size: Some("1280x720".to_string()),
status: aether_data::repository::video_tasks::VideoTaskStatus::Completed,
progress_percent: 100,
progress_message: None,
retry_count: 0,
poll_interval_seconds: 10,
next_poll_at_unix_secs: None,
poll_count: 1,
max_poll_count: 360,
created_at_unix_secs: 123,
submitted_at_unix_secs: Some(123),
completed_at_unix_secs: Some(124),
updated_at_unix_secs: 124,
error_code: None,
error_message: None,
video_url: Some("https://cdn.example.com/original.mp4".to_string()),
request_metadata: Some(json!({
"rust_local_snapshot": {
"OpenAi": {
"local_task_id": "task-local-123",
"upstream_task_id": "ext-video-task-123",
"created_at_unix_secs": 123,
"user_id": "user-openai-video-remix-local-123",
"api_key_id": "key-openai-video-remix-local-123",
"model": "sora-2",
"prompt": "original prompt",
"size": "1280x720",
"seconds": "4",
"remixed_from_video_id": null,
"status": "Completed",
"progress_percent": 100,
"completed_at_unix_secs": 124,
"expires_at_unix_secs": null,
"error_code": null,
"error_message": null,
"video_url": "https://cdn.example.com/original.mp4",
"persistence": {
"request_id": "request-openai-video-remix-local-123",
"username": "video-user",
"api_key_name": "video-key",
"client_api_format": "openai:video",
"provider_api_format": "openai:video",
"original_request_body": {
"prompt": "original prompt"
},
"format_converted": false
},
"transport": {
"upstream_base_url": "https://api.openai.example",
"provider_name": "openai-video",
"provider_id": "provider-openai-video-local-1",
"endpoint_id": "endpoint-openai-video-local-1",
"key_id": "key-openai-video-local-1",
"headers": {
"authorization": "Bearer sk-upstream-openai-video",
"content-type": "application/json"
},
"content_type": "application/json",
"model_name": "sora-2-upstream",
"proxy": null,
"tls_profile": null,
"timeouts": null
}
}
}
})),
})
.await
.expect("upsert should succeed");
let (upstream_url, upstream_handle) = start_server(upstream).await;
let (execution_runtime_url, execution_runtime_handle) = start_server(execution_runtime).await;
let gateway_state =
build_state_with_execution_runtime_override(execution_runtime_url)
.with_data_state_for_tests(
crate::data::GatewayDataState::with_video_task_and_request_candidate_repository_for_tests(
repository,
Arc::clone(&request_candidate_repository),
),
);
let gateway = build_router_with_state(gateway_state);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.post(format!("{gateway_url}/v1/videos/task-local-123/remix"))
.header(http::header::CONTENT_TYPE, "application/json")
.header(TRACE_ID_HEADER, "trace-openai-video-remix-local-123")
.body("{\"prompt\":\"remix this\",\"model\":\"sora-2\"}")
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let body: serde_json::Value = response.json().await.expect("body should parse");
assert_eq!(body.get("object"), Some(&json!("video")));
assert_eq!(body.get("status"), Some(&json!("queued")));
assert_eq!(body.get("prompt"), Some(&json!("remix this")));
assert_eq!(
body.get("remixed_from_video_id"),
Some(&json!("task-local-123"))
);
let seen_execution_runtime_request = seen_execution_runtime
.lock()
.expect("mutex should lock")
.clone()
.expect("execution runtime sync should be captured");
assert_eq!(seen_execution_runtime_request.method, "POST");
assert_eq!(
seen_execution_runtime_request.url,
"https://api.openai.example/v1/videos/ext-video-task-123/remix"
);
assert_eq!(
seen_execution_runtime_request.authorization,
"Bearer sk-upstream-openai-video"
);
assert_eq!(seen_execution_runtime_request.prompt, "remix this");
let stored_candidates = request_candidate_repository
.list_by_request_id("request-openai-video-remix-local-123")
.await
.expect("request candidate trace should read");
assert_eq!(stored_candidates.len(), 1);
assert_eq!(stored_candidates[0].status, RequestCandidateStatus::Success);
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
assert_eq!(*report_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*decision_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*plan_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
execution_runtime_handle.abort();
upstream_handle.abort();
}

View File

@@ -0,0 +1,353 @@
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
use aether_data::repository::candidates::{
InMemoryRequestCandidateRepository, RequestCandidateReadRepository, RequestCandidateStatus,
};
use aether_data::repository::provider_catalog::{
InMemoryProviderCatalogReadRepository, StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
StoredProviderCatalogProvider,
};
use aether_data::repository::video_tasks::{
InMemoryVideoTaskRepository, UpsertVideoTask, VideoTaskWriteRepository,
};
use axum::body::{to_bytes, Body};
use axum::response::Response;
use axum::routing::any;
use axum::{extract::Request, Json, Router};
use http::header::{HeaderName, HeaderValue};
use http::StatusCode;
use serde_json::json;
use std::sync::{Arc, Mutex};
use crate::constants::{
CONTROL_EXECUTED_HEADER, CONTROL_EXECUTE_FALLBACK_HEADER, TRACE_ID_HEADER,
};
use super::{
build_router_with_state, build_state_with_execution_runtime_override, start_server,
VideoTaskTruthSourceMode,
};
#[tokio::test]
async fn gateway_executes_openai_video_delete_via_reconstructed_data_backed_local_follow_up_with_local_follow_up_routing(
) {
#[derive(Debug, Clone, PartialEq, Eq)]
struct SeenExecutionRuntimeSyncRequest {
method: String,
url: String,
authorization: String,
}
fn sample_provider() -> StoredProviderCatalogProvider {
StoredProviderCatalogProvider::new(
"provider-openai-video-followup-1".to_string(),
"openai".to_string(),
Some("https://example.com".to_string()),
"custom".to_string(),
)
.expect("provider should build")
.with_transport_fields(
true,
false,
false,
None,
Some(2),
None,
Some(20.0),
None,
None,
)
}
fn sample_endpoint() -> StoredProviderCatalogEndpoint {
StoredProviderCatalogEndpoint::new(
"endpoint-openai-video-followup-1".to_string(),
"provider-openai-video-followup-1".to_string(),
"openai:video".to_string(),
Some("openai".to_string()),
Some("video".to_string()),
true,
)
.expect("endpoint should build")
.with_transport_fields(
"https://api.openai.example".to_string(),
None,
None,
Some(2),
None,
None,
None,
None,
)
.expect("endpoint transport should build")
}
fn sample_key() -> StoredProviderCatalogKey {
StoredProviderCatalogKey::new(
"key-openai-video-followup-1".to_string(),
"provider-openai-video-followup-1".to_string(),
"prod".to_string(),
"api_key".to_string(),
None,
true,
)
.expect("key should build")
.with_transport_fields(
Some(json!(["openai:video"])),
encrypt_python_fernet_plaintext(DEVELOPMENT_ENCRYPTION_KEY, "sk-upstream-openai-video")
.expect("api key should encrypt"),
None,
None,
Some(json!({"openai:video": 1})),
None,
None,
None,
None,
)
.expect("key transport should build")
}
let decision_hits = Arc::new(Mutex::new(0usize));
let decision_hits_clone = Arc::clone(&decision_hits);
let execute_hits = Arc::new(Mutex::new(0usize));
let execute_hits_clone = Arc::clone(&execute_hits);
let public_hits = Arc::new(Mutex::new(0usize));
let public_hits_clone = Arc::clone(&public_hits);
let report_hits = Arc::new(Mutex::new(0usize));
let report_hits_clone = Arc::clone(&report_hits);
let seen_execution_runtime = Arc::new(Mutex::new(None::<SeenExecutionRuntimeSyncRequest>));
let seen_execution_runtime_clone = Arc::clone(&seen_execution_runtime);
let upstream = Router::new()
.route(
"/api/internal/gateway/resolve",
any(|_request: Request| async move {
Json(json!({
"action": "proxy_public",
"route_class": "ai_public",
"route_family": "openai",
"route_kind": "video",
"auth_endpoint_signature": "openai:video",
"execution_runtime_candidate": true,
"auth_context": {
"user_id": "user-openai-video-delete-local-123",
"api_key_id": "key-openai-video-delete-local-123",
"access_allowed": true
},
"public_path": "/v1/videos/task-local-followup-123"
}))
}),
)
.route(
"/api/internal/gateway/decision-sync",
any(move |_request: Request| {
let decision_hits_inner = Arc::clone(&decision_hits_clone);
async move {
*decision_hits_inner.lock().expect("mutex should lock") += 1;
Json(json!({
"action": "execution_runtime_sync_decision",
"decision_kind": "openai_video_delete_sync",
"request_id": "unexpected-decision-hit"
}))
}
}),
)
.route(
"/api/internal/gateway/report-sync",
any(move |_request: Request| {
let report_hits_inner = Arc::clone(&report_hits_clone);
async move {
*report_hits_inner.lock().expect("mutex should lock") += 1;
Json(json!({"ok": true}))
}
}),
)
.route(
"/api/internal/gateway/execute-sync",
any(move |_request: Request| {
let execute_hits_inner = Arc::clone(&execute_hits_clone);
async move {
*execute_hits_inner.lock().expect("mutex should lock") += 1;
let mut response = Response::builder()
.status(StatusCode::OK)
.body(Body::from("{\"fallback\":true}"))
.expect("response should build");
response.headers_mut().insert(
HeaderName::from_static(CONTROL_EXECUTED_HEADER),
HeaderValue::from_static("true"),
);
response
}
}),
)
.route(
"/v1/videos/task-local-followup-123",
any(move |_request: Request| {
let public_hits_inner = Arc::clone(&public_hits_clone);
async move {
*public_hits_inner.lock().expect("mutex should lock") += 1;
(StatusCode::IM_A_TEAPOT, Body::from("public-route-hit"))
}
}),
);
let execution_runtime = Router::new().route(
"/v1/execute/sync",
any(move |request: Request| {
let seen_execution_runtime_inner = Arc::clone(&seen_execution_runtime_clone);
async move {
let (_parts, body) = request.into_parts();
let raw_body = to_bytes(body, usize::MAX).await.expect("body should read");
let payload: serde_json::Value = serde_json::from_slice(&raw_body)
.expect("execution runtime payload should parse");
*seen_execution_runtime_inner
.lock()
.expect("mutex should lock") = Some(SeenExecutionRuntimeSyncRequest {
method: payload
.get("method")
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
url: payload
.get("url")
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
authorization: payload
.get("headers")
.and_then(|value| value.get("x-api-key"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
});
Json(json!({
"request_id": "trace-openai-video-delete-local-123",
"status_code": 404,
"headers": {
"content-type": "application/json"
}
}))
}
}),
);
let (upstream_url, upstream_handle) = start_server(upstream).await;
let (execution_runtime_url, execution_runtime_handle) = start_server(execution_runtime).await;
let repository = Arc::new(InMemoryVideoTaskRepository::default());
repository
.upsert(UpsertVideoTask {
id: "task-local-followup-123".to_string(),
short_id: None,
request_id: "request-openai-video-delete-local-123".to_string(),
user_id: Some("user-openai-video-delete-local-123".to_string()),
api_key_id: Some("key-openai-video-delete-local-123".to_string()),
username: Some("video-user".to_string()),
api_key_name: Some("video-key".to_string()),
external_task_id: Some("ext-video-task-followup-123".to_string()),
provider_id: Some("provider-openai-video-followup-1".to_string()),
endpoint_id: Some("endpoint-openai-video-followup-1".to_string()),
key_id: Some("key-openai-video-followup-1".to_string()),
client_api_format: Some("openai:video".to_string()),
provider_api_format: Some("openai:video".to_string()),
format_converted: false,
model: Some("sora-2".to_string()),
prompt: Some("video delete".to_string()),
original_request_body: Some(json!({
"model": "sora-2",
"prompt": "video delete"
})),
duration_seconds: Some(4),
resolution: Some("720p".to_string()),
aspect_ratio: Some("16:9".to_string()),
size: Some("1280x720".to_string()),
status: aether_data::repository::video_tasks::VideoTaskStatus::Completed,
progress_percent: 100,
progress_message: None,
retry_count: 0,
poll_interval_seconds: 10,
next_poll_at_unix_secs: None,
poll_count: 0,
max_poll_count: 360,
created_at_unix_secs: 123,
submitted_at_unix_secs: Some(123),
completed_at_unix_secs: Some(456),
updated_at_unix_secs: 456,
error_code: None,
error_message: None,
video_url: Some("https://cdn.example.com/video-delete.mp4".to_string()),
request_metadata: None,
})
.await
.expect("upsert should succeed");
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider()],
vec![sample_endpoint()],
vec![sample_key()],
));
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
let gateway = build_router_with_state(
build_state_with_execution_runtime_override(execution_runtime_url)
.with_video_task_truth_source_mode(VideoTaskTruthSourceMode::RustAuthoritative)
.with_data_state_for_tests(
crate::data::GatewayDataState::with_video_task_provider_transport_and_request_candidate_repository_for_tests(
repository,
provider_catalog_repository,
Arc::clone(&request_candidate_repository),
DEVELOPMENT_ENCRYPTION_KEY,
),
),
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.delete(format!("{gateway_url}/v1/videos/task-local-followup-123"))
.header(CONTROL_EXECUTE_FALLBACK_HEADER, "true")
.header(TRACE_ID_HEADER, "trace-openai-video-delete-local-123")
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let response_json: serde_json::Value = response.json().await.expect("body should parse");
assert_eq!(
response_json,
json!({
"id": "task-local-followup-123",
"object": "video",
"deleted": true
})
);
let seen_execution_runtime_request = seen_execution_runtime
.lock()
.expect("mutex should lock")
.clone()
.expect("execution runtime sync should be captured");
assert_eq!(seen_execution_runtime_request.method, "DELETE");
assert_eq!(
seen_execution_runtime_request.url,
"https://api.openai.example/v1/videos/ext-video-task-followup-123"
);
assert_eq!(
seen_execution_runtime_request.authorization,
"sk-upstream-openai-video"
);
let stored_candidates = request_candidate_repository
.list_by_request_id("request-openai-video-delete-local-123")
.await
.expect("request candidate trace should read");
assert_eq!(stored_candidates.len(), 1);
assert_eq!(stored_candidates[0].status, RequestCandidateStatus::Success);
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
assert_eq!(*decision_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*execute_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*report_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
execution_runtime_handle.abort();
upstream_handle.abort();
}

View File

@@ -0,0 +1,333 @@
use std::sync::{Arc, Mutex};
use aether_data::repository::video_tasks::{
InMemoryVideoTaskRepository, UpsertVideoTask, VideoTaskLookupKey, VideoTaskReadRepository,
VideoTaskWriteRepository,
};
use axum::body::to_bytes;
use axum::routing::any;
use axum::{extract::Request, Json, Router};
use serde_json::json;
use super::{
build_state_with_execution_runtime_override, start_server, AppState, VideoTaskTruthSourceMode,
};
fn sample_due_openai_task(upstream_base_url: &str) -> UpsertVideoTask {
UpsertVideoTask {
id: "task-local-123".to_string(),
short_id: Some("task-local-123".to_string()),
request_id: "request-video-poller-local-123".to_string(),
user_id: Some("user-video-poller-123".to_string()),
api_key_id: Some("key-video-poller-123".to_string()),
username: Some("video-user".to_string()),
api_key_name: Some("video-key".to_string()),
external_task_id: Some("ext-video-task-123".to_string()),
provider_id: Some("provider-openai-video-local-1".to_string()),
endpoint_id: Some("endpoint-openai-video-local-1".to_string()),
key_id: Some("key-openai-video-local-1".to_string()),
client_api_format: Some("openai:video".to_string()),
provider_api_format: Some("openai:video".to_string()),
format_converted: false,
model: Some("sora-2".to_string()),
prompt: Some("hello".to_string()),
original_request_body: Some(json!({"prompt": "hello"})),
duration_seconds: Some(4),
resolution: Some("720p".to_string()),
aspect_ratio: Some("16:9".to_string()),
size: Some("1280x720".to_string()),
status: aether_data::repository::video_tasks::VideoTaskStatus::Submitted,
progress_percent: 0,
progress_message: None,
retry_count: 0,
poll_interval_seconds: 10,
next_poll_at_unix_secs: Some(0),
poll_count: 0,
max_poll_count: 360,
created_at_unix_secs: 123,
submitted_at_unix_secs: Some(123),
completed_at_unix_secs: None,
updated_at_unix_secs: 123,
error_code: None,
error_message: None,
video_url: None,
request_metadata: Some(json!({
"rust_local_snapshot": {
"OpenAi": {
"local_task_id": "task-local-123",
"upstream_task_id": "ext-video-task-123",
"created_at_unix_secs": 123,
"user_id": "user-video-poller-123",
"api_key_id": "key-video-poller-123",
"model": "sora-2",
"prompt": "hello",
"size": "1280x720",
"seconds": "4",
"remixed_from_video_id": null,
"status": "Submitted",
"progress_percent": 0,
"completed_at_unix_secs": null,
"expires_at_unix_secs": null,
"error_code": null,
"error_message": null,
"video_url": null,
"persistence": {
"request_id": "request-video-poller-local-123",
"username": "video-user",
"api_key_name": "video-key",
"client_api_format": "openai:video",
"provider_api_format": "openai:video",
"original_request_body": {
"prompt": "hello"
},
"format_converted": false
},
"transport": {
"upstream_base_url": upstream_base_url,
"provider_name": "openai-video",
"provider_id": "provider-openai-video-local-1",
"endpoint_id": "endpoint-openai-video-local-1",
"key_id": "key-openai-video-local-1",
"headers": {
"authorization": "Bearer sk-upstream-openai-video",
"content-type": "application/json"
},
"content_type": "application/json",
"model_name": "sora-2-upstream",
"proxy": null,
"tls_profile": null,
"timeouts": null
}
}
}
})),
}
}
#[tokio::test]
async fn gateway_background_video_task_poller_refreshes_due_openai_task_from_repository() {
#[derive(Debug, Clone, PartialEq, Eq)]
struct SeenExecutionRuntimeRequest {
method: String,
url: String,
}
let seen_execution_runtime_requests =
Arc::new(Mutex::new(Vec::<SeenExecutionRuntimeRequest>::new()));
let seen_execution_runtime_requests_clone = Arc::clone(&seen_execution_runtime_requests);
let execution_runtime = Router::new().route(
"/v1/execute/sync",
any(move |request: Request| {
let seen_execution_runtime_requests_inner =
Arc::clone(&seen_execution_runtime_requests_clone);
async move {
let (_parts, body) = request.into_parts();
let raw_body = to_bytes(body, usize::MAX).await.expect("body should read");
let payload: serde_json::Value = serde_json::from_slice(&raw_body)
.expect("execution runtime payload should parse");
let method = payload
.get("method")
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string();
let url = payload
.get("url")
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string();
seen_execution_runtime_requests_inner
.lock()
.expect("mutex should lock")
.push(SeenExecutionRuntimeRequest {
method: method.clone(),
url: url.clone(),
});
Json(json!({
"request_id": "req-openai-video-poller-refresh-123",
"status_code": 200,
"headers": {
"content-type": "application/json"
},
"body": {
"json_body": {
"id": "ext-video-task-123",
"status": "processing",
"progress": 37
}
}
}))
}
}),
);
let (execution_runtime_url, execution_runtime_handle) = start_server(execution_runtime).await;
let repository = Arc::new(InMemoryVideoTaskRepository::default());
repository
.upsert(sample_due_openai_task("https://api.openai.example"))
.await
.expect("task upsert should succeed");
let gateway_state = build_state_with_execution_runtime_override(execution_runtime_url)
.with_video_task_data_repository_for_tests(Arc::clone(&repository))
.with_video_task_truth_source_mode(VideoTaskTruthSourceMode::RustAuthoritative)
.with_video_task_poller_config(std::time::Duration::from_millis(25), 8);
let background_tasks = gateway_state.spawn_background_tasks();
assert!(!background_tasks.is_empty(), "poller task should spawn");
let stored = {
let deadline = tokio::time::Instant::now() + std::time::Duration::from_millis(500);
loop {
let stored = repository
.find(VideoTaskLookupKey::Id("task-local-123"))
.await
.expect("video task lookup should succeed")
.expect("video task should exist");
if stored.progress_percent == 37 {
break stored;
}
assert!(
tokio::time::Instant::now() < deadline,
"poller did not refresh task within 500ms"
);
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
};
assert_eq!(
stored.status,
aether_data::repository::video_tasks::VideoTaskStatus::Processing
);
assert_eq!(stored.progress_percent, 37);
assert_eq!(stored.poll_count, 1);
assert!(
stored.next_poll_at_unix_secs.is_some_and(|value| value > 0),
"poller should push next poll into the future"
);
assert_eq!(
stored
.request_metadata
.as_ref()
.and_then(|value| value.get("rust_owner"))
.and_then(serde_json::Value::as_str),
Some("async_task")
);
assert_eq!(
stored
.request_metadata
.as_ref()
.and_then(|value| value.get("poll_raw_response"))
.and_then(|value| value.get("status"))
.and_then(serde_json::Value::as_str),
Some("processing")
);
assert_eq!(
seen_execution_runtime_requests
.lock()
.expect("mutex should lock")
.clone(),
vec![SeenExecutionRuntimeRequest {
method: "GET".to_string(),
url: "https://api.openai.example/v1/videos/ext-video-task-123".to_string(),
}]
);
for handle in background_tasks {
handle.abort();
}
execution_runtime_handle.abort();
}
#[tokio::test]
async fn gateway_background_video_task_poller_refreshes_due_openai_task_from_repository_without_execution_runtime_override(
) {
#[derive(Debug, Clone, PartialEq, Eq)]
struct SeenUpstreamRequest {
method: String,
path: String,
}
let seen_upstream_requests = Arc::new(Mutex::new(Vec::<SeenUpstreamRequest>::new()));
let seen_upstream_requests_clone = Arc::clone(&seen_upstream_requests);
let upstream = Router::new().route(
"/v1/videos/ext-video-task-123",
any(move |request: Request| {
let seen_upstream_requests_inner = Arc::clone(&seen_upstream_requests_clone);
async move {
seen_upstream_requests_inner
.lock()
.expect("mutex should lock")
.push(SeenUpstreamRequest {
method: request.method().as_str().to_string(),
path: request.uri().path().to_string(),
});
Json(json!({
"id": "ext-video-task-123",
"status": "processing",
"progress": 37
}))
}
}),
);
let (upstream_url, upstream_handle) = start_server(upstream).await;
let repository = Arc::new(InMemoryVideoTaskRepository::default());
repository
.upsert(sample_due_openai_task(&upstream_url))
.await
.expect("task upsert should succeed");
let gateway_state = AppState::new()
.expect("gateway state should build")
.with_video_task_data_repository_for_tests(Arc::clone(&repository))
.with_video_task_truth_source_mode(VideoTaskTruthSourceMode::RustAuthoritative)
.with_video_task_poller_config(std::time::Duration::from_millis(25), 8);
let background_tasks = gateway_state.spawn_background_tasks();
assert!(!background_tasks.is_empty(), "poller task should spawn");
let stored = {
let deadline = tokio::time::Instant::now() + std::time::Duration::from_millis(500);
loop {
let stored = repository
.find(VideoTaskLookupKey::Id("task-local-123"))
.await
.expect("video task lookup should succeed")
.expect("video task should exist");
if stored.progress_percent == 37 {
break stored;
}
assert!(
tokio::time::Instant::now() < deadline,
"poller did not refresh task within 500ms"
);
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
};
assert_eq!(
stored.status,
aether_data::repository::video_tasks::VideoTaskStatus::Processing
);
assert_eq!(stored.progress_percent, 37);
assert_eq!(stored.poll_count, 1);
assert!(
stored.next_poll_at_unix_secs.is_some_and(|value| value > 0),
"poller should push next poll into the future"
);
assert_eq!(
seen_upstream_requests
.lock()
.expect("mutex should lock")
.clone(),
vec![SeenUpstreamRequest {
method: "GET".to_string(),
path: "/v1/videos/ext-video-task-123".to_string(),
}]
);
for handle in background_tasks {
handle.abort();
}
upstream_handle.abort();
}

View File

@@ -0,0 +1,226 @@
use std::sync::{Arc, Mutex};
use axum::body::Body;
use axum::response::Response;
use axum::routing::any;
use axum::{extract::Request, Router};
use http::header::{HeaderName, HeaderValue};
use http::StatusCode;
use crate::constants::{
CONTROL_EXECUTED_HEADER, CONTROL_EXECUTE_FALLBACK_HEADER, EXECUTION_PATH_HEADER,
};
use super::{build_router, start_server};
#[tokio::test]
async fn gateway_locally_denies_video_control_sync_even_with_opt_in_headers_when_execution_runtime_missing(
) {
let execute_hits = Arc::new(Mutex::new(0usize));
let execute_hits_clone = Arc::clone(&execute_hits);
let public_hits = Arc::new(Mutex::new(0usize));
let public_hits_clone = Arc::clone(&public_hits);
let upstream = Router::new()
.route(
"/api/internal/gateway/execute-sync",
any(move |_request: Request| {
let execute_hits_inner = Arc::clone(&execute_hits_clone);
async move {
*execute_hits_inner.lock().expect("mutex should lock") += 1;
let mut response = Response::builder()
.status(StatusCode::OK)
.body(Body::from("{\"status\":\"queued\"}"))
.expect("response should build");
response.headers_mut().insert(
http::header::CONTENT_TYPE,
HeaderValue::from_static("application/json"),
);
response.headers_mut().insert(
HeaderName::from_static(CONTROL_EXECUTED_HEADER),
HeaderValue::from_static("true"),
);
response
}
}),
)
.route(
"/v1/videos/task-123",
any(move |_request: Request| {
let public_hits_inner = Arc::clone(&public_hits_clone);
async move {
*public_hits_inner.lock().expect("mutex should lock") += 1;
(StatusCode::IM_A_TEAPOT, Body::from("public-route-hit"))
}
}),
);
let (upstream_url, upstream_handle) = start_server(upstream).await;
let gateway = build_router().expect("gateway should build");
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.get(format!("{gateway_url}/v1/videos/task-123"))
.header(CONTROL_EXECUTE_FALLBACK_HEADER, "true")
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
let payload: serde_json::Value = response.json().await.expect("body should parse");
assert_eq!(payload["error"]["type"], "http_error");
assert_eq!(
payload["error"]["message"],
"OpenAI video execution runtime miss did not match a Rust execution path"
);
assert_eq!(*execute_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_locally_denies_video_control_sync_without_opt_in_header_when_execution_runtime_missing(
) {
let execute_hits = Arc::new(Mutex::new(0usize));
let execute_hits_clone = Arc::clone(&execute_hits);
let public_hits = Arc::new(Mutex::new(0usize));
let public_hits_clone = Arc::clone(&public_hits);
let public_execution_path = Arc::new(Mutex::new(None::<String>));
let public_execution_path_clone = Arc::clone(&public_execution_path);
let upstream = Router::new()
.route(
"/api/internal/gateway/execute-sync",
any(move |_request: Request| {
let execute_hits_inner = Arc::clone(&execute_hits_clone);
async move {
*execute_hits_inner.lock().expect("mutex should lock") += 1;
let mut response = Response::builder()
.status(StatusCode::OK)
.body(Body::from("{\"unexpected\":true}"))
.expect("response should build");
response.headers_mut().insert(
HeaderName::from_static(CONTROL_EXECUTED_HEADER),
HeaderValue::from_static("true"),
);
response
}
}),
)
.route(
"/v1/videos/task-123",
any(move |request: Request| {
let public_hits_inner = Arc::clone(&public_hits_clone);
let public_execution_path_inner = Arc::clone(&public_execution_path_clone);
async move {
*public_hits_inner.lock().expect("mutex should lock") += 1;
*public_execution_path_inner
.lock()
.expect("mutex should lock") = Some(
request
.headers()
.get(EXECUTION_PATH_HEADER)
.and_then(|value| value.to_str().ok())
.unwrap_or_default()
.to_string(),
);
(StatusCode::IM_A_TEAPOT, Body::from("public-route-hit"))
}
}),
);
let (upstream_url, upstream_handle) = start_server(upstream).await;
let gateway = build_router().expect("gateway should build");
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.get(format!("{gateway_url}/v1/videos/task-123"))
.header(CONTROL_EXECUTE_FALLBACK_HEADER, "true")
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
let payload: serde_json::Value = response.json().await.expect("body should parse");
assert_eq!(payload["error"]["type"], "http_error");
assert_eq!(
payload["error"]["message"],
"OpenAI video execution runtime miss did not match a Rust execution path"
);
assert_eq!(*execute_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
assert_eq!(
public_execution_path
.lock()
.expect("mutex should lock")
.clone()
.as_deref(),
None
);
gateway_handle.abort();
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_skips_video_get_control_sync_without_opt_in_header() {
let execute_hits = Arc::new(Mutex::new(0usize));
let execute_hits_clone = Arc::clone(&execute_hits);
let public_hits = Arc::new(Mutex::new(0usize));
let public_hits_clone = Arc::clone(&public_hits);
let upstream = Router::new()
.route(
"/api/internal/gateway/execute-sync",
any(move |_request: Request| {
let execute_hits_inner = Arc::clone(&execute_hits_clone);
async move {
*execute_hits_inner.lock().expect("mutex should lock") += 1;
let mut response = Response::builder()
.status(StatusCode::OK)
.body(Body::from("{\"status\":\"queued\"}"))
.expect("response should build");
response.headers_mut().insert(
HeaderName::from_static(CONTROL_EXECUTED_HEADER),
HeaderValue::from_static("true"),
);
response
}
}),
)
.route(
"/v1/videos/task-123",
any(move |_request: Request| {
let public_hits_inner = Arc::clone(&public_hits_clone);
async move {
*public_hits_inner.lock().expect("mutex should lock") += 1;
(StatusCode::IM_A_TEAPOT, Body::from("public-route-hit"))
}
}),
);
let (upstream_url, upstream_handle) = start_server(upstream).await;
let gateway = build_router().expect("gateway should build");
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.get(format!("{gateway_url}/v1/videos/task-123"))
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
let payload: serde_json::Value = response.json().await.expect("body should parse");
assert_eq!(payload["error"]["type"], "http_error");
assert_eq!(
payload["error"]["message"],
"OpenAI video execution runtime miss did not match a Rust execution path"
);
assert_eq!(*execute_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
upstream_handle.abort();
}

View File

@@ -0,0 +1,359 @@
use aether_contracts::{StreamFrame, StreamFramePayload, StreamFrameType};
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
use aether_data::repository::provider_catalog::{
InMemoryProviderCatalogReadRepository, StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
StoredProviderCatalogProvider,
};
use aether_data::repository::video_tasks::{
InMemoryVideoTaskRepository, UpsertVideoTask, VideoTaskWriteRepository,
};
use axum::body::{to_bytes, Body, Bytes};
use axum::response::Response;
use axum::routing::any;
use axum::{extract::Request, Json, Router};
use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
use http::header::{HeaderName, HeaderValue};
use http::StatusCode;
use serde_json::json;
use std::sync::{Arc, Mutex};
use crate::constants::{
CONTROL_EXECUTED_HEADER, CONTROL_EXECUTE_FALLBACK_HEADER, TRACE_ID_HEADER,
};
use super::{
build_router_with_state, build_state_with_execution_runtime_override, start_server,
VideoTaskTruthSourceMode,
};
#[tokio::test]
async fn gateway_executes_openai_video_content_from_reconstructed_data_task_without_decision_stream(
) {
#[derive(Debug, Clone, PartialEq, Eq)]
struct SeenExecutionRuntimeStreamRequest {
method: String,
url: String,
}
fn sample_provider() -> StoredProviderCatalogProvider {
StoredProviderCatalogProvider::new(
"provider-openai-video-content-followup-1".to_string(),
"openai".to_string(),
Some("https://example.com".to_string()),
"custom".to_string(),
)
.expect("provider should build")
.with_transport_fields(
true,
false,
false,
None,
Some(2),
None,
Some(20.0),
None,
None,
)
}
fn sample_endpoint() -> StoredProviderCatalogEndpoint {
StoredProviderCatalogEndpoint::new(
"endpoint-openai-video-content-followup-1".to_string(),
"provider-openai-video-content-followup-1".to_string(),
"openai:video".to_string(),
Some("openai".to_string()),
Some("video".to_string()),
true,
)
.expect("endpoint should build")
.with_transport_fields(
"https://api.openai.example".to_string(),
None,
None,
Some(2),
None,
None,
None,
None,
)
.expect("endpoint transport should build")
}
fn sample_key() -> StoredProviderCatalogKey {
StoredProviderCatalogKey::new(
"key-openai-video-content-followup-1".to_string(),
"provider-openai-video-content-followup-1".to_string(),
"prod".to_string(),
"api_key".to_string(),
None,
true,
)
.expect("key should build")
.with_transport_fields(
Some(json!(["openai:video"])),
encrypt_python_fernet_plaintext(DEVELOPMENT_ENCRYPTION_KEY, "sk-upstream-openai-video")
.expect("api key should encrypt"),
None,
None,
Some(json!({"openai:video": 1})),
None,
None,
None,
None,
)
.expect("key transport should build")
}
let decision_stream_hits = Arc::new(Mutex::new(0usize));
let decision_stream_hits_clone = Arc::clone(&decision_stream_hits);
let execute_stream_hits = Arc::new(Mutex::new(0usize));
let execute_stream_hits_clone = Arc::clone(&execute_stream_hits);
let public_hits = Arc::new(Mutex::new(0usize));
let public_hits_clone = Arc::clone(&public_hits);
let seen_execution_runtime_stream =
Arc::new(Mutex::new(None::<SeenExecutionRuntimeStreamRequest>));
let seen_execution_runtime_stream_clone = Arc::clone(&seen_execution_runtime_stream);
let upstream = Router::new()
.route(
"/api/internal/gateway/resolve",
any(|request: Request| async move {
Json(json!({
"action": "proxy_public",
"route_class": "ai_public",
"route_family": "openai",
"route_kind": "video",
"auth_endpoint_signature": "openai:video",
"execution_runtime_candidate": true,
"auth_context": {
"user_id": "user-video-content-local-123",
"api_key_id": "key-video-content-local-123",
"access_allowed": true
},
"public_path": request.uri().path()
}))
}),
)
.route(
"/api/internal/gateway/decision-stream",
any(move |_request: Request| {
let decision_stream_hits_inner = Arc::clone(&decision_stream_hits_clone);
async move {
*decision_stream_hits_inner
.lock()
.expect("mutex should lock") += 1;
Json(json!({
"action": "execution_runtime_stream_decision",
"decision_kind": "openai_video_content",
"request_id": "unexpected-decision-stream-hit"
}))
}
}),
)
.route(
"/api/internal/gateway/execute-stream",
any(move |_request: Request| {
let execute_stream_hits_inner = Arc::clone(&execute_stream_hits_clone);
async move {
*execute_stream_hits_inner.lock().expect("mutex should lock") += 1;
let mut response = Response::builder()
.status(StatusCode::OK)
.body(Body::from("fallback"))
.expect("response should build");
response.headers_mut().insert(
HeaderName::from_static(CONTROL_EXECUTED_HEADER),
HeaderValue::from_static("true"),
);
response
}
}),
)
.route(
"/v1/videos/task-content-local-123/content",
any(move |_request: Request| {
let public_hits_inner = Arc::clone(&public_hits_clone);
async move {
*public_hits_inner.lock().expect("mutex should lock") += 1;
(StatusCode::IM_A_TEAPOT, Body::from("public-route-hit"))
}
}),
);
let execution_runtime = Router::new().route(
"/v1/execute/stream",
any(move |request: Request| {
let seen_execution_runtime_stream_inner =
Arc::clone(&seen_execution_runtime_stream_clone);
async move {
let (_parts, body) = request.into_parts();
let raw_body = to_bytes(body, usize::MAX).await.expect("body should read");
let payload: serde_json::Value = serde_json::from_slice(&raw_body)
.expect("execution runtime payload should parse");
*seen_execution_runtime_stream_inner
.lock()
.expect("mutex should lock") = Some(SeenExecutionRuntimeStreamRequest {
method: payload
.get("method")
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
url: payload
.get("url")
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
});
let frames = [
StreamFrame {
frame_type: StreamFrameType::Headers,
payload: StreamFramePayload::Headers {
status_code: 200,
headers: std::collections::BTreeMap::from([(
"content-type".to_string(),
"video/mp4".to_string(),
)]),
},
},
StreamFrame {
frame_type: StreamFrameType::Data,
payload: StreamFramePayload::Data {
chunk_b64: Some(BASE64_STANDARD.encode(b"video-")),
text: None,
},
},
StreamFrame {
frame_type: StreamFrameType::Data,
payload: StreamFramePayload::Data {
chunk_b64: Some(BASE64_STANDARD.encode(b"content")),
text: None,
},
},
StreamFrame::eof(),
];
let body = frames
.into_iter()
.map(|frame| serde_json::to_string(&frame).expect("frame should serialize"))
.collect::<Vec<_>>()
.join("\n")
+ "\n";
let mut response = Response::builder()
.status(StatusCode::OK)
.body(Body::from(body))
.expect("response should build");
response.headers_mut().insert(
http::header::CONTENT_TYPE,
HeaderValue::from_static("application/x-ndjson"),
);
response
}
}),
);
let (upstream_url, upstream_handle) = start_server(upstream).await;
let (execution_runtime_url, execution_runtime_handle) = start_server(execution_runtime).await;
let repository = Arc::new(InMemoryVideoTaskRepository::default());
repository
.upsert(UpsertVideoTask {
id: "task-content-local-123".to_string(),
short_id: None,
request_id: "request-openai-video-content-local-123".to_string(),
user_id: Some("user-video-content-local-123".to_string()),
api_key_id: Some("key-video-content-local-123".to_string()),
username: Some("video-user".to_string()),
api_key_name: Some("video-key".to_string()),
external_task_id: Some("ext-video-content-followup-123".to_string()),
provider_id: Some("provider-openai-video-content-followup-1".to_string()),
endpoint_id: Some("endpoint-openai-video-content-followup-1".to_string()),
key_id: Some("key-openai-video-content-followup-1".to_string()),
client_api_format: Some("openai:video".to_string()),
provider_api_format: Some("openai:video".to_string()),
format_converted: false,
model: Some("sora-2".to_string()),
prompt: Some("video content".to_string()),
original_request_body: Some(json!({
"model": "sora-2",
"prompt": "video content"
})),
duration_seconds: Some(4),
resolution: Some("720p".to_string()),
aspect_ratio: Some("16:9".to_string()),
size: Some("1280x720".to_string()),
status: aether_data::repository::video_tasks::VideoTaskStatus::Completed,
progress_percent: 100,
progress_message: None,
retry_count: 0,
poll_interval_seconds: 10,
next_poll_at_unix_secs: None,
poll_count: 0,
max_poll_count: 360,
created_at_unix_secs: 123,
submitted_at_unix_secs: Some(123),
completed_at_unix_secs: Some(456),
updated_at_unix_secs: 456,
error_code: None,
error_message: None,
video_url: Some("https://cdn.example.com/video-content.mp4".to_string()),
request_metadata: None,
})
.await
.expect("upsert should succeed");
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider()],
vec![sample_endpoint()],
vec![sample_key()],
));
let gateway = build_router_with_state(
build_state_with_execution_runtime_override(execution_runtime_url)
.with_video_task_truth_source_mode(VideoTaskTruthSourceMode::RustAuthoritative)
.with_video_task_repository_and_provider_transport_for_tests(
repository,
provider_catalog_repository,
DEVELOPMENT_ENCRYPTION_KEY,
),
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.get(format!(
"{gateway_url}/v1/videos/task-content-local-123/content?variant=video"
))
.header(CONTROL_EXECUTE_FALLBACK_HEADER, "true")
.header(TRACE_ID_HEADER, "trace-openai-video-content-local-123")
.send()
.await
.expect("content request should succeed");
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response
.headers()
.get(http::header::CONTENT_TYPE)
.and_then(|value| value.to_str().ok()),
Some("video/mp4")
);
assert_eq!(
response.bytes().await.expect("body should read"),
Bytes::from_static(b"video-content")
);
let seen_stream_request = seen_execution_runtime_stream
.lock()
.expect("mutex should lock")
.clone()
.expect("execution runtime stream should be captured");
assert_eq!(seen_stream_request.method, "GET");
assert_eq!(
seen_stream_request.url,
"https://cdn.example.com/video-content.mp4"
);
assert_eq!(*decision_stream_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*execute_stream_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*public_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
execution_runtime_handle.abort();
upstream_handle.abort();
}