Merge remote-tracking branch 'origin/main' into codex/pool-key-bulk-management-20260714

# Conflicts:
#	apps/aether-gateway/src/handlers/admin/request/provider/tasks.rs
#	frontend/src/api/endpoints/pool.ts
This commit is contained in:
MMEXA
2026-07-16 23:43:04 +08:00
1257 changed files with 80521 additions and 35495 deletions
@@ -70,7 +70,7 @@ async fn gateway_refreshes_admin_provider_quota_locally_for_codex_with_trusted_a
}),
);
let seen_execution_runtime = Arc::new(Mutex::new(None::<SeenExecutionRuntimeRequest>));
let seen_execution_runtime = Arc::new(Mutex::new(Vec::<SeenExecutionRuntimeRequest>::new()));
let seen_execution_runtime_clone = Arc::clone(&seen_execution_runtime);
let execution_runtime = Router::new().route(
"/v1/execute/sync",
@@ -83,21 +83,22 @@ async fn gateway_refreshes_admin_provider_quota_locally_for_codex_with_trusted_a
.expect("body should read"),
)
.expect("plan should parse");
*seen_execution_runtime_inner
seen_execution_runtime_inner
.lock()
.expect("mutex should lock") = Some(SeenExecutionRuntimeRequest {
url: plan.url.clone(),
authorization: plan
.headers
.get("authorization")
.cloned()
.unwrap_or_default(),
provider_api_format: plan.provider_api_format.clone(),
total_ms: plan
.timeouts
.as_ref()
.and_then(|timeouts| timeouts.total_ms),
});
.expect("mutex should lock")
.push(SeenExecutionRuntimeRequest {
url: plan.url.clone(),
authorization: plan
.headers
.get("authorization")
.cloned()
.unwrap_or_default(),
provider_api_format: plan.provider_api_format.clone(),
total_ms: plan
.timeouts
.as_ref()
.and_then(|timeouts| timeouts.total_ms),
});
let result = aether_contracts::ExecutionResult {
request_id: plan.request_id,
candidate_id: None,
@@ -223,24 +224,24 @@ async fn gateway_refreshes_admin_provider_quota_locally_for_codex_with_trusted_a
);
assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0);
let seen_execution_runtime_request = seen_execution_runtime
let seen_execution_runtime_requests = seen_execution_runtime
.lock()
.expect("mutex should lock")
.clone()
.expect("execution runtime request should be captured");
.clone();
assert_eq!(seen_execution_runtime_requests.len(), 2);
assert_eq!(
seen_execution_runtime_request.url,
seen_execution_runtime_requests[0].url,
"https://chatgpt.com/backend-api/wham/usage"
);
assert_eq!(
seen_execution_runtime_request.authorization,
"Bearer sk-codex-123"
seen_execution_runtime_requests[1].url,
"https://chatgpt.com/backend-api/wham/rate-limit-reset-credits"
);
assert_eq!(
seen_execution_runtime_request.provider_api_format,
"openai:responses"
);
assert_eq!(seen_execution_runtime_request.total_ms, Some(30_000));
for request in seen_execution_runtime_requests {
assert_eq!(request.authorization, "Bearer sk-codex-123");
assert_eq!(request.provider_api_format, "openai:responses");
assert_eq!(request.total_ms, Some(30_000));
}
let reloaded = provider_catalog_repository
.list_keys_by_ids(&["key-codex-a".to_string()])
@@ -679,7 +680,10 @@ async fn gateway_refreshes_admin_provider_quota_locally_for_requested_codex_keys
.lock()
.expect("mutex should lock")
.clone(),
vec!["Bearer sk-codex-a".to_string()]
vec![
"Bearer sk-codex-a".to_string(),
"Bearer sk-codex-a".to_string(),
]
);
let reloaded = provider_catalog_repository
@@ -559,6 +559,103 @@ async fn gateway_creates_admin_provider_endpoint_locally_with_trusted_admin_prin
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_rejects_streaming_policy_for_search_endpoint_before_catalog_write() {
let mut create_provider = sample_provider("provider-search-create", "search-create", 10);
create_provider.provider_type = "custom".to_string();
let mut update_provider = sample_provider("provider-search-update", "search-update", 20);
update_provider.provider_type = "custom".to_string();
let mut existing_endpoint = sample_endpoint(
"endpoint-search-update",
"provider-search-update",
"openai:search",
"https://search.example/v1",
);
existing_endpoint.config = Some(json!({"marker": "kept"}));
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![create_provider, update_provider],
vec![existing_endpoint],
vec![],
));
let gateway = build_router_with_state(
AppState::new()
.expect("gateway should build")
.with_data_state_for_tests(
GatewayDataState::with_provider_catalog_repository_for_tests(
provider_catalog_repository.clone(),
),
),
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let client = reqwest::Client::new();
let create_response = client
.post(format!(
"{gateway_url}/api/admin/endpoints/providers/provider-search-create/endpoints"
))
.header(GATEWAY_HEADER, "rust-phase3b")
.header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123")
.header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin")
.header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123")
.json(&json!({
"provider_id": "provider-search-create",
"api_format": "openai:search",
"base_url": "https://search.example/v1",
"config": {"upstream_stream_policy": "force_stream"}
}))
.send()
.await
.expect("request should succeed");
assert_eq!(create_response.status(), StatusCode::BAD_REQUEST);
let create_payload: serde_json::Value = create_response
.json()
.await
.expect("json body should parse");
assert_eq!(
create_payload["detail"],
"OpenAI Search 端点仅支持非流式上游请求"
);
let update_response = client
.put(format!(
"{gateway_url}/api/admin/endpoints/endpoint-search-update"
))
.header(GATEWAY_HEADER, "rust-phase3b")
.header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123")
.header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin")
.header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123")
.json(&json!({
"config": {"upstreamStreamPolicy": true}
}))
.send()
.await
.expect("request should succeed");
assert_eq!(update_response.status(), StatusCode::BAD_REQUEST);
let update_payload: serde_json::Value = update_response
.json()
.await
.expect("json body should parse");
assert_eq!(
update_payload["detail"],
"OpenAI Search 端点仅支持非流式上游请求"
);
let created = provider_catalog_repository
.list_endpoints_by_provider_ids(&["provider-search-create".to_string()])
.await
.expect("endpoints should read");
assert!(created.is_empty());
let unchanged = provider_catalog_repository
.list_endpoints_by_ids(&["endpoint-search-update".to_string()])
.await
.expect("endpoint should read");
assert_eq!(unchanged.len(), 1);
assert_eq!(unchanged[0].config, Some(json!({"marker": "kept"})));
gateway_handle.abort();
}
#[tokio::test]
async fn gateway_updates_admin_provider_endpoint_locally_with_trusted_admin_principal() {
let upstream_hits = Arc::new(Mutex::new(0usize));
@@ -6086,6 +6086,9 @@ async fn gateway_manual_codex_oauth_refresh_reconciles_missing_fixed_endpoint_im
.iter()
.find(|endpoint| endpoint.api_format == "openai:responses")
.expect("openai responses endpoint should be reconciled");
assert!(endpoints
.iter()
.any(|endpoint| endpoint.api_format == "openai:search"));
assert_eq!(
responses_endpoint.base_url,
"https://chatgpt.com/backend-api/codex"
@@ -10,7 +10,7 @@ use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadReposi
use aether_data::repository::proxy_nodes::InMemoryProxyNodeRepository;
use aether_data_contracts::repository::provider_catalog::ProviderCatalogReadRepository;
use aether_runtime_state::{RedisClientConfig, RuntimeState};
use aether_testkit::ManagedRedisServer;
use aether_test_support::ManagedRedisServer;
use axum::body::to_bytes;
use axum::body::Body;
use axum::routing::{any, get, post};
@@ -559,7 +559,7 @@ async fn gateway_handles_admin_provider_query_models_falls_back_to_codex_preset_
.expect("mutex should lock") += 1;
assert_eq!(
plan.url,
"https://chatgpt.com/backend-api/codex/models?client_version=0.128.0-alpha.1"
"https://chatgpt.com/backend-api/codex/models?client_version=0.144.1"
);
Json(json!({
"request_id": "req-provider-query-codex-invalidated",
@@ -625,6 +625,11 @@ async fn gateway_handles_admin_provider_query_models_falls_back_to_codex_preset_
let payload: serde_json::Value = response.json().await.expect("json body should parse");
assert_eq!(payload["success"], json!(true));
assert_eq!(payload["data"]["error"], serde_json::Value::Null);
let warning = payload["data"]["warning"]
.as_str()
.expect("Codex fallback warning should be present");
assert!(warning.contains("Codex 动态模型目录不可用"));
assert!(warning.contains("invalidated"));
let model_ids = payload["data"]["models"]
.as_array()
.expect("models should be an array")
@@ -634,11 +639,14 @@ async fn gateway_handles_admin_provider_query_models_falls_back_to_codex_preset_
assert_eq!(
model_ids,
vec![
"gpt-5.3-codex",
"gpt-5.3-codex-spark",
"codex-auto-review",
"gpt-5.2",
"gpt-5.4",
"gpt-5.4-mini",
"gpt-5.5",
"gpt-5.6-luna",
"gpt-5.6-sol",
"gpt-5.6-terra",
]
);
assert_eq!(
@@ -2391,6 +2399,188 @@ async fn gateway_streams_codex_openai_responses_upstream_for_admin_pool_model_te
execution_runtime_handle.abort();
}
#[test]
fn gateway_executes_codex_search_admin_pool_model_test_with_search_contract() {
run_provider_query_test(
"gateway_executes_codex_search_admin_pool_model_test_with_search_contract",
gateway_executes_codex_search_admin_pool_model_test_with_search_contract_impl,
);
}
async fn gateway_executes_codex_search_admin_pool_model_test_with_search_contract_impl() {
let execution_runtime = Router::new().route(
"/v1/execute/sync",
any(move |Json(plan): Json<ExecutionPlan>| async move {
assert_eq!(plan.provider_id, "provider-codex-search");
assert_eq!(plan.endpoint_id, "endpoint-codex-search");
assert_eq!(plan.key_id, "key-codex-search");
assert_eq!(plan.client_api_format, "openai:search");
assert_eq!(plan.provider_api_format, "openai:search");
assert_eq!(
plan.url,
"https://chatgpt.com/backend-api/codex/alpha/search"
);
assert_eq!(plan.model_name.as_deref(), Some("gpt-5.6-sol"));
assert!(!plan.stream, "Codex Search is a synchronous JSON protocol");
assert_eq!(
plan.timeouts
.as_ref()
.and_then(|timeouts| timeouts.total_ms),
Some(900_000)
);
assert_eq!(
plan.headers.get("authorization").map(String::as_str),
Some("Bearer codex-search-access-token")
);
assert_eq!(
plan.headers.get("chatgpt-account-id").map(String::as_str),
Some("account-search-admin")
);
assert_eq!(
plan.headers.get("x-openai-fedramp").map(String::as_str),
Some("true")
);
assert_eq!(
plan.headers.get("originator").map(String::as_str),
Some("codex_cli_rs")
);
assert!(plan
.headers
.get("user-agent")
.is_some_and(|value| value.starts_with("codex_cli_rs/")));
assert!(!plan.headers.contains_key("openai-beta"));
assert!(!plan
.headers
.contains_key("x-openai-internal-codex-responses-lite"));
assert_ne!(
plan.headers.get("accept").map(String::as_str),
Some("text/event-stream")
);
let body = plan.body.json_body.as_ref().expect("search json body");
assert_eq!(
body["id"],
json!("aether-model-test-provider-query-search-trace")
);
assert_eq!(body["model"], json!("gpt-5.6-sol"));
assert_eq!(body["input"], json!("find current OpenAI documentation"));
assert_eq!(
body["commands"]["search_query"][0]["q"],
json!("OpenAI Codex Search")
);
assert!(body.get("stream").is_none());
assert!(body.get("store").is_none());
assert!(body.get("service_tier").is_none());
assert!(body.get("unknown_field").is_none());
Json(json!({
"request_id": plan.request_id,
"candidate_id": plan.candidate_id,
"status_code": 200,
"headers": {
"content-type": "application/json"
},
"body": {
"json_body": {
"output": "search result"
}
},
"telemetry": {
"elapsed_ms": 21
}
}))
}),
);
let (execution_runtime_url, execution_runtime_handle) = start_server(execution_runtime).await;
let mut provider = sample_provider("provider-codex-search", "Codex Search", 10);
provider.provider_type = "codex".to_string();
provider.request_timeout_secs = Some(900.0);
let mut endpoint = sample_endpoint(
"endpoint-codex-search",
"provider-codex-search",
"openai:search",
"https://chatgpt.com/backend-api/codex",
);
endpoint.config = Some(json!({"upstream_stream_policy": "force_stream"}));
let mut key = sample_key(
"key-codex-search",
"provider-codex-search",
"openai:search",
"codex-search-access-token",
);
key.auth_type = "oauth".to_string();
key.encrypted_auth_config = Some(
aether_crypto::encrypt_python_fernet_plaintext(
DEVELOPMENT_ENCRYPTION_KEY,
r#"{"provider_type":"codex","account_id":"account-search-admin","is_fedramp":true}"#,
)
.expect("auth config should encrypt"),
);
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![provider],
vec![endpoint],
vec![key],
));
let gateway = build_router_with_state(
build_state_with_execution_runtime_override(execution_runtime_url)
.with_data_state_for_tests(GatewayDataState::with_provider_transport_reader_for_tests(
provider_catalog_repository,
DEVELOPMENT_ENCRYPTION_KEY.to_string(),
)),
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.post(format!(
"{gateway_url}/api/admin/provider-query/test-model-failover"
))
.header(GATEWAY_HEADER, "rust-phase3b")
.header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123")
.header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin")
.header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123")
.json(&json!({
"provider_id": "provider-codex-search",
"mode": "pool",
"model": "gpt-5.6-sol",
"failover_models": ["gpt-5.6-sol"],
"api_format": "openai:search",
"endpoint_id": "endpoint-codex-search",
"request_id": "provider-query-search-trace",
"request_body": {
"model": "gpt-5.6-sol",
"input": "find current OpenAI documentation",
"commands": {
"search_query": [{"q": "OpenAI Codex Search"}]
},
"max_output_tokens": 256,
"stream": true,
"store": false,
"service_tier": "priority",
"unknown_field": true
}
}))
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let payload: serde_json::Value = response.json().await.expect("json body should parse");
assert_eq!(payload["success"], json!(true), "payload={payload}");
assert_eq!(
payload["attempts"][0]["request_body"]["id"],
json!("aether-model-test-provider-query-search-trace")
);
assert_eq!(
payload["attempts"][0]["response_body"]["output"],
json!("search result")
);
gateway_handle.abort();
execution_runtime_handle.abort();
}
#[test]
fn gateway_routes_grok_responses_admin_pool_model_test_through_grok_runtime() {
run_provider_query_test(
@@ -3837,13 +4027,12 @@ async fn gateway_handles_openai_responses_test_model_locally_impl() {
.and_then(|value| value.as_str()),
Some(prompt)
);
assert_eq!(
plan.body
.json_body
.as_ref()
.and_then(|body| body.get("instructions")),
Some(&json!(""))
);
assert!(plan
.body
.json_body
.as_ref()
.and_then(|body| body.get("instructions"))
.is_none());
assert_eq!(
plan.body
.json_body
@@ -3856,7 +4045,7 @@ async fn gateway_handles_openai_responses_test_model_locally_impl() {
.json_body
.as_ref()
.and_then(|body| body.get("prompt_cache_key"))
.is_some());
.is_none());
Json(json!({
"request_id": plan.request_id,
"candidate_id": plan.candidate_id,
@@ -3960,8 +4149,8 @@ async fn gateway_handles_openai_image_test_model_locally_impl() {
assert_eq!(plan.client_api_format, "openai:image");
assert_eq!(plan.provider_api_format, "openai:image");
assert_eq!(plan.model_name.as_deref(), Some("gpt-image-1"));
assert_eq!(plan.url, "https://api.openai.example/v1/responses");
assert!(plan.stream);
assert_eq!(plan.url, "https://api.openai.example/v1/images/generations");
assert!(!plan.stream);
assert_eq!(
plan.headers.get("authorization").map(String::as_str),
Some("Bearer sk-test-image")
@@ -3971,38 +4160,42 @@ async fn gateway_handles_openai_image_test_model_locally_impl() {
.json_body
.as_ref()
.and_then(|body| body.get("model")),
Some(&json!(crate::ai_serving::CODEX_OPENAI_IMAGE_INTERNAL_MODEL))
Some(&json!("gpt-image-1"))
);
assert_eq!(
plan.body
.json_body
.as_ref()
.and_then(|body| body.get("input"))
.and_then(|input| input.as_array())
.and_then(|items| items.first())
.and_then(|item| item.get("content"))
.and_then(|body| body.get("prompt"))
.and_then(|value| value.as_str()),
Some("Draw a small blue square")
);
assert!(plan
.body
.json_body
.as_ref()
.is_some_and(|body| body.get("stream").is_none()));
Json(json!({
"request_id": plan.request_id,
"candidate_id": plan.candidate_id,
"status_code": 200,
"headers": {
"content-type": "text/event-stream"
"content-type": "application/json"
},
"body": {
"body_bytes_b64": base64::engine::general_purpose::STANDARD.encode(
concat!(
"event: response.created\n",
"data: {\"type\":\"response.created\",\"response\":{\"created_at\":1776839946}}\n\n",
"event: response.output_item.done\n",
"data: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"type\":\"image_generation_call\",\"output_format\":\"png\",\"revised_prompt\":\"revised prompt\",\"result\":\"aGVsbG8=\"}}\n\n",
"event: response.completed\n",
"data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_img_123\",\"model\":\"gpt-image-1\",\"status\":\"completed\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":171,\"output_tokens\":1372,\"total_tokens\":1543}}}}\n\n"
)
.as_bytes()
)
"json_body": {
"created": 1776839946,
"model": "gpt-image-1",
"data": [{
"b64_json": "aGVsbG8=",
"revised_prompt": "revised prompt"
}],
"usage": {
"input_tokens": 171,
"output_tokens": 1372,
"total_tokens": 1543
}
}
},
"telemetry": {
"elapsed_ms": 19
@@ -832,7 +832,7 @@ async fn gateway_updates_admin_provider_locally_with_trusted_admin_principal() {
"is_active": false,
"concurrent_limit": 8,
"max_retries": 6,
"request_timeout": 55.0,
"request_timeout": aether_contracts::MAX_EXECUTION_REQUEST_TIMEOUT_SECS,
"stream_first_byte_timeout": 11.0,
"enable_format_conversion": false,
"config": {
@@ -860,7 +860,10 @@ async fn gateway_updates_admin_provider_locally_with_trusted_admin_principal() {
assert_eq!(payload["enable_format_conversion"], false);
assert_eq!(payload["is_active"], false);
assert_eq!(payload["max_retries"], 6);
assert_eq!(payload["request_timeout"], 55.0);
assert_eq!(
payload["request_timeout"].as_f64(),
Some(aether_contracts::MAX_EXECUTION_REQUEST_TIMEOUT_SECS as f64)
);
assert_eq!(payload["stream_first_byte_timeout"], 11.0);
assert_eq!(payload["proxy"], json!({"url": "https://proxy.example"}));
assert_eq!(payload["claude_code_advanced"], json!({"pool_size": 2}));
@@ -870,6 +873,21 @@ async fn gateway_updates_admin_provider_locally_with_trusted_admin_principal() {
assert_eq!(payload["ops_configured"], true);
assert_eq!(payload["ops_architecture_id"], "cubence");
let invalid_timeout_response = reqwest::Client::new()
.patch(format!("{gateway_url}/api/admin/providers/provider-openai"))
.header(crate::constants::GATEWAY_HEADER, "rust-phase3b")
.header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123")
.header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin")
.header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123")
.json(&json!({
"request_timeout":
aether_contracts::MAX_EXECUTION_REQUEST_TIMEOUT_SECS + 1
}))
.send()
.await
.expect("request should succeed");
assert_eq!(invalid_timeout_response.status(), StatusCode::BAD_REQUEST);
let disable_response = reqwest::Client::new()
.patch(format!("{gateway_url}/api/admin/providers/provider-openai"))
.header(crate::constants::GATEWAY_HEADER, "rust-phase3b")
@@ -924,6 +942,10 @@ async fn gateway_updates_admin_provider_locally_with_trusted_admin_principal() {
.iter()
.find(|provider| provider.id == "provider-openai")
.expect("provider should exist");
assert_eq!(
updated_provider.request_timeout_secs,
Some(aether_contracts::MAX_EXECUTION_REQUEST_TIMEOUT_SECS as f64)
);
assert_eq!(
updated_provider
.config
@@ -1000,6 +1022,7 @@ async fn gateway_creates_admin_provider_locally_with_trusted_admin_principal() {
"website": "codex.example",
"keep_priority_on_conversion": true,
"max_retries": 7,
"request_timeout": aether_contracts::MAX_EXECUTION_REQUEST_TIMEOUT_SECS,
"config": {"chat_pii_redaction": {"enabled": true}},
"pool_advanced": {},
"failover_rules": {"strategy": "ordered"},
@@ -1034,6 +1057,10 @@ async fn gateway_creates_admin_provider_locally_with_trusted_admin_principal() {
assert_eq!(created.website.as_deref(), Some("https://codex.example"));
assert!(created.enable_format_conversion);
assert_eq!(created.max_retries, Some(7));
assert_eq!(
created.request_timeout_secs,
Some(aether_contracts::MAX_EXECUTION_REQUEST_TIMEOUT_SECS as f64)
);
assert_eq!(created.keep_priority_on_conversion, true);
assert_eq!(
created
@@ -1080,7 +1107,7 @@ async fn gateway_creates_admin_provider_locally_with_trusted_admin_principal() {
.list_endpoints_by_provider_ids(std::slice::from_ref(&created.id))
.await
.expect("endpoints should list");
assert_eq!(endpoints.len(), 3);
assert_eq!(endpoints.len(), 4);
let responses_endpoint = endpoints
.iter()
.find(|endpoint| endpoint.api_format == "openai:responses")
@@ -1089,6 +1116,10 @@ async fn gateway_creates_admin_provider_locally_with_trusted_admin_principal() {
.iter()
.find(|endpoint| endpoint.api_format == "openai:responses:compact")
.expect("compact endpoint should exist");
let search_endpoint = endpoints
.iter()
.find(|endpoint| endpoint.api_format == "openai:search")
.expect("search endpoint should exist");
let image_endpoint = endpoints
.iter()
.find(|endpoint| endpoint.api_format == "openai:image")
@@ -1101,12 +1132,17 @@ async fn gateway_creates_admin_provider_locally_with_trusted_admin_principal() {
compact_endpoint.base_url,
"https://chatgpt.com/backend-api/codex"
);
assert_eq!(
search_endpoint.base_url,
"https://chatgpt.com/backend-api/codex"
);
assert_eq!(
image_endpoint.base_url,
"https://chatgpt.com/backend-api/codex"
);
assert_eq!(responses_endpoint.max_retries, Some(7));
assert_eq!(compact_endpoint.max_retries, Some(7));
assert_eq!(search_endpoint.max_retries, Some(7));
assert_eq!(image_endpoint.max_retries, Some(7));
assert_eq!(
responses_endpoint
@@ -1116,16 +1152,25 @@ async fn gateway_creates_admin_provider_locally_with_trusted_admin_principal() {
.and_then(serde_json::Value::as_str),
Some("force_stream")
);
assert_eq!(
search_endpoint
.config
.as_ref()
.and_then(|value| value.get("upstream_stream_policy"))
.and_then(serde_json::Value::as_str),
None
);
assert_eq!(
image_endpoint
.config
.as_ref()
.and_then(|value| value.get("upstream_stream_policy"))
.and_then(serde_json::Value::as_str),
Some("force_stream")
None
);
assert!(responses_endpoint.body_rules.is_none());
assert!(compact_endpoint.body_rules.is_none());
assert!(search_endpoint.body_rules.is_none());
assert!(image_endpoint.body_rules.is_none());
assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0);
@@ -1216,7 +1261,7 @@ async fn gateway_updates_fixed_provider_and_reconciles_template_managed_endpoint
.list_endpoints_by_provider_ids(&["provider-codex".to_string()])
.await
.expect("endpoints should list");
assert_eq!(endpoints.len(), 3);
assert_eq!(endpoints.len(), 4);
let responses_endpoint = endpoints
.iter()
.find(|endpoint| endpoint.api_format == "openai:responses")
@@ -1225,6 +1270,10 @@ async fn gateway_updates_fixed_provider_and_reconciles_template_managed_endpoint
.iter()
.find(|endpoint| endpoint.api_format == "openai:responses:compact")
.expect("compact endpoint should exist");
let search_endpoint = endpoints
.iter()
.find(|endpoint| endpoint.api_format == "openai:search")
.expect("search endpoint should exist");
let image_endpoint = endpoints
.iter()
.find(|endpoint| endpoint.api_format == "openai:image")
@@ -1232,6 +1281,7 @@ async fn gateway_updates_fixed_provider_and_reconciles_template_managed_endpoint
assert_eq!(responses_endpoint.max_retries, Some(9));
assert_eq!(compact_endpoint.max_retries, Some(9));
assert_eq!(search_endpoint.max_retries, Some(9));
assert_eq!(image_endpoint.max_retries, Some(9));
assert_eq!(
responses_endpoint
@@ -1242,20 +1292,37 @@ async fn gateway_updates_fixed_provider_and_reconciles_template_managed_endpoint
.and_then(serde_json::Value::as_bool),
Some(true)
);
assert_eq!(
search_endpoint
.config
.as_ref()
.and_then(|value| value.get("_aether_fixed_provider_template"))
.and_then(|value| value.get("managed"))
.and_then(serde_json::Value::as_bool),
Some(true)
);
assert_eq!(
search_endpoint
.config
.as_ref()
.and_then(|value| value.get("upstream_stream_policy"))
.and_then(serde_json::Value::as_str),
None
);
assert_eq!(
image_endpoint
.config
.as_ref()
.and_then(|value| value.get("upstream_stream_policy"))
.and_then(serde_json::Value::as_str),
Some("force_stream")
None
);
let keys = provider_catalog_repository
.list_keys_by_provider_ids(&["provider-codex".to_string()])
.await
.expect("keys should list");
assert_eq!(keys.len(), 1);
assert!(keys[0].api_formats.is_none());
assert_eq!(keys[0].api_formats, Some(json!(["openai:responses"])));
assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
@@ -1009,6 +1009,26 @@ async fn gateway_handles_admin_stats_provider_performance_locally_with_trusted_a
payload["timeline"][1]["avg_first_byte_time_ms"],
serde_json::Value::Null
);
let without_timeline_response = admin_request(reqwest::Client::new().get(format!(
"{gateway_url}/api/admin/stats/performance/providers?start_date=2024-03-21&end_date=2024-03-21&granularity=hour&limit=2&tz_offset_minutes=0&include_timeline=false"
)))
.send()
.await
.expect("request without timeline should succeed");
assert_eq!(without_timeline_response.status(), StatusCode::OK);
let without_timeline_payload: serde_json::Value = without_timeline_response
.json()
.await
.expect("json body without timeline should parse");
assert_eq!(without_timeline_payload["summary"], payload["summary"]);
assert_eq!(without_timeline_payload["providers"], payload["providers"]);
assert_eq!(
without_timeline_payload["timeline"]
.as_array()
.map(Vec::len),
Some(0)
);
assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0);
gateway_handle.abort();
@@ -1379,7 +1399,7 @@ async fn gateway_handles_admin_stats_leaderboard_models_locally_with_trusted_adm
assert_eq!(payload["metric"], "tokens");
assert_eq!(payload["items"][0]["rank"], 1);
assert_eq!(payload["items"][0]["id"], "gpt-5");
assert_eq!(payload["items"][0]["value"], 160);
assert_eq!(payload["items"][0]["value"], 150);
assert_eq!(payload["items"][1]["id"], "claude-3-5-sonnet");
assert_eq!(payload["items"][1]["value"], 100);
assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0);
@@ -641,7 +641,7 @@ async fn gateway_handles_admin_usage_aggregation_stats_locally_with_trusted_admi
assert_eq!(items[0]["model"], "gpt-5");
assert_eq!(items[0]["request_count"], 2);
assert_eq!(items[0]["output_tokens"], 40);
assert_eq!(items[0]["effective_input_tokens"], 150);
assert_eq!(items[0]["effective_input_tokens"], 120);
assert_eq!(items[0]["total_input_context"], 160);
assert_eq!(items[0]["cache_creation_tokens"], 30);
assert_eq!(items[0]["cache_creation_ephemeral_5m_tokens"], 12);
@@ -1026,7 +1026,7 @@ async fn gateway_handles_admin_usage_active_locally_with_trusted_admin_principal
let payload: serde_json::Value = response.json().await.expect("json body should parse");
assert_eq!(payload["requests"].as_array().expect("array").len(), 1);
assert_eq!(payload["requests"][0]["id"], "usage-pending");
assert_eq!(payload["requests"][0]["effective_input_tokens"], 5);
assert_eq!(payload["requests"][0]["effective_input_tokens"], 0);
assert_eq!(payload["requests"][0]["provider"], "OpenAI");
assert_eq!(payload["requests"][0]["api_key_name"], "fresh-primary");
assert_eq!(payload["requests"][0]["has_fallback"], true);
@@ -1326,7 +1326,7 @@ async fn gateway_handles_admin_usage_records_locally_with_trusted_admin_principa
payload["records"][0]["provider_key_name"],
"upstream-primary"
);
assert_eq!(payload["records"][0]["effective_input_tokens"], 35);
assert_eq!(payload["records"][0]["effective_input_tokens"], 20);
assert_eq!(payload["records"][0]["first_byte_time_ms"], 120);
assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0);
@@ -2053,8 +2053,8 @@ async fn gateway_handles_admin_usage_detail_locally_with_trusted_admin_principal
assert_eq!(payload["api_key"]["name"], "primary");
assert_eq!(payload["provider"], "OpenAI");
assert_eq!(payload["model"], "gpt-5");
assert_eq!(payload["effective_input_tokens"], 115);
assert_eq!(payload["total_tokens"], 165);
assert_eq!(payload["effective_input_tokens"], 100);
assert_eq!(payload["total_tokens"], 150);
assert_eq!(payload["cache_creation_cost"], 0.0);
assert_eq!(payload["cache_read_cost"], 0.0);
assert_eq!(
+128 -18
View File
@@ -1,6 +1,7 @@
use std::io;
use std::sync::{Arc, Mutex};
use aether_contracts::tunnel::RequestMeta;
use aether_data::repository::proxy_nodes::ProxyNodeReadRepository;
use axum::body::Body;
use axum::routing::{any, post};
@@ -10,12 +11,45 @@ use futures_util::stream;
use http::header::HeaderValue;
use http::StatusCode;
use serde_json::json;
use std::collections::HashMap;
use std::time::Duration;
use super::{
build_router_with_state, sample_proxy_node, start_server, AppState, GatewayDataState,
InMemoryProxyNodeRepository, TRACE_ID_HEADER,
};
fn relay_request_meta(
stream: bool,
request_timeout_ms: Option<u64>,
stream_first_byte_timeout_ms: Option<u64>,
) -> RequestMeta {
RequestMeta {
provider_id: Some("provider-1".to_string()),
endpoint_id: Some("endpoint-1".to_string()),
key_id: Some("key-1".to_string()),
method: "POST".to_string(),
url: "https://example.com/responses".to_string(),
headers: HashMap::new(),
stream,
request_timeout_ms,
stream_first_byte_timeout_ms,
timeout: 60,
follow_redirects: None,
http1_only: false,
transport_profile: None,
}
}
fn relay_envelope(meta: &RequestMeta, body: &[u8]) -> Vec<u8> {
let encoded_meta = serde_json::to_vec(meta).expect("metadata should encode");
let mut envelope = Vec::with_capacity(4 + encoded_meta.len() + body.len());
envelope.extend_from_slice(&(encoded_meta.len() as u32).to_be_bytes());
envelope.extend_from_slice(&encoded_meta);
envelope.extend_from_slice(body);
envelope
}
#[tokio::test]
async fn gateway_handles_internal_tunnel_heartbeat_locally_with_loopback() {
let upstream_hits = Arc::new(Mutex::new(0usize));
@@ -292,10 +326,13 @@ async fn gateway_forwards_tunnel_relay_to_attachment_owner() {
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let envelope = relay_envelope(&relay_request_meta(false, None, None), b"relay-envelope");
let response = reqwest::Client::new()
.post(format!("{gateway_url}/api/internal/tunnel/relay/node-123"))
.header(TRACE_ID_HEADER, "trace-owner-forward")
.body("relay-envelope")
.header(http::header::CONTENT_TYPE, "application/octet-stream")
.body(envelope.clone())
.send()
.await
.expect("request should succeed");
@@ -309,8 +346,8 @@ async fn gateway_forwards_tunnel_relay_to_attachment_owner() {
Some("trace-owner-forward")
);
assert_eq!(
response.text().await.expect("body should read"),
"relay-envelope"
response.bytes().await.expect("body should read"),
Bytes::from(envelope)
);
assert_eq!(*owner_hits.lock().expect("mutex should lock"), 1);
@@ -318,6 +355,71 @@ async fn gateway_forwards_tunnel_relay_to_attachment_owner() {
owner_handle.abort();
}
#[tokio::test]
async fn gateway_owner_relay_uses_non_stream_timeout_from_envelope() {
let owner = Router::new().route(
"/api/internal/tunnel/relay/node-123",
post(|body: Body| async move {
let body = axum::body::to_bytes(body, usize::MAX)
.await
.expect("body should read");
tokio::time::sleep(Duration::from_millis(40)).await;
(StatusCode::OK, Body::from(body))
}),
);
let (owner_url, owner_handle) = start_server(owner).await;
let data_state = GatewayDataState::disabled().with_system_config_values_for_tests(vec![(
"tunnel.attachments.node-123".to_string(),
json!({
"gateway_instance_id": "gateway-b",
"relay_base_url": owner_url,
"conn_count": 1,
"observed_at_unix_secs": 4_102_444_800u64,
}),
)]);
let mut state = AppState::new()
.expect("gateway should build")
.with_data_state_for_tests(data_state)
.with_tunnel_identity_for_tests("gateway-a", Some("http://gateway-a.internal"));
let short_timeout_client = reqwest::Client::builder()
.timeout(Duration::from_millis(10))
.build()
.expect("test client should build");
state.client = short_timeout_client.clone();
state.owner_forward_client = short_timeout_client;
let gateway = build_router_with_state(state);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let meta = relay_request_meta(false, Some(100), None);
let envelope = relay_envelope(&meta, b"relay-body");
let encoded_meta = serde_json::to_vec(&meta).expect("metadata should encode");
let split_at = 4 + encoded_meta.len() / 2;
let request_body = reqwest::Body::wrap_stream(stream::iter(vec![
Ok::<Bytes, io::Error>(Bytes::copy_from_slice(&envelope[..split_at])),
Ok::<Bytes, io::Error>(Bytes::copy_from_slice(&envelope[split_at..])),
]));
let response = reqwest::Client::builder()
.timeout(Duration::from_secs(1))
.build()
.expect("request client should build")
.post(format!("{gateway_url}/api/internal/tunnel/relay/node-123"))
.body(request_body)
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response.bytes().await.expect("response body should read"),
Bytes::from(envelope)
);
gateway_handle.abort();
owner_handle.abort();
}
#[tokio::test]
async fn gateway_streams_tunnel_relay_body_to_attachment_owner() {
let owner_hits = Arc::new(Mutex::new(0usize));
@@ -331,8 +433,12 @@ async fn gateway_streams_tunnel_relay_body_to_attachment_owner() {
let body = axum::body::to_bytes(body, usize::MAX)
.await
.expect("body should read");
assert_eq!(body, Bytes::from_static(b"relay-stream-envelope"));
(StatusCode::OK, Body::from("stream-ok"))
let response_body = Body::from_stream(async_stream::stream! {
yield Ok::<_, io::Error>(Bytes::from_static(b"stream-"));
tokio::time::sleep(Duration::from_millis(40)).await;
yield Ok::<_, io::Error>(Bytes::from_static(b"ok"));
});
(StatusCode::OK, response_body)
}
}),
);
@@ -347,19 +453,23 @@ async fn gateway_streams_tunnel_relay_body_to_attachment_owner() {
"observed_at_unix_secs": 4_102_444_800u64,
}),
)]);
let gateway = build_router_with_state(
AppState::new()
.expect("gateway should build")
.with_data_state_for_tests(data_state)
.with_tunnel_identity_for_tests("gateway-a", Some("http://gateway-a.internal")),
);
let mut state = AppState::new()
.expect("gateway should build")
.with_data_state_for_tests(data_state)
.with_tunnel_identity_for_tests("gateway-a", Some("http://gateway-a.internal"));
state.client = reqwest::Client::builder()
.timeout(Duration::from_millis(10))
.build()
.expect("short shared client should build");
let gateway = build_router_with_state(state);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let request_body = reqwest::Body::wrap_stream(stream::iter(vec![
Ok::<Bytes, io::Error>(Bytes::from_static(b"relay-")),
Ok::<Bytes, io::Error>(Bytes::from_static(b"stream-")),
Ok::<Bytes, io::Error>(Bytes::from_static(b"envelope")),
]));
let meta = relay_request_meta(true, Some(900_000), Some(100));
let envelope = relay_envelope(&meta, b"relay-stream-envelope");
let expected_envelope = Bytes::copy_from_slice(&envelope);
let request_body = reqwest::Body::wrap_stream(stream::iter(vec![Ok::<Bytes, io::Error>(
expected_envelope.clone(),
)]));
let response = reqwest::Client::new()
.post(format!("{gateway_url}/api/internal/tunnel/relay/node-123"))
.body(request_body)
@@ -369,8 +479,8 @@ async fn gateway_streams_tunnel_relay_body_to_attachment_owner() {
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response.text().await.expect("body should read"),
"stream-ok"
response.bytes().await.expect("body should read"),
Bytes::from_static(b"stream-ok")
);
assert_eq!(*owner_hits.lock().expect("mutex should lock"), 1);