i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 09:20:22 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(proxy): record tunnel stability metrics

Browse Source
This commit is contained in:
fawney19
2026-05-08 22:03:17 +08:00
parent 9a84a6ff6c
commit a703acd1fe
81 changed files with 7229 additions and 1110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

715
apps/aether-gateway/src/tests/ai_execute/sync/image.rs
View File

@@ -19,6 +19,721 @@ use aether_data_contracts::repository::provider_catalog::{
use base64::Engine as _;
use sha2::{Digest, Sha256};
#[tokio::test]
async fn gateway_converts_openai_image_sync_to_gemini_image_provider() {
#[derive(Debug, Clone)]
struct SeenExecutionRuntimeSyncRequest {
trace_id: String,
url: String,
auth_header_value: String,
has_model_field: bool,
prompt: String,
response_modalities: Vec<String>,
image_size: String,
}
fn hash_api_key(value: &str) -> String {
let mut hasher = Sha256::new();
hasher.update(value.as_bytes());
format!("{:x}", hasher.finalize())
}
fn sample_auth_snapshot(api_key_id: &str, user_id: &str) -> StoredAuthApiKeySnapshot {
StoredAuthApiKeySnapshot::new(
user_id.to_string(),
"alice".to_string(),
Some("alice@example.com".to_string()),
"user".to_string(),
"local".to_string(),
true,
false,
Some(serde_json::json!(["openai", "google"])),
Some(serde_json::json!(["openai:image"])),
Some(serde_json::json!(["gpt-image-2"])),
api_key_id.to_string(),
Some("default".to_string()),
true,
false,
false,
Some(60),
Some(5),
Some(4_102_444_800_i64),
Some(serde_json::json!(["openai", "google"])),
Some(serde_json::json!(["openai:image"])),
Some(serde_json::json!(["gpt-image-2"])),
)
.expect("auth snapshot should build")
}
fn sample_candidate_row() -> StoredMinimalCandidateSelectionRow {
StoredMinimalCandidateSelectionRow {
provider_id: "provider-gemini-image-bridge-1".to_string(),
provider_name: "google".to_string(),
provider_type: "google".to_string(),
provider_priority: 10,
provider_is_active: true,
endpoint_id: "endpoint-gemini-image-bridge-1".to_string(),
endpoint_api_format: "gemini:generate_content".to_string(),
endpoint_api_family: Some("gemini".to_string()),
endpoint_kind: Some("chat".to_string()),
endpoint_is_active: true,
key_id: "key-gemini-image-bridge-1".to_string(),
key_name: "prod".to_string(),
key_auth_type: "api_key".to_string(),
key_is_active: true,
key_api_formats: Some(vec!["gemini:generate_content".to_string()]),
key_allowed_models: None,
key_capabilities: None,
key_internal_priority: 5,
key_global_priority_by_format: Some(serde_json::json!({
"gemini:generate_content": 1
})),
model_id: "model-gemini-image-bridge-1".to_string(),
global_model_id: "global-model-gemini-image-bridge-1".to_string(),
global_model_name: "gpt-image-2".to_string(),
global_model_mappings: None,
global_model_supports_streaming: Some(true),
model_provider_model_name: "gemini-2.5-flash-image-upstream".to_string(),
model_provider_model_mappings: Some(vec![StoredProviderModelMapping {
name: "gemini-2.5-flash-image-upstream".to_string(),
priority: 1,
api_formats: Some(vec!["gemini:generate_content".to_string()]),
endpoint_ids: None,
}]),
model_supports_streaming: Some(true),
model_is_active: true,
model_is_available: true,
}
}
fn sample_provider_catalog_provider() -> StoredProviderCatalogProvider {
StoredProviderCatalogProvider::new(
"provider-gemini-image-bridge-1".to_string(),
"google".to_string(),
Some("https://generativelanguage.googleapis.com".to_string()),
"google".to_string(),
)
.expect("provider should build")
.with_transport_fields(
true,
false,
false,
None,
Some(2),
None,
Some(20.0),
None,
None,
)
}
fn sample_provider_catalog_endpoint() -> StoredProviderCatalogEndpoint {
StoredProviderCatalogEndpoint::new(
"endpoint-gemini-image-bridge-1".to_string(),
"provider-gemini-image-bridge-1".to_string(),
"gemini:generate_content".to_string(),
Some("gemini".to_string()),
Some("chat".to_string()),
true,
)
.expect("endpoint should build")
.with_transport_fields(
"https://generativelanguage.googleapis.com".to_string(),
None,
Some(serde_json::json!([
{"action":"drop","path":"model"}
])),
Some(2),
None,
None,
None,
None,
)
.expect("endpoint transport should build")
}
fn sample_provider_catalog_key() -> StoredProviderCatalogKey {
StoredProviderCatalogKey::new(
"key-gemini-image-bridge-1".to_string(),
"provider-gemini-image-bridge-1".to_string(),
"prod".to_string(),
"api_key".to_string(),
None,
true,
)
.expect("key should build")
.with_transport_fields(
Some(serde_json::json!(["gemini:generate_content"])),
encrypt_python_fernet_plaintext(DEVELOPMENT_ENCRYPTION_KEY, "sk-upstream-gemini-image")
.expect("api key should encrypt"),
None,
None,
Some(serde_json::json!({"gemini:generate_content": 1})),
None,
None,
None,
None,
)
.expect("key transport should build")
}
let seen_execution_runtime = Arc::new(Mutex::new(None::<SeenExecutionRuntimeSyncRequest>));
let seen_execution_runtime_clone = Arc::clone(&seen_execution_runtime);
let execution_runtime = Router::new().route(
"/v1/execute/sync",
any(move |request: Request| {
let seen_execution_runtime_inner = Arc::clone(&seen_execution_runtime_clone);
async move {
let (parts, body) = request.into_parts();
let raw_body = to_bytes(body, usize::MAX).await.expect("body should read");
let payload: serde_json::Value = serde_json::from_slice(&raw_body)
.expect("execution runtime payload should parse");
let body_json = payload
.get("body")
.and_then(|value| value.get("json_body"))
.cloned()
.unwrap_or_else(|| json!({}));
*seen_execution_runtime_inner
.lock()
.expect("mutex should lock") = Some(SeenExecutionRuntimeSyncRequest {
trace_id: parts
.headers
.get(TRACE_ID_HEADER)
.and_then(|value| value.to_str().ok())
.unwrap_or_default()
.to_string(),
url: payload
.get("url")
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
auth_header_value: payload
.get("headers")
.and_then(|value| value.get("x-goog-api-key"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
has_model_field: body_json.get("model").is_some(),
prompt: body_json
.get("contents")
.and_then(|value| value.get(0))
.and_then(|value| value.get("parts"))
.and_then(|value| value.get(0))
.and_then(|value| value.get("text"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
response_modalities: body_json
.get("generationConfig")
.and_then(|value| value.get("responseModalities"))
.and_then(|value| value.as_array())
.map(|items| {
items
.iter()
.filter_map(|item| item.as_str().map(ToOwned::to_owned))
.collect()
})
.unwrap_or_default(),
image_size: body_json
.get("generationConfig")
.and_then(|value| value.get("imageSize"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
});
Json(json!({
"request_id": "trace-openai-image-to-gemini-123",
"status_code": 200,
"headers": {
"content-type": "application/json"
},
"body": {
"json_body": {
"modelVersion": "gemini-2.5-flash-image-upstream",
"usageMetadata": {
"promptTokenCount": 11,
"candidatesTokenCount": 22,
"totalTokenCount": 33
},
"candidates": [{
"content": {
"role": "model",
"parts": [
{"text": "revised kite prompt"},
{"inlineData": {"mimeType": "image/png", "data": "aGVsbG8="}}
]
},
"finishReason": "STOP"
}]
}
},
"telemetry": {
"elapsed_ms": 37
}
}))
}
}),
);
let client_api_key = "sk-client-openai-image-to-gemini";
let auth_repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
Some(hash_api_key(client_api_key)),
sample_auth_snapshot(
"key-openai-image-client-bridge-1",
"user-openai-image-bridge-1",
),
)]));
let candidate_selection_repository =
Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(vec![
sample_candidate_row(),
]));
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider_catalog_provider()],
vec![sample_provider_catalog_endpoint()],
vec![sample_provider_catalog_key()],
));
let (execution_runtime_url, execution_runtime_handle) = start_server(execution_runtime).await;
let gateway_state = build_state_with_execution_runtime_override(execution_runtime_url)
.with_data_state_for_tests(
crate::data::GatewayDataState::with_auth_candidate_selection_provider_catalog_and_request_candidate_repository_for_tests(
auth_repository,
candidate_selection_repository,
provider_catalog_repository,
Arc::new(InMemoryRequestCandidateRepository::default()),
DEVELOPMENT_ENCRYPTION_KEY,
),
);
let gateway = build_router_with_state(gateway_state);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.post(format!("{gateway_url}/v1/images/generations"))
.header(http::header::CONTENT_TYPE, "application/json")
.header(
http::header::AUTHORIZATION,
format!("Bearer {client_api_key}"),
)
.header(TRACE_ID_HEADER, "trace-openai-image-to-gemini-123")
.body(
"{\"model\":\"gpt-image-2\",\"prompt\":\"Draw a red kite\",\"size\":\"1024x1024\",\"response_format\":\"b64_json\"}",
)
.send()
.await
.expect("request should succeed");
let response_status = response.status();
let response_body = response.text().await.expect("body should read");
assert_eq!(response_status, StatusCode::OK, "{response_body}");
let response_json: serde_json::Value =
serde_json::from_str(&response_body).expect("body should parse");
assert_eq!(response_json["data"][0]["b64_json"], "aGVsbG8=");
assert_eq!(
response_json["data"][0]["revised_prompt"],
"revised kite prompt"
);
assert_eq!(response_json["model"], "gemini-2.5-flash-image-upstream");
assert_eq!(response_json["usage"]["input_tokens"], 11);
assert_eq!(response_json["usage"]["output_tokens"], 22);
let seen_execution_runtime_request = seen_execution_runtime
.lock()
.expect("mutex should lock")
.clone()
.expect("execution runtime sync should be captured");
assert_eq!(
seen_execution_runtime_request.trace_id,
"trace-openai-image-to-gemini-123"
);
assert_eq!(
seen_execution_runtime_request.url,
"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash-image-upstream:generateContent"
);
assert_eq!(
seen_execution_runtime_request.auth_header_value,
"sk-upstream-gemini-image"
);
assert!(!seen_execution_runtime_request.has_model_field);
assert_eq!(seen_execution_runtime_request.prompt, "Draw a red kite");
assert_eq!(
seen_execution_runtime_request.response_modalities,
vec!["TEXT".to_string(), "IMAGE".to_string()]
);
assert_eq!(seen_execution_runtime_request.image_size, "1024x1024");
gateway_handle.abort();
execution_runtime_handle.abort();
}
#[tokio::test]
async fn gateway_converts_gemini_image_sync_to_openai_image_provider() {
#[derive(Debug, Clone)]
struct SeenExecutionRuntimeSyncRequest {
trace_id: String,
url: String,
authorization: String,
model: String,
action: String,
prompt: String,
image_url: String,
request_stream: bool,
}
fn hash_api_key(value: &str) -> String {
let mut hasher = Sha256::new();
hasher.update(value.as_bytes());
format!("{:x}", hasher.finalize())
}
fn sample_auth_snapshot(api_key_id: &str, user_id: &str) -> StoredAuthApiKeySnapshot {
StoredAuthApiKeySnapshot::new(
user_id.to_string(),
"alice".to_string(),
Some("alice@example.com".to_string()),
"user".to_string(),
"local".to_string(),
true,
false,
Some(serde_json::json!(["gemini", "openai"])),
Some(serde_json::json!(["gemini:generate_content"])),
Some(serde_json::json!(["gemini-image"])),
api_key_id.to_string(),
Some("default".to_string()),
true,
false,
false,
Some(60),
Some(5),
Some(4_102_444_800_i64),
Some(serde_json::json!(["gemini", "openai"])),
Some(serde_json::json!(["gemini:generate_content"])),
Some(serde_json::json!(["gemini-image"])),
)
.expect("auth snapshot should build")
}
fn sample_candidate_row() -> StoredMinimalCandidateSelectionRow {
StoredMinimalCandidateSelectionRow {
provider_id: "provider-openai-image-bridge-1".to_string(),
provider_name: "openai".to_string(),
provider_type: "openai".to_string(),
provider_priority: 10,
provider_is_active: true,
endpoint_id: "endpoint-openai-image-bridge-1".to_string(),
endpoint_api_format: "openai:image".to_string(),
endpoint_api_family: Some("openai".to_string()),
endpoint_kind: Some("image".to_string()),
endpoint_is_active: true,
key_id: "key-openai-image-bridge-1".to_string(),
key_name: "prod".to_string(),
key_auth_type: "api_key".to_string(),
key_is_active: true,
key_api_formats: Some(vec!["openai:image".to_string()]),
key_allowed_models: None,
key_capabilities: None,
key_internal_priority: 5,
key_global_priority_by_format: Some(serde_json::json!({"openai:image": 1})),
model_id: "model-openai-image-bridge-1".to_string(),
global_model_id: "global-model-openai-image-bridge-1".to_string(),
global_model_name: "gemini-image".to_string(),
global_model_mappings: None,
global_model_supports_streaming: Some(true),
model_provider_model_name: "gpt-image-2-upstream".to_string(),
model_provider_model_mappings: Some(vec![StoredProviderModelMapping {
name: "gpt-image-2-upstream".to_string(),
priority: 1,
api_formats: Some(vec!["openai:image".to_string()]),
endpoint_ids: None,
}]),
model_supports_streaming: Some(true),
model_is_active: true,
model_is_available: true,
}
}
fn sample_provider_catalog_provider() -> StoredProviderCatalogProvider {
StoredProviderCatalogProvider::new(
"provider-openai-image-bridge-1".to_string(),
"openai".to_string(),
Some("https://api.openai.com".to_string()),
"openai".to_string(),
)
.expect("provider should build")
.with_transport_fields(
true,
false,
false,
None,
Some(2),
None,
Some(20.0),
None,
None,
)
}
fn sample_provider_catalog_endpoint() -> StoredProviderCatalogEndpoint {
StoredProviderCatalogEndpoint::new(
"endpoint-openai-image-bridge-1".to_string(),
"provider-openai-image-bridge-1".to_string(),
"openai:image".to_string(),
Some("openai".to_string()),
Some("image".to_string()),
true,
)
.expect("endpoint should build")
.with_transport_fields(
"https://api.openai.com".to_string(),
None,
None,
Some(2),
None,
None,
None,
None,
)
.expect("endpoint transport should build")
}
fn sample_provider_catalog_key() -> StoredProviderCatalogKey {
StoredProviderCatalogKey::new(
"key-openai-image-bridge-1".to_string(),
"provider-openai-image-bridge-1".to_string(),
"prod".to_string(),
"api_key".to_string(),
None,
true,
)
.expect("key should build")
.with_transport_fields(
Some(serde_json::json!(["openai:image"])),
encrypt_python_fernet_plaintext(DEVELOPMENT_ENCRYPTION_KEY, "sk-upstream-openai-image")
.expect("api key should encrypt"),
None,
None,
Some(serde_json::json!({"openai:image": 1})),
None,
None,
None,
None,
)
.expect("key transport should build")
}
let seen_execution_runtime = Arc::new(Mutex::new(None::<SeenExecutionRuntimeSyncRequest>));
let seen_execution_runtime_clone = Arc::clone(&seen_execution_runtime);
let execution_runtime = Router::new().route(
"/v1/execute/sync",
any(move |request: Request| {
let seen_execution_runtime_inner = Arc::clone(&seen_execution_runtime_clone);
async move {
let (parts, body) = request.into_parts();
let raw_body = to_bytes(body, usize::MAX).await.expect("body should read");
let payload: serde_json::Value = serde_json::from_slice(&raw_body)
.expect("execution runtime payload should parse");
let body_json = payload
.get("body")
.and_then(|value| value.get("json_body"))
.cloned()
.unwrap_or_else(|| json!({}));
let content = body_json
.get("input")
.and_then(|value| value.get(0))
.and_then(|value| value.get("content"))
.cloned()
.unwrap_or_else(|| json!([]));
*seen_execution_runtime_inner
.lock()
.expect("mutex should lock") = Some(SeenExecutionRuntimeSyncRequest {
trace_id: parts
.headers
.get(TRACE_ID_HEADER)
.and_then(|value| value.to_str().ok())
.unwrap_or_default()
.to_string(),
url: payload
.get("url")
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
authorization: payload
.get("headers")
.and_then(|value| value.get("authorization"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
model: body_json
.get("model")
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
action: body_json
.get("tools")
.and_then(|value| value.get(0))
.and_then(|value| value.get("action"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
prompt: content
.as_array()
.into_iter()
.flatten()
.find(|item| {
item.get("type").and_then(|value| value.as_str()) == Some("input_text")
})
.and_then(|item| item.get("text"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
image_url: content
.as_array()
.into_iter()
.flatten()
.find(|item| {
item.get("type").and_then(|value| value.as_str()) == Some("input_image")
})
.and_then(|item| item.get("image_url"))
.and_then(|value| value.as_str())
.unwrap_or_default()
.to_string(),
request_stream: body_json
.get("stream")
.and_then(|value| value.as_bool())
.unwrap_or(true),
});
Json(json!({
"request_id": "trace-gemini-image-to-openai-123",
"status_code": 200,
"headers": {
"content-type": "application/json"
},
"body": {
"json_body": {
"id": "resp_img_bridge_123",
"object": "response",
"model": "gpt-image-2-upstream",
"status": "completed",
"usage": {
"input_tokens": 3,
"output_tokens": 4,
"total_tokens": 7
},
"output": [{
"type": "image_generation_call",
"status": "completed",
"output_format": "png",
"revised_prompt": "converted gemini prompt",
"result": "aGVsbG8="
}]
}
},
"telemetry": {
"elapsed_ms": 43
}
}))
}
}),
);
let client_api_key = "client-gemini-image-to-openai";
let auth_repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
Some(hash_api_key(client_api_key)),
sample_auth_snapshot(
"key-gemini-image-client-bridge-1",
"user-gemini-image-bridge-1",
),
)]));
let candidate_selection_repository =
Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(vec![
sample_candidate_row(),
]));
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider_catalog_provider()],
vec![sample_provider_catalog_endpoint()],
vec![sample_provider_catalog_key()],
));
let (execution_runtime_url, execution_runtime_handle) = start_server(execution_runtime).await;
let gateway_state = build_state_with_execution_runtime_override(execution_runtime_url)
.with_data_state_for_tests(
crate::data::GatewayDataState::with_auth_candidate_selection_provider_catalog_and_request_candidate_repository_for_tests(
auth_repository,
candidate_selection_repository,
provider_catalog_repository,
Arc::new(InMemoryRequestCandidateRepository::default()),
DEVELOPMENT_ENCRYPTION_KEY,
),
);
let gateway = build_router_with_state(gateway_state);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.post(format!(
"{gateway_url}/v1beta/models/gemini-image:generateContent?key={client_api_key}"
))
.header(http::header::CONTENT_TYPE, "application/json")
.header(TRACE_ID_HEADER, "trace-gemini-image-to-openai-123")
.body(
"{\"generationConfig\":{\"responseModalities\":[\"TEXT\",\"IMAGE\"]},\"contents\":[{\"role\":\"user\",\"parts\":[{\"text\":\"Change the background\"},{\"inlineData\":{\"mimeType\":\"image/png\",\"data\":\"aGVsbG8=\"}}]}]}",
)
.send()
.await
.expect("request should succeed");
let response_status = response.status();
let response_body = response.text().await.expect("body should read");
assert_eq!(response_status, StatusCode::OK, "{response_body}");
let response_json: serde_json::Value =
serde_json::from_str(&response_body).expect("body should parse");
assert_eq!(response_json["modelVersion"], "gpt-image-2-upstream");
assert_eq!(
response_json["candidates"][0]["content"]["parts"][0]["text"],
"converted gemini prompt"
);
assert_eq!(
response_json["candidates"][0]["content"]["parts"][1]["inlineData"]["mimeType"],
"image/png"
);
assert_eq!(
response_json["candidates"][0]["content"]["parts"][1]["inlineData"]["data"],
"aGVsbG8="
);
assert_eq!(response_json["usageMetadata"]["promptTokenCount"], 3);
assert_eq!(response_json["usageMetadata"]["candidatesTokenCount"], 4);
let seen_execution_runtime_request = seen_execution_runtime
.lock()
.expect("mutex should lock")
.clone()
.expect("execution runtime sync should be captured");
assert_eq!(
seen_execution_runtime_request.trace_id,
"trace-gemini-image-to-openai-123"
);
assert_eq!(
seen_execution_runtime_request.url,
"https://api.openai.com/v1/responses"
);
assert_eq!(
seen_execution_runtime_request.authorization,
"Bearer sk-upstream-openai-image"
);
assert_eq!(seen_execution_runtime_request.model, "gpt-image-2-upstream");
assert_eq!(seen_execution_runtime_request.action, "edit");
assert_eq!(
seen_execution_runtime_request.prompt,
"Change the background"
);
assert_eq!(
seen_execution_runtime_request.image_url,
"data:image/png;base64,aGVsbG8="
);
assert!(!seen_execution_runtime_request.request_stream);
gateway_handle.abort();
execution_runtime_handle.abort();
}
#[tokio::test]
async fn gateway_executes_codex_image_sync_via_local_decision_gate_after_oauth_refresh() {
#[derive(Debug, Clone)]

147
apps/aether-gateway/src/tests/control/admin/proxy_nodes.rs
View File

@@ -1,4 +1,5 @@
use std::sync::{Arc, Mutex};
use std::time::{SystemTime, UNIX_EPOCH};
use aether_data::repository::management_tokens::InMemoryManagementTokenRepository;
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
@@ -1442,6 +1443,7 @@ async fn gateway_handles_admin_proxy_node_events_locally_with_trusted_admin_prin
node_id: "node-1".to_string(),
event_type: "connected".to_string(),
detail: Some("older".to_string()),
event_metadata: None,
created_at_unix_ms: Some(1_710_000_000),
},
StoredProxyNodeEvent {
@@ -1449,6 +1451,7 @@ async fn gateway_handles_admin_proxy_node_events_locally_with_trusted_admin_prin
node_id: "node-1".to_string(),
event_type: "disconnected".to_string(),
detail: Some("newer".to_string()),
event_metadata: None,
created_at_unix_ms: Some(1_710_000_100),
},
],
@@ -1490,6 +1493,150 @@ async fn gateway_handles_admin_proxy_node_events_locally_with_trusted_admin_prin
upstream_handle.abort();
}
#[tokio::test]
async fn gateway_reports_proxy_node_metrics_and_filters_events_locally() {
let proxy_node_repository =
Arc::new(InMemoryProxyNodeRepository::seed(vec![sample_proxy_node(
"node-1",
)]));
let gateway = build_router_with_state(
AppState::new()
.expect("gateway should build")
.with_data_state_for_tests(GatewayDataState::with_proxy_node_repository_for_tests(
proxy_node_repository,
)),
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let client = reqwest::Client::new();
let now_unix_secs = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("system time should be after epoch")
.as_secs();
let heartbeat_response = client
.post(format!("{gateway_url}/api/internal/tunnel/heartbeat"))
.json(&json!({
"node_id": "node-1",
"heartbeat_id": 91,
"heartbeat_interval": 30,
"active_connections": 7,
"proxy_metadata": {
"tunnel_metrics": {
"connect_errors": 3,
"disconnects": 1,
"error_events_total": 1,
"ws_in_bytes": 1000,
"ws_out_bytes": 2000,
"ws_in_frames": 10,
"ws_out_frames": 20,
"heartbeat_rtt_last_ms": 42
},
"recent_tunnel_errors": [{
"timestamp_unix_secs": now_unix_secs,
"category": "tcp_connect_timeout",
"message": "tunnel TCP connect timeout"
}]
},
"proxy_version": "2.0.0"
}))
.send()
.await
.expect("heartbeat request should succeed");
assert_eq!(heartbeat_response.status(), StatusCode::OK);
let from = now_unix_secs.saturating_sub(120);
let to = now_unix_secs.saturating_add(120);
let metrics_response = client
.get(format!(
"{gateway_url}/api/admin/proxy-nodes/node-1/metrics?from={from}&to={to}&step=1m"
))
.header(GATEWAY_HEADER, "rust-phase3b")
.header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123")
.header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin")
.header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123")
.send()
.await
.expect("metrics request should succeed");
assert_eq!(metrics_response.status(), StatusCode::OK);
let metrics_payload: serde_json::Value = metrics_response
.json()
.await
.expect("metrics json should parse");
assert_eq!(metrics_payload["step"], "1m");
assert_eq!(metrics_payload["summary"]["samples"], 1);
assert_eq!(metrics_payload["summary"]["uptime_samples"], 1);
assert_eq!(metrics_payload["summary"]["active_connections_max"], 7);
assert_eq!(metrics_payload["summary"]["heartbeat_rtt_ms_avg"], 42.0);
assert_eq!(metrics_payload["summary"]["connect_errors_delta"], 3);
assert_eq!(metrics_payload["summary"]["ws_out_frames_delta"], 20);
let metric_items = metrics_payload["items"]
.as_array()
.expect("metrics items should be array");
assert_eq!(metric_items.len(), 1);
assert_eq!(metric_items[0]["node_id"], "node-1");
assert!(metric_items[0]["bucket_start"].is_string());
let fleet_response = client
.get(format!(
"{gateway_url}/api/admin/proxy-nodes/metrics/fleet?from={from}&to={to}&step=1m"
))
.header(GATEWAY_HEADER, "rust-phase3b")
.header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123")
.header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin")
.header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123")
.send()
.await
.expect("fleet metrics request should succeed");
assert_eq!(fleet_response.status(), StatusCode::OK);
let fleet_payload: serde_json::Value = fleet_response
.json()
.await
.expect("fleet json should parse");
assert_eq!(fleet_payload["summary"]["samples"], 1);
assert_eq!(fleet_payload["summary"]["error_events_delta"], 1);
let events_response = client
.get(format!(
"{gateway_url}/api/admin/proxy-nodes/node-1/events?from={from}&to={to}&event_type=tunnel_err"
))
.header(GATEWAY_HEADER, "rust-phase3b")
.header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123")
.header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin")
.header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123")
.send()
.await
.expect("events request should succeed");
assert_eq!(events_response.status(), StatusCode::OK);
let events_payload: serde_json::Value = events_response
.json()
.await
.expect("events json should parse");
let event_items = events_payload["items"]
.as_array()
.expect("event items should be array");
assert_eq!(event_items.len(), 1);
assert_eq!(event_items[0]["event_type"], "tunnel_err");
assert_eq!(
event_items[0]["event_metadata"]["category"],
"tcp_connect_timeout"
);
let invalid_metrics_response = client
.get(format!(
"{gateway_url}/api/admin/proxy-nodes/node-1/metrics?from={from}&to={to}&step=5m"
))
.header(GATEWAY_HEADER, "rust-phase3b")
.header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123")
.header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin")
.header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123")
.send()
.await
.expect("invalid metrics request should succeed");
assert_eq!(invalid_metrics_response.status(), StatusCode::BAD_REQUEST);
gateway_handle.abort();
}
#[tokio::test]
async fn gateway_updates_proxy_node_config_and_dispatches_upgrade_targets_locally() {
let upstream_hits = Arc::new(Mutex::new(0usize));