use std::sync::{Arc, Mutex}; use std::time::{SystemTime, UNIX_EPOCH}; use aether_data::repository::candidates::InMemoryRequestCandidateRepository; use aether_data::repository::global_models::InMemoryGlobalModelReadRepository; use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository; use aether_data::repository::quota::InMemoryProviderQuotaRepository; use aether_data_contracts::repository::candidates::RequestCandidateStatus; use aether_data_contracts::repository::global_models::{ GlobalModelReadRepository, StoredAdminGlobalModel, StoredAdminProviderModel, StoredProviderActiveGlobalModel, StoredProviderModelStats, StoredPublicGlobalModel, }; use aether_data_contracts::repository::provider_catalog::{ ProviderCatalogReadRepository, StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider, }; use axum::body::{Body, Bytes}; use axum::routing::any; use axum::{extract::Request, Router}; use http::{HeaderMap, HeaderValue, StatusCode}; use serde_json::json; use super::super::{ build_router_with_state, issue_test_admin_access_token, sample_admin_provider_model, sample_bound_key, sample_bound_key as sample_key, sample_bound_provider_proxy, sample_endpoint, sample_provider, sample_provider_active_global_model, sample_provider_model_stats, sample_provider_quota, sample_public_global_model_with_mappings, sample_request_candidate, start_server, AppState, }; use crate::admin_api::{ maybe_build_local_admin_providers_response, AdminAppState, AdminRequestContext, }; use crate::audit::AdminAuditEvent; use crate::constants::{ GATEWAY_HEADER, TRUSTED_ADMIN_MANAGEMENT_TOKEN_ID_HEADER, TRUSTED_ADMIN_SESSION_ID_HEADER, TRUSTED_ADMIN_USER_ID_HEADER, TRUSTED_ADMIN_USER_ROLE_HEADER, }; use crate::control::resolve_public_request_context; use crate::data::GatewayDataState; const ADMIN_PROVIDERS_DATA_UNAVAILABLE_DETAIL: &str = "Admin provider catalog data unavailable"; async fn provider_health_summary( endpoints: &[StoredProviderCatalogEndpoint], keys: &[StoredProviderCatalogKey], ) -> serde_json::Value { let repository = Arc::new(InMemoryProviderCatalogReadRepository::seed( vec![sample_provider("provider-openai", "openai", 10)], endpoints.to_vec(), keys.to_vec(), )); let state = AppState::new() .expect("gateway should build") .with_data_state_for_tests(GatewayDataState::with_provider_catalog_reader_for_tests( repository, )); let response = local_admin_providers_response( &state, http::Method::GET, "/api/admin/providers/summary", None, ) .await; assert_eq!(response.status(), StatusCode::OK); let body = axum::body::to_bytes(response.into_body(), 1024 * 1024) .await .expect("summary body should read"); let payload: serde_json::Value = serde_json::from_slice(&body).expect("summary should parse"); assert_eq!(payload["items"].as_array().map(Vec::len), Some(1)); payload["items"][0].clone() } #[tokio::test] async fn admin_provider_summary_health_preserves_redacted_key_summaries() { let endpoint = sample_endpoint( "endpoint-chat", "provider-openai", "openai:chat", "https://api.openai.example", ); let keys = [ ("key-api", "api_key", None, 0.25), ("key-oauth", "oauth", Some("{}"), 0.75), ] .into_iter() .map(|(key_id, auth_type, auth_config, score)| { let mut key = sample_key(key_id, "provider-openai", "openai:chat", "test") .with_health_fields(Some(json!({"openai:chat": {"health_score": score}})), None); key.auth_type = auth_type.to_string(); key.encrypted_api_key = Some("summary".to_string()); key.encrypted_auth_config = auth_config.map(ToOwned::to_owned); key }) .collect(); let repository = Arc::new(InMemoryProviderCatalogReadRepository::seed( vec![sample_provider("provider-openai", "openai", 10)], vec![endpoint], keys, )); let state = AppState::new() .expect("gateway should build") .with_data_state_for_tests(GatewayDataState::with_provider_catalog_reader_for_tests( repository, )); for uri in [ "/api/admin/providers/summary", "/api/admin/providers/provider-openai/summary", ] { let response = local_admin_providers_response(&state, http::Method::GET, uri, None).await; assert_eq!(response.status(), StatusCode::OK); let body = axum::body::to_bytes(response.into_body(), 1024 * 1024) .await .expect("summary body should read"); let payload: serde_json::Value = serde_json::from_slice(&body).expect("summary should parse"); let summary = if uri == "/api/admin/providers/summary" { &payload["items"][0] } else { &payload }; assert_eq!(summary["total_keys"], 2); assert_eq!(summary["active_keys"], 2); assert_eq!(summary["endpoint_health_details"][0]["total_keys"], 2); assert_eq!(summary["endpoint_health_details"][0]["active_keys"], 2); assert_eq!(summary["endpoint_health_details"][0]["health_score"], 0.5); assert_eq!(summary["avg_health_score"], 0.5); assert_eq!(summary["unhealthy_endpoints"], 0); } } #[tokio::test] async fn admin_provider_summary_health_ignores_disabled_keys() { let endpoint = sample_endpoint( "endpoint-chat", "provider-openai", "openai:chat", "https://api.openai.example", ); for (active_score, disabled_score) in [(0.2, 1.0), (1.0, 0.2), (0.0, 1.0)] { let active_key = sample_key("key-active", "provider-openai", "openai:chat", "test") .with_health_fields( Some(json!({"openai:chat": {"health_score": active_score}})), None, ); let mut disabled_key = sample_key("key-disabled", "provider-openai", "openai:chat", "test") .with_health_fields( Some(json!({"openai:chat": {"health_score": disabled_score}})), None, ); disabled_key.is_active = false; let payload = provider_health_summary(std::slice::from_ref(&endpoint), &[active_key, disabled_key]) .await; assert_eq!( payload["endpoint_health_details"][0]["health_score"], active_score ); assert_eq!(payload["endpoint_health_details"][0]["total_keys"], 2); assert_eq!(payload["endpoint_health_details"][0]["active_keys"], 1); assert_eq!(payload["avg_health_score"], active_score); assert_eq!( payload["unhealthy_endpoints"], usize::from(active_score < 0.5) ); } } #[tokio::test] async fn admin_provider_summary_health_does_not_inflate_observed_scores_with_missing_data() { let endpoint = sample_endpoint( "endpoint-chat", "provider-openai", "openai:chat", "https://api.openai.example", ); let keys = [ sample_key("key-observed", "provider-openai", "openai:chat", "test") .with_health_fields(Some(json!({"openai:chat": {"health_score": 0.2}})), None), sample_key("key-unobserved", "provider-openai", "openai:chat", "test"), sample_key("key-other-format", "provider-openai", "openai:chat", "test") .with_health_fields( Some(json!({"openai:responses": {"health_score": 1.0}})), None, ), ]; let payload = provider_health_summary(&[endpoint], &keys).await; assert_eq!(payload["endpoint_health_details"][0]["health_score"], 0.2); assert_eq!(payload["endpoint_health_details"][0]["active_keys"], 3); assert_eq!(payload["avg_health_score"], 0.2); assert_eq!(payload["unhealthy_endpoints"], 1); } #[tokio::test] async fn admin_provider_summary_health_is_unknown_without_active_observations() { let endpoint = sample_endpoint( "endpoint-chat", "provider-openai", "openai:chat", "https://api.openai.example", ); let mut disabled_key = sample_key("key-disabled", "provider-openai", "openai:chat", "test") .with_health_fields(Some(json!({"openai:chat": {"health_score": 0.2}})), None); disabled_key.is_active = false; for keys in [ Vec::new(), vec![sample_key( "key-unobserved", "provider-openai", "openai:chat", "test", )], vec![disabled_key], ] { let payload = provider_health_summary(std::slice::from_ref(&endpoint), &keys).await; assert_eq!( payload["endpoint_health_details"][0]["health_score"], json!(null) ); assert_eq!(payload["avg_health_score"], json!(null)); assert_eq!(payload["unhealthy_endpoints"], 0); } let payload = provider_health_summary(&[], &[]).await; assert_eq!(payload["avg_health_score"], json!(null)); assert_eq!(payload["unhealthy_endpoints"], 0); } #[tokio::test] async fn admin_provider_summary_health_excludes_disabled_and_unobserved_endpoints() { let mut disabled_endpoint = sample_endpoint( "endpoint-disabled", "provider-openai", "openai:responses", "https://api.openai.example", ); disabled_endpoint.is_active = false; let endpoints = [ sample_endpoint( "endpoint-chat", "provider-openai", "openai:chat", "https://api.openai.example", ), disabled_endpoint, sample_endpoint( "endpoint-unobserved", "provider-openai", "openai:embedding", "https://api.openai.example", ), ]; let keys = [ sample_key("key-chat", "provider-openai", "openai:chat", "test") .with_health_fields(Some(json!({"openai:chat": {"health_score": 0.8}})), None), sample_key( "key-responses", "provider-openai", "openai:responses", "test", ) .with_health_fields( Some(json!({"openai:responses": {"health_score": 0.2}})), None, ), sample_key( "key-unobserved", "provider-openai", "openai:embedding", "test", ), ]; let payload = provider_health_summary(&endpoints, &keys).await; assert_eq!(payload["endpoint_health_details"][0]["health_score"], 0.8); assert_eq!( payload["endpoint_health_details"][1]["health_score"], json!(null) ); assert_eq!( payload["endpoint_health_details"][2]["health_score"], json!(null) ); assert_eq!(payload["avg_health_score"], 0.8); assert_eq!(payload["unhealthy_endpoints"], 0); } fn trusted_admin_headers() -> HeaderMap { let mut headers = HeaderMap::new(); headers.insert(GATEWAY_HEADER, HeaderValue::from_static("rust-phase3b")); headers.insert( TRUSTED_ADMIN_USER_ID_HEADER, HeaderValue::from_static("admin-user-123"), ); headers.insert( TRUSTED_ADMIN_USER_ROLE_HEADER, HeaderValue::from_static("admin"), ); headers.insert( TRUSTED_ADMIN_SESSION_ID_HEADER, HeaderValue::from_static("session-123"), ); headers.insert( TRUSTED_ADMIN_MANAGEMENT_TOKEN_ID_HEADER, HeaderValue::from_static("management-token-123"), ); headers } async fn local_admin_providers_response( state: &AppState, method: http::Method, uri: &str, body: Option, ) -> axum::response::Response { let headers = trusted_admin_headers(); let request_context = resolve_public_request_context( state, &method, &uri.parse().expect("uri should parse"), &headers, "trace-123", ) .await .expect("request context should resolve"); let body_bytes = body.map(|value| Bytes::from(value.to_string())); maybe_build_local_admin_providers_response( &AdminAppState::new(state), &AdminRequestContext::new(&request_context), body_bytes.as_ref(), ) .await .expect("local providers response should build") .expect("providers route should resolve locally") } #[tokio::test] async fn gateway_handles_admin_providers_locally_with_trusted_admin_principal() { let upstream_hits = Arc::new(Mutex::new(0usize)); let upstream_hits_clone = Arc::clone(&upstream_hits); let upstream = Router::new().route( "/api/admin/providers/", any(move |_request: Request| { let upstream_hits_inner = Arc::clone(&upstream_hits_clone); async move { *upstream_hits_inner.lock().expect("mutex should lock") += 1; (StatusCode::OK, Body::from("unexpected upstream hit")) } }), ); let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed( vec![ sample_provider("provider-openai", "openai", 10) .with_timestamps(Some(1_711_000_000), Some(1_711_000_100)), sample_provider("provider-anthropic", "anthropic", 20) .with_timestamps(Some(1_710_000_000), Some(1_710_000_100)), ], vec![sample_endpoint( "endpoint-openai-chat", "provider-openai", "openai:chat", "https://api.openai.example", )], vec![sample_key( "key-openai-a", "provider-openai", "openai:chat", "sk-test-a", )], )); let (upstream_url, upstream_handle) = start_server(upstream).await; let gateway = build_router_with_state( AppState::new() .expect("gateway should build") .with_data_state_for_tests( GatewayDataState::with_provider_catalog_repository_for_tests( provider_catalog_repository.clone(), ), ), ); let (gateway_url, gateway_handle) = start_server(gateway).await; let response = reqwest::Client::new() .get(format!( "{gateway_url}/api/admin/providers/?skip=0&limit=50" )) .header(crate::constants::GATEWAY_HEADER, "rust-phase3b") .header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123") .header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin") .header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123") .send() .await .expect("request should succeed"); assert_eq!(response.status(), StatusCode::OK); let payload: serde_json::Value = response.json().await.expect("json body should parse"); let items = payload.as_array().expect("payload should be an array"); assert_eq!(items.len(), 2); assert_eq!(items[0]["id"], "provider-openai"); assert_eq!(items[0]["name"], "openai"); assert_eq!(items[0]["api_format"], "openai:chat"); assert_eq!(items[0]["base_url"], "https://api.openai.example"); assert_eq!(items[0]["api_key"], "***"); assert_eq!(items[0]["priority"], 10); assert_eq!(items[0]["created_at"], "2024-03-21T05:46:40Z"); assert_eq!(items[0]["updated_at"], "2024-03-21T05:48:20Z"); assert_eq!(items[1]["id"], "provider-anthropic"); assert_eq!(items[1]["api_format"], serde_json::Value::Null); assert_eq!(items[1]["base_url"], serde_json::Value::Null); assert_eq!(items[1]["api_key"], serde_json::Value::Null); assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0); gateway_handle.abort(); upstream_handle.abort(); } #[tokio::test] async fn gateway_handles_admin_providers_locally_with_local_503_when_catalog_reader_unavailable() { let upstream_hits = Arc::new(Mutex::new(0usize)); let upstream_hits_clone = Arc::clone(&upstream_hits); let upstream = Router::new().route( "/api/admin/providers/", any(move |_request: Request| { let upstream_hits_inner = Arc::clone(&upstream_hits_clone); async move { *upstream_hits_inner.lock().expect("mutex should lock") += 1; (StatusCode::OK, Body::from("unexpected upstream hit")) } }), ); let (upstream_url, upstream_handle) = start_server(upstream).await; let gateway = build_router_with_state(AppState::new().expect("gateway should build")); let (gateway_url, gateway_handle) = start_server(gateway).await; let response = reqwest::Client::new() .get(format!( "{gateway_url}/api/admin/providers/?skip=0&limit=50" )) .header(crate::constants::GATEWAY_HEADER, "rust-phase3b") .header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123") .header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin") .header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123") .send() .await .expect("request should succeed"); assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); let payload: serde_json::Value = response.json().await.expect("json body should parse"); assert_eq!(payload["detail"], ADMIN_PROVIDERS_DATA_UNAVAILABLE_DETAIL); assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0); gateway_handle.abort(); upstream_handle.abort(); } #[tokio::test] async fn gateway_handles_admin_provider_summary_locally_with_trusted_admin_principal() { let upstream_hits = Arc::new(Mutex::new(0usize)); let upstream_hits_clone = Arc::clone(&upstream_hits); let upstream = Router::new().route( "/api/admin/providers/provider-openai/summary", any(move |_request: Request| { let upstream_hits_inner = Arc::clone(&upstream_hits_clone); async move { *upstream_hits_inner.lock().expect("mutex should lock") += 1; (StatusCode::OK, Body::from("unexpected upstream hit")) } }), ); let provider = sample_provider("provider-openai", "openai", 10) .with_description(Some("OpenAI primary provider".to_string())) .with_transport_fields( true, false, true, None, Some(4), Some(sample_bound_provider_proxy( "provider-openai", "proxy.example", "secret", )), Some(45.0), Some(12.0), Some(json!({ "claude_code_advanced": {"pool_size": 3}, "pool_advanced": {"enabled": true}, "failover_rules": {"strategy": "ordered"}, "chat_pii_redaction": {"enabled": true}, "kiro": {"simulated_cache_enabled": true}, "provider_ops": {"architecture_id": "anyrouter"} })), ); let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed( vec![provider.with_timestamps(Some(1_711_000_000), Some(1_711_000_100))], vec![ sample_endpoint( "endpoint-openai-chat", "provider-openai", "openai:chat", "https://api.openai.example", ), sample_endpoint( "endpoint-openai-cli", "provider-openai", "openai:responses", "https://api.openai.example", ), ], vec![ sample_key( "key-openai-chat", "provider-openai", "openai:chat", "sk-test-chat", ) .with_health_fields(Some(json!({"openai:chat": {"health_score": 0.25}})), None), sample_bound_key( "key-openai-cli", "provider-openai", "openai:responses", "sk-test-cli-2", ) .with_health_fields( Some(json!({"openai:responses": {"health_score": 0.75}})), None, ), ], )); let global_model_repository = Arc::new( InMemoryGlobalModelReadRepository::seed(Vec::new()) .with_provider_model_stats(vec![sample_provider_model_stats("provider-openai", 5, 3)]) .with_active_global_model_refs(vec![ sample_provider_active_global_model("provider-openai", "gpt-5"), sample_provider_active_global_model("provider-openai", "gpt-5-mini"), ]), ); let quota_repository = Arc::new(InMemoryProviderQuotaRepository::seed(vec![ sample_provider_quota("provider-openai"), ])); let (upstream_url, upstream_handle) = start_server(upstream).await; let gateway = build_router_with_state( AppState::new() .expect("gateway should build") .with_data_state_for_tests( GatewayDataState::with_provider_catalog_global_model_and_quota_readers_for_tests( provider_catalog_repository, global_model_repository, quota_repository, ), ), ); let (gateway_url, gateway_handle) = start_server(gateway).await; let response = reqwest::Client::new() .get(format!( "{gateway_url}/api/admin/providers/provider-openai/summary" )) .header(crate::constants::GATEWAY_HEADER, "rust-phase3b") .header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123") .header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin") .header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123") .send() .await .expect("request should succeed"); assert_eq!(response.status(), StatusCode::OK); let payload: serde_json::Value = response.json().await.expect("json body should parse"); assert_eq!(payload["id"], "provider-openai"); assert_eq!(payload["name"], "openai"); assert_eq!(payload["description"], "OpenAI primary provider"); assert_eq!(payload["billing_type"], "monthly_quota"); assert_eq!(payload["monthly_quota_usd"], 100.0); assert_eq!(payload["monthly_used_usd"], 12.5); assert_eq!(payload["max_transfer_count"], 0); assert_eq!(payload["max_transfer_timeout_seconds"], 0); assert_eq!(payload["total_endpoints"], 2); assert_eq!(payload["active_endpoints"], 2); assert_eq!(payload["total_keys"], 2); assert_eq!(payload["active_keys"], 2); assert_eq!(payload["total_models"], 5); assert_eq!(payload["active_models"], 3); assert_eq!(payload["global_model_ids"], json!(["gpt-5", "gpt-5-mini"])); assert_eq!( payload["api_formats"], json!(["openai:chat", "openai:responses"]) ); assert_eq!(payload["ops_configured"], true); assert_eq!(payload["ops_architecture_id"], "anyrouter"); assert_eq!(payload["chat_pii_redaction"], json!({"enabled": true})); assert_eq!(payload["kiro_simulated_cache_enabled"], true); assert_eq!(payload["created_at"], "2024-03-21T05:46:40Z"); assert_eq!(payload["updated_at"], "2024-03-21T05:48:20Z"); assert_eq!( payload["endpoint_health_details"], json!([ { "api_format": "openai:chat", "health_score": 0.25, "is_active": true, "total_keys": 1, "active_keys": 1 }, { "api_format": "openai:responses", "health_score": 0.75, "is_active": true, "total_keys": 1, "active_keys": 1 } ]) ); assert_eq!(payload["avg_health_score"], 0.5); assert_eq!(payload["unhealthy_endpoints"], 1); assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0); gateway_handle.abort(); upstream_handle.abort(); } #[tokio::test] async fn gateway_handles_admin_providers_summary_list_locally_with_bearer_admin_session() { let upstream_hits = Arc::new(Mutex::new(0usize)); let upstream_hits_clone = Arc::clone(&upstream_hits); let upstream = Router::new().route( "/api/admin/providers/summary", any(move |_request: Request| { let upstream_hits_inner = Arc::clone(&upstream_hits_clone); async move { *upstream_hits_inner.lock().expect("mutex should lock") += 1; (StatusCode::OK, Body::from("unexpected upstream hit")) } }), ); let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed( vec![ sample_provider("provider-openai", "openai", 10) .with_timestamps(Some(1_711_000_000), Some(1_711_000_100)), sample_provider("provider-anthropic", "anthropic", 20) .with_timestamps(Some(1_710_000_000), Some(1_710_000_100)), ], vec![], vec![], )); let (upstream_url, upstream_handle) = start_server(upstream).await; let state = AppState::new() .expect("gateway should build") .with_data_state_for_tests(GatewayDataState::with_provider_catalog_reader_for_tests( provider_catalog_repository, )); let access_token = issue_test_admin_access_token(&state, "device-admin-providers").await; let gateway = build_router_with_state(state); let (gateway_url, gateway_handle) = start_server(gateway).await; let response = reqwest::Client::new() .get(format!( "{gateway_url}/api/admin/providers/summary?page=1&page_size=20" )) .header("authorization", format!("Bearer {access_token}")) .header("x-client-device-id", "device-admin-providers") .send() .await .expect("request should succeed"); assert_eq!(response.status(), StatusCode::OK); let payload: serde_json::Value = response.json().await.expect("json body should parse"); let items = payload["items"].as_array().expect("items should be array"); assert_eq!(items.len(), 2); let mut ids = items .iter() .filter_map(|item| item.get("id").and_then(serde_json::Value::as_str)) .collect::>(); ids.sort_unstable(); assert_eq!(ids, vec!["provider-anthropic", "provider-openai"]); assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0); gateway_handle.abort(); upstream_handle.abort(); } #[tokio::test] async fn gateway_handles_admin_provider_summary_locally_with_bearer_admin_session() { let upstream_hits = Arc::new(Mutex::new(0usize)); let upstream_hits_clone = Arc::clone(&upstream_hits); let upstream = Router::new().route( "/api/admin/providers/provider-openai/summary", any(move |_request: Request| { let upstream_hits_inner = Arc::clone(&upstream_hits_clone); async move { *upstream_hits_inner.lock().expect("mutex should lock") += 1; (StatusCode::OK, Body::from("unexpected upstream hit")) } }), ); let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed( vec![sample_provider("provider-openai", "openai", 10) .with_description(Some("OpenAI primary provider".to_string())) .with_timestamps(Some(1_711_000_000), Some(1_711_000_100))], vec![sample_endpoint( "endpoint-openai-chat", "provider-openai", "openai:chat", "https://api.openai.example", )], vec![sample_key( "key-openai-chat", "provider-openai", "openai:chat", "sk-test-chat", )], )); let (upstream_url, upstream_handle) = start_server(upstream).await; let state = AppState::new() .expect("gateway should build") .with_data_state_for_tests(GatewayDataState::with_provider_catalog_reader_for_tests( provider_catalog_repository, )); let access_token = issue_test_admin_access_token(&state, "device-admin-provider-summary").await; let gateway = build_router_with_state(state); let (gateway_url, gateway_handle) = start_server(gateway).await; let response = reqwest::Client::new() .get(format!( "{gateway_url}/api/admin/providers/provider-openai/summary" )) .header("authorization", format!("Bearer {access_token}")) .header("x-client-device-id", "device-admin-provider-summary") .send() .await .expect("request should succeed"); assert_eq!(response.status(), StatusCode::OK); let payload: serde_json::Value = response.json().await.expect("json body should parse"); assert_eq!(payload["id"], "provider-openai"); assert_eq!(payload["name"], "openai"); assert_eq!(payload["description"], "OpenAI primary provider"); assert_eq!(payload["total_endpoints"], 1); assert_eq!(payload["total_keys"], 1); assert_eq!(payload["api_formats"], json!(["openai:chat"])); assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0); gateway_handle.abort(); upstream_handle.abort(); } #[tokio::test] async fn gateway_handles_admin_providers_summary_list_locally_with_local_503_when_catalog_reader_unavailable( ) { let upstream_hits = Arc::new(Mutex::new(0usize)); let upstream_hits_clone = Arc::clone(&upstream_hits); let upstream = Router::new().route( "/api/admin/providers/summary", any(move |_request: Request| { let upstream_hits_inner = Arc::clone(&upstream_hits_clone); async move { *upstream_hits_inner.lock().expect("mutex should lock") += 1; (StatusCode::OK, Body::from("unexpected upstream hit")) } }), ); let (upstream_url, upstream_handle) = start_server(upstream).await; let state = AppState::new().expect("gateway should build"); let access_token = issue_test_admin_access_token(&state, "device-admin-providers-503").await; let gateway = build_router_with_state(state); let (gateway_url, gateway_handle) = start_server(gateway).await; let response = reqwest::Client::new() .get(format!( "{gateway_url}/api/admin/providers/summary?page=1&page_size=20" )) .header("authorization", format!("Bearer {access_token}")) .header("x-client-device-id", "device-admin-providers-503") .send() .await .expect("request should succeed"); assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); let payload: serde_json::Value = response.json().await.expect("json body should parse"); assert_eq!(payload["detail"], ADMIN_PROVIDERS_DATA_UNAVAILABLE_DETAIL); assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0); gateway_handle.abort(); upstream_handle.abort(); } #[tokio::test] async fn gateway_handles_admin_providers_summary_list_locally_with_trusted_admin_principal() { let upstream_hits = Arc::new(Mutex::new(0usize)); let upstream_hits_clone = Arc::clone(&upstream_hits); let upstream = Router::new().route( "/api/admin/providers/summary", any(move |_request: Request| { let upstream_hits_inner = Arc::clone(&upstream_hits_clone); async move { *upstream_hits_inner.lock().expect("mutex should lock") += 1; (StatusCode::OK, Body::from("unexpected upstream hit")) } }), ); let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed( vec![ sample_provider("provider-openai", "openai", 10) .with_description(Some("OpenAI primary provider".to_string())) .with_timestamps(Some(1_711_000_000), Some(1_711_000_100)), sample_provider("provider-anthropic", "anthropic", 20) .with_description(Some("Anthropic backup provider".to_string())) .with_transport_fields(false, false, true, None, Some(2), None, None, None, None) .with_timestamps(Some(1_710_000_000), Some(1_710_000_100)), ], vec![ sample_endpoint( "endpoint-openai-chat", "provider-openai", "openai:chat", "https://api.openai.example", ), sample_endpoint( "endpoint-anthropic-chat", "provider-anthropic", "claude:messages", "https://api.anthropic.example", ), ], vec![sample_key( "key-openai-chat", "provider-openai", "openai:chat", "sk-test-chat", ) .with_health_fields(Some(json!({"openai:chat": {"health_score": 0.8}})), None)], )); let global_model_repository = Arc::new( InMemoryGlobalModelReadRepository::seed(Vec::new()) .with_provider_model_stats(vec![ sample_provider_model_stats("provider-openai", 5, 3), sample_provider_model_stats("provider-anthropic", 2, 1), ]) .with_active_global_model_refs(vec![ sample_provider_active_global_model("provider-openai", "gpt-5"), sample_provider_active_global_model("provider-anthropic", "claude-sonnet-4-5"), ]), ); let quota_repository = Arc::new(InMemoryProviderQuotaRepository::seed(vec![ sample_provider_quota("provider-openai"), sample_provider_quota("provider-anthropic"), ])); let (upstream_url, upstream_handle) = start_server(upstream).await; let gateway = build_router_with_state( AppState::new() .expect("gateway should build") .with_data_state_for_tests( GatewayDataState::with_provider_catalog_global_model_and_quota_readers_for_tests( provider_catalog_repository, global_model_repository, quota_repository, ), ), ); let (gateway_url, gateway_handle) = start_server(gateway).await; let response = reqwest::Client::new() .get(format!( "{gateway_url}/api/admin/providers/summary?page=1&page_size=20&search=open&status=active&api_format=openai:chat&model_id=gpt-5" )) .header(crate::constants::GATEWAY_HEADER, "rust-phase3b") .header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123") .header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin") .header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123") .send() .await .expect("request should succeed"); assert_eq!(response.status(), StatusCode::OK); let payload: serde_json::Value = response.json().await.expect("json body should parse"); assert_eq!(payload["total"], 1); assert_eq!(payload["page"], 1); assert_eq!(payload["page_size"], 20); let items = payload["items"] .as_array() .expect("items should be an array"); assert_eq!(items.len(), 1); assert_eq!(items[0]["id"], "provider-openai"); assert_eq!(items[0]["name"], "openai"); assert_eq!(items[0]["total_models"], 5); assert_eq!(items[0]["active_models"], 3); assert_eq!(items[0]["global_model_ids"], json!(["gpt-5"])); assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0); gateway_handle.abort(); upstream_handle.abort(); } #[tokio::test] async fn gateway_returns_service_unavailable_for_admin_providers_summary_without_provider_catalog_reader( ) { let upstream_hits = Arc::new(Mutex::new(0usize)); let upstream_hits_clone = Arc::clone(&upstream_hits); let upstream = Router::new().route( "/api/admin/providers/summary", any(move |_request: Request| { let upstream_hits_inner = Arc::clone(&upstream_hits_clone); async move { *upstream_hits_inner.lock().expect("mutex should lock") += 1; (StatusCode::OK, Body::from("unexpected upstream hit")) } }), ); let (upstream_url, upstream_handle) = start_server(upstream).await; let gateway = build_router_with_state(AppState::new().expect("gateway should build")); let (gateway_url, gateway_handle) = start_server(gateway).await; let response = reqwest::Client::new() .get(format!( "{gateway_url}/api/admin/providers/summary?page=1&page_size=20" )) .header(crate::constants::GATEWAY_HEADER, "rust-phase3b") .header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123") .header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin") .header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123") .send() .await .expect("request should succeed"); assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); let payload: serde_json::Value = response.json().await.expect("json body should parse"); assert_eq!(payload["detail"], "Admin provider catalog data unavailable"); assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0); gateway_handle.abort(); upstream_handle.abort(); } #[tokio::test] async fn gateway_returns_service_unavailable_for_admin_provider_create_without_provider_catalog_writer( ) { let upstream_hits = Arc::new(Mutex::new(0usize)); let upstream_hits_clone = Arc::clone(&upstream_hits); let upstream = Router::new().route( "/api/admin/providers", any(move |_request: Request| { let upstream_hits_inner = Arc::clone(&upstream_hits_clone); async move { *upstream_hits_inner.lock().expect("mutex should lock") += 1; (StatusCode::OK, Body::from("unexpected upstream hit")) } }), ); let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed( Vec::new(), Vec::new(), Vec::new(), )); let (upstream_url, upstream_handle) = start_server(upstream).await; let gateway = build_router_with_state( AppState::new() .expect("gateway should build") .with_data_state_for_tests(GatewayDataState::with_provider_catalog_reader_for_tests( provider_catalog_repository, )), ); let (gateway_url, gateway_handle) = start_server(gateway).await; let response = reqwest::Client::new() .post(format!("{gateway_url}/api/admin/providers")) .header(crate::constants::GATEWAY_HEADER, "rust-phase3b") .header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123") .header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin") .header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123") .json(&json!({ "name": "openai", "provider_type": "openai", "provider_priority": 10 })) .send() .await .expect("request should succeed"); assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); let payload: serde_json::Value = response.json().await.expect("json body should parse"); assert_eq!(payload["detail"], "Admin provider catalog data unavailable"); assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0); gateway_handle.abort(); upstream_handle.abort(); } #[tokio::test] async fn gateway_updates_admin_provider_locally_with_trusted_admin_principal() { let upstream_hits = Arc::new(Mutex::new(0usize)); let upstream_hits_clone = Arc::clone(&upstream_hits); let upstream = Router::new().route( "/api/admin/providers/provider-openai", any(move |_request: Request| { let upstream_hits_inner = Arc::clone(&upstream_hits_clone); async move { *upstream_hits_inner.lock().expect("mutex should lock") += 1; (StatusCode::OK, Body::from("unexpected upstream hit")) } }), ); let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed( vec![ sample_provider("provider-openai", "openai", 10) .with_billing_fields( Some("free_tier".to_string()), None, None, Some(30), None, None, ) .with_transport_fields( true, false, false, None, None, None, None, None, Some(json!({ "pool_advanced": {}, "failover_rules": {"strategy": "ordered"} })), ) .with_timestamps(Some(1_711_000_000), Some(1_711_000_100)), sample_provider("provider-other", "other", 20), ], vec![sample_endpoint( "endpoint-openai-chat", "provider-openai", "openai:chat", "https://api.openai.example", )], Vec::new(), )); let (upstream_url, upstream_handle) = start_server(upstream).await; let gateway = build_router_with_state( AppState::new() .expect("gateway should build") .with_data_state_for_tests( GatewayDataState::with_provider_catalog_repository_for_tests( provider_catalog_repository.clone(), ), ), ); let (gateway_url, gateway_handle) = start_server(gateway).await; let response = reqwest::Client::new() .patch(format!("{gateway_url}/api/admin/providers/provider-openai")) .header(crate::constants::GATEWAY_HEADER, "rust-phase3b") .header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123") .header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin") .header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123") .json(&json!({ "name": "openai-renamed", "provider_type": "claude_code", "description": "Updated provider", "website": "https://updated.example", "provider_priority": 3, "keep_priority_on_conversion": true, "is_active": false, "concurrent_limit": 8, "max_retries": 6, "max_transfer_count": 10, "max_transfer_timeout_seconds": 60, "request_timeout": aether_contracts::MAX_EXECUTION_REQUEST_TIMEOUT_SECS, "stream_first_byte_timeout": 11.0, "enable_format_conversion": false, "config": { "provider_ops": {"architecture_id": "cubence"}, "chat_pii_redaction": {"enabled": true} }, "claude_code_advanced": {"pool_size": 2}, "proxy": {"url": "https://proxy.example"} })) .send() .await .expect("request should succeed"); let status = response.status(); let body = response.text().await.expect("body should read"); assert_eq!(status, StatusCode::OK, "body={body}"); let payload: serde_json::Value = serde_json::from_str(&body).expect("json body should parse"); assert_eq!(payload["id"], "provider-openai"); assert_eq!(payload["name"], "openai-renamed"); assert_eq!(payload["provider_type"], "claude_code"); assert_eq!(payload["description"], "Updated provider"); assert_eq!(payload["website"], "https://updated.example"); assert_eq!(payload["provider_priority"], 3); assert_eq!(payload["keep_priority_on_conversion"], true); assert_eq!(payload["enable_format_conversion"], false); assert_eq!(payload["is_active"], false); assert_eq!(payload["max_retries"], 6); assert_eq!(payload["max_transfer_count"], 10); assert_eq!(payload["max_transfer_timeout_seconds"], 60); assert_eq!( payload["request_timeout"].as_f64(), Some(aether_contracts::MAX_EXECUTION_REQUEST_TIMEOUT_SECS as f64) ); assert_eq!(payload["stream_first_byte_timeout"], 11.0); assert_eq!(payload["proxy"], json!({"url": "https://proxy.example/"})); assert_eq!(payload["claude_code_advanced"], json!({"pool_size": 2})); assert_eq!(payload["pool_advanced"], json!({})); assert_eq!(payload["failover_rules"], json!({"strategy": "ordered"})); assert_eq!(payload["chat_pii_redaction"], json!({"enabled": true})); assert_eq!(payload["ops_configured"], true); assert_eq!(payload["ops_architecture_id"], "cubence"); let invalid_timeout_response = reqwest::Client::new() .patch(format!("{gateway_url}/api/admin/providers/provider-openai")) .header(crate::constants::GATEWAY_HEADER, "rust-phase3b") .header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123") .header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin") .header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123") .json(&json!({ "request_timeout": aether_contracts::MAX_EXECUTION_REQUEST_TIMEOUT_SECS + 1 })) .send() .await .expect("request should succeed"); assert_eq!(invalid_timeout_response.status(), StatusCode::BAD_REQUEST); for (field_name, value) in [ ("max_transfer_count", -1), ("max_transfer_timeout_seconds", -1), ] { let invalid_transfer_response = reqwest::Client::new() .patch(format!("{gateway_url}/api/admin/providers/provider-openai")) .header(crate::constants::GATEWAY_HEADER, "rust-phase3b") .header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123") .header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin") .header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123") .json(&json!({ (field_name): value })) .send() .await .expect("request should succeed"); let status = invalid_transfer_response.status(); let payload: serde_json::Value = invalid_transfer_response .json() .await .expect("json body should parse"); assert_eq!(status, StatusCode::BAD_REQUEST); assert_eq!(payload["detail"], format!("{field_name} 必须是非负整数")); } let disable_response = reqwest::Client::new() .patch(format!("{gateway_url}/api/admin/providers/provider-openai")) .header(crate::constants::GATEWAY_HEADER, "rust-phase3b") .header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123") .header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin") .header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123") .json(&json!({ "max_transfer_count": null, "config": { "chat_pii_redaction": {"enabled": false} } })) .send() .await .expect("request should succeed"); let disable_status = disable_response.status(); let disable_body = disable_response.text().await.expect("body should read"); assert_eq!(disable_status, StatusCode::OK, "body={disable_body}"); let disable_payload: serde_json::Value = serde_json::from_str(&disable_body).expect("json body should parse"); assert_eq!( disable_payload["chat_pii_redaction"], json!({"enabled": false}) ); assert_eq!(disable_payload["pool_advanced"], json!({})); assert_eq!(disable_payload["max_transfer_count"], 0); assert_eq!(disable_payload["max_transfer_timeout_seconds"], 60); assert_eq!( disable_payload["failover_rules"], json!({"strategy": "ordered"}) ); assert_eq!(disable_payload["ops_architecture_id"], "cubence"); let invalid_response = reqwest::Client::new() .patch(format!("{gateway_url}/api/admin/providers/provider-openai")) .header(crate::constants::GATEWAY_HEADER, "rust-phase3b") .header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123") .header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin") .header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123") .json(&json!({ "config": { "chat_pii_redaction": {"enabled": true, "entities": ["email"]} } })) .send() .await .expect("request should succeed"); assert_eq!(invalid_response.status(), StatusCode::BAD_REQUEST); let providers = provider_catalog_repository .list_providers(false) .await .expect("providers should list"); let updated_provider = providers .iter() .find(|provider| provider.id == "provider-openai") .expect("provider should exist"); assert_eq!(updated_provider.billing_type.as_deref(), Some("free_tier")); assert_eq!( updated_provider.request_timeout_secs, Some(aether_contracts::MAX_EXECUTION_REQUEST_TIMEOUT_SECS as f64) ); assert_eq!( updated_provider .config .as_ref() .and_then(|value| value.get("max_transfer_count")) .and_then(serde_json::Value::as_u64), Some(0) ); assert_eq!( updated_provider .config .as_ref() .and_then(|value| value.get("max_transfer_timeout_seconds")) .and_then(serde_json::Value::as_u64), Some(60) ); assert_eq!( updated_provider .config .as_ref() .and_then(|value| value.get("chat_pii_redaction")) .cloned(), Some(json!({"enabled": false})) ); assert_eq!( updated_provider .config .as_ref() .and_then(|value| value.get("pool_advanced")) .cloned(), Some(json!({})) ); assert_eq!( updated_provider .config .as_ref() .and_then(|value| value.get("failover_rules")) .cloned(), Some(json!({"strategy": "ordered"})) ); assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0); gateway_handle.abort(); upstream_handle.abort(); } #[tokio::test] async fn gateway_creates_admin_provider_locally_with_trusted_admin_principal() { let upstream_hits = Arc::new(Mutex::new(0usize)); let upstream_hits_clone = Arc::clone(&upstream_hits); let upstream = Router::new().route( "/api/admin/providers/", any(move |_request: Request| { let upstream_hits_inner = Arc::clone(&upstream_hits_clone); async move { *upstream_hits_inner.lock().expect("mutex should lock") += 1; (StatusCode::OK, Body::from("unexpected upstream hit")) } }), ); let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed( vec![sample_provider("provider-existing", "existing", 0)], Vec::new(), Vec::new(), )); let (upstream_url, upstream_handle) = start_server(upstream).await; let gateway = build_router_with_state( AppState::new() .expect("gateway should build") .with_data_state_for_tests( GatewayDataState::with_provider_catalog_repository_for_tests( provider_catalog_repository.clone(), ), ), ); let (gateway_url, gateway_handle) = start_server(gateway).await; let response = reqwest::Client::new() .post(format!("{gateway_url}/api/admin/providers/")) .header(crate::constants::GATEWAY_HEADER, "rust-phase3b") .header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123") .header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin") .header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123") .json(&json!({ "name": "codex-provider", "provider_type": "codex", "description": "Codex managed provider", "website": "codex.example", "keep_priority_on_conversion": true, "max_retries": 7, "max_transfer_count": 12, "max_transfer_timeout_seconds": 90, "request_timeout": aether_contracts::MAX_EXECUTION_REQUEST_TIMEOUT_SECS, "config": { "chat_pii_redaction": {"enabled": true}, "codex": {"pass_through_cyber_flag_interrupt": true} }, "codex_fingerprint_convergence_enabled": true, "pool_advanced": {}, "failover_rules": {"strategy": "ordered"}, "proxy": {"url": "https://proxy.example"} })) .send() .await .expect("request should succeed"); let status = response.status(); let body = response.text().await.expect("body should read"); assert_eq!(status, StatusCode::OK, "body={body}"); let payload: serde_json::Value = serde_json::from_str(&body).expect("json body should parse"); assert_eq!(payload["name"], "codex-provider"); assert_eq!(payload["message"], "提供商创建成功"); let providers = provider_catalog_repository .list_providers(false) .await .expect("providers should list"); let created = providers .iter() .find(|provider| provider.name == "codex-provider") .expect("created provider should exist"); let existing = providers .iter() .find(|provider| provider.id == "provider-existing") .expect("existing provider should remain"); assert_eq!(created.provider_type, "codex"); assert_eq!(created.billing_type.as_deref(), Some("pay_as_you_go")); assert_eq!(created.provider_priority, 0); assert_eq!(existing.provider_priority, 1); assert_eq!(created.website.as_deref(), Some("https://codex.example")); assert!(created.enable_format_conversion); assert_eq!(created.max_retries, Some(7)); assert_eq!( created .config .as_ref() .and_then(|value| value.get("max_transfer_count")) .and_then(serde_json::Value::as_u64), Some(12) ); assert_eq!( created .config .as_ref() .and_then(|value| value.get("max_transfer_timeout_seconds")) .and_then(serde_json::Value::as_u64), Some(90) ); assert_eq!( created.request_timeout_secs, Some(aether_contracts::MAX_EXECUTION_REQUEST_TIMEOUT_SECS as f64) ); assert_eq!(created.keep_priority_on_conversion, true); assert_eq!( created .config .as_ref() .and_then(|value| value.get("pool_advanced")) .cloned(), Some(json!({})) ); assert_eq!( created .config .as_ref() .and_then(|value| value.get("chat_pii_redaction")) .cloned(), Some(json!({"enabled": true})) ); assert_eq!( created .config .as_ref() .and_then(|value| value.get("failover_rules")) .cloned(), Some(json!({"strategy": "ordered"})) ); assert_eq!( created .config .as_ref() .and_then(|value| value.get("codex")) .cloned(), Some(json!({ "fingerprint_convergence_enabled": true, "pass_through_cyber_flag_interrupt": true })) ); let invalid_response = reqwest::Client::new() .post(format!("{gateway_url}/api/admin/providers/")) .header(crate::constants::GATEWAY_HEADER, "rust-phase3b") .header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123") .header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin") .header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123") .json(&json!({ "name": "invalid-redaction-provider", "provider_type": "custom", "config": {"chat_pii_redaction": {"enabled": true, "entities": ["email"]}} })) .send() .await .expect("request should succeed"); assert_eq!(invalid_response.status(), StatusCode::BAD_REQUEST); let endpoints = provider_catalog_repository .list_endpoints_by_provider_ids(std::slice::from_ref(&created.id)) .await .expect("endpoints should list"); assert_eq!(endpoints.len(), 5); let responses_endpoint = endpoints .iter() .find(|endpoint| endpoint.api_format == "openai:responses") .expect("responses endpoint should exist"); let compact_endpoint = endpoints .iter() .find(|endpoint| endpoint.api_format == "openai:responses:compact") .expect("compact endpoint should exist"); let search_endpoint = endpoints .iter() .find(|endpoint| endpoint.api_format == "openai:search") .expect("search endpoint should exist"); let image_endpoint = endpoints .iter() .find(|endpoint| endpoint.api_format == "openai:image") .expect("image endpoint should exist"); let live_endpoint = endpoints .iter() .find(|endpoint| endpoint.api_format == "codex:live") .expect("Codex Live endpoint should exist"); assert_eq!( responses_endpoint.base_url, "https://chatgpt.com/backend-api/codex" ); assert_eq!( compact_endpoint.base_url, "https://chatgpt.com/backend-api/codex" ); assert_eq!( search_endpoint.base_url, "https://chatgpt.com/backend-api/codex" ); assert_eq!( image_endpoint.base_url, "https://chatgpt.com/backend-api/codex" ); assert_eq!(responses_endpoint.max_retries, Some(7)); assert_eq!(compact_endpoint.max_retries, Some(7)); assert_eq!(search_endpoint.max_retries, Some(7)); assert_eq!(image_endpoint.max_retries, Some(7)); assert_eq!(live_endpoint.api_family.as_deref(), Some("codex")); assert_eq!(live_endpoint.endpoint_kind.as_deref(), Some("live")); assert_eq!( crate::api::ai::public_api_format_local_path(&live_endpoint.api_format), "/v1/live" ); assert!(live_endpoint.custom_path.is_none()); assert_eq!(live_endpoint.max_retries, Some(7)); assert_eq!( responses_endpoint .config .as_ref() .and_then(|value| value.get("upstream_stream_policy")) .and_then(serde_json::Value::as_str), Some("force_stream") ); assert_eq!( search_endpoint .config .as_ref() .and_then(|value| value.get("upstream_stream_policy")) .and_then(serde_json::Value::as_str), None ); assert_eq!( image_endpoint .config .as_ref() .and_then(|value| value.get("upstream_stream_policy")) .and_then(serde_json::Value::as_str), None ); assert!(responses_endpoint.body_rules.is_none()); assert!(compact_endpoint.body_rules.is_none()); assert!(search_endpoint.body_rules.is_none()); assert!(image_endpoint.body_rules.is_none()); assert!(live_endpoint.body_rules.is_none()); assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0); gateway_handle.abort(); upstream_handle.abort(); } #[tokio::test] async fn gateway_updates_fixed_provider_and_reconciles_template_managed_endpoints() { let upstream_hits = Arc::new(Mutex::new(0usize)); let upstream_hits_clone = Arc::clone(&upstream_hits); let upstream = Router::new().route( "/api/admin/providers/provider-codex", any(move |_request: Request| { let upstream_hits_inner = Arc::clone(&upstream_hits_clone); async move { *upstream_hits_inner.lock().expect("mutex should lock") += 1; (StatusCode::OK, Body::from("unexpected upstream hit")) } }), ); let mut provider = sample_provider("provider-codex", "codex", 10).with_transport_fields( true, false, true, None, Some(2), None, None, None, Some(json!({ "codex": {"pass_through_cyber_flag_interrupt": true} })), ); provider.provider_type = "codex".to_string(); let mut cli_endpoint = sample_endpoint( "endpoint-codex-cli", "provider-codex", "openai:responses", "https://chatgpt.com/backend-api/codex", ); cli_endpoint.max_retries = Some(2); cli_endpoint.config = Some(json!({"upstream_stream_policy": "force_stream"})); let mut live_endpoint = sample_endpoint( "endpoint-codex-live", "provider-codex", "codex:live", "https://chatgpt.com/backend-api/codex", ); live_endpoint.max_retries = Some(2); live_endpoint.custom_path = Some("/custom/live".to_string()); let mut key = sample_key( "key-codex-oauth", "provider-codex", "openai:responses", "oauth-placeholder", ); key.auth_type = "oauth".to_string(); key.api_formats = Some(json!(["openai:responses"])); let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed( vec![provider], vec![cli_endpoint, live_endpoint], vec![key], )); let (_upstream_url, upstream_handle) = start_server(upstream).await; let gateway = build_router_with_state( AppState::new() .expect("gateway should build") .with_data_state_for_tests( GatewayDataState::with_provider_catalog_repository_for_tests( provider_catalog_repository.clone(), ), ), ); let (gateway_url, gateway_handle) = start_server(gateway).await; let response = reqwest::Client::new() .patch(format!("{gateway_url}/api/admin/providers/provider-codex")) .header(crate::constants::GATEWAY_HEADER, "rust-phase3b") .header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123") .header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin") .header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123") .json(&json!({ "max_retries": 9, "codex_fingerprint_convergence_enabled": true })) .send() .await .expect("request should succeed"); let status = response.status(); let body = response.text().await.expect("body should read"); assert_eq!(status, StatusCode::OK, "body={body}"); let payload: serde_json::Value = serde_json::from_str(&body).expect("json body should parse"); assert_eq!(payload["codex_fingerprint_convergence_enabled"], true); let providers = provider_catalog_repository .list_providers(false) .await .expect("providers should list"); let updated_provider = providers .iter() .find(|provider| provider.id == "provider-codex") .expect("updated provider should exist"); assert_eq!( updated_provider .config .as_ref() .and_then(|value| value.get("codex")) .cloned(), Some(json!({ "fingerprint_convergence_enabled": true, "pass_through_cyber_flag_interrupt": true })) ); let endpoints = provider_catalog_repository .list_endpoints_by_provider_ids(&["provider-codex".to_string()]) .await .expect("endpoints should list"); assert_eq!(endpoints.len(), 5); let responses_endpoint = endpoints .iter() .find(|endpoint| endpoint.api_format == "openai:responses") .expect("responses endpoint should exist"); let compact_endpoint = endpoints .iter() .find(|endpoint| endpoint.api_format == "openai:responses:compact") .expect("compact endpoint should exist"); let search_endpoint = endpoints .iter() .find(|endpoint| endpoint.api_format == "openai:search") .expect("search endpoint should exist"); let image_endpoint = endpoints .iter() .find(|endpoint| endpoint.api_format == "openai:image") .expect("image endpoint should exist"); let live_endpoint = endpoints .iter() .find(|endpoint| endpoint.api_format == "codex:live") .expect("Codex Live endpoint should exist"); assert_eq!(responses_endpoint.max_retries, Some(9)); assert_eq!(compact_endpoint.max_retries, Some(9)); assert_eq!(search_endpoint.max_retries, Some(9)); assert_eq!(image_endpoint.max_retries, Some(9)); assert_eq!(live_endpoint.api_family.as_deref(), Some("codex")); assert_eq!(live_endpoint.endpoint_kind.as_deref(), Some("live")); assert_eq!( crate::api::ai::public_api_format_local_path(&live_endpoint.api_format), "/v1/live" ); assert_eq!(live_endpoint.custom_path.as_deref(), Some("/custom/live")); assert_eq!(live_endpoint.max_retries, Some(9)); assert_eq!( responses_endpoint .config .as_ref() .and_then(|value| value.get("_aether_fixed_provider_template")) .and_then(|value| value.get("managed")) .and_then(serde_json::Value::as_bool), Some(true) ); assert_eq!( search_endpoint .config .as_ref() .and_then(|value| value.get("_aether_fixed_provider_template")) .and_then(|value| value.get("managed")) .and_then(serde_json::Value::as_bool), Some(true) ); assert_eq!( search_endpoint .config .as_ref() .and_then(|value| value.get("upstream_stream_policy")) .and_then(serde_json::Value::as_str), None ); assert_eq!( image_endpoint .config .as_ref() .and_then(|value| value.get("upstream_stream_policy")) .and_then(serde_json::Value::as_str), None ); let keys = provider_catalog_repository .list_keys_by_provider_ids(&["provider-codex".to_string()]) .await .expect("keys should list"); assert_eq!(keys.len(), 1); assert_eq!(keys[0].api_formats, Some(json!(["openai:responses"]))); assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0); gateway_handle.abort(); upstream_handle.abort(); } #[tokio::test] async fn gateway_lists_effective_api_formats_for_fixed_oauth_provider_keys() { let mut provider = sample_provider("provider-codex", "codex", 10) .with_transport_fields(true, false, true, None, None, None, None, None, None); provider.provider_type = "codex".to_string(); let mut key = sample_key( "key-codex-legacy", "provider-codex", "openai:responses", "oauth-placeholder", ); key.name = "codex legacy".to_string(); key.auth_type = "oauth".to_string(); key.api_formats = Some(json!(["openai:responses", "openai:responses:compact"])); let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed( vec![provider], vec![ sample_endpoint( "endpoint-codex-cli", "provider-codex", "openai:responses", "https://chatgpt.com/backend-api/codex", ), sample_endpoint( "endpoint-codex-image", "provider-codex", "openai:image", "https://chatgpt.com/backend-api/codex", ), ], vec![key], )); let gateway = build_router_with_state( AppState::new() .expect("gateway should build") .with_data_state_for_tests(GatewayDataState::with_provider_catalog_reader_for_tests( provider_catalog_repository, )), ); let (gateway_url, gateway_handle) = start_server(gateway).await; let response = reqwest::Client::new() .get(format!( "{gateway_url}/api/admin/endpoints/providers/provider-codex/keys?skip=0&limit=100" )) .header(crate::constants::GATEWAY_HEADER, "rust-phase3b") .header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123") .header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin") .header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123") .send() .await .expect("request should succeed"); assert_eq!(response.status(), StatusCode::OK); let payload: serde_json::Value = response.json().await.expect("json body should parse"); let keys = payload.as_array().expect("keys payload should be array"); assert_eq!(keys.len(), 1); assert_eq!( keys[0]["api_formats"], json!(["openai:image", "openai:responses"]) ); gateway_handle.abort(); } #[tokio::test] async fn gateway_handles_admin_provider_health_monitor_locally_with_trusted_admin_principal() { let upstream_hits = Arc::new(Mutex::new(0usize)); let upstream_hits_clone = Arc::clone(&upstream_hits); let upstream = Router::new().route( "/api/admin/providers/provider-openai/health-monitor", any(move |_request: Request| { let upstream_hits_inner = Arc::clone(&upstream_hits_clone); async move { *upstream_hits_inner.lock().expect("mutex should lock") += 1; (StatusCode::OK, Body::from("unexpected upstream hit")) } }), ); let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed( vec![sample_provider("provider-openai", "openai", 10)], vec![ sample_endpoint( "endpoint-openai-chat", "provider-openai", "openai:chat", "https://api.openai.example", ), sample_endpoint( "endpoint-openai-cli", "provider-openai", "openai:responses", "https://api.openai.example", ), ], Vec::new(), )); let now_unix_secs = SystemTime::now() .duration_since(UNIX_EPOCH) .expect("current time should be after epoch") .as_secs() as i64; let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::seed(vec![ sample_request_candidate( "cand-chat-success", "req-chat-success", "endpoint-openai-chat", RequestCandidateStatus::Success, now_unix_secs - 3_000, Some(now_unix_secs - 2_980), ), sample_request_candidate( "cand-chat-failed", "req-chat-failed", "endpoint-openai-chat", RequestCandidateStatus::Failed, now_unix_secs - 2_000, Some(now_unix_secs - 1_980), ), sample_request_candidate( "cand-cli-skipped", "req-cli-skipped", "endpoint-openai-cli", RequestCandidateStatus::Skipped, now_unix_secs - 1_500, Some(now_unix_secs - 1_490), ), ])); let (upstream_url, upstream_handle) = start_server(upstream).await; let gateway = build_router_with_state( AppState::new() .expect("gateway should build") .with_data_state_for_tests( GatewayDataState::with_provider_catalog_and_request_candidate_reader_for_tests( provider_catalog_repository, request_candidate_repository, ), ), ); let (gateway_url, gateway_handle) = start_server(gateway).await; let response = reqwest::Client::new() .get(format!( "{gateway_url}/api/admin/providers/provider-openai/health-monitor?lookback_hours=6&per_endpoint_limit=48" )) .header(crate::constants::GATEWAY_HEADER, "rust-phase3b") .header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123") .header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin") .header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123") .send() .await .expect("request should succeed"); assert_eq!(response.status(), StatusCode::OK); let payload: serde_json::Value = response.json().await.expect("json body should parse"); assert_eq!(payload["provider_id"], "provider-openai"); assert_eq!(payload["provider_name"], "openai"); assert!(payload["generated_at"].as_str().is_some()); let endpoints = payload["endpoints"] .as_array() .expect("endpoints should be an array"); assert_eq!(endpoints.len(), 2); assert_eq!(endpoints[0]["endpoint_id"], "endpoint-openai-chat"); assert_eq!(endpoints[0]["api_format"], "openai:chat"); assert_eq!(endpoints[0]["total_attempts"], 2); assert_eq!(endpoints[0]["success_count"], 1); assert_eq!(endpoints[0]["failed_count"], 1); assert_eq!(endpoints[0]["skipped_count"], 0); assert_eq!(endpoints[0]["success_rate"], 0.5); assert!(endpoints[0]["last_event_at"] .as_str() .is_some_and(|value| value.ends_with(".000Z"))); assert_eq!( endpoints[0]["events"] .as_array() .expect("events should be an array") .len(), 2 ); assert!(endpoints[0]["events"][0]["timestamp"] .as_str() .is_some_and(|value| value.ends_with(".000Z"))); assert_eq!(endpoints[1]["endpoint_id"], "endpoint-openai-cli"); assert_eq!(endpoints[1]["api_format"], "openai:responses"); assert_eq!(endpoints[1]["total_attempts"], 1); assert_eq!(endpoints[1]["success_count"], 0); assert_eq!(endpoints[1]["failed_count"], 0); assert_eq!(endpoints[1]["skipped_count"], 1); assert_eq!(endpoints[1]["success_rate"], 0.0); assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0); gateway_handle.abort(); upstream_handle.abort(); } #[tokio::test] async fn gateway_handles_admin_provider_mapping_preview_locally_with_trusted_admin_principal() { let upstream_hits = Arc::new(Mutex::new(0usize)); let upstream_hits_clone = Arc::clone(&upstream_hits); let upstream = Router::new().route( "/api/admin/providers/provider-openai/mapping-preview", any(move |_request: Request| { let upstream_hits_inner = Arc::clone(&upstream_hits_clone); async move { *upstream_hits_inner.lock().expect("mutex should lock") += 1; (StatusCode::OK, Body::from("unexpected upstream hit")) } }), ); let mut mapping_key = sample_key( "key-openai-preview", "provider-openai", "openai:chat", "sk-preview-1234", ); mapping_key.allowed_models = Some(json!(["gpt-5", "gpt-4.1-mini"])); let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed( vec![sample_provider("provider-openai", "openai", 10)], Vec::new(), vec![mapping_key], )); let global_model_repository = Arc::new(InMemoryGlobalModelReadRepository::seed(vec![ sample_public_global_model_with_mappings("global-gpt-5", "gpt-5", "GPT 5", &["gpt-5"]), sample_public_global_model_with_mappings( "global-gpt-4.1-mini", "gpt-4.1-mini", "GPT 4.1 mini", &["gpt-4\\.1-.*"], ), ])); let (upstream_url, upstream_handle) = start_server(upstream).await; let gateway = build_router_with_state( AppState::new() .expect("gateway should build") .with_data_state_for_tests( GatewayDataState::with_provider_catalog_reader_for_tests( provider_catalog_repository, ) .with_global_model_repository_for_tests(global_model_repository), ), ); let (gateway_url, gateway_handle) = start_server(gateway).await; let response = reqwest::Client::new() .get(format!( "{gateway_url}/api/admin/providers/provider-openai/mapping-preview" )) .header(crate::constants::GATEWAY_HEADER, "rust-phase3b") .header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123") .header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin") .header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123") .send() .await .expect("request should succeed"); assert_eq!(response.status(), StatusCode::OK); let payload: serde_json::Value = response.json().await.expect("json body should parse"); assert_eq!(payload["provider_id"], "provider-openai"); assert_eq!(payload["provider_name"], "openai"); assert_eq!(payload["total_keys"], 1); assert_eq!(payload["total_matches"], 2); assert_eq!(payload["truncated"], false); let keys = payload["keys"].as_array().expect("keys should be an array"); assert_eq!(keys.len(), 1); assert_eq!(keys[0]["key_id"], "key-openai-preview"); assert_eq!(keys[0]["masked_key"], "sk-p***234"); assert_eq!(keys[0]["allowed_models"], json!(["gpt-5", "gpt-4.1-mini"])); let matches = keys[0]["matching_global_models"] .as_array() .expect("matching models should be an array"); assert_eq!(matches.len(), 2); let gpt5 = matches .iter() .find(|item| item["global_model_name"] == "gpt-5") .expect("gpt-5 match should exist"); let gpt41mini = matches .iter() .find(|item| item["global_model_name"] == "gpt-4.1-mini") .expect("gpt-4.1-mini match should exist"); assert_eq!(gpt5["global_model_name"], "gpt-5"); assert_eq!(gpt5["matched_models"][0]["mapping_pattern"], "gpt-5"); assert_eq!(gpt41mini["global_model_name"], "gpt-4.1-mini"); assert_eq!( gpt41mini["matched_models"][0]["mapping_pattern"], "gpt-4\\.1-.*" ); assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0); gateway_handle.abort(); upstream_handle.abort(); } #[tokio::test] async fn gateway_submits_admin_provider_delete_task_locally_with_trusted_admin_principal() { let upstream_hits = Arc::new(Mutex::new(0usize)); let upstream_hits_clone = Arc::clone(&upstream_hits); let upstream = Router::new() .route( "/api/admin/providers/provider-openai", any(move |_request: Request| { let upstream_hits_inner = Arc::clone(&upstream_hits_clone); async move { *upstream_hits_inner.lock().expect("mutex should lock") += 1; (StatusCode::OK, Body::from("unexpected upstream hit")) } }), ) .route( "/api/admin/providers/provider-openai/delete-task/{task_id}", any(|_request: Request| async move { (StatusCode::OK, Body::from("unexpected upstream hit")) }), ); let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed( vec![sample_provider("provider-openai", "openai", 10)], vec![sample_endpoint( "endpoint-openai-chat", "provider-openai", "openai:chat", "https://api.openai.example", )], vec![sample_key( "key-openai-delete", "provider-openai", "openai:chat", "sk-delete", )], )); let global_model_repository = Arc::new( InMemoryGlobalModelReadRepository::seed(Vec::new()).with_admin_provider_models(vec![ sample_admin_provider_model( "model-openai-delete", "provider-openai", "global-gpt-5", "gpt-5", ), ]), ); let (upstream_url, upstream_handle) = start_server(upstream).await; let gateway = build_router_with_state( AppState::new() .expect("gateway should build") .with_data_state_for_tests( GatewayDataState::with_provider_catalog_repository_for_tests( provider_catalog_repository.clone(), ) .with_global_model_repository_for_tests(global_model_repository.clone()), ), ); let (gateway_url, gateway_handle) = start_server(gateway).await; let response = reqwest::Client::new() .delete(format!("{gateway_url}/api/admin/providers/provider-openai")) .header(crate::constants::GATEWAY_HEADER, "rust-phase3b") .header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123") .header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin") .header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123") .send() .await .expect("request should succeed"); assert_eq!(response.status(), StatusCode::OK); let submit_payload: serde_json::Value = response.json().await.expect("json body should parse"); let task_id = submit_payload["task_id"] .as_str() .expect("task id should be present") .to_string(); assert_eq!(submit_payload["status"], "pending"); assert_eq!( submit_payload["message"], "删除任务已提交，提供商已进入后台删除队列" ); let task_response = reqwest::Client::new() .get(format!( "{gateway_url}/api/admin/providers/provider-openai/delete-task/{task_id}" )) .header(crate::constants::GATEWAY_HEADER, "rust-phase3b") .header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123") .header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin") .header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123") .send() .await .expect("request should succeed"); assert_eq!(task_response.status(), StatusCode::OK); let task_payload: serde_json::Value = task_response.json().await.expect("json body should parse"); assert_eq!(task_payload["task_id"], task_id); assert_eq!(task_payload["provider_id"], "provider-openai"); assert_eq!(task_payload["status"], "completed"); assert_eq!(task_payload["stage"], "completed"); assert_eq!(task_payload["total_keys"], 1); assert_eq!(task_payload["deleted_keys"], 1); assert_eq!(task_payload["total_endpoints"], 1); assert_eq!(task_payload["deleted_endpoints"], 1); let remaining_providers = provider_catalog_repository .list_providers(false) .await .expect("provider list should load"); assert!(remaining_providers.is_empty()); let remaining_endpoints = provider_catalog_repository .list_endpoints_by_provider_ids(&["provider-openai".to_string()]) .await .expect("endpoint list should load"); assert!(remaining_endpoints.is_empty()); let remaining_keys = provider_catalog_repository .list_keys_by_provider_ids(&["provider-openai".to_string()]) .await .expect("key list should load"); assert!(remaining_keys.is_empty()); let remaining_models = global_model_repository .list_admin_provider_models( &aether_data_contracts::repository::global_models::AdminProviderModelListQuery { provider_id: "provider-openai".to_string(), offset: 0, limit: 100, is_active: None, }, ) .await .expect("provider model list should load"); assert!(remaining_models.is_empty()); assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0); gateway_handle.abort(); upstream_handle.abort(); } #[test] fn provider_delete_task_reservation_reuses_active_provider_task() { let state = AppState::new().expect("gateway should build"); let first = crate::LocalProviderDeleteTaskState { task_id: "task-first".to_string(), provider_id: "provider-openai".to_string(), status: "pending".to_string(), stage: "queued".to_string(), total_keys: 0, deleted_keys: 0, total_endpoints: 0, deleted_endpoints: 0, message: "delete task submitted".to_string(), }; let second = crate::LocalProviderDeleteTaskState { task_id: "task-second".to_string(), provider_id: "provider-openai".to_string(), status: "pending".to_string(), stage: "queued".to_string(), total_keys: 0, deleted_keys: 0, total_endpoints: 0, deleted_endpoints: 0, message: "delete task submitted".to_string(), }; assert_eq!( state.reserve_provider_delete_task(first.clone()).task_id, "task-first" ); assert_eq!( state.reserve_provider_delete_task(second.clone()).task_id, "task-first" ); state.put_provider_delete_task(crate::LocalProviderDeleteTaskState { status: "completed".to_string(), stage: "completed".to_string(), ..first }); assert_eq!( state.reserve_provider_delete_task(second).task_id, "task-second" ); } #[tokio::test] async fn local_admin_provider_delete_task_status_attaches_audit_only_for_terminal_states() { let mut completed_state = AppState::new().expect("gateway should build"); completed_state.put_provider_delete_task(crate::LocalProviderDeleteTaskState { task_id: "task-completed".to_string(), provider_id: "provider-openai".to_string(), status: "completed".to_string(), stage: "completed".to_string(), total_keys: 3, deleted_keys: 3, total_endpoints: 2, deleted_endpoints: 2, message: "provider deleted".to_string(), }); let completed_response = local_admin_providers_response( &completed_state, http::Method::GET, "/api/admin/providers/provider-openai/delete-task/task-completed", None, ) .await; assert_eq!(completed_response.status(), StatusCode::OK); let completed_audit = completed_response .extensions() .get::() .expect("completed delete task should attach audit"); assert_eq!( completed_audit.event_name, "admin_provider_delete_task_completed_viewed" ); assert_eq!( completed_audit.action, "view_provider_delete_task_terminal_state" ); assert_eq!(completed_audit.target_type, "provider_delete_task"); assert_eq!(completed_audit.target_id, "provider-openai:task-completed"); let mut running_state = AppState::new().expect("gateway should build"); running_state.put_provider_delete_task(crate::LocalProviderDeleteTaskState { task_id: "task-running".to_string(), provider_id: "provider-openai".to_string(), status: "running".to_string(), stage: "deleting_keys".to_string(), total_keys: 3, deleted_keys: 1, total_endpoints: 2, deleted_endpoints: 0, message: "deleted 1 / 3 keys".to_string(), }); let running_response = local_admin_providers_response( &running_state, http::Method::GET, "/api/admin/providers/provider-openai/delete-task/task-running", None, ) .await; assert_eq!(running_response.status(), StatusCode::OK); assert!( running_response .extensions() .get::() .is_none(), "running delete task should not attach audit" ); } #[tokio::test] async fn gateway_handles_admin_provider_pool_status_locally_with_trusted_admin_principal() { let upstream_hits = Arc::new(Mutex::new(0usize)); let upstream_hits_clone = Arc::clone(&upstream_hits); let upstream = Router::new().route( "/api/admin/providers/provider-openai/pool-status", any(move |_request: Request| { let upstream_hits_inner = Arc::clone(&upstream_hits_clone); async move { *upstream_hits_inner.lock().expect("mutex should lock") += 1; (StatusCode::OK, Body::from("unexpected upstream hit")) } }), ); let provider = sample_provider("provider-openai", "openai", 10).with_transport_fields( true, false, true, None, None, None, None, None, Some(json!({ "pool_advanced": { "lru_enabled": true, "cost_window_seconds": 7200, "cost_limit_per_key_tokens": 12000 } })), ); let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed( vec![provider], Vec::new(), vec![sample_key( "key-openai-pool", "provider-openai", "openai:chat", "sk-test", )], )); let (upstream_url, upstream_handle) = start_server(upstream).await; let gateway = build_router_with_state( AppState::new() .expect("gateway should build") .with_data_state_for_tests(GatewayDataState::with_provider_catalog_reader_for_tests( provider_catalog_repository, )), ); let (gateway_url, gateway_handle) = start_server(gateway).await; let response = reqwest::Client::new() .get(format!( "{gateway_url}/api/admin/providers/provider-openai/pool-status" )) .header(crate::constants::GATEWAY_HEADER, "rust-phase3b") .header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123") .header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin") .header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123") .send() .await .expect("request should succeed"); assert_eq!(response.status(), StatusCode::OK); let payload: serde_json::Value = response.json().await.expect("json body should parse"); assert_eq!(payload["provider_id"], "provider-openai"); assert_eq!(payload["provider_name"], "openai"); assert_eq!(payload["pool_enabled"], true); assert_eq!(payload["total_keys"], 1); assert_eq!(payload["total_sticky_sessions"], 0); assert_eq!(payload["provider_hot_count"], 0); assert_eq!(payload["provider_desired_hot"], 0); assert_eq!(payload["provider_in_flight"], 0); assert_eq!(payload["provider_ema_in_flight"], 0.0); assert_eq!(payload["provider_burst_pending"], false); let keys = payload["keys"].as_array().expect("keys should be an array"); assert_eq!(keys.len(), 1); assert_eq!(keys[0]["key_id"], "key-openai-pool"); assert_eq!(keys[0]["key_name"], "default"); assert_eq!(keys[0]["cost_window_usage"], 0); assert_eq!(keys[0]["cost_limit"], 12000); assert_eq!(keys[0]["sticky_sessions"], 0); assert!(keys[0]["lru_score"].is_null()); assert!(keys[0]["cooldown_reason"].is_null()); assert!(keys[0]["cooldown_ttl_seconds"].is_null()); assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0); gateway_handle.abort(); upstream_handle.abort(); } #[tokio::test] async fn gateway_clears_admin_provider_pool_cooldown_locally_with_trusted_admin_principal() { let upstream_hits = Arc::new(Mutex::new(0usize)); let upstream_hits_clone = Arc::clone(&upstream_hits); let upstream = Router::new().route( "/api/admin/providers/provider-openai/pool/clear-cooldown/key-openai-pool", any(move |_request: Request| { let upstream_hits_inner = Arc::clone(&upstream_hits_clone); async move { *upstream_hits_inner.lock().expect("mutex should lock") += 1; (StatusCode::OK, Body::from("unexpected upstream hit")) } }), ); let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed( vec![sample_provider("provider-openai", "openai", 10)], Vec::new(), vec![sample_key( "key-openai-pool", "provider-openai", "openai:chat", "sk-test", )], )); let (upstream_url, upstream_handle) = start_server(upstream).await; let gateway = build_router_with_state( AppState::new() .expect("gateway should build") .with_data_state_for_tests(GatewayDataState::with_provider_catalog_reader_for_tests( provider_catalog_repository, )), ); let (gateway_url, gateway_handle) = start_server(gateway).await; let response = reqwest::Client::new() .post(format!( "{gateway_url}/api/admin/providers/provider-openai/pool/clear-cooldown/key-openai-pool" )) .header(crate::constants::GATEWAY_HEADER, "rust-phase3b") .header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123") .header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin") .header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123") .send() .await .expect("request should succeed"); assert_eq!(response.status(), StatusCode::OK); let payload: serde_json::Value = response.json().await.expect("json body should parse"); assert_eq!(payload["message"], "已清除 Key default 的冷却状态"); assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0); gateway_handle.abort(); upstream_handle.abort(); } #[tokio::test] async fn gateway_resets_admin_provider_pool_cost_locally_with_trusted_admin_principal() { let upstream_hits = Arc::new(Mutex::new(0usize)); let upstream_hits_clone = Arc::clone(&upstream_hits); let upstream = Router::new().route( "/api/admin/providers/provider-openai/pool/reset-cost/key-openai-pool", any(move |_request: Request| { let upstream_hits_inner = Arc::clone(&upstream_hits_clone); async move { *upstream_hits_inner.lock().expect("mutex should lock") += 1; (StatusCode::OK, Body::from("unexpected upstream hit")) } }), ); let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed( vec![sample_provider("provider-openai", "openai", 10)], Vec::new(), vec![sample_key( "key-openai-pool", "provider-openai", "openai:chat", "sk-test", )], )); let (upstream_url, upstream_handle) = start_server(upstream).await; let gateway = build_router_with_state( AppState::new() .expect("gateway should build") .with_data_state_for_tests(GatewayDataState::with_provider_catalog_reader_for_tests( provider_catalog_repository, )), ); let (gateway_url, gateway_handle) = start_server(gateway).await; let response = reqwest::Client::new() .post(format!( "{gateway_url}/api/admin/providers/provider-openai/pool/reset-cost/key-openai-pool" )) .header(crate::constants::GATEWAY_HEADER, "rust-phase3b") .header(TRUSTED_ADMIN_USER_ID_HEADER, "admin-user-123") .header(TRUSTED_ADMIN_USER_ROLE_HEADER, "admin") .header(TRUSTED_ADMIN_SESSION_ID_HEADER, "session-123") .send() .await .expect("request should succeed"); assert_eq!(response.status(), StatusCode::OK); let payload: serde_json::Value = response.json().await.expect("json body should parse"); assert_eq!(payload["message"], "已重置 Key default 的成本窗口"); assert_eq!(*upstream_hits.lock().expect("mutex should lock"), 0); gateway_handle.abort(); upstream_handle.abort(); }