fix(gateway): harden Gemini endpoint routing

This commit is contained in:
MMEXA
2026-05-18 00:53:34 +00:00
parent 81ff375bfd
commit b004a02e4a
20 changed files with 1348 additions and 83 deletions
@@ -129,6 +129,56 @@ fn gemini_embedding_success_state(
.with_data_state_for_tests(data_state)
}
fn vertex_gemini_embedding_success_state(execution_runtime_url: String) -> AppState {
let mut snapshot = sample_currently_usable_auth_snapshot(
"key-vertex-gemini-embedding-success",
"user-vertex-gemini-embedding-success",
);
snapshot.user_allowed_providers = None;
snapshot.api_key_allowed_providers = Some(vec!["openai".to_string(), "vertex_ai".to_string()]);
snapshot.user_allowed_api_formats = Some(vec!["openai:embedding".to_string()]);
snapshot.api_key_allowed_api_formats = Some(vec!["openai:embedding".to_string()]);
snapshot.user_allowed_models = Some(vec!["gemini-embedding-2-preview".to_string()]);
snapshot.api_key_allowed_models = Some(vec!["gemini-embedding-2-preview".to_string()]);
let auth_repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
Some(hash_api_key("sk-vertex-gemini-embedding-success")),
snapshot,
)]));
let candidate_repository =
Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(vec![
vertex_gemini_embedding_candidate_row(),
]));
let mut provider = sample_provider("provider-vertex-gemini-embedding", "Vertex AI", 1);
provider.provider_type = "vertex_ai".to_string();
let mut key = sample_key(
"key-upstream-vertex-gemini-embedding",
"provider-vertex-gemini-embedding",
"gemini:embedding",
"sk-upstream-vertex-gemini-embedding",
);
key.allowed_models = Some(json!(["gemini-embedding-2"]));
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![provider],
vec![sample_endpoint(
"endpoint-vertex-gemini-embedding",
"provider-vertex-gemini-embedding",
"gemini:embedding",
"https://aiplatform.googleapis.com",
)],
vec![key],
));
let data_state =
GatewayDataState::with_provider_catalog_and_minimal_candidate_selection_for_tests(
provider_catalog_repository,
candidate_repository,
)
.with_auth_api_key_reader(auth_repository)
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY);
build_state_with_execution_runtime_override(execution_runtime_url)
.with_data_state_for_tests(data_state)
}
fn gemini_embedding_conversion_execution_runtime() -> Router {
Router::new().route(
"/v1/execute/sync",
@@ -139,6 +189,16 @@ fn gemini_embedding_conversion_execution_runtime() -> Router {
)
}
fn vertex_gemini_embedding_conversion_execution_runtime() -> Router {
Router::new().route(
"/v1/execute/sync",
any(|Json(plan): Json<ExecutionPlan>| async move {
assert_openai_to_vertex_gemini_embedding_execution_plan(&plan);
Json(gemini_embedding_execution_result(&plan))
}),
)
}
fn gemini_embedding_batch_conversion_execution_runtime() -> Router {
Router::new().route(
"/v1/execute/sync",
@@ -227,6 +287,19 @@ fn gemini_embedding_candidate_row() -> StoredMinimalCandidateSelectionRow {
}
}
fn vertex_gemini_embedding_candidate_row() -> StoredMinimalCandidateSelectionRow {
let mut row = gemini_embedding_candidate_row();
row.provider_id = "provider-vertex-gemini-embedding".to_string();
row.provider_name = "Vertex AI".to_string();
row.provider_type = "vertex_ai".to_string();
row.endpoint_id = "endpoint-vertex-gemini-embedding".to_string();
row.key_id = "key-upstream-vertex-gemini-embedding".to_string();
row.key_name = "default".to_string();
row.key_allowed_models = Some(vec!["gemini-embedding-2".to_string()]);
row.model_provider_model_name = "gemini-embedding-2".to_string();
row
}
fn assert_embedding_execution_plan(plan: &ExecutionPlan) {
assert_eq!(plan.client_api_format, "openai:embedding");
assert_eq!(plan.provider_api_format, "openai:embedding");
@@ -262,6 +335,30 @@ fn assert_openai_to_gemini_embedding_execution_plan(plan: &ExecutionPlan) {
assert!(body.get("messages").is_none());
}
fn assert_openai_to_vertex_gemini_embedding_execution_plan(plan: &ExecutionPlan) {
assert_eq!(plan.provider_id, "provider-vertex-gemini-embedding");
assert_eq!(plan.client_api_format, "openai:embedding");
assert_eq!(plan.provider_api_format, "gemini:embedding");
assert_eq!(plan.method, "POST");
assert_eq!(
plan.url,
"https://aiplatform.googleapis.com/v1/publishers/google/models/gemini-embedding-2:embedContent?key=sk-upstream-vertex-gemini-embedding"
);
assert_eq!(
plan.model_name.as_deref(),
Some("gemini-embedding-2-preview")
);
assert!(!plan.stream);
let body = plan.body.json_body.as_ref().expect("json request body");
assert!(
body.get("model").is_none(),
"Vertex embedContent carries the model in the path; the body must not repeat it"
);
assert_eq!(body["content"]["parts"][0]["text"], "hello");
assert!(body.get("input").is_none());
assert!(body.get("messages").is_none());
}
fn assert_openai_to_gemini_batch_embedding_execution_plan(plan: &ExecutionPlan) {
assert_eq!(plan.client_api_format, "openai:embedding");
assert_eq!(plan.provider_api_format, "gemini:embedding");
@@ -502,6 +599,50 @@ async fn embeddings_route_converts_openai_payload_to_gemini_embedding_provider()
execution_runtime_handle.abort();
}
#[tokio::test]
async fn embeddings_route_converts_openai_payload_to_vertex_gemini_embedding_provider() {
let (execution_runtime_url, execution_runtime_handle) =
start_server(vertex_gemini_embedding_conversion_execution_runtime()).await;
let gateway =
build_router_with_state(vertex_gemini_embedding_success_state(execution_runtime_url));
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.post(format!("{gateway_url}/v1/embeddings"))
.header(
http::header::AUTHORIZATION,
"Bearer sk-vertex-gemini-embedding-success",
)
.json(&json!({
"model": "gemini-embedding-2-preview",
"input": "hello"
}))
.send()
.await
.expect("request should succeed");
let endpoint_signature = response
.headers()
.get(CONTROL_ENDPOINT_SIGNATURE_HEADER)
.and_then(|value| value.to_str().ok())
.map(str::to_string);
let status = response.status();
let body_text = response.text().await.expect("body should read");
assert_eq!(
status,
StatusCode::OK,
"unexpected response body: {body_text}"
);
assert_eq!(endpoint_signature.as_deref(), Some("openai:embedding"));
let payload: serde_json::Value = serde_json::from_str(&body_text).expect("body should parse");
assert_eq!(payload["object"], "list");
assert_eq!(payload["model"], "gemini-embedding-2-preview");
assert_eq!(payload["data"][0]["embedding"], json!([0.1, 0.2, 0.3]));
gateway_handle.abort();
execution_runtime_handle.abort();
}
#[tokio::test]
async fn embeddings_route_converts_openai_batch_payload_to_gemini_batch_endpoint() {
let (execution_runtime_url, execution_runtime_handle) =
@@ -1978,6 +1978,91 @@ async fn gateway_handles_public_test_connection_without_hitting_fallback_probe()
provider_handle.abort();
}
#[tokio::test]
async fn gateway_gemini_test_connection_does_not_force_low_max_output_tokens() {
let provider_hits = Arc::new(Mutex::new(0usize));
let provider_hits_clone = Arc::clone(&provider_hits);
let provider = Router::new().route(
"/{*path}",
any(move |request: Request| {
let provider_hits_inner = Arc::clone(&provider_hits_clone);
async move {
*provider_hits_inner.lock().expect("mutex should lock") += 1;
let body = to_bytes(request.into_body(), usize::MAX)
.await
.expect("body should read");
let body_json: serde_json::Value =
serde_json::from_slice(&body).expect("json body should parse");
assert_eq!(body_json["contents"][0]["parts"][0]["text"], "Health check");
assert!(
body_json
.get("generationConfig")
.and_then(|config| config.get("maxOutputTokens"))
.is_none(),
"Gemini test connection must not force a tiny maxOutputTokens value"
);
Json(json!({
"candidates": [{
"content": {
"role": "model",
"parts": [{"text": "ok"}]
},
"finishReason": "STOP"
}],
"responseId": "gemini_test_connection_ok"
}))
.into_response()
}
}),
);
let (provider_url, provider_handle) = start_server(provider).await;
let provider_catalog_repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider("provider-gemini", "google", 10)],
vec![sample_endpoint(
"endpoint-gemini",
"provider-gemini",
"gemini:generate_content",
&provider_url,
)],
vec![sample_key(
"key-gemini",
"provider-gemini",
"gemini:generate_content",
"google-api-key",
)],
));
let gateway = build_router_with_state(
AppState::new()
.expect("gateway should build")
.with_data_state_for_tests(GatewayDataState::with_provider_transport_reader_for_tests(
provider_catalog_repository,
DEVELOPMENT_ENCRYPTION_KEY,
)),
);
let (gateway_url, gateway_handle) = start_server(gateway).await;
let response = reqwest::Client::new()
.get(format!(
"{gateway_url}/v1/test-connection?provider=provider-gemini&model=gemini-3-flash-preview&api_format=gemini:generate_content"
))
.send()
.await
.expect("request should succeed");
assert_eq!(response.status(), StatusCode::OK);
let payload: serde_json::Value = response.json().await.expect("json body should parse");
assert_eq!(payload["status"], "success");
assert_eq!(payload["provider_id"], "provider-gemini");
assert_eq!(payload["endpoint_id"], "endpoint-gemini");
assert_eq!(payload["api_format"], "gemini:generate_content");
assert_eq!(*provider_hits.lock().expect("mutex should lock"), 1);
gateway_handle.abort();
provider_handle.abort();
}
async fn assert_public_support_route_returns_local_503(
method: reqwest::Method,
path: &str,