修复 Gemini 空输出按候选重试处理

This commit is contained in:
MMEXA
2026-07-09 23:10:30 +08:00
parent 7bc7d0f8d8
commit d2ea437c1c
5 changed files with 385 additions and 30 deletions
@@ -1439,6 +1439,57 @@ fn local_finalize_handles_openai_responses_cross_format_stream_response_from_gem
);
}
#[test]
fn local_finalize_rejects_antigravity_usage_only_gemini_wrapper() {
let payload = GatewaySyncReportRequest {
trace_id: "trace-antigravity-empty-gemini-wrapper".to_string(),
report_kind: "gemini_chat_sync_finalize".to_string(),
report_context: Some(json!({
"client_api_format": "gemini:generate_content",
"provider_api_format": "gemini:generate_content",
"model": "gemini-3.5-flash",
"mapped_model": "gemini-3-flash-agent",
"needs_conversion": false,
"has_envelope": true,
"envelope_name": "antigravity:v1internal",
"upstream_is_stream": true,
})),
status_code: 200,
headers: BTreeMap::from([("content-type".to_string(), "application/json".to_string())]),
body_json: Some(json!({
"chunks": [{
"response": {
"responseId": "resp-usage-only",
"modelVersion": "gemini-3-flash-agent",
"usageMetadata": {
"promptTokenCount": 5528,
"totalTokenCount": 5528
}
},
"metadata": {},
"traceId": "trace-antigravity-empty-gemini-wrapper"
}],
"metadata": {
"stream": true,
"stored_chunks": 1,
"total_chunks": 1
}
})),
client_body_json: None,
body_base64: None,
telemetry: None,
};
let outcome = maybe_build_local_core_sync_finalize_response(
"trace-antigravity-empty-gemini-wrapper",
&test_decision(),
&payload,
)
.expect("local finalize should evaluate payload");
assert!(outcome.is_none());
}
#[test]
fn local_finalize_handles_openai_responses_compact_openai_family_stream_response_even_when_conversion_flagged(
) {
@@ -105,6 +105,7 @@ const SYNC_EXECUTION_IDLE_LOG_INTERVAL: Duration = Duration::from_secs(60);
const OPENAI_IMAGE_SYNC_JSON_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(15);
const OPENAI_IMAGE_SYNC_JSON_HEARTBEAT_BYTES: &[u8] = b"\n";
const OPENAI_IMAGE_SYNC_PROGRESS_WRITE_INTERVAL: Duration = Duration::from_secs(5);
const INVALID_GEMINI_PROVIDER_SUCCESS_MESSAGE: &str = "Provider returned HTTP 200 but the Gemini response did not contain visible model output; refusing to finalize it as a successful response.";
#[derive(Debug)]
struct SyncExecutionFailure {
@@ -628,13 +629,7 @@ fn invalid_gemini_provider_success_message(
if status_code >= 400 {
return None;
}
let provider_api_format = report_context
.and_then(|value| value.get("provider_api_format"))
.and_then(Value::as_str)
.unwrap_or(plan.provider_api_format.as_str());
if crate::ai_serving::normalize_api_format_alias(provider_api_format)
!= "gemini:generate_content"
{
if !provider_api_format_is_gemini_generate_content(plan, report_context) {
return None;
}
let body_json = body_json?;
@@ -658,7 +653,53 @@ fn invalid_gemini_provider_success_message(
if crate::ai_serving::gemini_generate_content_response_has_visible_output(body_json) {
return None;
}
Some("Provider returned HTTP 200 but the Gemini response did not contain visible model output; refusing to finalize it as a successful response.")
Some(INVALID_GEMINI_PROVIDER_SUCCESS_MESSAGE)
}
fn invalid_gemini_provider_stream_success_message(
plan: &ExecutionPlan,
report_context: Option<&Value>,
status_code: u16,
body_json: Option<&Value>,
body_bytes: &[u8],
has_body_bytes: bool,
) -> Option<&'static str> {
if status_code >= 400 || body_json.is_some() || !has_body_bytes {
return None;
}
if !provider_api_format_is_gemini_generate_content(plan, report_context) {
return None;
}
let Some(body_json) = crate::ai_serving::aggregate_gemini_stream_sync_response(body_bytes)
else {
return Some(INVALID_GEMINI_PROVIDER_SUCCESS_MESSAGE);
};
if crate::ai_serving::gemini_generate_content_response_has_visible_output(&body_json) {
return None;
}
Some(INVALID_GEMINI_PROVIDER_SUCCESS_MESSAGE)
}
fn provider_api_format_is_gemini_generate_content(
plan: &ExecutionPlan,
report_context: Option<&Value>,
) -> bool {
let provider_api_format = report_context
.and_then(|value| value.get("provider_api_format"))
.and_then(Value::as_str)
.unwrap_or(plan.provider_api_format.as_str());
crate::ai_serving::normalize_api_format_alias(provider_api_format) == "gemini:generate_content"
}
fn invalid_gemini_provider_success_execution_error(message: &str) -> ExecutionError {
ExecutionError {
kind: ExecutionErrorKind::Upstream5xx,
phase: ExecutionPhase::Finalize,
message: message.to_string(),
upstream_status: Some(StatusCode::OK.as_u16()),
retryable: true,
failover_recommended: true,
}
}
fn build_invalid_provider_success_body(
@@ -2031,16 +2072,19 @@ async fn execute_execution_runtime_sync_impl(
report_context.as_ref(),
result.status_code,
body_json.as_ref(),
) {
)
.or_else(|| {
invalid_gemini_provider_stream_success_message(
&plan,
report_context.as_ref(),
result.status_code,
body_json.as_ref(),
&body_bytes,
body_base64.is_some(),
)
}) {
result.status_code = StatusCode::BAD_GATEWAY.as_u16();
result.error = Some(ExecutionError {
kind: ExecutionErrorKind::Upstream5xx,
phase: ExecutionPhase::Finalize,
message: message.to_string(),
upstream_status: Some(StatusCode::OK.as_u16()),
retryable: false,
failover_recommended: false,
});
result.error = Some(invalid_gemini_provider_success_execution_error(message));
if let Some(error_body) =
build_invalid_provider_success_body(&plan, report_context.as_ref(), message)
{
@@ -2913,6 +2957,65 @@ mod tests {
assert!(message.contains("visible model output"));
}
#[test]
fn invalid_gemini_provider_success_error_is_retryable_candidate_failure() {
let error = invalid_gemini_provider_success_execution_error(
INVALID_GEMINI_PROVIDER_SUCCESS_MESSAGE,
);
assert_eq!(error.kind, ExecutionErrorKind::Upstream5xx);
assert_eq!(error.phase, ExecutionPhase::Finalize);
assert_eq!(error.upstream_status, Some(StatusCode::OK.as_u16()));
assert!(error.retryable);
assert!(error.failover_recommended);
}
#[test]
fn invalid_gemini_provider_success_accepts_antigravity_chunks_with_visible_output() {
let plan = test_gemini_chat_plan();
let report_context = json!({
"has_envelope": true,
"envelope_name": "antigravity:v1internal",
"provider_api_format": "gemini:generate_content",
});
let body = json!({
"chunks": [{
"response": {
"responseId": "resp_antigravity_chunks_123",
"candidates": [{
"content": {
"parts": [{"text": "Hello Gemini"}],
"role": "model"
},
"finishReason": "STOP",
"index": 0
}],
"modelVersion": "gemini-3-flash-agent",
"usageMetadata": {
"promptTokenCount": 2,
"candidatesTokenCount": 2,
"totalTokenCount": 4
}
},
"traceId": "trace-antigravity-chunks"
}],
"metadata": {
"stream": true,
"stored_chunks": 1,
"total_chunks": 1
}
});
let message = invalid_gemini_provider_success_message(
&plan,
Some(&report_context),
StatusCode::OK.as_u16(),
Some(&body),
);
assert!(message.is_none());
}
#[test]
fn invalid_gemini_provider_success_unwraps_gemini_cli_v1internal_envelope() {
let plan = test_gemini_chat_plan();
@@ -305,6 +305,9 @@ pub(super) fn admin_usage_terminal_candidate_state_override(
candidates: &[StoredRequestCandidate],
) -> Option<serde_json::Value> {
let candidate = admin_usage_current_candidate(candidates)?;
if admin_usage_candidate_failure_is_retryable_transition(candidate) {
return None;
}
let status = match candidate.status {
RequestCandidateStatus::Success => "completed",
@@ -343,6 +346,30 @@ pub(super) fn admin_usage_terminal_candidate_state_override(
Some(payload)
}
fn admin_usage_candidate_failure_is_retryable_transition(
candidate: &StoredRequestCandidate,
) -> bool {
if candidate.status != RequestCandidateStatus::Failed {
return false;
}
let Some(error_flow) = candidate
.extra_data
.as_ref()
.and_then(|value| value.get("error_flow"))
else {
return false;
};
let retryable = error_flow
.get("retryable")
.and_then(serde_json::Value::as_bool)
.unwrap_or(false);
let retry_next_candidate = error_flow
.get("decision")
.and_then(serde_json::Value::as_str)
.is_some_and(|value| value == "retry_next_candidate");
retryable && retry_next_candidate
}
pub(super) fn apply_admin_usage_state_override(
item: &mut StoredRequestUsageAudit,
override_payload: &serde_json::Value,
@@ -998,6 +1025,7 @@ mod tests {
use aether_data_contracts::repository::candidates::{
RequestCandidateStatus, StoredRequestCandidate,
};
use serde_json::json;
use super::admin_usage_terminal_candidate_state_override;
@@ -1076,4 +1104,27 @@ mod tests {
assert!(payload.is_none());
}
#[test]
fn admin_usage_active_override_ignores_retryable_candidate_failure() {
let mut failed = sample_candidate(
0,
RequestCandidateStatus::Failed,
Some(502),
Some(1_000),
Some("provider returned HTTP 200 without visible model output"),
);
failed.extra_data = Some(json!({
"error_flow": {
"decision": "retry_next_candidate",
"retryable": true,
"propagation": "suppressed",
"classification": "retry_upstream_failure"
}
}));
let payload = admin_usage_terminal_candidate_state_override(&[failed]);
assert!(payload.is_none());
}
}
@@ -510,6 +510,11 @@ fn maybe_build_standard_same_format_sync_body(
if is_error_like_sync_body(body_json) {
return None;
}
if api_format_is_gemini_generate_content(expected_api_format)
&& !gemini_generate_content_body_has_visible_output(body_json)
{
return None;
}
let mut body_json = body_json.clone();
if expected_api_format == "claude:messages" {
@@ -578,6 +583,11 @@ fn maybe_build_standard_same_format_stream_sync_body(
else {
return Ok(None);
};
if api_format_is_gemini_generate_content(&provider_stream_event_api_format)
&& !gemini_generate_content_body_has_visible_output(&body)
{
return Ok(None);
}
let Some(client_body) = convert_aggregated_stream_body_to_client_sync_response(
report_kind,
body,
@@ -752,6 +762,14 @@ fn is_error_like_sync_body(value: &Value) -> bool {
})
}
fn gemini_generate_content_body_has_visible_output(value: &Value) -> bool {
crate::formats::gemini::generate_content::response::from_raw(value).is_some()
}
fn api_format_is_gemini_generate_content(api_format: &str) -> bool {
aether_ai_formats::normalize_api_format_alias(api_format) == "gemini:generate_content"
}
pub fn maybe_build_standard_cross_format_sync_product(
report_kind: &str,
provider_api_format: &str,
@@ -5415,6 +5433,34 @@ mod tests {
assert_eq!(body_json["choices"][0]["message"]["content"], "pong");
}
#[test]
fn standard_same_format_gemini_finalize_rejects_usage_only_body() {
let report_context = json!({
"provider_api_format": "gemini:generate_content",
"client_api_format": "gemini:generate_content",
"needs_conversion": false,
});
let provider_body_json = json!({
"responseId": "resp-empty-visible-output",
"modelVersion": "gemini-3-flash-preview",
"usageMetadata": {
"promptTokenCount": 8,
"totalTokenCount": 8
}
});
let product = maybe_build_standard_sync_finalize_product_from_normalized_payload(
"gemini_chat_sync_finalize",
200,
Some(&report_context),
Some(&provider_body_json),
None,
)
.expect("dispatch should succeed");
assert_eq!(product, None);
}
#[test]
fn standard_cross_format_finalize_uses_explicit_stream_event_format_for_provider_body() {
let body = concat!(
@@ -101,19 +101,7 @@ pub fn normalize_provider_private_response_value(
}
}
Some(ANTIGRAVITY_V1INTERNAL_ENVELOPE_NAME) => {
if let Some(response) = data
.get("response")
.and_then(Value::as_object)
.filter(|response| !response.contains_key("response"))
{
let mut unwrapped = response.clone();
if let Some(response_id) = data.get("responseId").cloned() {
unwrapped.insert("_v1internal_response_id".to_string(), response_id);
}
Value::Object(unwrapped)
} else {
data
}
normalize_antigravity_sync_response_value(data, provider_api_format)
}
Some(WINDSURF_ENVELOPE_NAME) => normalize_windsurf_sync_response_value(data)?,
_ => return None,
@@ -122,6 +110,59 @@ pub fn normalize_provider_private_response_value(
Some(unwrapped)
}
fn normalize_antigravity_sync_response_value(data: Value, provider_api_format: &str) -> Value {
if crate::normalize_api_format_alias(provider_api_format) == "gemini:generate_content" {
if let Some(aggregated) = aggregate_antigravity_gemini_chunks(&data) {
return aggregated;
}
}
if let Some(response) = data
.get("response")
.and_then(Value::as_object)
.filter(|response| !response.contains_key("response"))
{
let mut unwrapped = response.clone();
if let Some(response_id) = data.get("responseId").cloned() {
unwrapped.insert("_v1internal_response_id".to_string(), response_id);
}
Value::Object(unwrapped)
} else {
data
}
}
fn aggregate_antigravity_gemini_chunks(data: &Value) -> Option<Value> {
let chunks = data.get("chunks").and_then(Value::as_array)?;
if chunks.is_empty() {
return None;
}
let body = serde_json::to_vec(chunks).ok()?;
let mut aggregated =
crate::formats::shared::sync_products::aggregate_gemini_stream_sync_response(&body)?;
if let Some(response_id) = antigravity_chunks_response_id(chunks) {
if let Some(object) = aggregated.as_object_mut() {
if object.get("responseId").is_none_or(|value| value.is_null()) {
object.insert("responseId".to_string(), response_id);
}
}
}
Some(aggregated)
}
fn antigravity_chunks_response_id(chunks: &[Value]) -> Option<Value> {
chunks.iter().find_map(|chunk| {
chunk
.get("responseId")
.or_else(|| {
chunk
.get("response")
.and_then(|response| response.get("responseId"))
})
.cloned()
.filter(|value| !value.is_null())
})
}
pub fn transform_provider_private_stream_line(
report_context: &Value,
line: Vec<u8>,
@@ -914,6 +955,69 @@ mod tests {
);
}
#[test]
fn unwraps_antigravity_chunks_sync_response() {
let report_context = json!({
"has_envelope": true,
"provider_api_format": "gemini:generate_content",
"envelope_name": "antigravity:v1internal",
"mapped_model": "gemini-3-flash-agent",
});
let body = json!({
"chunks": [{
"response": {
"responseId": "resp_antigravity_chunks_123",
"candidates": [{
"content": {
"parts": [{"text": "Hello "}],
"role": "model"
},
"index": 0
}],
"modelVersion": "gemini-3-flash-agent"
},
"traceId": "trace-antigravity-chunks"
}, {
"response": {
"responseId": "resp_antigravity_chunks_123",
"candidates": [{
"content": {
"parts": [{"text": "Gemini"}],
"role": "model"
},
"finishReason": "STOP",
"index": 0
}],
"modelVersion": "gemini-3-flash-agent",
"usageMetadata": {
"promptTokenCount": 2,
"candidatesTokenCount": 2,
"totalTokenCount": 4
}
},
"traceId": "trace-antigravity-chunks"
}],
"metadata": {
"stream": true,
"stored_chunks": 2,
"total_chunks": 2
}
});
let normalized = normalize_provider_private_response_value(body, &report_context)
.expect("chunks should normalize");
assert_eq!(
normalized["candidates"][0]["content"]["parts"][0]["text"],
json!("Hello Gemini")
);
assert_eq!(
normalized["_v1internal_response_id"],
json!("resp_antigravity_chunks_123")
);
assert_eq!(normalized["usageMetadata"]["totalTokenCount"], json!(4));
}
#[test]
fn unwraps_antigravity_stream_line_and_injects_ids() {
let report_context = json!({