mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-10 13:10:21 +08:00
修复 Gemini 空输出按候选重试处理
This commit is contained in:
@@ -1439,6 +1439,57 @@ fn local_finalize_handles_openai_responses_cross_format_stream_response_from_gem
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_finalize_rejects_antigravity_usage_only_gemini_wrapper() {
|
||||
let payload = GatewaySyncReportRequest {
|
||||
trace_id: "trace-antigravity-empty-gemini-wrapper".to_string(),
|
||||
report_kind: "gemini_chat_sync_finalize".to_string(),
|
||||
report_context: Some(json!({
|
||||
"client_api_format": "gemini:generate_content",
|
||||
"provider_api_format": "gemini:generate_content",
|
||||
"model": "gemini-3.5-flash",
|
||||
"mapped_model": "gemini-3-flash-agent",
|
||||
"needs_conversion": false,
|
||||
"has_envelope": true,
|
||||
"envelope_name": "antigravity:v1internal",
|
||||
"upstream_is_stream": true,
|
||||
})),
|
||||
status_code: 200,
|
||||
headers: BTreeMap::from([("content-type".to_string(), "application/json".to_string())]),
|
||||
body_json: Some(json!({
|
||||
"chunks": [{
|
||||
"response": {
|
||||
"responseId": "resp-usage-only",
|
||||
"modelVersion": "gemini-3-flash-agent",
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 5528,
|
||||
"totalTokenCount": 5528
|
||||
}
|
||||
},
|
||||
"metadata": {},
|
||||
"traceId": "trace-antigravity-empty-gemini-wrapper"
|
||||
}],
|
||||
"metadata": {
|
||||
"stream": true,
|
||||
"stored_chunks": 1,
|
||||
"total_chunks": 1
|
||||
}
|
||||
})),
|
||||
client_body_json: None,
|
||||
body_base64: None,
|
||||
telemetry: None,
|
||||
};
|
||||
|
||||
let outcome = maybe_build_local_core_sync_finalize_response(
|
||||
"trace-antigravity-empty-gemini-wrapper",
|
||||
&test_decision(),
|
||||
&payload,
|
||||
)
|
||||
.expect("local finalize should evaluate payload");
|
||||
|
||||
assert!(outcome.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_finalize_handles_openai_responses_compact_openai_family_stream_response_even_when_conversion_flagged(
|
||||
) {
|
||||
|
||||
@@ -105,6 +105,7 @@ const SYNC_EXECUTION_IDLE_LOG_INTERVAL: Duration = Duration::from_secs(60);
|
||||
const OPENAI_IMAGE_SYNC_JSON_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(15);
|
||||
const OPENAI_IMAGE_SYNC_JSON_HEARTBEAT_BYTES: &[u8] = b"\n";
|
||||
const OPENAI_IMAGE_SYNC_PROGRESS_WRITE_INTERVAL: Duration = Duration::from_secs(5);
|
||||
const INVALID_GEMINI_PROVIDER_SUCCESS_MESSAGE: &str = "Provider returned HTTP 200 but the Gemini response did not contain visible model output; refusing to finalize it as a successful response.";
|
||||
|
||||
#[derive(Debug)]
|
||||
struct SyncExecutionFailure {
|
||||
@@ -628,13 +629,7 @@ fn invalid_gemini_provider_success_message(
|
||||
if status_code >= 400 {
|
||||
return None;
|
||||
}
|
||||
let provider_api_format = report_context
|
||||
.and_then(|value| value.get("provider_api_format"))
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or(plan.provider_api_format.as_str());
|
||||
if crate::ai_serving::normalize_api_format_alias(provider_api_format)
|
||||
!= "gemini:generate_content"
|
||||
{
|
||||
if !provider_api_format_is_gemini_generate_content(plan, report_context) {
|
||||
return None;
|
||||
}
|
||||
let body_json = body_json?;
|
||||
@@ -658,7 +653,53 @@ fn invalid_gemini_provider_success_message(
|
||||
if crate::ai_serving::gemini_generate_content_response_has_visible_output(body_json) {
|
||||
return None;
|
||||
}
|
||||
Some("Provider returned HTTP 200 but the Gemini response did not contain visible model output; refusing to finalize it as a successful response.")
|
||||
Some(INVALID_GEMINI_PROVIDER_SUCCESS_MESSAGE)
|
||||
}
|
||||
|
||||
fn invalid_gemini_provider_stream_success_message(
|
||||
plan: &ExecutionPlan,
|
||||
report_context: Option<&Value>,
|
||||
status_code: u16,
|
||||
body_json: Option<&Value>,
|
||||
body_bytes: &[u8],
|
||||
has_body_bytes: bool,
|
||||
) -> Option<&'static str> {
|
||||
if status_code >= 400 || body_json.is_some() || !has_body_bytes {
|
||||
return None;
|
||||
}
|
||||
if !provider_api_format_is_gemini_generate_content(plan, report_context) {
|
||||
return None;
|
||||
}
|
||||
let Some(body_json) = crate::ai_serving::aggregate_gemini_stream_sync_response(body_bytes)
|
||||
else {
|
||||
return Some(INVALID_GEMINI_PROVIDER_SUCCESS_MESSAGE);
|
||||
};
|
||||
if crate::ai_serving::gemini_generate_content_response_has_visible_output(&body_json) {
|
||||
return None;
|
||||
}
|
||||
Some(INVALID_GEMINI_PROVIDER_SUCCESS_MESSAGE)
|
||||
}
|
||||
|
||||
fn provider_api_format_is_gemini_generate_content(
|
||||
plan: &ExecutionPlan,
|
||||
report_context: Option<&Value>,
|
||||
) -> bool {
|
||||
let provider_api_format = report_context
|
||||
.and_then(|value| value.get("provider_api_format"))
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or(plan.provider_api_format.as_str());
|
||||
crate::ai_serving::normalize_api_format_alias(provider_api_format) == "gemini:generate_content"
|
||||
}
|
||||
|
||||
fn invalid_gemini_provider_success_execution_error(message: &str) -> ExecutionError {
|
||||
ExecutionError {
|
||||
kind: ExecutionErrorKind::Upstream5xx,
|
||||
phase: ExecutionPhase::Finalize,
|
||||
message: message.to_string(),
|
||||
upstream_status: Some(StatusCode::OK.as_u16()),
|
||||
retryable: true,
|
||||
failover_recommended: true,
|
||||
}
|
||||
}
|
||||
|
||||
fn build_invalid_provider_success_body(
|
||||
@@ -2031,16 +2072,19 @@ async fn execute_execution_runtime_sync_impl(
|
||||
report_context.as_ref(),
|
||||
result.status_code,
|
||||
body_json.as_ref(),
|
||||
) {
|
||||
)
|
||||
.or_else(|| {
|
||||
invalid_gemini_provider_stream_success_message(
|
||||
&plan,
|
||||
report_context.as_ref(),
|
||||
result.status_code,
|
||||
body_json.as_ref(),
|
||||
&body_bytes,
|
||||
body_base64.is_some(),
|
||||
)
|
||||
}) {
|
||||
result.status_code = StatusCode::BAD_GATEWAY.as_u16();
|
||||
result.error = Some(ExecutionError {
|
||||
kind: ExecutionErrorKind::Upstream5xx,
|
||||
phase: ExecutionPhase::Finalize,
|
||||
message: message.to_string(),
|
||||
upstream_status: Some(StatusCode::OK.as_u16()),
|
||||
retryable: false,
|
||||
failover_recommended: false,
|
||||
});
|
||||
result.error = Some(invalid_gemini_provider_success_execution_error(message));
|
||||
if let Some(error_body) =
|
||||
build_invalid_provider_success_body(&plan, report_context.as_ref(), message)
|
||||
{
|
||||
@@ -2913,6 +2957,65 @@ mod tests {
|
||||
assert!(message.contains("visible model output"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_gemini_provider_success_error_is_retryable_candidate_failure() {
|
||||
let error = invalid_gemini_provider_success_execution_error(
|
||||
INVALID_GEMINI_PROVIDER_SUCCESS_MESSAGE,
|
||||
);
|
||||
|
||||
assert_eq!(error.kind, ExecutionErrorKind::Upstream5xx);
|
||||
assert_eq!(error.phase, ExecutionPhase::Finalize);
|
||||
assert_eq!(error.upstream_status, Some(StatusCode::OK.as_u16()));
|
||||
assert!(error.retryable);
|
||||
assert!(error.failover_recommended);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_gemini_provider_success_accepts_antigravity_chunks_with_visible_output() {
|
||||
let plan = test_gemini_chat_plan();
|
||||
let report_context = json!({
|
||||
"has_envelope": true,
|
||||
"envelope_name": "antigravity:v1internal",
|
||||
"provider_api_format": "gemini:generate_content",
|
||||
});
|
||||
let body = json!({
|
||||
"chunks": [{
|
||||
"response": {
|
||||
"responseId": "resp_antigravity_chunks_123",
|
||||
"candidates": [{
|
||||
"content": {
|
||||
"parts": [{"text": "Hello Gemini"}],
|
||||
"role": "model"
|
||||
},
|
||||
"finishReason": "STOP",
|
||||
"index": 0
|
||||
}],
|
||||
"modelVersion": "gemini-3-flash-agent",
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 2,
|
||||
"candidatesTokenCount": 2,
|
||||
"totalTokenCount": 4
|
||||
}
|
||||
},
|
||||
"traceId": "trace-antigravity-chunks"
|
||||
}],
|
||||
"metadata": {
|
||||
"stream": true,
|
||||
"stored_chunks": 1,
|
||||
"total_chunks": 1
|
||||
}
|
||||
});
|
||||
|
||||
let message = invalid_gemini_provider_success_message(
|
||||
&plan,
|
||||
Some(&report_context),
|
||||
StatusCode::OK.as_u16(),
|
||||
Some(&body),
|
||||
);
|
||||
|
||||
assert!(message.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_gemini_provider_success_unwraps_gemini_cli_v1internal_envelope() {
|
||||
let plan = test_gemini_chat_plan();
|
||||
|
||||
@@ -305,6 +305,9 @@ pub(super) fn admin_usage_terminal_candidate_state_override(
|
||||
candidates: &[StoredRequestCandidate],
|
||||
) -> Option<serde_json::Value> {
|
||||
let candidate = admin_usage_current_candidate(candidates)?;
|
||||
if admin_usage_candidate_failure_is_retryable_transition(candidate) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let status = match candidate.status {
|
||||
RequestCandidateStatus::Success => "completed",
|
||||
@@ -343,6 +346,30 @@ pub(super) fn admin_usage_terminal_candidate_state_override(
|
||||
Some(payload)
|
||||
}
|
||||
|
||||
fn admin_usage_candidate_failure_is_retryable_transition(
|
||||
candidate: &StoredRequestCandidate,
|
||||
) -> bool {
|
||||
if candidate.status != RequestCandidateStatus::Failed {
|
||||
return false;
|
||||
}
|
||||
let Some(error_flow) = candidate
|
||||
.extra_data
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("error_flow"))
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
let retryable = error_flow
|
||||
.get("retryable")
|
||||
.and_then(serde_json::Value::as_bool)
|
||||
.unwrap_or(false);
|
||||
let retry_next_candidate = error_flow
|
||||
.get("decision")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.is_some_and(|value| value == "retry_next_candidate");
|
||||
retryable && retry_next_candidate
|
||||
}
|
||||
|
||||
pub(super) fn apply_admin_usage_state_override(
|
||||
item: &mut StoredRequestUsageAudit,
|
||||
override_payload: &serde_json::Value,
|
||||
@@ -998,6 +1025,7 @@ mod tests {
|
||||
use aether_data_contracts::repository::candidates::{
|
||||
RequestCandidateStatus, StoredRequestCandidate,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
use super::admin_usage_terminal_candidate_state_override;
|
||||
|
||||
@@ -1076,4 +1104,27 @@ mod tests {
|
||||
|
||||
assert!(payload.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn admin_usage_active_override_ignores_retryable_candidate_failure() {
|
||||
let mut failed = sample_candidate(
|
||||
0,
|
||||
RequestCandidateStatus::Failed,
|
||||
Some(502),
|
||||
Some(1_000),
|
||||
Some("provider returned HTTP 200 without visible model output"),
|
||||
);
|
||||
failed.extra_data = Some(json!({
|
||||
"error_flow": {
|
||||
"decision": "retry_next_candidate",
|
||||
"retryable": true,
|
||||
"propagation": "suppressed",
|
||||
"classification": "retry_upstream_failure"
|
||||
}
|
||||
}));
|
||||
|
||||
let payload = admin_usage_terminal_candidate_state_override(&[failed]);
|
||||
|
||||
assert!(payload.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -510,6 +510,11 @@ fn maybe_build_standard_same_format_sync_body(
|
||||
if is_error_like_sync_body(body_json) {
|
||||
return None;
|
||||
}
|
||||
if api_format_is_gemini_generate_content(expected_api_format)
|
||||
&& !gemini_generate_content_body_has_visible_output(body_json)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut body_json = body_json.clone();
|
||||
if expected_api_format == "claude:messages" {
|
||||
@@ -578,6 +583,11 @@ fn maybe_build_standard_same_format_stream_sync_body(
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
if api_format_is_gemini_generate_content(&provider_stream_event_api_format)
|
||||
&& !gemini_generate_content_body_has_visible_output(&body)
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let Some(client_body) = convert_aggregated_stream_body_to_client_sync_response(
|
||||
report_kind,
|
||||
body,
|
||||
@@ -752,6 +762,14 @@ fn is_error_like_sync_body(value: &Value) -> bool {
|
||||
})
|
||||
}
|
||||
|
||||
fn gemini_generate_content_body_has_visible_output(value: &Value) -> bool {
|
||||
crate::formats::gemini::generate_content::response::from_raw(value).is_some()
|
||||
}
|
||||
|
||||
fn api_format_is_gemini_generate_content(api_format: &str) -> bool {
|
||||
aether_ai_formats::normalize_api_format_alias(api_format) == "gemini:generate_content"
|
||||
}
|
||||
|
||||
pub fn maybe_build_standard_cross_format_sync_product(
|
||||
report_kind: &str,
|
||||
provider_api_format: &str,
|
||||
@@ -5415,6 +5433,34 @@ mod tests {
|
||||
assert_eq!(body_json["choices"][0]["message"]["content"], "pong");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn standard_same_format_gemini_finalize_rejects_usage_only_body() {
|
||||
let report_context = json!({
|
||||
"provider_api_format": "gemini:generate_content",
|
||||
"client_api_format": "gemini:generate_content",
|
||||
"needs_conversion": false,
|
||||
});
|
||||
let provider_body_json = json!({
|
||||
"responseId": "resp-empty-visible-output",
|
||||
"modelVersion": "gemini-3-flash-preview",
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 8,
|
||||
"totalTokenCount": 8
|
||||
}
|
||||
});
|
||||
|
||||
let product = maybe_build_standard_sync_finalize_product_from_normalized_payload(
|
||||
"gemini_chat_sync_finalize",
|
||||
200,
|
||||
Some(&report_context),
|
||||
Some(&provider_body_json),
|
||||
None,
|
||||
)
|
||||
.expect("dispatch should succeed");
|
||||
|
||||
assert_eq!(product, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn standard_cross_format_finalize_uses_explicit_stream_event_format_for_provider_body() {
|
||||
let body = concat!(
|
||||
|
||||
@@ -101,19 +101,7 @@ pub fn normalize_provider_private_response_value(
|
||||
}
|
||||
}
|
||||
Some(ANTIGRAVITY_V1INTERNAL_ENVELOPE_NAME) => {
|
||||
if let Some(response) = data
|
||||
.get("response")
|
||||
.and_then(Value::as_object)
|
||||
.filter(|response| !response.contains_key("response"))
|
||||
{
|
||||
let mut unwrapped = response.clone();
|
||||
if let Some(response_id) = data.get("responseId").cloned() {
|
||||
unwrapped.insert("_v1internal_response_id".to_string(), response_id);
|
||||
}
|
||||
Value::Object(unwrapped)
|
||||
} else {
|
||||
data
|
||||
}
|
||||
normalize_antigravity_sync_response_value(data, provider_api_format)
|
||||
}
|
||||
Some(WINDSURF_ENVELOPE_NAME) => normalize_windsurf_sync_response_value(data)?,
|
||||
_ => return None,
|
||||
@@ -122,6 +110,59 @@ pub fn normalize_provider_private_response_value(
|
||||
Some(unwrapped)
|
||||
}
|
||||
|
||||
fn normalize_antigravity_sync_response_value(data: Value, provider_api_format: &str) -> Value {
|
||||
if crate::normalize_api_format_alias(provider_api_format) == "gemini:generate_content" {
|
||||
if let Some(aggregated) = aggregate_antigravity_gemini_chunks(&data) {
|
||||
return aggregated;
|
||||
}
|
||||
}
|
||||
if let Some(response) = data
|
||||
.get("response")
|
||||
.and_then(Value::as_object)
|
||||
.filter(|response| !response.contains_key("response"))
|
||||
{
|
||||
let mut unwrapped = response.clone();
|
||||
if let Some(response_id) = data.get("responseId").cloned() {
|
||||
unwrapped.insert("_v1internal_response_id".to_string(), response_id);
|
||||
}
|
||||
Value::Object(unwrapped)
|
||||
} else {
|
||||
data
|
||||
}
|
||||
}
|
||||
|
||||
fn aggregate_antigravity_gemini_chunks(data: &Value) -> Option<Value> {
|
||||
let chunks = data.get("chunks").and_then(Value::as_array)?;
|
||||
if chunks.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let body = serde_json::to_vec(chunks).ok()?;
|
||||
let mut aggregated =
|
||||
crate::formats::shared::sync_products::aggregate_gemini_stream_sync_response(&body)?;
|
||||
if let Some(response_id) = antigravity_chunks_response_id(chunks) {
|
||||
if let Some(object) = aggregated.as_object_mut() {
|
||||
if object.get("responseId").is_none_or(|value| value.is_null()) {
|
||||
object.insert("responseId".to_string(), response_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(aggregated)
|
||||
}
|
||||
|
||||
fn antigravity_chunks_response_id(chunks: &[Value]) -> Option<Value> {
|
||||
chunks.iter().find_map(|chunk| {
|
||||
chunk
|
||||
.get("responseId")
|
||||
.or_else(|| {
|
||||
chunk
|
||||
.get("response")
|
||||
.and_then(|response| response.get("responseId"))
|
||||
})
|
||||
.cloned()
|
||||
.filter(|value| !value.is_null())
|
||||
})
|
||||
}
|
||||
|
||||
pub fn transform_provider_private_stream_line(
|
||||
report_context: &Value,
|
||||
line: Vec<u8>,
|
||||
@@ -914,6 +955,69 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unwraps_antigravity_chunks_sync_response() {
|
||||
let report_context = json!({
|
||||
"has_envelope": true,
|
||||
"provider_api_format": "gemini:generate_content",
|
||||
"envelope_name": "antigravity:v1internal",
|
||||
"mapped_model": "gemini-3-flash-agent",
|
||||
});
|
||||
let body = json!({
|
||||
"chunks": [{
|
||||
"response": {
|
||||
"responseId": "resp_antigravity_chunks_123",
|
||||
"candidates": [{
|
||||
"content": {
|
||||
"parts": [{"text": "Hello "}],
|
||||
"role": "model"
|
||||
},
|
||||
"index": 0
|
||||
}],
|
||||
"modelVersion": "gemini-3-flash-agent"
|
||||
},
|
||||
"traceId": "trace-antigravity-chunks"
|
||||
}, {
|
||||
"response": {
|
||||
"responseId": "resp_antigravity_chunks_123",
|
||||
"candidates": [{
|
||||
"content": {
|
||||
"parts": [{"text": "Gemini"}],
|
||||
"role": "model"
|
||||
},
|
||||
"finishReason": "STOP",
|
||||
"index": 0
|
||||
}],
|
||||
"modelVersion": "gemini-3-flash-agent",
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 2,
|
||||
"candidatesTokenCount": 2,
|
||||
"totalTokenCount": 4
|
||||
}
|
||||
},
|
||||
"traceId": "trace-antigravity-chunks"
|
||||
}],
|
||||
"metadata": {
|
||||
"stream": true,
|
||||
"stored_chunks": 2,
|
||||
"total_chunks": 2
|
||||
}
|
||||
});
|
||||
|
||||
let normalized = normalize_provider_private_response_value(body, &report_context)
|
||||
.expect("chunks should normalize");
|
||||
|
||||
assert_eq!(
|
||||
normalized["candidates"][0]["content"]["parts"][0]["text"],
|
||||
json!("Hello Gemini")
|
||||
);
|
||||
assert_eq!(
|
||||
normalized["_v1internal_response_id"],
|
||||
json!("resp_antigravity_chunks_123")
|
||||
);
|
||||
assert_eq!(normalized["usageMetadata"]["totalTokenCount"], json!(4));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unwraps_antigravity_stream_line_and_injects_ids() {
|
||||
let report_context = json!({
|
||||
|
||||
Reference in New Issue
Block a user