feat(gateway): harden failover and payload handling

Retry pre-response transport failures across candidates with an explicit stop policy, and propagate end-to-end timing into usage records and UI diagnostics.

Remove legacy body, import, cookie, PII, and tunnel replay caps while preserving optional operator-configured gateway limits.
This commit is contained in:
elky
2026-07-30 01:03:27 +08:00
parent a97acc07fc
commit a04673a90d
80 changed files with 3640 additions and 1236 deletions
@@ -54,6 +54,31 @@ impl LocalFailoverClassification {
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum LocalTransportFailoverClassification {
StopTransportError,
RetryTransportError,
}
impl LocalTransportFailoverClassification {
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::StopTransportError => "stop_transport_error",
Self::RetryTransportError => "retry_transport_error",
}
}
}
pub(crate) const fn classify_local_transport_error(
policy: &LocalFailoverPolicy,
) -> LocalTransportFailoverClassification {
if policy.stop_on_transport_errors {
LocalTransportFailoverClassification::StopTransportError
} else {
LocalTransportFailoverClassification::RetryTransportError
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum FailureRetryAction {
Stop,
@@ -486,8 +511,9 @@ mod tests {
use super::{
classify_anthropic_failure_disposition, classify_local_failover,
failure_disposition_from_local_classification, FailureDisposition, FailureRetryAction,
FailureScope, FailureTokenAction, LocalFailoverClassification, LocalFailoverInput,
classify_local_transport_error, failure_disposition_from_local_classification,
FailureDisposition, FailureRetryAction, FailureScope, FailureTokenAction,
LocalFailoverClassification, LocalFailoverInput, LocalTransportFailoverClassification,
};
use crate::orchestration::{LocalFailoverPolicy, LocalFailoverRegexRule};
@@ -504,6 +530,27 @@ mod tests {
);
}
#[test]
fn classifier_retries_transport_errors_by_default_and_honors_explicit_stop() {
assert_eq!(
classify_local_transport_error(&LocalFailoverPolicy::default()),
LocalTransportFailoverClassification::RetryTransportError
);
let stop_policy = LocalFailoverPolicy {
stop_on_transport_errors: true,
..LocalFailoverPolicy::default()
};
assert_eq!(
classify_local_transport_error(&stop_policy),
LocalTransportFailoverClassification::StopTransportError
);
assert_eq!(
classify_local_transport_error(&stop_policy).as_str(),
"stop_transport_error"
);
}
#[test]
fn classifier_detects_success_failover_pattern() {
let policy = LocalFailoverPolicy {
+16 -5
View File
@@ -27,9 +27,10 @@ pub(crate) use self::attempt::{
};
pub(crate) use self::classifier::{
classify_anthropic_failure_disposition, classify_failure_disposition, classify_local_failover,
failure_disposition_from_local_classification, local_failover_error_message,
FailureDisposition, FailureRetryAction, FailureScope, FailureTokenAction,
LocalFailoverClassification, LocalFailoverInput,
classify_local_transport_error, failure_disposition_from_local_classification,
local_failover_error_message, FailureDisposition, FailureRetryAction, FailureScope,
FailureTokenAction, LocalFailoverClassification, LocalFailoverInput,
LocalTransportFailoverClassification,
};
pub(crate) use self::effects::{
apply_local_execution_effect, LocalAdaptiveRateLimitEffect, LocalAdaptiveSuccessEffect,
@@ -51,8 +52,9 @@ pub(crate) use self::policy::{
LocalFailoverRegexRule, CYBER_CONTINUE_FAILOVER_CONFIG_KEY,
};
pub(crate) use self::recovery::{
analyze_local_failover, apply_provider_failure_disposition, recover_local_failover_decision,
LocalFailoverAnalysis, LocalFailoverDecision,
analyze_local_failover, analyze_local_transport_error, apply_provider_failure_disposition,
recover_local_failover_decision, LocalFailoverAnalysis, LocalFailoverDecision,
LocalTransportFailoverAnalysis,
};
#[cfg(test)]
pub(crate) use self::report_effects::clear_local_report_effect_caches_for_tests;
@@ -95,6 +97,15 @@ pub(crate) async fn resolve_local_failover_decision_for_attempt(
.decision
}
pub(crate) async fn resolve_local_transport_failover_analysis_for_attempt(
state: &AppState,
plan: &ExecutionPlan,
report_context: Option<&serde_json::Value>,
) -> LocalTransportFailoverAnalysis {
let policy = resolve_local_failover_policy(state, plan, report_context).await;
analyze_local_transport_error(&policy)
}
pub(crate) fn build_local_error_flow_metadata(
status_code: u16,
response_text: Option<&str>,
@@ -16,6 +16,7 @@ pub(crate) struct LocalFailoverPolicy {
pub(crate) max_transfer_timeout_seconds: u64,
pub(crate) stop_status_codes: BTreeSet<u16>,
pub(crate) continue_status_codes: BTreeSet<u16>,
pub(crate) stop_on_transport_errors: bool,
pub(crate) success_failover_patterns: Vec<LocalFailoverRegexRule>,
pub(crate) error_stop_patterns: Vec<LocalFailoverRegexRule>,
pub(crate) stop_cyber_policy_errors: bool,
@@ -30,6 +31,7 @@ impl Default for LocalFailoverPolicy {
max_transfer_timeout_seconds: 0,
stop_status_codes: BTreeSet::new(),
continue_status_codes: BTreeSet::new(),
stop_on_transport_errors: false,
success_failover_patterns: Vec::new(),
error_stop_patterns: Vec::new(),
stop_cyber_policy_errors: true,
@@ -71,6 +73,7 @@ pub(crate) async fn resolve_local_failover_policy(
max_transfer_timeout_seconds = policy.max_transfer_timeout_seconds,
stop_status_code_count = policy.stop_status_codes.len(),
continue_status_code_count = policy.continue_status_codes.len(),
stop_on_transport_errors = policy.stop_on_transport_errors,
success_failover_pattern_count = policy.success_failover_patterns.len(),
error_stop_pattern_count = policy.error_stop_patterns.len(),
cyber_continue_failover,
@@ -157,6 +160,10 @@ pub(crate) fn local_failover_policy_from_transport(
)
})
.unwrap_or_default(),
stop_on_transport_errors: rules
.and_then(|value| value.get("stop_on_transport_errors"))
.and_then(Value::as_bool)
.unwrap_or(false),
success_failover_patterns: rules
.map(|value| parse_regex_rules(value, "success_failover_patterns"))
.unwrap_or_default(),
@@ -192,6 +199,10 @@ pub(crate) fn local_failover_policy_from_report_context(
.get("continue_status_codes")
.map(parse_status_code_list)
.unwrap_or_default(),
stop_on_transport_errors: object
.get("stop_on_transport_errors")
.and_then(Value::as_bool)
.unwrap_or(false),
success_failover_patterns: parse_regex_rules(object, "success_failover_patterns"),
error_stop_patterns: parse_regex_rules(object, "error_stop_patterns"),
stop_cyber_policy_errors: object
@@ -235,6 +246,7 @@ fn local_failover_policy_to_value(policy: &LocalFailoverPolicy) -> Value {
"max_transfer_timeout_seconds": policy.max_transfer_timeout_seconds,
"stop_status_codes": policy.stop_status_codes.iter().copied().collect::<Vec<_>>(),
"continue_status_codes": policy.continue_status_codes.iter().copied().collect::<Vec<_>>(),
"stop_on_transport_errors": policy.stop_on_transport_errors,
"success_failover_patterns": policy.success_failover_patterns.iter().map(local_failover_regex_rule_to_value).collect::<Vec<_>>(),
"error_stop_patterns": policy.error_stop_patterns.iter().map(local_failover_regex_rule_to_value).collect::<Vec<_>>(),
"stop_cyber_policy_errors": policy.stop_cyber_policy_errors,
@@ -416,6 +428,7 @@ mod tests {
"max_retries": 2,
"continue_status_codes": [429],
"stop_status_codes": [400],
"stop_on_transport_errors": true,
"success_failover_patterns": [{"pattern": "quota", "status_codes": [200]}],
"error_stop_patterns": [{"pattern": "validation", "status_codes": [422]}]
}
@@ -431,6 +444,7 @@ mod tests {
max_transfer_timeout_seconds: 60,
stop_status_codes: [400].into_iter().collect(),
continue_status_codes: [429].into_iter().collect(),
stop_on_transport_errors: true,
success_failover_patterns: vec![LocalFailoverRegexRule {
pattern: "quota".to_string(),
status_codes: [200].into_iter().collect(),
@@ -445,6 +459,35 @@ mod tests {
);
}
#[test]
fn transport_error_failover_defaults_to_continue_and_accepts_explicit_stop() {
let default_policy =
local_failover_policy_from_transport(&sample_transport(None, None, None));
assert!(!default_policy.stop_on_transport_errors);
let stop_policy = local_failover_policy_from_transport(&sample_transport(
None,
None,
Some(json!({
"failover_rules": {
"stop_on_transport_errors": true,
}
})),
));
assert!(stop_policy.stop_on_transport_errors);
let invalid_policy = local_failover_policy_from_transport(&sample_transport(
None,
None,
Some(json!({
"failover_rules": {
"stop_on_transport_errors": "true",
}
})),
));
assert!(!invalid_policy.stop_on_transport_errors);
}
#[test]
fn transfer_limits_are_read_only_from_top_level_provider_config() {
let top_level = local_failover_policy_from_transport(&sample_transport(
@@ -1,6 +1,7 @@
use super::classifier::{
classify_failure_disposition, classify_local_failover, FailureRetryAction,
LocalFailoverClassification, LocalFailoverInput,
classify_failure_disposition, classify_local_failover, classify_local_transport_error,
FailureRetryAction, LocalFailoverClassification, LocalFailoverInput,
LocalTransportFailoverClassification,
};
use super::LocalFailoverPolicy;
@@ -27,6 +28,12 @@ pub(crate) struct LocalFailoverAnalysis {
pub(crate) decision: LocalFailoverDecision,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct LocalTransportFailoverAnalysis {
pub(crate) classification: LocalTransportFailoverClassification,
pub(crate) decision: LocalFailoverDecision,
}
impl LocalFailoverAnalysis {
pub(crate) const fn use_default() -> Self {
Self {
@@ -47,6 +54,24 @@ pub(crate) fn analyze_local_failover(
}
}
pub(crate) fn analyze_local_transport_error(
policy: &LocalFailoverPolicy,
) -> LocalTransportFailoverAnalysis {
let classification = classify_local_transport_error(policy);
let decision = match classification {
LocalTransportFailoverClassification::StopTransportError => {
LocalFailoverDecision::StopLocalFailover
}
LocalTransportFailoverClassification::RetryTransportError => {
LocalFailoverDecision::RetryNextCandidate
}
};
LocalTransportFailoverAnalysis {
classification,
decision,
}
}
pub(crate) fn apply_provider_failure_disposition(
provider_api_format: &str,
status_code: u16,
@@ -105,7 +130,7 @@ const fn decision_from_classification(
#[cfg(test)]
mod tests {
use super::{
analyze_local_failover, apply_provider_failure_disposition,
analyze_local_failover, analyze_local_transport_error, apply_provider_failure_disposition,
recover_local_failover_decision, LocalFailoverAnalysis, LocalFailoverDecision,
};
use crate::orchestration::{
@@ -136,6 +161,23 @@ mod tests {
);
}
#[test]
fn transport_error_recovery_defaults_to_retry_and_can_stop() {
assert_eq!(
analyze_local_transport_error(&LocalFailoverPolicy::default()).decision,
LocalFailoverDecision::RetryNextCandidate
);
let stop_policy = LocalFailoverPolicy {
stop_on_transport_errors: true,
..LocalFailoverPolicy::default()
};
assert_eq!(
analyze_local_transport_error(&stop_policy).decision,
LocalFailoverDecision::StopLocalFailover
);
}
#[test]
fn recovery_retries_default_client_error_without_custom_rule() {
assert_eq!(