mirror of
https://github.com/fawney19/Aether.git
synced 2026-10-07 18:07:47 +08:00
feat(gateway): harden failover and payload handling
Retry pre-response transport failures across candidates with an explicit stop policy, and propagate end-to-end timing into usage records and UI diagnostics. Remove legacy body, import, cookie, PII, and tunnel replay caps while preserving optional operator-configured gateway limits.
This commit is contained in:
@@ -54,6 +54,31 @@ impl LocalFailoverClassification {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum LocalTransportFailoverClassification {
|
||||
StopTransportError,
|
||||
RetryTransportError,
|
||||
}
|
||||
|
||||
impl LocalTransportFailoverClassification {
|
||||
pub(crate) const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::StopTransportError => "stop_transport_error",
|
||||
Self::RetryTransportError => "retry_transport_error",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) const fn classify_local_transport_error(
|
||||
policy: &LocalFailoverPolicy,
|
||||
) -> LocalTransportFailoverClassification {
|
||||
if policy.stop_on_transport_errors {
|
||||
LocalTransportFailoverClassification::StopTransportError
|
||||
} else {
|
||||
LocalTransportFailoverClassification::RetryTransportError
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum FailureRetryAction {
|
||||
Stop,
|
||||
@@ -486,8 +511,9 @@ mod tests {
|
||||
|
||||
use super::{
|
||||
classify_anthropic_failure_disposition, classify_local_failover,
|
||||
failure_disposition_from_local_classification, FailureDisposition, FailureRetryAction,
|
||||
FailureScope, FailureTokenAction, LocalFailoverClassification, LocalFailoverInput,
|
||||
classify_local_transport_error, failure_disposition_from_local_classification,
|
||||
FailureDisposition, FailureRetryAction, FailureScope, FailureTokenAction,
|
||||
LocalFailoverClassification, LocalFailoverInput, LocalTransportFailoverClassification,
|
||||
};
|
||||
use crate::orchestration::{LocalFailoverPolicy, LocalFailoverRegexRule};
|
||||
|
||||
@@ -504,6 +530,27 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classifier_retries_transport_errors_by_default_and_honors_explicit_stop() {
|
||||
assert_eq!(
|
||||
classify_local_transport_error(&LocalFailoverPolicy::default()),
|
||||
LocalTransportFailoverClassification::RetryTransportError
|
||||
);
|
||||
|
||||
let stop_policy = LocalFailoverPolicy {
|
||||
stop_on_transport_errors: true,
|
||||
..LocalFailoverPolicy::default()
|
||||
};
|
||||
assert_eq!(
|
||||
classify_local_transport_error(&stop_policy),
|
||||
LocalTransportFailoverClassification::StopTransportError
|
||||
);
|
||||
assert_eq!(
|
||||
classify_local_transport_error(&stop_policy).as_str(),
|
||||
"stop_transport_error"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classifier_detects_success_failover_pattern() {
|
||||
let policy = LocalFailoverPolicy {
|
||||
|
||||
@@ -27,9 +27,10 @@ pub(crate) use self::attempt::{
|
||||
};
|
||||
pub(crate) use self::classifier::{
|
||||
classify_anthropic_failure_disposition, classify_failure_disposition, classify_local_failover,
|
||||
failure_disposition_from_local_classification, local_failover_error_message,
|
||||
FailureDisposition, FailureRetryAction, FailureScope, FailureTokenAction,
|
||||
LocalFailoverClassification, LocalFailoverInput,
|
||||
classify_local_transport_error, failure_disposition_from_local_classification,
|
||||
local_failover_error_message, FailureDisposition, FailureRetryAction, FailureScope,
|
||||
FailureTokenAction, LocalFailoverClassification, LocalFailoverInput,
|
||||
LocalTransportFailoverClassification,
|
||||
};
|
||||
pub(crate) use self::effects::{
|
||||
apply_local_execution_effect, LocalAdaptiveRateLimitEffect, LocalAdaptiveSuccessEffect,
|
||||
@@ -51,8 +52,9 @@ pub(crate) use self::policy::{
|
||||
LocalFailoverRegexRule, CYBER_CONTINUE_FAILOVER_CONFIG_KEY,
|
||||
};
|
||||
pub(crate) use self::recovery::{
|
||||
analyze_local_failover, apply_provider_failure_disposition, recover_local_failover_decision,
|
||||
LocalFailoverAnalysis, LocalFailoverDecision,
|
||||
analyze_local_failover, analyze_local_transport_error, apply_provider_failure_disposition,
|
||||
recover_local_failover_decision, LocalFailoverAnalysis, LocalFailoverDecision,
|
||||
LocalTransportFailoverAnalysis,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use self::report_effects::clear_local_report_effect_caches_for_tests;
|
||||
@@ -95,6 +97,15 @@ pub(crate) async fn resolve_local_failover_decision_for_attempt(
|
||||
.decision
|
||||
}
|
||||
|
||||
pub(crate) async fn resolve_local_transport_failover_analysis_for_attempt(
|
||||
state: &AppState,
|
||||
plan: &ExecutionPlan,
|
||||
report_context: Option<&serde_json::Value>,
|
||||
) -> LocalTransportFailoverAnalysis {
|
||||
let policy = resolve_local_failover_policy(state, plan, report_context).await;
|
||||
analyze_local_transport_error(&policy)
|
||||
}
|
||||
|
||||
pub(crate) fn build_local_error_flow_metadata(
|
||||
status_code: u16,
|
||||
response_text: Option<&str>,
|
||||
|
||||
@@ -16,6 +16,7 @@ pub(crate) struct LocalFailoverPolicy {
|
||||
pub(crate) max_transfer_timeout_seconds: u64,
|
||||
pub(crate) stop_status_codes: BTreeSet<u16>,
|
||||
pub(crate) continue_status_codes: BTreeSet<u16>,
|
||||
pub(crate) stop_on_transport_errors: bool,
|
||||
pub(crate) success_failover_patterns: Vec<LocalFailoverRegexRule>,
|
||||
pub(crate) error_stop_patterns: Vec<LocalFailoverRegexRule>,
|
||||
pub(crate) stop_cyber_policy_errors: bool,
|
||||
@@ -30,6 +31,7 @@ impl Default for LocalFailoverPolicy {
|
||||
max_transfer_timeout_seconds: 0,
|
||||
stop_status_codes: BTreeSet::new(),
|
||||
continue_status_codes: BTreeSet::new(),
|
||||
stop_on_transport_errors: false,
|
||||
success_failover_patterns: Vec::new(),
|
||||
error_stop_patterns: Vec::new(),
|
||||
stop_cyber_policy_errors: true,
|
||||
@@ -71,6 +73,7 @@ pub(crate) async fn resolve_local_failover_policy(
|
||||
max_transfer_timeout_seconds = policy.max_transfer_timeout_seconds,
|
||||
stop_status_code_count = policy.stop_status_codes.len(),
|
||||
continue_status_code_count = policy.continue_status_codes.len(),
|
||||
stop_on_transport_errors = policy.stop_on_transport_errors,
|
||||
success_failover_pattern_count = policy.success_failover_patterns.len(),
|
||||
error_stop_pattern_count = policy.error_stop_patterns.len(),
|
||||
cyber_continue_failover,
|
||||
@@ -157,6 +160,10 @@ pub(crate) fn local_failover_policy_from_transport(
|
||||
)
|
||||
})
|
||||
.unwrap_or_default(),
|
||||
stop_on_transport_errors: rules
|
||||
.and_then(|value| value.get("stop_on_transport_errors"))
|
||||
.and_then(Value::as_bool)
|
||||
.unwrap_or(false),
|
||||
success_failover_patterns: rules
|
||||
.map(|value| parse_regex_rules(value, "success_failover_patterns"))
|
||||
.unwrap_or_default(),
|
||||
@@ -192,6 +199,10 @@ pub(crate) fn local_failover_policy_from_report_context(
|
||||
.get("continue_status_codes")
|
||||
.map(parse_status_code_list)
|
||||
.unwrap_or_default(),
|
||||
stop_on_transport_errors: object
|
||||
.get("stop_on_transport_errors")
|
||||
.and_then(Value::as_bool)
|
||||
.unwrap_or(false),
|
||||
success_failover_patterns: parse_regex_rules(object, "success_failover_patterns"),
|
||||
error_stop_patterns: parse_regex_rules(object, "error_stop_patterns"),
|
||||
stop_cyber_policy_errors: object
|
||||
@@ -235,6 +246,7 @@ fn local_failover_policy_to_value(policy: &LocalFailoverPolicy) -> Value {
|
||||
"max_transfer_timeout_seconds": policy.max_transfer_timeout_seconds,
|
||||
"stop_status_codes": policy.stop_status_codes.iter().copied().collect::<Vec<_>>(),
|
||||
"continue_status_codes": policy.continue_status_codes.iter().copied().collect::<Vec<_>>(),
|
||||
"stop_on_transport_errors": policy.stop_on_transport_errors,
|
||||
"success_failover_patterns": policy.success_failover_patterns.iter().map(local_failover_regex_rule_to_value).collect::<Vec<_>>(),
|
||||
"error_stop_patterns": policy.error_stop_patterns.iter().map(local_failover_regex_rule_to_value).collect::<Vec<_>>(),
|
||||
"stop_cyber_policy_errors": policy.stop_cyber_policy_errors,
|
||||
@@ -416,6 +428,7 @@ mod tests {
|
||||
"max_retries": 2,
|
||||
"continue_status_codes": [429],
|
||||
"stop_status_codes": [400],
|
||||
"stop_on_transport_errors": true,
|
||||
"success_failover_patterns": [{"pattern": "quota", "status_codes": [200]}],
|
||||
"error_stop_patterns": [{"pattern": "validation", "status_codes": [422]}]
|
||||
}
|
||||
@@ -431,6 +444,7 @@ mod tests {
|
||||
max_transfer_timeout_seconds: 60,
|
||||
stop_status_codes: [400].into_iter().collect(),
|
||||
continue_status_codes: [429].into_iter().collect(),
|
||||
stop_on_transport_errors: true,
|
||||
success_failover_patterns: vec![LocalFailoverRegexRule {
|
||||
pattern: "quota".to_string(),
|
||||
status_codes: [200].into_iter().collect(),
|
||||
@@ -445,6 +459,35 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transport_error_failover_defaults_to_continue_and_accepts_explicit_stop() {
|
||||
let default_policy =
|
||||
local_failover_policy_from_transport(&sample_transport(None, None, None));
|
||||
assert!(!default_policy.stop_on_transport_errors);
|
||||
|
||||
let stop_policy = local_failover_policy_from_transport(&sample_transport(
|
||||
None,
|
||||
None,
|
||||
Some(json!({
|
||||
"failover_rules": {
|
||||
"stop_on_transport_errors": true,
|
||||
}
|
||||
})),
|
||||
));
|
||||
assert!(stop_policy.stop_on_transport_errors);
|
||||
|
||||
let invalid_policy = local_failover_policy_from_transport(&sample_transport(
|
||||
None,
|
||||
None,
|
||||
Some(json!({
|
||||
"failover_rules": {
|
||||
"stop_on_transport_errors": "true",
|
||||
}
|
||||
})),
|
||||
));
|
||||
assert!(!invalid_policy.stop_on_transport_errors);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transfer_limits_are_read_only_from_top_level_provider_config() {
|
||||
let top_level = local_failover_policy_from_transport(&sample_transport(
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use super::classifier::{
|
||||
classify_failure_disposition, classify_local_failover, FailureRetryAction,
|
||||
LocalFailoverClassification, LocalFailoverInput,
|
||||
classify_failure_disposition, classify_local_failover, classify_local_transport_error,
|
||||
FailureRetryAction, LocalFailoverClassification, LocalFailoverInput,
|
||||
LocalTransportFailoverClassification,
|
||||
};
|
||||
use super::LocalFailoverPolicy;
|
||||
|
||||
@@ -27,6 +28,12 @@ pub(crate) struct LocalFailoverAnalysis {
|
||||
pub(crate) decision: LocalFailoverDecision,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) struct LocalTransportFailoverAnalysis {
|
||||
pub(crate) classification: LocalTransportFailoverClassification,
|
||||
pub(crate) decision: LocalFailoverDecision,
|
||||
}
|
||||
|
||||
impl LocalFailoverAnalysis {
|
||||
pub(crate) const fn use_default() -> Self {
|
||||
Self {
|
||||
@@ -47,6 +54,24 @@ pub(crate) fn analyze_local_failover(
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn analyze_local_transport_error(
|
||||
policy: &LocalFailoverPolicy,
|
||||
) -> LocalTransportFailoverAnalysis {
|
||||
let classification = classify_local_transport_error(policy);
|
||||
let decision = match classification {
|
||||
LocalTransportFailoverClassification::StopTransportError => {
|
||||
LocalFailoverDecision::StopLocalFailover
|
||||
}
|
||||
LocalTransportFailoverClassification::RetryTransportError => {
|
||||
LocalFailoverDecision::RetryNextCandidate
|
||||
}
|
||||
};
|
||||
LocalTransportFailoverAnalysis {
|
||||
classification,
|
||||
decision,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn apply_provider_failure_disposition(
|
||||
provider_api_format: &str,
|
||||
status_code: u16,
|
||||
@@ -105,7 +130,7 @@ const fn decision_from_classification(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
analyze_local_failover, apply_provider_failure_disposition,
|
||||
analyze_local_failover, analyze_local_transport_error, apply_provider_failure_disposition,
|
||||
recover_local_failover_decision, LocalFailoverAnalysis, LocalFailoverDecision,
|
||||
};
|
||||
use crate::orchestration::{
|
||||
@@ -136,6 +161,23 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transport_error_recovery_defaults_to_retry_and_can_stop() {
|
||||
assert_eq!(
|
||||
analyze_local_transport_error(&LocalFailoverPolicy::default()).decision,
|
||||
LocalFailoverDecision::RetryNextCandidate
|
||||
);
|
||||
|
||||
let stop_policy = LocalFailoverPolicy {
|
||||
stop_on_transport_errors: true,
|
||||
..LocalFailoverPolicy::default()
|
||||
};
|
||||
assert_eq!(
|
||||
analyze_local_transport_error(&stop_policy).decision,
|
||||
LocalFailoverDecision::StopLocalFailover
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recovery_retries_default_client_error_without_custom_rule() {
|
||||
assert_eq!(
|
||||
|
||||
Reference in New Issue
Block a user