i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-01 17:00:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor gateway orchestration and failover effects

Browse Source
This commit is contained in:
fawney19
2026-04-18 11:35:11 +08:00
parent 3321bb3ccc
commit 569242d72f
42 changed files with 4365 additions and 1486 deletions
Download Patch File Download Diff File Expand all files Collapse all files

452
apps/aether-gateway/src/execution_runtime/fallback.rs
View File

@@ -1,24 +1,11 @@
use std::collections::BTreeSet;
use aether_contracts::{ExecutionPlan, ExecutionResult};
use regex::Regex;
use serde_json::{json, Value};
use tracing::debug;
use serde_json::Value;
use crate::provider_transport::GatewayProviderTransportSnapshot;
use crate::orchestration::{
resolve_local_failover_analysis_for_attempt, LocalFailoverAnalysis, LocalFailoverDecision,
};
use crate::AppState;
fn local_candidate_index(report_context: Option<&serde_json::Value>) -> Option<u64> {
report_context
.and_then(serde_json::Value::as_object)
.and_then(|context| context.get("candidate_index"))
.and_then(serde_json::Value::as_u64)
}
fn should_failover_local_upstream_status(status_code: u16) -> bool {
status_code >= 400
}
fn sync_plan_kind_disables_local_candidate_failover(plan_kind: &str) -> bool {
matches!(
plan_kind,
@@ -26,38 +13,6 @@ fn sync_plan_kind_disables_local_candidate_failover(plan_kind: &str) -> bool {
)
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
struct LocalFailoverPolicy {
max_retries: Option<u64>,
stop_status_codes: BTreeSet<u16>,
continue_status_codes: BTreeSet<u16>,
success_failover_patterns: Vec<LocalFailoverRegexRule>,
error_stop_patterns: Vec<LocalFailoverRegexRule>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct LocalFailoverRegexRule {
pattern: String,
status_codes: BTreeSet<u16>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum LocalFailoverDecision {
UseDefault,
RetryNextCandidate,
StopLocalFailover,
}
impl LocalFailoverDecision {
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::UseDefault => "use_default",
Self::RetryNextCandidate => "retry_next_candidate",
Self::StopLocalFailover => "stop_local_failover",
}
}
}
pub(crate) async fn should_retry_next_local_candidate_sync(
state: &AppState,
plan: &ExecutionPlan,
@@ -66,22 +21,43 @@ pub(crate) async fn should_retry_next_local_candidate_sync(
result: &ExecutionResult,
response_text: Option<&str>,
) -> bool {
if sync_plan_kind_disables_local_candidate_failover(plan_kind) {
return false;
}
matches!(
resolve_local_failover_decision(
analyze_local_candidate_failover_sync(
state,
plan,
plan_kind,
report_context,
result.status_code,
result,
response_text,
)
.await,
.await
.decision,
LocalFailoverDecision::RetryNextCandidate
)
}
pub(crate) async fn analyze_local_candidate_failover_sync(
state: &AppState,
plan: &ExecutionPlan,
plan_kind: &str,
report_context: Option<&serde_json::Value>,
result: &ExecutionResult,
response_text: Option<&str>,
) -> LocalFailoverAnalysis {
if sync_plan_kind_disables_local_candidate_failover(plan_kind) {
return LocalFailoverAnalysis::use_default();
}
resolve_local_failover_analysis_for_attempt(
state,
plan,
report_context,
result.status_code,
response_text,
)
.await
}
pub(crate) async fn should_stop_local_candidate_failover_sync(
state: &AppState,
plan: &ExecutionPlan,
@@ -90,19 +66,20 @@ pub(crate) async fn should_stop_local_candidate_failover_sync(
result: &ExecutionResult,
response_text: Option<&str>,
) -> bool {
if sync_plan_kind_disables_local_candidate_failover(plan_kind) {
return false;
}
matches!(
resolve_local_failover_decision(
analyze_local_candidate_failover_sync(
state,
plan,
plan_kind,
report_context,
result.status_code,
result,
response_text,
)
.await,
LocalFailoverDecision::StopLocalFailover
LocalFailoverAnalysis {
decision: LocalFailoverDecision::StopLocalFailover,
..
}
)
}
@@ -195,7 +172,7 @@ pub(crate) async fn should_retry_next_local_candidate_stream(
response_text: Option<&str>,
) -> bool {
matches!(
resolve_local_candidate_failover_decision_stream(
resolve_local_candidate_failover_analysis_stream(
state,
plan,
report_context,
@@ -203,7 +180,10 @@ pub(crate) async fn should_retry_next_local_candidate_stream(
response_text,
)
.await,
LocalFailoverDecision::RetryNextCandidate
LocalFailoverAnalysis {
decision: LocalFailoverDecision::RetryNextCandidate,
..
}
)
}
@@ -216,7 +196,7 @@ pub(crate) async fn should_stop_local_candidate_failover_stream(
response_text: Option<&str>,
) -> bool {
matches!(
resolve_local_candidate_failover_decision_stream(
resolve_local_candidate_failover_analysis_stream(
state,
plan,
report_context,
@@ -224,10 +204,30 @@ pub(crate) async fn should_stop_local_candidate_failover_stream(
response_text,
)
.await,
LocalFailoverDecision::StopLocalFailover
LocalFailoverAnalysis {
decision: LocalFailoverDecision::StopLocalFailover,
..
}
)
}
pub(crate) async fn resolve_local_candidate_failover_analysis_stream(
state: &AppState,
plan: &ExecutionPlan,
report_context: Option<&serde_json::Value>,
status_code: u16,
response_text: Option<&str>,
) -> LocalFailoverAnalysis {
resolve_local_failover_analysis_for_attempt(
state,
plan,
report_context,
status_code,
response_text,
)
.await
}
pub(crate) async fn resolve_local_candidate_failover_decision_stream(
state: &AppState,
plan: &ExecutionPlan,
@@ -235,7 +235,15 @@ pub(crate) async fn resolve_local_candidate_failover_decision_stream(
status_code: u16,
response_text: Option<&str>,
) -> LocalFailoverDecision {
resolve_local_failover_decision(state, plan, report_context, status_code, response_text).await
resolve_local_candidate_failover_analysis_stream(
state,
plan,
report_context,
status_code,
response_text,
)
.await
.decision
}
pub(crate) fn local_failover_response_text(
@@ -255,304 +263,6 @@ pub(crate) fn local_failover_response_text(
.map(ToOwned::to_owned)
}
async fn resolve_local_failover_decision(
state: &AppState,
plan: &ExecutionPlan,
report_context: Option<&serde_json::Value>,
status_code: u16,
response_text: Option<&str>,
) -> LocalFailoverDecision {
let Some(candidate_index) = local_candidate_index(report_context) else {
return LocalFailoverDecision::UseDefault;
};
let policy = resolve_local_failover_policy(state, plan, report_context).await;
let response_text = response_text
.map(str::trim)
.filter(|value| !value.is_empty());
if policy.stop_status_codes.contains(&status_code) {
return LocalFailoverDecision::StopLocalFailover;
}
if status_code >= 400
&& response_text.is_some_and(|text| {
policy
.error_stop_patterns
.iter()
.any(|rule| local_failover_regex_rule_matches(rule, text, status_code))
})
{
return LocalFailoverDecision::StopLocalFailover;
}
if policy
.max_retries
.is_some_and(|max_retries| candidate_index >= max_retries)
{
return LocalFailoverDecision::UseDefault;
}
if status_code == 200
&& response_text.is_some_and(|text| {
policy
.success_failover_patterns
.iter()
.any(|rule| local_failover_regex_rule_matches(rule, text, status_code))
})
{
return LocalFailoverDecision::RetryNextCandidate;
}
if policy.continue_status_codes.contains(&status_code) {
return LocalFailoverDecision::RetryNextCandidate;
}
if should_failover_local_upstream_status(status_code) {
return LocalFailoverDecision::RetryNextCandidate;
}
LocalFailoverDecision::UseDefault
}
async fn resolve_local_failover_policy(
state: &AppState,
plan: &ExecutionPlan,
report_context: Option<&serde_json::Value>,
) -> LocalFailoverPolicy {
if let Some(policy) = local_failover_policy_from_report_context(report_context) {
debug!(
event_name = "local_failover_policy_loaded",
log_type = "debug",
request_id = %plan.request_id,
provider_id = %plan.provider_id,
endpoint_id = %plan.endpoint_id,
key_id = %plan.key_id,
source = "report_context",
max_retries = ?policy.max_retries,
stop_status_code_count = policy.stop_status_codes.len(),
continue_status_code_count = policy.continue_status_codes.len(),
success_failover_pattern_count = policy.success_failover_patterns.len(),
error_stop_pattern_count = policy.error_stop_patterns.len(),
"gateway loaded local failover policy from report context"
);
return policy;
}
let transport = match state
.read_provider_transport_snapshot(&plan.provider_id, &plan.endpoint_id, &plan.key_id)
.await
{
Ok(Some(transport)) => transport,
Ok(None) | Err(_) => return LocalFailoverPolicy::default(),
};
let policy = local_failover_policy_from_transport(&transport);
debug!(
event_name = "local_failover_policy_loaded",
log_type = "debug",
request_id = %plan.request_id,
provider_id = %plan.provider_id,
endpoint_id = %plan.endpoint_id,
key_id = %plan.key_id,
source = "transport_snapshot",
max_retries = ?policy.max_retries,
stop_status_code_count = policy.stop_status_codes.len(),
continue_status_code_count = policy.continue_status_codes.len(),
success_failover_pattern_count = policy.success_failover_patterns.len(),
error_stop_pattern_count = policy.error_stop_patterns.len(),
"gateway loaded local failover policy from transport snapshot"
);
policy
}
fn local_failover_policy_from_transport(
transport: &GatewayProviderTransportSnapshot,
) -> LocalFailoverPolicy {
let rules = transport
.provider
.config
.as_ref()
.and_then(|config| config.get("failover_rules"))
.and_then(serde_json::Value::as_object);
let max_retries = rules
.and_then(|value| value.get("max_retries"))
.and_then(parse_u64_value)
.or_else(|| {
transport
.endpoint
.max_retries
.and_then(|value| u64::try_from(value).ok())
})
.or_else(|| {
transport
.provider
.max_retries
.and_then(|value| u64::try_from(value).ok())
});
LocalFailoverPolicy {
max_retries,
stop_status_codes: rules
.map(|value| {
parse_status_code_set(
value,
&[
"stop_on_status_codes",
"early_stop_status_codes",
"non_retryable_status_codes",
"stop_status_codes",
],
)
})
.unwrap_or_default(),
continue_status_codes: rules
.map(|value| {
parse_status_code_set(
value,
&[
"continue_on_status_codes",
"retryable_status_codes",
"retry_on_status_codes",
"continue_status_codes",
],
)
})
.unwrap_or_default(),
success_failover_patterns: rules
.map(|value| parse_regex_rules(value, "success_failover_patterns"))
.unwrap_or_default(),
error_stop_patterns: rules
.map(|value| parse_regex_rules(value, "error_stop_patterns"))
.unwrap_or_default(),
}
}
fn local_failover_policy_from_report_context(
report_context: Option<&Value>,
) -> Option<LocalFailoverPolicy> {
let object = report_context
.and_then(Value::as_object)?
.get("local_failover_policy")?
.as_object()?;
Some(LocalFailoverPolicy {
max_retries: object.get("max_retries").and_then(parse_u64_value),
stop_status_codes: object
.get("stop_status_codes")
.map(parse_status_code_list)
.unwrap_or_default(),
continue_status_codes: object
.get("continue_status_codes")
.map(parse_status_code_list)
.unwrap_or_default(),
success_failover_patterns: parse_regex_rules(object, "success_failover_patterns"),
error_stop_patterns: parse_regex_rules(object, "error_stop_patterns"),
})
}
fn parse_status_code_list(value: &Value) -> BTreeSet<u16> {
value
.as_array()
.into_iter()
.flat_map(|values| values.iter())
.filter_map(|value| parse_u64_value(value).and_then(|value| u16::try_from(value).ok()))
.collect()
}
fn local_failover_policy_to_value(policy: &LocalFailoverPolicy) -> Value {
json!({
"max_retries": policy.max_retries,
"stop_status_codes": policy.stop_status_codes.iter().copied().collect::<Vec<_>>(),
"continue_status_codes": policy.continue_status_codes.iter().copied().collect::<Vec<_>>(),
"success_failover_patterns": policy.success_failover_patterns.iter().map(local_failover_regex_rule_to_value).collect::<Vec<_>>(),
"error_stop_patterns": policy.error_stop_patterns.iter().map(local_failover_regex_rule_to_value).collect::<Vec<_>>(),
})
}
fn local_failover_regex_rule_to_value(rule: &LocalFailoverRegexRule) -> Value {
json!({
"pattern": rule.pattern,
"status_codes": rule.status_codes.iter().copied().collect::<Vec<_>>(),
})
}
pub(crate) fn append_local_failover_policy_to_value(
value: Value,
transport: &GatewayProviderTransportSnapshot,
) -> Value {
let Value::Object(mut object) = value else {
return value;
};
object.insert(
"local_failover_policy".to_string(),
local_failover_policy_to_value(&local_failover_policy_from_transport(transport)),
);
Value::Object(object)
}
fn parse_regex_rules(
rules: &serde_json::Map<String, serde_json::Value>,
key: &str,
) -> Vec<LocalFailoverRegexRule> {
rules
.get(key)
.and_then(serde_json::Value::as_array)
.into_iter()
.flat_map(|items| items.iter())
.filter_map(parse_regex_rule)
.collect()
}
fn parse_regex_rule(value: &serde_json::Value) -> Option<LocalFailoverRegexRule> {
let object = value.as_object()?;
let pattern = object
.get("pattern")
.and_then(serde_json::Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())?;
Some(LocalFailoverRegexRule {
pattern: pattern.to_string(),
status_codes: object
.get("status_codes")
.and_then(serde_json::Value::as_array)
.into_iter()
.flat_map(|values| values.iter())
.filter_map(|value| parse_u64_value(value).and_then(|value| u16::try_from(value).ok()))
.collect(),
})
}
fn local_failover_regex_rule_matches(
rule: &LocalFailoverRegexRule,
response_text: &str,
status_code: u16,
) -> bool {
if !rule.status_codes.is_empty() && !rule.status_codes.contains(&status_code) {
return false;
}
Regex::new(&rule.pattern)
.ok()
.is_some_and(|regex| regex.is_match(response_text))
}
fn parse_status_code_set(
rules: &serde_json::Map<String, serde_json::Value>,
keys: &[&str],
) -> BTreeSet<u16> {
keys.iter()
.filter_map(|key| rules.get(*key))
.filter_map(serde_json::Value::as_array)
.flat_map(|values| values.iter())
.filter_map(|value| parse_u64_value(value).and_then(|value| u16::try_from(value).ok()))
.collect()
}
fn parse_u64_value(value: &serde_json::Value) -> Option<u64> {
value
.as_u64()
.or_else(|| value.as_i64().and_then(|value| u64::try_from(value).ok()))
}
pub(crate) fn should_fallback_to_control_stream(
plan_kind: &str,
status_code: u16,
@@ -623,13 +333,15 @@ mod tests {
use super::{
resolve_core_stream_error_finalize_report_kind,
resolve_core_sync_error_finalize_report_kind, resolve_local_failover_policy,
should_fallback_to_control_stream, should_fallback_to_control_sync,
should_retry_next_local_candidate_stream, should_retry_next_local_candidate_sync,
should_stop_local_candidate_failover_stream, should_stop_local_candidate_failover_sync,
LocalFailoverPolicy, LocalFailoverRegexRule,
resolve_core_sync_error_finalize_report_kind, should_fallback_to_control_stream,
should_fallback_to_control_sync, should_retry_next_local_candidate_stream,
should_retry_next_local_candidate_sync, should_stop_local_candidate_failover_stream,
should_stop_local_candidate_failover_sync,
};
use crate::data::GatewayDataState;
use crate::orchestration::{
resolve_local_failover_policy, LocalFailoverPolicy, LocalFailoverRegexRule,
};
use crate::AppState;
fn sample_plan() -> aether_contracts::ExecutionPlan {
@@ -1122,7 +834,7 @@ mod tests {
.await
);
assert!(
!should_retry_next_local_candidate_stream(
should_retry_next_local_candidate_stream(
&state,
&plan,
"openai_chat_stream",

10
apps/aether-gateway/src/execution_runtime/mod.rs
View File

@@ -6,7 +6,6 @@ use serde_json::{Map, Value};
mod constants;
mod fallback;
pub(crate) mod ndjson;
mod pool_feedback;
#[cfg(test)]
pub(crate) mod remote_compat;
mod server;
@@ -20,18 +19,17 @@ pub(crate) use self::constants::{
MAX_ERROR_BODY_BYTES, MAX_STREAM_PREFETCH_BYTES, MAX_STREAM_PREFETCH_FRAMES,
};
pub(crate) use self::fallback::{
append_local_failover_policy_to_value, local_failover_response_text,
analyze_local_candidate_failover_sync, local_failover_response_text,
resolve_core_stream_direct_finalize_report_kind,
resolve_core_stream_error_finalize_report_kind, resolve_core_sync_error_finalize_report_kind,
resolve_local_candidate_failover_analysis_stream,
resolve_local_candidate_failover_decision_stream, should_fallback_to_control_stream,
should_fallback_to_control_sync, should_finalize_sync_response,
should_retry_next_local_candidate_stream, should_retry_next_local_candidate_sync,
should_stop_local_candidate_failover_stream, should_stop_local_candidate_failover_sync,
LocalFailoverDecision,
};
pub(crate) use pool_feedback::{
record_pool_error_feedback, record_pool_stream_timeout_feedback,
record_stream_pool_success_feedback, record_sync_pool_success_feedback,
pub(crate) use crate::orchestration::{
append_local_failover_policy_to_value, LocalFailoverAnalysis, LocalFailoverDecision,
};
pub use server::{
build_execution_runtime_router, build_execution_runtime_router_with_request_concurrency_limit,

188
apps/aether-gateway/src/execution_runtime/pool_feedback.rs
View File

@@ -1,188 +0,0 @@
use aether_contracts::{ExecutionPlan, ExecutionTelemetry};
use aether_usage_runtime::{
build_stream_terminal_usage_outcome, build_sync_terminal_usage_outcome, TerminalUsageOutcome,
};
use serde_json::Value;
use std::collections::BTreeMap;
use tracing::warn;
use crate::ai_pipeline::extract_pool_sticky_session_token;
use crate::handlers::shared::provider_pool::admin_provider_pool_config_from_config_value;
use crate::handlers::shared::provider_pool::{
record_admin_provider_pool_error, record_admin_provider_pool_stream_timeout,
record_admin_provider_pool_success, AdminProviderPoolConfig,
};
use crate::usage::{GatewayStreamReportRequest, GatewaySyncReportRequest};
use crate::AppState;
struct PoolFeedbackContext {
runner: aether_data::redis::RedisKvRunner,
pool_config: AdminProviderPoolConfig,
sticky_session_token: Option<String>,
}
fn pool_feedback_request_body<'a>(
plan: &'a ExecutionPlan,
report_context: Option<&'a Value>,
) -> Option<&'a Value> {
report_context
.and_then(Value::as_object)
.and_then(|object| object.get("original_request_body"))
.filter(|value| !value.is_null())
.or(plan.body.json_body.as_ref())
}
async fn resolve_pool_feedback_context(
state: &AppState,
plan: &ExecutionPlan,
report_context: Option<&Value>,
) -> Option<PoolFeedbackContext> {
let Some(runner) = state.redis_kv_runner() else {
return None;
};
let transport = match state
.read_provider_transport_snapshot(&plan.provider_id, &plan.endpoint_id, &plan.key_id)
.await
{
Ok(Some(transport)) => transport,
Ok(None) => return None,
Err(err) => {
warn!(
"gateway execution runtime pool feedback: failed to read transport snapshot for provider {} endpoint {} key {}: {:?}",
plan.provider_id, plan.endpoint_id, plan.key_id, err
);
return None;
}
};
let Some(pool_config) =
admin_provider_pool_config_from_config_value(transport.provider.config.as_ref())
else {
return None;
};
let sticky_session_token = pool_feedback_request_body(plan, report_context)
.and_then(extract_pool_sticky_session_token);
Some(PoolFeedbackContext {
runner,
pool_config,
sticky_session_token,
})
}
fn total_tokens_used(outcome: &TerminalUsageOutcome) -> u64 {
outcome
.standardized_usage
.as_ref()
.map(|usage| {
usage
.input_tokens
.saturating_add(usage.output_tokens)
.max(0) as u64
})
.unwrap_or(0)
}
fn resolve_ttfb_ms(telemetry: Option<&ExecutionTelemetry>) -> Option<u64> {
telemetry.and_then(|telemetry| telemetry.ttfb_ms.or(telemetry.elapsed_ms))
}
pub(crate) async fn record_sync_pool_success_feedback(
state: &AppState,
plan: &ExecutionPlan,
report_context: Option<&Value>,
payload: &GatewaySyncReportRequest,
) {
let Some(context) = resolve_pool_feedback_context(state, plan, report_context).await else {
return;
};
let usage_outcome = build_sync_terminal_usage_outcome(plan, report_context, payload);
record_admin_provider_pool_success(
&context.runner,
&plan.provider_id,
&plan.key_id,
&context.pool_config,
context.sticky_session_token.as_deref(),
total_tokens_used(&usage_outcome),
resolve_ttfb_ms(payload.telemetry.as_ref()),
)
.await;
}
pub(crate) async fn record_stream_pool_success_feedback(
state: &AppState,
plan: &ExecutionPlan,
report_context: Option<&Value>,
payload: &GatewayStreamReportRequest,
) {
let Some(context) = resolve_pool_feedback_context(state, plan, report_context).await else {
return;
};
let usage_outcome = build_stream_terminal_usage_outcome(plan, report_context, payload);
record_admin_provider_pool_success(
&context.runner,
&plan.provider_id,
&plan.key_id,
&context.pool_config,
context.sticky_session_token.as_deref(),
total_tokens_used(&usage_outcome),
resolve_ttfb_ms(payload.telemetry.as_ref()),
)
.await;
}
pub(crate) async fn record_pool_error_feedback(
state: &AppState,
plan: &ExecutionPlan,
report_context: Option<&Value>,
status_code: u16,
headers: &BTreeMap<String, String>,
error_body: Option<&str>,
) {
if status_code < 400 {
return;
}
let Some(context) = resolve_pool_feedback_context(state, plan, report_context).await else {
return;
};
if status_code == 401 {
let _ = state
.invalidate_local_oauth_refresh_entry(&plan.key_id)
.await;
}
record_admin_provider_pool_error(
&context.runner,
&plan.provider_id,
&plan.key_id,
&context.pool_config,
status_code,
error_body,
Some(headers),
)
.await;
}
pub(crate) async fn record_pool_stream_timeout_feedback(
state: &AppState,
plan: &ExecutionPlan,
report_context: Option<&Value>,
) {
let Some(context) = resolve_pool_feedback_context(state, plan, report_context).await else {
return;
};
record_admin_provider_pool_stream_timeout(
&context.runner,
&plan.provider_id,
&plan.key_id,
&context.pool_config,
)
.await;
}

106
apps/aether-gateway/src/execution_runtime/stream/execution.rs
View File

@@ -53,14 +53,18 @@ use crate::execution_runtime::transport::{
DirectUpstreamStreamExecution, ExecutionRuntimeTransportError,
};
use crate::execution_runtime::{
local_failover_response_text, record_pool_error_feedback, record_stream_pool_success_feedback,
resolve_core_stream_direct_finalize_report_kind,
local_failover_response_text, resolve_core_stream_direct_finalize_report_kind,
resolve_core_stream_error_finalize_report_kind,
resolve_local_candidate_failover_decision_stream, should_fallback_to_control_stream,
resolve_local_candidate_failover_analysis_stream, should_fallback_to_control_stream,
should_retry_next_local_candidate_stream, LocalFailoverDecision,
};
use crate::execution_runtime::{MAX_STREAM_PREFETCH_BYTES, MAX_STREAM_PREFETCH_FRAMES};
use crate::log_ids::short_request_id;
use crate::orchestration::{
apply_local_execution_effect, LocalAdaptiveRateLimitEffect, LocalAttemptFailureEffect,
LocalExecutionEffect, LocalExecutionEffectContext, LocalHealthFailureEffect,
LocalHealthSuccessEffect, LocalOAuthInvalidationEffect, LocalPoolErrorEffect,
};
use crate::request_candidate_runtime::{
ensure_execution_request_candidate_slot, record_local_request_candidate_status,
};
@@ -566,16 +570,7 @@ async fn execute_stream_from_frame_stream(
let (body_json, body_base64) = decode_stream_error_body(&headers, &error_body);
let error_response_text =
local_failover_response_text(body_json.as_ref(), &error_body, None);
record_pool_error_feedback(
state,
&plan,
report_context.as_ref(),
status_code,
&headers,
error_response_text.as_deref(),
)
.await;
let failover_decision = resolve_local_candidate_failover_decision_stream(
let failover_analysis = resolve_local_candidate_failover_analysis_stream(
state,
&plan,
report_context.as_ref(),
@@ -583,6 +578,70 @@ async fn execute_stream_from_frame_stream(
error_response_text.as_deref(),
)
.await;
apply_local_execution_effect(
state,
LocalExecutionEffectContext {
plan: &plan,
report_context: report_context.as_ref(),
},
LocalExecutionEffect::AttemptFailure(LocalAttemptFailureEffect {
status_code,
classification: failover_analysis.classification,
}),
)
.await;
apply_local_execution_effect(
state,
LocalExecutionEffectContext {
plan: &plan,
report_context: report_context.as_ref(),
},
LocalExecutionEffect::AdaptiveRateLimit(LocalAdaptiveRateLimitEffect {
status_code,
classification: failover_analysis.classification,
headers: Some(&headers),
}),
)
.await;
apply_local_execution_effect(
state,
LocalExecutionEffectContext {
plan: &plan,
report_context: report_context.as_ref(),
},
LocalExecutionEffect::HealthFailure(LocalHealthFailureEffect {
status_code,
classification: failover_analysis.classification,
}),
)
.await;
apply_local_execution_effect(
state,
LocalExecutionEffectContext {
plan: &plan,
report_context: report_context.as_ref(),
},
LocalExecutionEffect::OauthInvalidation(LocalOAuthInvalidationEffect {
status_code,
response_text: error_response_text.as_deref(),
}),
)
.await;
apply_local_execution_effect(
state,
LocalExecutionEffectContext {
plan: &plan,
report_context: report_context.as_ref(),
},
LocalExecutionEffect::PoolError(LocalPoolErrorEffect {
status_code,
classification: failover_analysis.classification,
headers: &headers,
error_body: error_response_text.as_deref(),
}),
)
.await;
let failover_decision = failover_analysis.decision;
debug!(
event_name = "execution_runtime_stream_failover_decided",
log_type = "debug",
@@ -1502,11 +1561,24 @@ async fn execute_stream_from_frame_stream(
}),
telemetry: telemetry.clone(),
};
record_stream_pool_success_feedback(
apply_local_execution_effect(
&state_for_report,
&plan_for_report,
report_context_owned.as_ref(),
&usage_payload,
LocalExecutionEffectContext {
plan: &plan_for_report,
report_context: report_context_owned.as_ref(),
},
LocalExecutionEffect::HealthSuccess(LocalHealthSuccessEffect),
)
.await;
apply_local_execution_effect(
&state_for_report,
LocalExecutionEffectContext {
plan: &plan_for_report,
report_context: report_context_owned.as_ref(),
},
LocalExecutionEffect::PoolSuccessStream {
payload: &usage_payload,
},
)
.await;
record_stream_terminal_usage(

93
apps/aether-gateway/src/execution_runtime/stream/execution_failures.rs
View File

@@ -16,8 +16,13 @@ use crate::control::GatewayControlDecision;
use crate::execution_runtime::submission::{
resolve_core_error_background_report_kind, submit_local_core_error_or_sync_finalize,
};
use crate::execution_runtime::{record_pool_error_feedback, record_pool_stream_timeout_feedback};
use crate::log_ids::short_request_id;
use crate::orchestration::{
apply_local_execution_effect, resolve_local_failover_analysis_for_attempt,
LocalAdaptiveRateLimitEffect, LocalAttemptFailureEffect, LocalExecutionEffect,
LocalExecutionEffectContext, LocalHealthFailureEffect, LocalOAuthInvalidationEffect,
LocalPoolErrorEffect,
};
use crate::request_candidate_runtime::record_report_request_candidate_status;
use crate::usage::submit_sync_report;
use crate::{usage::GatewaySyncReportRequest, AppState, GatewayError};
@@ -123,22 +128,92 @@ async fn record_stream_sync_failure(
failure: &StreamFailureReport,
started_at_unix_ms: Option<u64>,
) {
if matches!(
failure.error_type.as_str(),
"first_byte_timeout" | "read_timeout"
) {
record_pool_stream_timeout_feedback(state, plan, report_context).await;
}
let error_body = serde_json::to_string(&failure.body_json).ok();
record_pool_error_feedback(
let failure_analysis = resolve_local_failover_analysis_for_attempt(
state,
plan,
report_context,
failure.status_code,
&payload.headers,
error_body.as_deref(),
)
.await;
if matches!(
failure.error_type.as_str(),
"first_byte_timeout" | "read_timeout"
) {
apply_local_execution_effect(
state,
LocalExecutionEffectContext {
plan,
report_context,
},
LocalExecutionEffect::PoolStreamTimeout,
)
.await;
}
apply_local_execution_effect(
state,
LocalExecutionEffectContext {
plan,
report_context,
},
LocalExecutionEffect::AttemptFailure(LocalAttemptFailureEffect {
status_code: failure.status_code,
classification: failure_analysis.classification,
}),
)
.await;
apply_local_execution_effect(
state,
LocalExecutionEffectContext {
plan,
report_context,
},
LocalExecutionEffect::AdaptiveRateLimit(LocalAdaptiveRateLimitEffect {
status_code: failure.status_code,
classification: failure_analysis.classification,
headers: Some(&payload.headers),
}),
)
.await;
apply_local_execution_effect(
state,
LocalExecutionEffectContext {
plan,
report_context,
},
LocalExecutionEffect::HealthFailure(LocalHealthFailureEffect {
status_code: failure.status_code,
classification: failure_analysis.classification,
}),
)
.await;
apply_local_execution_effect(
state,
LocalExecutionEffectContext {
plan,
report_context,
},
LocalExecutionEffect::OauthInvalidation(LocalOAuthInvalidationEffect {
status_code: failure.status_code,
response_text: error_body.as_deref(),
}),
)
.await;
apply_local_execution_effect(
state,
LocalExecutionEffectContext {
plan,
report_context,
},
LocalExecutionEffect::PoolError(LocalPoolErrorEffect {
status_code: failure.status_code,
classification: failure_analysis.classification,
headers: &payload.headers,
error_body: error_body.as_deref(),
}),
)
.await;
let context_seed = build_terminal_usage_context_seed(plan, report_context);
let payload_seed = build_sync_terminal_usage_payload_seed(payload);
state

135
apps/aether-gateway/src/execution_runtime/sync/execution.rs
View File

@@ -30,12 +30,16 @@ use crate::execution_runtime::remote_compat::post_sync_plan_to_remote_execution_
use crate::execution_runtime::submission::submit_local_core_error_or_sync_finalize;
use crate::execution_runtime::transport::DirectSyncExecutionRuntime;
use crate::execution_runtime::{
local_failover_response_text, record_pool_error_feedback, record_sync_pool_success_feedback,
analyze_local_candidate_failover_sync, local_failover_response_text,
resolve_core_sync_error_finalize_report_kind, should_fallback_to_control_sync,
should_finalize_sync_response, should_retry_next_local_candidate_sync,
should_stop_local_candidate_failover_sync,
should_finalize_sync_response, LocalFailoverDecision,
};
use crate::log_ids::short_request_id;
use crate::orchestration::{
apply_local_execution_effect, LocalAdaptiveRateLimitEffect, LocalAttemptFailureEffect,
LocalExecutionEffect, LocalExecutionEffectContext, LocalHealthFailureEffect,
LocalHealthSuccessEffect, LocalOAuthInvalidationEffect, LocalPoolErrorEffect,
};
use crate::request_candidate_runtime::{
ensure_execution_request_candidate_slot, record_local_request_candidate_status,
};
@@ -247,7 +251,7 @@ pub(crate) async fn execute_execution_runtime_sync(
&body_bytes,
result.error.as_ref().map(|error| error.message.as_str()),
);
let stop_local_failover = should_stop_local_candidate_failover_sync(
let local_failover_analysis = analyze_local_candidate_failover_sync(
state,
&plan,
plan_kind,
@@ -257,27 +261,74 @@ pub(crate) async fn execute_execution_runtime_sync(
)
.await;
if result.status_code >= 400 {
record_pool_error_feedback(
apply_local_execution_effect(
state,
&plan,
report_context.as_ref(),
result.status_code,
&headers,
local_failover_response_text.as_deref(),
LocalExecutionEffectContext {
plan: &plan,
report_context: report_context.as_ref(),
},
LocalExecutionEffect::AttemptFailure(LocalAttemptFailureEffect {
status_code: result.status_code,
classification: local_failover_analysis.classification,
}),
)
.await;
apply_local_execution_effect(
state,
LocalExecutionEffectContext {
plan: &plan,
report_context: report_context.as_ref(),
},
LocalExecutionEffect::AdaptiveRateLimit(LocalAdaptiveRateLimitEffect {
status_code: result.status_code,
classification: local_failover_analysis.classification,
headers: Some(&headers),
}),
)
.await;
apply_local_execution_effect(
state,
LocalExecutionEffectContext {
plan: &plan,
report_context: report_context.as_ref(),
},
LocalExecutionEffect::HealthFailure(LocalHealthFailureEffect {
status_code: result.status_code,
classification: local_failover_analysis.classification,
}),
)
.await;
apply_local_execution_effect(
state,
LocalExecutionEffectContext {
plan: &plan,
report_context: report_context.as_ref(),
},
LocalExecutionEffect::OauthInvalidation(LocalOAuthInvalidationEffect {
status_code: result.status_code,
response_text: local_failover_response_text.as_deref(),
}),
)
.await;
apply_local_execution_effect(
state,
LocalExecutionEffectContext {
plan: &plan,
report_context: report_context.as_ref(),
},
LocalExecutionEffect::PoolError(LocalPoolErrorEffect {
status_code: result.status_code,
classification: local_failover_analysis.classification,
headers: &headers,
error_body: local_failover_response_text.as_deref(),
}),
)
.await;
}
if should_retry_next_local_candidate_sync(
state,
&plan,
plan_kind,
report_context.as_ref(),
&result,
local_failover_response_text.as_deref(),
)
.await
&& !stop_local_failover
{
if matches!(
local_failover_analysis.decision,
LocalFailoverDecision::RetryNextCandidate
) {
let terminal_unix_secs = current_request_candidate_unix_ms();
record_local_request_candidate_status(
state,
@@ -341,16 +392,17 @@ pub(crate) async fn execute_execution_runtime_sync(
mapped_error_finalize_kind.clone()
};
if !stop_local_failover
&& should_fallback_to_control_sync(
plan_kind,
&result,
body_json.as_ref(),
has_body_bytes,
explicit_finalize || implicit_finalize.is_some(),
mapped_error_finalize_kind.is_some(),
)
{
if !matches!(
local_failover_analysis.decision,
LocalFailoverDecision::StopLocalFailover
) && should_fallback_to_control_sync(
plan_kind,
&result,
body_json.as_ref(),
has_body_bytes,
explicit_finalize || implicit_finalize.is_some(),
mapped_error_finalize_kind.is_some(),
) {
let terminal_unix_secs = current_request_candidate_unix_ms();
record_local_request_candidate_status(
state,
@@ -406,11 +458,24 @@ pub(crate) async fn execute_execution_runtime_sync(
telemetry: result.telemetry.clone(),
};
if result.status_code < 400 {
record_sync_pool_success_feedback(
apply_local_execution_effect(
state,
&plan,
report_context.as_ref(),
&base_usage_payload,
LocalExecutionEffectContext {
plan: &plan,
report_context: report_context.as_ref(),
},
LocalExecutionEffect::HealthSuccess(LocalHealthSuccessEffect),
)
.await;
apply_local_execution_effect(
state,
LocalExecutionEffectContext {
plan: &plan,
report_context: report_context.as_ref(),
},
LocalExecutionEffect::PoolSuccessSync {
payload: &base_usage_payload,
},
)
.await;
}