i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 01:10:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: 抽离 AI pipeline 与调度共享能力逻辑

Browse Source
This commit is contained in:
fawney19
2026-04-10 01:46:14 +08:00
parent b901a6ffc7
commit 5014e2f5fd
255 changed files with 15057 additions and 3115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

810
apps/aether-gateway/src/execution_runtime/fallback.rs
View File

@@ -1,25 +1,83 @@
use aether_contracts::ExecutionResult;
use std::collections::BTreeSet;
fn is_local_candidate_attempt(report_context: Option<&serde_json::Value>) -> bool {
use aether_contracts::{ExecutionPlan, ExecutionResult};
use regex::Regex;
use crate::AppState;
fn local_candidate_index(report_context: Option<&serde_json::Value>) -> Option<u64> {
report_context
.and_then(serde_json::Value::as_object)
.and_then(|context| context.get("candidate_index"))
.and_then(serde_json::Value::as_u64)
.is_some()
}
fn is_retryable_local_upstream_status(status_code: u16) -> bool {
status_code == 429 || status_code >= 500
}
pub(crate) fn should_retry_next_local_candidate_sync(
plan_kind: &str,
#[derive(Debug, Clone, Default, PartialEq, Eq)]
struct LocalFailoverPolicy {
max_retries: Option<u64>,
stop_status_codes: BTreeSet<u16>,
continue_status_codes: BTreeSet<u16>,
success_failover_patterns: Vec<LocalFailoverRegexRule>,
error_stop_patterns: Vec<LocalFailoverRegexRule>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct LocalFailoverRegexRule {
pattern: String,
status_codes: BTreeSet<u16>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum LocalFailoverDecision {
UseDefault,
RetryNextCandidate,
StopLocalFailover,
}
pub(crate) async fn should_retry_next_local_candidate_sync(
state: &AppState,
plan: &ExecutionPlan,
_plan_kind: &str,
report_context: Option<&serde_json::Value>,
result: &ExecutionResult,
response_text: Option<&str>,
) -> bool {
is_local_candidate_attempt(report_context)
&& plan_kind == "openai_chat_sync"
&& is_retryable_local_upstream_status(result.status_code)
matches!(
resolve_local_failover_decision(
state,
plan,
report_context,
result.status_code,
response_text,
)
.await,
LocalFailoverDecision::RetryNextCandidate
)
}
pub(crate) async fn should_stop_local_candidate_failover_sync(
state: &AppState,
plan: &ExecutionPlan,
_plan_kind: &str,
report_context: Option<&serde_json::Value>,
result: &ExecutionResult,
response_text: Option<&str>,
) -> bool {
matches!(
resolve_local_failover_decision(
state,
plan,
report_context,
result.status_code,
response_text,
)
.await,
LocalFailoverDecision::StopLocalFailover
)
}
pub(crate) fn should_fallback_to_control_sync(
@@ -102,14 +160,245 @@ pub(crate) fn resolve_core_sync_error_finalize_report_kind(
Some(report_kind.to_string())
}
pub(crate) fn should_retry_next_local_candidate_stream(
plan_kind: &str,
pub(crate) async fn should_retry_next_local_candidate_stream(
state: &AppState,
plan: &ExecutionPlan,
_plan_kind: &str,
report_context: Option<&serde_json::Value>,
status_code: u16,
response_text: Option<&str>,
) -> bool {
is_local_candidate_attempt(report_context)
&& plan_kind == "openai_chat_stream"
&& is_retryable_local_upstream_status(status_code)
matches!(
resolve_local_failover_decision(state, plan, report_context, status_code, response_text)
.await,
LocalFailoverDecision::RetryNextCandidate
)
}
pub(crate) async fn should_stop_local_candidate_failover_stream(
state: &AppState,
plan: &ExecutionPlan,
_plan_kind: &str,
report_context: Option<&serde_json::Value>,
status_code: u16,
response_text: Option<&str>,
) -> bool {
matches!(
resolve_local_failover_decision(state, plan, report_context, status_code, response_text)
.await,
LocalFailoverDecision::StopLocalFailover
)
}
pub(crate) fn local_failover_response_text(
body_json: Option<&serde_json::Value>,
body_bytes: &[u8],
fallback_text: Option<&str>,
) -> Option<String> {
if let Some(body_json) = body_json {
return serde_json::to_string(body_json).ok();
}
if !body_bytes.is_empty() {
return Some(String::from_utf8_lossy(body_bytes).into_owned());
}
fallback_text
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
}
async fn resolve_local_failover_decision(
state: &AppState,
plan: &ExecutionPlan,
report_context: Option<&serde_json::Value>,
status_code: u16,
response_text: Option<&str>,
) -> LocalFailoverDecision {
let Some(candidate_index) = local_candidate_index(report_context) else {
return LocalFailoverDecision::UseDefault;
};
let policy = resolve_local_failover_policy(state, plan).await;
let response_text = response_text
.map(str::trim)
.filter(|value| !value.is_empty());
if policy.stop_status_codes.contains(&status_code) {
return LocalFailoverDecision::StopLocalFailover;
}
if status_code >= 400
&& response_text.is_some_and(|text| {
policy
.error_stop_patterns
.iter()
.any(|rule| local_failover_regex_rule_matches(rule, text, status_code))
})
{
return LocalFailoverDecision::StopLocalFailover;
}
if policy
.max_retries
.is_some_and(|max_retries| candidate_index >= max_retries)
{
return LocalFailoverDecision::UseDefault;
}
if status_code == 200
&& response_text.is_some_and(|text| {
policy
.success_failover_patterns
.iter()
.any(|rule| local_failover_regex_rule_matches(rule, text, status_code))
})
{
return LocalFailoverDecision::RetryNextCandidate;
}
if policy.continue_status_codes.contains(&status_code) {
return LocalFailoverDecision::RetryNextCandidate;
}
if is_retryable_local_upstream_status(status_code) {
return LocalFailoverDecision::RetryNextCandidate;
}
LocalFailoverDecision::UseDefault
}
async fn resolve_local_failover_policy(
state: &AppState,
plan: &ExecutionPlan,
) -> LocalFailoverPolicy {
let transport = match state
.read_provider_transport_snapshot(&plan.provider_id, &plan.endpoint_id, &plan.key_id)
.await
{
Ok(Some(transport)) => transport,
Ok(None) | Err(_) => return LocalFailoverPolicy::default(),
};
let rules = transport
.provider
.config
.as_ref()
.and_then(|config| config.get("failover_rules"))
.and_then(serde_json::Value::as_object);
let max_retries = rules
.and_then(|value| value.get("max_retries"))
.and_then(parse_u64_value)
.or_else(|| {
transport
.endpoint
.max_retries
.and_then(|value| u64::try_from(value).ok())
})
.or_else(|| {
transport
.provider
.max_retries
.and_then(|value| u64::try_from(value).ok())
});
LocalFailoverPolicy {
max_retries,
stop_status_codes: rules
.map(|value| {
parse_status_code_set(
value,
&[
"stop_on_status_codes",
"early_stop_status_codes",
"non_retryable_status_codes",
"stop_status_codes",
],
)
})
.unwrap_or_default(),
continue_status_codes: rules
.map(|value| {
parse_status_code_set(
value,
&[
"continue_on_status_codes",
"retryable_status_codes",
"retry_on_status_codes",
"continue_status_codes",
],
)
})
.unwrap_or_default(),
success_failover_patterns: rules
.map(|value| parse_regex_rules(value, "success_failover_patterns"))
.unwrap_or_default(),
error_stop_patterns: rules
.map(|value| parse_regex_rules(value, "error_stop_patterns"))
.unwrap_or_default(),
}
}
fn parse_regex_rules(
rules: &serde_json::Map<String, serde_json::Value>,
key: &str,
) -> Vec<LocalFailoverRegexRule> {
rules
.get(key)
.and_then(serde_json::Value::as_array)
.into_iter()
.flat_map(|items| items.iter())
.filter_map(parse_regex_rule)
.collect()
}
fn parse_regex_rule(value: &serde_json::Value) -> Option<LocalFailoverRegexRule> {
let object = value.as_object()?;
let pattern = object
.get("pattern")
.and_then(serde_json::Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())?;
Some(LocalFailoverRegexRule {
pattern: pattern.to_string(),
status_codes: object
.get("status_codes")
.and_then(serde_json::Value::as_array)
.into_iter()
.flat_map(|values| values.iter())
.filter_map(|value| parse_u64_value(value).and_then(|value| u16::try_from(value).ok()))
.collect(),
})
}
fn local_failover_regex_rule_matches(
rule: &LocalFailoverRegexRule,
response_text: &str,
status_code: u16,
) -> bool {
if !rule.status_codes.is_empty() && !rule.status_codes.contains(&status_code) {
return false;
}
Regex::new(&rule.pattern)
.ok()
.is_some_and(|regex| regex.is_match(response_text))
}
fn parse_status_code_set(
rules: &serde_json::Map<String, serde_json::Value>,
keys: &[&str],
) -> BTreeSet<u16> {
keys.iter()
.filter_map(|key| rules.get(*key))
.filter_map(serde_json::Value::as_array)
.flat_map(|values| values.iter())
.filter_map(|value| parse_u64_value(value).and_then(|value| u16::try_from(value).ok()))
.collect()
}
fn parse_u64_value(value: &serde_json::Value) -> Option<u64> {
value
.as_u64()
.or_else(|| value.as_i64().and_then(|value| u64::try_from(value).ok()))
}
pub(crate) fn should_fallback_to_control_stream(
@@ -172,14 +461,121 @@ pub(crate) fn resolve_core_stream_direct_finalize_report_kind(plan_kind: &str) -
#[cfg(test)]
mod tests {
use std::collections::BTreeSet;
use aether_contracts::ExecutionResult;
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
use aether_data_contracts::repository::provider_catalog::{
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
};
use super::{
resolve_core_stream_error_finalize_report_kind,
resolve_core_sync_error_finalize_report_kind, should_fallback_to_control_stream,
should_fallback_to_control_sync, should_retry_next_local_candidate_stream,
should_retry_next_local_candidate_sync,
resolve_core_sync_error_finalize_report_kind, resolve_local_failover_policy,
should_fallback_to_control_stream, should_fallback_to_control_sync,
should_retry_next_local_candidate_stream, should_retry_next_local_candidate_sync,
should_stop_local_candidate_failover_stream, should_stop_local_candidate_failover_sync,
LocalFailoverPolicy, LocalFailoverRegexRule,
};
use crate::data::GatewayDataState;
use crate::AppState;
fn sample_plan() -> aether_contracts::ExecutionPlan {
aether_contracts::ExecutionPlan {
request_id: "req-1".to_string(),
candidate_id: Some("cand-1".to_string()),
provider_name: Some("provider-1".to_string()),
provider_id: "provider-1".to_string(),
endpoint_id: "endpoint-1".to_string(),
key_id: "key-1".to_string(),
method: "POST".to_string(),
url: "https://example.com/v1/chat/completions".to_string(),
headers: Default::default(),
content_type: Some("application/json".to_string()),
content_encoding: None,
body: aether_contracts::RequestBody::from_json(serde_json::json!({"model":"gpt-5"})),
stream: false,
client_api_format: "openai:chat".to_string(),
provider_api_format: "openai:chat".to_string(),
model_name: Some("gpt-5".to_string()),
proxy: None,
tls_profile: None,
timeouts: None,
}
}
fn sample_provider(config: Option<serde_json::Value>) -> StoredProviderCatalogProvider {
StoredProviderCatalogProvider::new(
"provider-1".to_string(),
"provider-1".to_string(),
Some("https://provider.example".to_string()),
"custom".to_string(),
)
.expect("provider should build")
.with_transport_fields(true, false, false, None, Some(3), None, None, None, config)
}
fn sample_endpoint() -> StoredProviderCatalogEndpoint {
StoredProviderCatalogEndpoint::new(
"endpoint-1".to_string(),
"provider-1".to_string(),
"openai:chat".to_string(),
Some("openai".to_string()),
Some("chat".to_string()),
true,
)
.expect("endpoint should build")
.with_transport_fields(
"https://api.provider.example".to_string(),
None,
None,
Some(2),
None,
None,
None,
None,
)
.expect("endpoint transport should build")
}
fn sample_key() -> StoredProviderCatalogKey {
StoredProviderCatalogKey::new(
"key-1".to_string(),
"provider-1".to_string(),
"key-1".to_string(),
"api_key".to_string(),
None,
true,
)
.expect("key should build")
.with_transport_fields(
Some(serde_json::json!(["openai:chat"])),
"plain-upstream-key".to_string(),
None,
None,
Some(serde_json::json!({"openai:chat": 1})),
None,
None,
None,
None,
)
.expect("key transport should build")
}
fn build_state_with_provider_config(config: Option<serde_json::Value>) -> AppState {
let provider_catalog = InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider(config)],
vec![sample_endpoint()],
vec![sample_key()],
);
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
std::sync::Arc::new(provider_catalog),
"development-key",
);
AppState::new()
.expect("state should build")
.with_data_state_for_tests(data_state)
}
#[test]
fn sync_failover_marks_chat_errors() {
@@ -220,8 +616,8 @@ mod tests {
);
}
#[test]
fn sync_retry_next_candidate_is_local_openai_chat_only() {
#[tokio::test]
async fn sync_retry_next_candidate_requires_local_candidate_context() {
let result = ExecutionResult {
request_id: "req-1".to_string(),
candidate_id: None,
@@ -235,26 +631,57 @@ mod tests {
"candidate_index": 0,
"retry_index": 0,
});
let state = build_state_with_provider_config(None);
let plan = sample_plan();
assert!(should_retry_next_local_candidate_sync(
"openai_chat_sync",
Some(&local_report_context),
&result,
));
assert!(!should_retry_next_local_candidate_sync(
"openai_chat_sync",
None,
&result,
));
assert!(!should_retry_next_local_candidate_sync(
"claude_chat_sync",
None,
&result,
));
assert!(
should_retry_next_local_candidate_sync(
&state,
&plan,
"openai_chat_sync",
Some(&local_report_context),
&result,
None,
)
.await
);
assert!(
should_retry_next_local_candidate_sync(
&state,
&plan,
"claude_cli_sync",
Some(&local_report_context),
&result,
None,
)
.await
);
assert!(
!should_retry_next_local_candidate_sync(
&state,
&plan,
"openai_chat_sync",
None,
&result,
None,
)
.await
);
assert!(
!should_retry_next_local_candidate_sync(
&state,
&plan,
"claude_chat_sync",
None,
&result,
None,
)
.await
);
}
#[test]
fn sync_retry_next_candidate_treats_rate_limit_as_retryable() {
#[tokio::test]
async fn sync_retry_next_candidate_treats_rate_limit_as_retryable() {
let result = ExecutionResult {
request_id: "req-1".to_string(),
candidate_id: None,
@@ -268,49 +695,304 @@ mod tests {
"candidate_index": 0,
"retry_index": 0,
});
let state = build_state_with_provider_config(None);
let plan = sample_plan();
assert!(should_retry_next_local_candidate_sync(
"openai_chat_sync",
Some(&local_report_context),
&result,
));
assert!(
should_retry_next_local_candidate_sync(
&state,
&plan,
"openai_chat_sync",
Some(&local_report_context),
&result,
None,
)
.await
);
}
#[test]
fn stream_retry_next_candidate_is_local_openai_chat_only() {
#[tokio::test]
async fn stream_retry_next_candidate_requires_local_candidate_context() {
let local_report_context = serde_json::json!({
"candidate_index": 0,
"retry_index": 0,
});
let state = build_state_with_provider_config(None);
let plan = sample_plan();
assert!(should_retry_next_local_candidate_stream(
"openai_chat_stream",
Some(&local_report_context),
502,
));
assert!(!should_retry_next_local_candidate_stream(
"openai_chat_stream",
None,
502,
));
assert!(!should_retry_next_local_candidate_stream(
"claude_chat_stream",
Some(&local_report_context),
502,
));
assert!(
should_retry_next_local_candidate_stream(
&state,
&plan,
"openai_chat_stream",
Some(&local_report_context),
502,
None,
)
.await
);
assert!(
should_retry_next_local_candidate_stream(
&state,
&plan,
"gemini_cli_stream",
Some(&local_report_context),
502,
None,
)
.await
);
assert!(
!should_retry_next_local_candidate_stream(
&state,
&plan,
"openai_chat_stream",
None,
502,
None,
)
.await
);
assert!(
!should_retry_next_local_candidate_stream(
&state,
&plan,
"claude_chat_stream",
None,
502,
None,
)
.await
);
}
#[test]
fn stream_retry_next_candidate_treats_rate_limit_as_retryable() {
#[tokio::test]
async fn stream_retry_next_candidate_treats_rate_limit_as_retryable() {
let local_report_context = serde_json::json!({
"candidate_index": 0,
"retry_index": 0,
});
let state = build_state_with_provider_config(None);
let plan = sample_plan();
assert!(should_retry_next_local_candidate_stream(
"openai_chat_stream",
Some(&local_report_context),
429,
));
assert!(
should_retry_next_local_candidate_stream(
&state,
&plan,
"openai_chat_stream",
Some(&local_report_context),
429,
None,
)
.await
);
}
#[test]
fn resolve_local_failover_policy_reads_provider_rules() {
let state = build_state_with_provider_config(Some(serde_json::json!({
"failover_rules": {
"max_retries": 1,
"stop_on_status_codes": [503],
"continue_on_status_codes": [409, 429]
}
})));
let plan = sample_plan();
let runtime = tokio::runtime::Runtime::new().expect("runtime should build");
let policy = runtime.block_on(resolve_local_failover_policy(&state, &plan));
assert_eq!(
policy,
LocalFailoverPolicy {
max_retries: Some(1),
stop_status_codes: [503].into_iter().collect(),
continue_status_codes: [409, 429].into_iter().collect(),
success_failover_patterns: Vec::new(),
error_stop_patterns: Vec::new(),
}
);
}
#[tokio::test]
async fn local_failover_policy_can_stop_retryable_statuses_and_continue_non_retryable_statuses()
{
let state = build_state_with_provider_config(Some(serde_json::json!({
"failover_rules": {
"max_retries": 2,
"stop_on_status_codes": [503],
"continue_on_status_codes": [409]
}
})));
let plan = sample_plan();
let first_candidate = serde_json::json!({
"candidate_index": 0,
"retry_index": 0,
});
let third_candidate = serde_json::json!({
"candidate_index": 2,
"retry_index": 0,
});
assert!(
!should_retry_next_local_candidate_stream(
&state,
&plan,
"openai_chat_stream",
Some(&first_candidate),
503,
None,
)
.await
);
assert!(
should_stop_local_candidate_failover_stream(
&state,
&plan,
"openai_chat_stream",
Some(&first_candidate),
503,
None,
)
.await
);
assert!(
should_retry_next_local_candidate_stream(
&state,
&plan,
"openai_chat_stream",
Some(&first_candidate),
409,
None,
)
.await
);
assert!(
!should_retry_next_local_candidate_stream(
&state,
&plan,
"openai_chat_stream",
Some(&third_candidate),
429,
None,
)
.await
);
}
#[test]
fn resolve_local_failover_policy_reads_regex_rules() {
let state = build_state_with_provider_config(Some(serde_json::json!({
"failover_rules": {
"success_failover_patterns": [
{"pattern": "relay:.*格式错误"}
],
"error_stop_patterns": [
{"pattern": "content_policy_violation", "status_codes": [400, 403]}
]
}
})));
let plan = sample_plan();
let runtime = tokio::runtime::Runtime::new().expect("runtime should build");
let policy = runtime.block_on(resolve_local_failover_policy(&state, &plan));
assert_eq!(
policy.success_failover_patterns,
vec![LocalFailoverRegexRule {
pattern: "relay:.*格式错误".to_string(),
status_codes: BTreeSet::new(),
}]
);
assert_eq!(
policy.error_stop_patterns,
vec![LocalFailoverRegexRule {
pattern: "content_policy_violation".to_string(),
status_codes: [400, 403].into_iter().collect(),
}]
);
}
#[tokio::test]
async fn success_failover_pattern_can_retry_sync_candidate() {
let result = ExecutionResult {
request_id: "req-1".to_string(),
candidate_id: None,
status_code: 200,
headers: Default::default(),
body: None,
telemetry: None,
error: None,
};
let local_report_context = serde_json::json!({
"candidate_index": 0,
"retry_index": 0,
});
let state = build_state_with_provider_config(Some(serde_json::json!({
"failover_rules": {
"success_failover_patterns": [
{"pattern": "relay:.*格式错误"}
]
}
})));
let plan = sample_plan();
assert!(
should_retry_next_local_candidate_sync(
&state,
&plan,
"openai_chat_sync",
Some(&local_report_context),
&result,
Some("{\"error\":\"relay: 返回格式错误\"}"),
)
.await
);
}
#[tokio::test]
async fn error_stop_pattern_can_stop_sync_failover() {
let result = ExecutionResult {
request_id: "req-1".to_string(),
candidate_id: None,
status_code: 400,
headers: Default::default(),
body: None,
telemetry: None,
error: None,
};
let local_report_context = serde_json::json!({
"candidate_index": 0,
"retry_index": 0,
});
let state = build_state_with_provider_config(Some(serde_json::json!({
"failover_rules": {
"error_stop_patterns": [
{"pattern": "content_policy_violation", "status_codes": [400]}
]
}
})));
let plan = sample_plan();
assert!(
should_stop_local_candidate_failover_sync(
&state,
&plan,
"openai_chat_sync",
Some(&local_report_context),
&result,
Some("{\"error\":\"content_policy_violation\"}"),
)
.await
);
assert!(
!should_retry_next_local_candidate_sync(
&state,
&plan,
"openai_chat_sync",
Some(&local_report_context),
&result,
Some("{\"error\":\"content_policy_violation\"}"),
)
.await
);
}
}

5
apps/aether-gateway/src/execution_runtime/mod.rs
View File

@@ -19,11 +19,12 @@ pub(crate) use self::constants::{
MAX_ERROR_BODY_BYTES, MAX_STREAM_PREFETCH_BYTES, MAX_STREAM_PREFETCH_FRAMES,
};
pub(crate) use self::fallback::{
resolve_core_stream_direct_finalize_report_kind,
local_failover_response_text, resolve_core_stream_direct_finalize_report_kind,
resolve_core_stream_error_finalize_report_kind, resolve_core_sync_error_finalize_report_kind,
should_fallback_to_control_stream, should_fallback_to_control_sync,
should_finalize_sync_response, should_retry_next_local_candidate_stream,
should_retry_next_local_candidate_sync,
should_retry_next_local_candidate_sync, should_stop_local_candidate_failover_stream,
should_stop_local_candidate_failover_sync,
};
pub use server::{
build_execution_runtime_router, build_execution_runtime_router_with_request_concurrency_limit,

296
apps/aether-gateway/src/execution_runtime/stream/execution.rs
View File

@@ -1,6 +1,7 @@
use std::collections::VecDeque;
use std::io::Error as IoError;
use aether_contracts::{ExecutionPlan, ExecutionTelemetry, StreamFramePayload};
use aether_contracts::{ExecutionPlan, ExecutionTelemetry, StreamFrame, StreamFramePayload};
use aether_data_contracts::repository::candidates::RequestCandidateStatus;
use aether_scheduler_core::SchedulerRequestCandidateStatusUpdate;
use async_stream::stream;
@@ -32,7 +33,7 @@ use crate::ai_pipeline_api::{
use crate::api::response::{
attach_control_metadata_headers, build_client_response, build_client_response_from_parts,
};
use crate::clock::current_unix_secs as current_request_candidate_unix_secs;
use crate::clock::current_unix_ms as current_request_candidate_unix_ms;
use crate::constants::{CONTROL_CANDIDATE_ID_HEADER, CONTROL_REQUEST_ID_HEADER};
use crate::control::GatewayControlDecision;
use crate::execution_runtime::build_direct_execution_frame_stream;
@@ -45,9 +46,9 @@ use crate::execution_runtime::transport::{
DirectSyncExecutionRuntime, DirectUpstreamStreamExecution,
};
use crate::execution_runtime::{
resolve_core_stream_direct_finalize_report_kind,
local_failover_response_text, resolve_core_stream_direct_finalize_report_kind,
resolve_core_stream_error_finalize_report_kind, should_fallback_to_control_stream,
should_retry_next_local_candidate_stream,
should_retry_next_local_candidate_stream, should_stop_local_candidate_failover_stream,
};
use crate::execution_runtime::{MAX_STREAM_PREFETCH_BYTES, MAX_STREAM_PREFETCH_FRAMES};
use crate::log_ids::short_request_id;
@@ -165,7 +166,7 @@ pub(crate) async fn execute_execution_runtime_stream(
};
if response.status() != http::StatusCode::OK {
let terminal_unix_secs = current_request_candidate_unix_secs();
let terminal_unix_secs = current_request_candidate_unix_ms();
record_local_request_candidate_status(
state,
&plan,
@@ -179,8 +180,8 @@ pub(crate) async fn execute_execution_runtime_stream(
response.status()
)),
latency_ms: None,
started_at_unix_secs: Some(terminal_unix_secs),
finished_at_unix_secs: Some(terminal_unix_secs),
started_at_unix_ms: Some(terminal_unix_secs),
finished_at_unix_ms: Some(terminal_unix_secs),
},
)
.await;
@@ -209,6 +210,60 @@ pub(crate) async fn execute_execution_runtime_stream(
}
}
fn decode_stream_data_chunk(
chunk_b64: Option<&str>,
text: Option<&str>,
) -> Result<Vec<u8>, GatewayError> {
if let Some(chunk_b64) = chunk_b64 {
return base64::engine::general_purpose::STANDARD
.decode(chunk_b64)
.map_err(|err| GatewayError::Internal(err.to_string()));
}
Ok(text.unwrap_or_default().as_bytes().to_vec())
}
async fn next_stream_frame<R>(
buffered_frames: &mut VecDeque<StreamFrame>,
lines: &mut FramedRead<R, LinesCodec>,
) -> Result<Option<StreamFrame>, GatewayError>
where
R: tokio::io::AsyncRead + Unpin,
{
if let Some(frame) = buffered_frames.pop_front() {
return Ok(Some(frame));
}
read_next_frame(lines).await
}
async fn probe_local_stream_success_failover_text<R>(
buffered_frames: &mut VecDeque<StreamFrame>,
lines: &mut FramedRead<R, LinesCodec>,
) -> Result<Option<String>, GatewayError>
where
R: tokio::io::AsyncRead + Unpin,
{
while let Some(frame) = read_next_frame(lines).await? {
let probe_text = match &frame.payload {
StreamFramePayload::Data { chunk_b64, text } => {
match decode_stream_data_chunk(chunk_b64.as_deref(), text.as_deref()) {
Ok(chunk) if !chunk.is_empty() => {
Some(String::from_utf8_lossy(&chunk).into_owned())
}
Ok(_) | Err(_) => None,
}
}
StreamFramePayload::Error { .. } | StreamFramePayload::Eof { .. } => None,
StreamFramePayload::Headers { .. } | StreamFramePayload::Telemetry { .. } => None,
};
buffered_frames.push_back(frame);
if probe_text.is_some() {
return Ok(probe_text);
}
}
Ok(None)
}
async fn execute_stream_from_frame_stream(
state: &AppState,
plan: ExecutionPlan,
@@ -237,69 +292,137 @@ async fn execute_stream_from_frame_stream(
"execution runtime stream must start with headers frame".to_string(),
));
};
let mut buffered_frames = VecDeque::new();
if should_retry_next_local_candidate_stream(plan_kind, report_context.as_ref(), status_code) {
let terminal_unix_secs = current_request_candidate_unix_secs();
record_local_request_candidate_status(
if status_code == 200 {
let success_probe_text =
probe_local_stream_success_failover_text(&mut buffered_frames, &mut lines).await?;
if should_retry_next_local_candidate_stream(
state,
&plan,
plan_kind,
report_context.as_ref(),
SchedulerRequestCandidateStatusUpdate {
status: RequestCandidateStatus::Failed,
status_code: Some(status_code),
error_type: Some("retryable_upstream_status".to_string()),
error_message: Some(format!(
"execution runtime stream returned retryable status {status_code}"
)),
latency_ms: None,
started_at_unix_secs: Some(terminal_unix_secs),
finished_at_unix_secs: Some(terminal_unix_secs),
},
)
.await;
warn!(
event_name = "local_stream_candidate_retry_scheduled",
log_type = "event",
trace_id = %trace_id,
request_id = %request_id_for_log,
status_code,
"gateway local stream decision retrying next candidate after retryable execution runtime status"
);
return Ok(None);
success_probe_text.as_deref(),
)
.await
{
let terminal_unix_secs = current_request_candidate_unix_ms();
record_local_request_candidate_status(
state,
&plan,
report_context.as_ref(),
SchedulerRequestCandidateStatusUpdate {
status: RequestCandidateStatus::Failed,
status_code: Some(status_code),
error_type: Some("success_failover_pattern".to_string()),
error_message: Some(
"execution runtime stream matched provider success failover rule"
.to_string(),
),
latency_ms: None,
started_at_unix_ms: Some(terminal_unix_secs),
finished_at_unix_ms: Some(terminal_unix_secs),
},
)
.await;
warn!(
event_name = "local_stream_candidate_retry_scheduled",
log_type = "event",
trace_id = %trace_id,
request_id = %request_id_for_log,
status_code,
"gateway local stream decision retrying next candidate after success failover rule match"
);
return Ok(None);
}
}
let stream_error_finalize_kind =
resolve_core_stream_error_finalize_report_kind(plan_kind, status_code);
if should_fallback_to_control_stream(
plan_kind,
status_code,
stream_error_finalize_kind.is_some(),
) {
let terminal_unix_secs = current_request_candidate_unix_secs();
record_local_request_candidate_status(
state,
&plan,
report_context.as_ref(),
SchedulerRequestCandidateStatusUpdate {
status: RequestCandidateStatus::Failed,
status_code: Some(status_code),
error_type: Some("control_fallback".to_string()),
error_message: Some(format!(
"stream decision fell back to control after status {status_code}"
)),
latency_ms: None,
started_at_unix_secs: Some(terminal_unix_secs),
finished_at_unix_secs: Some(terminal_unix_secs),
},
)
.await;
return Ok(None);
}
if status_code >= 400 {
let error_body = collect_error_body(&mut lines).await?;
let (body_json, body_base64) = decode_stream_error_body(&headers, &error_body);
let error_response_text =
local_failover_response_text(body_json.as_ref(), &error_body, None);
let stop_local_failover = should_stop_local_candidate_failover_stream(
state,
&plan,
plan_kind,
report_context.as_ref(),
status_code,
error_response_text.as_deref(),
)
.await;
if !stop_local_failover
&& should_retry_next_local_candidate_stream(
state,
&plan,
plan_kind,
report_context.as_ref(),
status_code,
error_response_text.as_deref(),
)
.await
{
let terminal_unix_secs = current_request_candidate_unix_ms();
record_local_request_candidate_status(
state,
&plan,
report_context.as_ref(),
SchedulerRequestCandidateStatusUpdate {
status: RequestCandidateStatus::Failed,
status_code: Some(status_code),
error_type: Some("retryable_upstream_status".to_string()),
error_message: Some(format!(
"execution runtime stream returned retryable status {status_code}"
)),
latency_ms: None,
started_at_unix_ms: Some(terminal_unix_secs),
finished_at_unix_ms: Some(terminal_unix_secs),
},
)
.await;
warn!(
event_name = "local_stream_candidate_retry_scheduled",
log_type = "event",
trace_id = %trace_id,
request_id = %request_id_for_log,
status_code,
"gateway local stream decision retrying next candidate after retryable execution runtime status"
);
return Ok(None);
}
if !stop_local_failover
&& should_fallback_to_control_stream(
plan_kind,
status_code,
stream_error_finalize_kind.is_some(),
)
{
let terminal_unix_secs = current_request_candidate_unix_ms();
record_local_request_candidate_status(
state,
&plan,
report_context.as_ref(),
SchedulerRequestCandidateStatusUpdate {
status: RequestCandidateStatus::Failed,
status_code: Some(status_code),
error_type: Some("control_fallback".to_string()),
error_message: Some(format!(
"stream decision fell back to control after status {status_code}"
)),
latency_ms: None,
started_at_unix_ms: Some(terminal_unix_secs),
finished_at_unix_ms: Some(terminal_unix_secs),
},
)
.await;
return Ok(None);
}
let usage_report_kind = stream_error_finalize_kind
.clone()
.or_else(|| report_kind.clone())
@@ -324,7 +447,7 @@ async fn execute_stream_from_frame_stream(
&usage_payload,
)
.await;
let terminal_unix_secs = current_request_candidate_unix_secs();
let terminal_unix_secs = current_request_candidate_unix_ms();
record_local_request_candidate_status(
state,
&plan,
@@ -337,8 +460,8 @@ async fn execute_stream_from_frame_stream(
"execution runtime stream returned error status {status_code}"
)),
latency_ms: None,
started_at_unix_secs: Some(terminal_unix_secs),
finished_at_unix_secs: Some(terminal_unix_secs),
started_at_unix_ms: Some(terminal_unix_secs),
finished_at_unix_ms: Some(terminal_unix_secs),
},
)
.await;
@@ -399,7 +522,7 @@ async fn execute_stream_from_frame_stream(
while prefetched_chunks.len() < MAX_STREAM_PREFETCH_FRAMES
&& prefetched_inspection_body.len() < MAX_STREAM_PREFETCH_BYTES
{
let Some(frame) = (match read_next_frame(&mut lines).await {
let Some(frame) = (match next_stream_frame(&mut buffered_frames, &mut lines).await {
Ok(frame) => frame,
Err(err) => {
let failure = build_stream_failure_report(
@@ -429,14 +552,14 @@ async fn execute_stream_from_frame_stream(
};
match frame.payload {
StreamFramePayload::Data { chunk_b64, text } => {
let chunk = if let Some(chunk_b64) = chunk_b64 {
match base64::engine::general_purpose::STANDARD.decode(chunk_b64) {
Ok(decoded) => decoded,
let chunk =
match decode_stream_data_chunk(chunk_b64.as_deref(), text.as_deref()) {
Ok(chunk) => chunk,
Err(err) => {
let failure = build_stream_failure_report(
"execution_runtime_stream_chunk_decode_error",
format!(
"failed to decode execution runtime stream chunk: {err}"
"failed to decode execution runtime stream chunk: {err:?}"
),
502,
);
@@ -456,12 +579,7 @@ async fn execute_stream_from_frame_stream(
)
.await;
}
}
} else if let Some(text) = text {
text.into_bytes()
} else {
Vec::new()
};
};
if chunk.is_empty() {
continue;
@@ -621,7 +739,7 @@ async fn execute_stream_from_frame_stream(
}
}
let candidate_started_unix_secs = current_request_candidate_unix_secs();
let candidate_started_unix_secs = current_request_candidate_unix_ms();
state
.usage_runtime
.record_pending(state.data.as_ref(), &plan, report_context.as_ref())
@@ -649,8 +767,8 @@ async fn execute_stream_from_frame_stream(
latency_ms: prefetched_telemetry
.as_ref()
.and_then(|telemetry| telemetry.elapsed_ms),
started_at_unix_secs: Some(candidate_started_unix_secs),
finished_at_unix_secs: None,
started_at_unix_ms: Some(candidate_started_unix_secs),
finished_at_unix_ms: None,
},
)
.await;
@@ -672,6 +790,7 @@ async fn execute_stream_from_frame_stream(
let request_id_for_report = request_id.to_string();
let request_id_for_report_log = short_request_id(request_id);
let candidate_id_for_report = candidate_id.map(ToOwned::to_owned);
let mut buffered_frames = buffered_frames;
tokio::spawn(async move {
let mut provider_buffered_body = provider_prefetched_body_for_report;
let mut buffered_body = prefetched_body_for_report;
@@ -682,7 +801,7 @@ async fn execute_stream_from_frame_stream(
if !reached_eof {
loop {
let next_frame = match read_next_frame(&mut lines).await {
let next_frame = match next_stream_frame(&mut buffered_frames, &mut lines).await {
Ok(frame) => frame,
Err(err) => {
warn!(
@@ -707,9 +826,9 @@ async fn execute_stream_from_frame_stream(
};
match frame.payload {
StreamFramePayload::Data { chunk_b64, text } => {
let chunk = if let Some(chunk_b64) = chunk_b64 {
match base64::engine::general_purpose::STANDARD.decode(chunk_b64) {
Ok(decoded) => decoded,
let chunk =
match decode_stream_data_chunk(chunk_b64.as_deref(), text.as_deref()) {
Ok(chunk) => chunk,
Err(err) => {
warn!(
event_name = "stream_execution_chunk_decode_failed",
@@ -717,22 +836,19 @@ async fn execute_stream_from_frame_stream(
trace_id = %trace_id_owned,
request_id = %request_id_for_report_log,
candidate_id = ?candidate_id_for_report.as_deref(),
error = %err,
error = ?err,
"gateway failed to decode execution runtime chunk"
);
terminal_failure = Some(build_stream_failure_report(
"execution_runtime_stream_chunk_decode_error",
format!("failed to decode execution runtime stream chunk: {err}"),
format!(
"failed to decode execution runtime stream chunk: {err:?}"
),
502,
));
break;
}
}
} else if let Some(text) = text {
text.into_bytes()
} else {
Vec::new()
};
};
if chunk.is_empty() {
continue;
@@ -998,8 +1114,8 @@ async fn execute_stream_from_frame_stream(
error_type: Some("downstream_disconnect".to_string()),
error_message: Some("client disconnected before stream completion".to_string()),
latency_ms: telemetry.as_ref().and_then(|value| value.elapsed_ms),
started_at_unix_secs: Some(candidate_started_unix_secs_for_report),
finished_at_unix_secs: Some(current_request_candidate_unix_secs()),
started_at_unix_ms: Some(candidate_started_unix_secs_for_report),
finished_at_unix_ms: Some(current_request_candidate_unix_ms()),
},
)
.await;
@@ -1055,8 +1171,8 @@ async fn execute_stream_from_frame_stream(
error_type: None,
error_message: None,
latency_ms: telemetry.as_ref().and_then(|value| value.elapsed_ms),
started_at_unix_secs: Some(candidate_started_unix_secs_for_report),
finished_at_unix_secs: Some(current_request_candidate_unix_secs()),
started_at_unix_ms: Some(candidate_started_unix_secs_for_report),
finished_at_unix_ms: Some(current_request_candidate_unix_ms()),
},
)
.await;

14
apps/aether-gateway/src/execution_runtime/stream/execution_failures.rs
View File

@@ -8,7 +8,7 @@ use serde_json::{Map, Value};
use tracing::warn;
use crate::api::response::attach_control_metadata_headers;
use crate::clock::current_unix_secs as current_request_candidate_unix_secs;
use crate::clock::current_unix_ms as current_request_candidate_unix_ms;
use crate::control::GatewayControlDecision;
use crate::execution_runtime::submission::{
resolve_core_error_background_report_kind, submit_local_core_error_or_sync_finalize,
@@ -117,13 +117,13 @@ async fn record_stream_sync_failure(
report_context: Option<&Value>,
payload: &GatewaySyncReportRequest,
failure: &StreamFailureReport,
started_at_unix_secs: Option<u64>,
started_at_unix_ms: Option<u64>,
) {
state
.usage_runtime
.record_sync_terminal(state.data.as_ref(), plan, report_context, payload)
.await;
let terminal_unix_secs = current_request_candidate_unix_secs();
let terminal_unix_secs = current_request_candidate_unix_ms();
record_report_request_candidate_status(
state,
report_context,
@@ -136,8 +136,8 @@ async fn record_stream_sync_failure(
.telemetry
.as_ref()
.and_then(|telemetry| telemetry.elapsed_ms),
started_at_unix_secs: started_at_unix_secs.or(Some(terminal_unix_secs)),
finished_at_unix_secs: Some(terminal_unix_secs),
started_at_unix_ms: started_at_unix_ms.or(Some(terminal_unix_secs)),
finished_at_unix_ms: Some(terminal_unix_secs),
},
)
.await;
@@ -195,7 +195,7 @@ pub(super) async fn submit_midstream_stream_failure(
headers: &std::collections::BTreeMap<String, String>,
telemetry: Option<ExecutionTelemetry>,
buffered_body: &[u8],
started_at_unix_secs: u64,
started_at_unix_ms: u64,
failure: StreamFailureReport,
) {
let Some(report_kind) =
@@ -219,7 +219,7 @@ pub(super) async fn submit_midstream_stream_failure(
report_context,
&payload,
&failure,
Some(started_at_unix_secs),
Some(started_at_unix_ms),
)
.await;
if let Err(err) = submit_sync_report(state, trace_id, payload).await {

183
apps/aether-gateway/src/execution_runtime/submission.rs
View File

@@ -297,29 +297,39 @@ fn resolve_local_sync_error_status_code(status_code: u16, body_json: &serde_json
return status_code;
}
let Some(error_object) = body_json.get("error").and_then(|value| value.as_object()) else {
return 400;
};
let body_object = body_json.as_object();
let error_object = body_object
.and_then(|object| object.get("error"))
.and_then(|value| value.as_object());
for key in ["code", "status"] {
let Some(value) = error_object.get(key) else {
continue;
};
if let Some(number) = value.as_u64() {
let raw_code = first_non_empty_error_text(error_object, body_object, &["code"]);
let raw_status = first_non_empty_error_text(error_object, body_object, &["status"]);
for numeric_hint in [raw_code.as_deref(), raw_status.as_deref()]
.into_iter()
.flatten()
{
if let Ok(number) = numeric_hint.parse::<u16>() {
if (400..600).contains(&number) {
return number as u16;
}
}
if let Some(text) = value.as_str() {
if let Ok(number) = text.parse::<u16>() {
if (400..600).contains(&number) {
return number;
}
return number;
}
}
}
400
let raw_type = first_non_empty_error_text(error_object, body_object, &["type", "__type"]);
let message = first_non_empty_error_text(
error_object,
body_object,
&["message", "detail", "reason", "status", "type", "__type"],
)
.unwrap_or_else(|| "HTTP 400".to_string());
let kind = classify_local_sync_error_kind(
status_code,
raw_type.as_deref(),
raw_status.as_deref(),
raw_code.as_deref(),
message.as_str(),
);
default_status_code_for_local_sync_error_kind(kind)
}
fn extract_local_sync_error_details(
@@ -436,6 +446,20 @@ fn classify_local_sync_error_kind(
LocalCoreSyncErrorKind::InvalidRequest
}
fn default_status_code_for_local_sync_error_kind(kind: LocalCoreSyncErrorKind) -> u16 {
match kind {
LocalCoreSyncErrorKind::InvalidRequest | LocalCoreSyncErrorKind::ContextLengthExceeded => {
400
}
LocalCoreSyncErrorKind::Authentication => 401,
LocalCoreSyncErrorKind::PermissionDenied => 403,
LocalCoreSyncErrorKind::NotFound => 404,
LocalCoreSyncErrorKind::RateLimit => 429,
LocalCoreSyncErrorKind::Overloaded => 503,
LocalCoreSyncErrorKind::ServerError => 500,
}
}
pub(crate) fn strip_utf8_bom_and_ws(mut body: &[u8]) -> &[u8] {
loop {
while let Some(first) = body.first() {
@@ -534,3 +558,128 @@ pub(crate) async fn submit_local_core_error_or_sync_finalize(
Ok(response)
}
#[cfg(test)]
mod tests {
use axum::body::to_bytes;
use serde_json::json;
use super::maybe_build_local_core_error_response;
use crate::control::GatewayControlDecision;
use crate::usage::GatewaySyncReportRequest;
fn test_decision() -> GatewayControlDecision {
GatewayControlDecision::synthetic(
"/v1/chat/completions",
Some("ai_public".to_string()),
Some("openai".to_string()),
Some("chat".to_string()),
Some("openai:chat".to_string()),
)
.with_execution_runtime_candidate(true)
}
fn core_finalize_payload(
report_kind: &str,
client_api_format: &str,
provider_api_format: &str,
status_code: u16,
body_json: serde_json::Value,
) -> GatewaySyncReportRequest {
GatewaySyncReportRequest {
trace_id: "trace-core-error-status-123".to_string(),
report_kind: report_kind.to_string(),
report_context: Some(json!({
"client_api_format": client_api_format,
"provider_api_format": provider_api_format,
})),
status_code,
headers: Default::default(),
body_json: Some(body_json),
client_body_json: None,
body_base64: None,
telemetry: None,
}
}
#[tokio::test]
async fn maybe_build_local_core_error_response_infers_status_from_semantic_error_type() {
let payload = core_finalize_payload(
"openai_chat_sync_finalize",
"openai:chat",
"claude:chat",
200,
json!({
"type": "error",
"error": {
"type": "rate_limit_error",
"message": "slow down"
}
}),
);
let response = maybe_build_local_core_error_response(
"trace-sync-status-type",
&test_decision(),
&payload,
)
.expect("response build should not error")
.expect("response should exist");
assert_eq!(response.status(), http::StatusCode::TOO_MANY_REQUESTS);
assert_eq!(
serde_json::from_slice::<serde_json::Value>(
&to_bytes(response.into_body(), usize::MAX)
.await
.expect("body should read"),
)
.expect("body should decode"),
json!({
"error": {
"message": "slow down",
"type": "rate_limit_error"
}
})
);
}
#[tokio::test]
async fn maybe_build_local_core_error_response_infers_status_from_gemini_status_text() {
let payload = core_finalize_payload(
"gemini_chat_sync_finalize",
"gemini:chat",
"gemini:chat",
200,
json!({
"error": {
"message": "quota reached",
"status": "RESOURCE_EXHAUSTED"
}
}),
);
let response = maybe_build_local_core_error_response(
"trace-sync-status-gemini",
&test_decision(),
&payload,
)
.expect("response build should not error")
.expect("response should exist");
assert_eq!(response.status(), http::StatusCode::TOO_MANY_REQUESTS);
assert_eq!(
serde_json::from_slice::<serde_json::Value>(
&to_bytes(response.into_body(), usize::MAX)
.await
.expect("body should read"),
)
.expect("body should decode"),
json!({
"error": {
"message": "quota reached",
"status": "RESOURCE_EXHAUSTED"
}
})
);
}
}

82
apps/aether-gateway/src/execution_runtime/sync/execution.rs
View File

@@ -15,7 +15,7 @@ use crate::ai_pipeline_api::{
use crate::api::response::{
attach_control_metadata_headers, build_client_response, build_client_response_from_parts,
};
use crate::clock::current_unix_secs as current_request_candidate_unix_secs;
use crate::clock::current_unix_ms as current_request_candidate_unix_ms;
use crate::constants::{CONTROL_CANDIDATE_ID_HEADER, CONTROL_REQUEST_ID_HEADER};
use crate::control::GatewayControlDecision;
#[cfg(test)]
@@ -23,8 +23,9 @@ use crate::execution_runtime::remote_compat::post_sync_plan_to_remote_execution_
use crate::execution_runtime::submission::submit_local_core_error_or_sync_finalize;
use crate::execution_runtime::transport::DirectSyncExecutionRuntime;
use crate::execution_runtime::{
resolve_core_sync_error_finalize_report_kind, should_fallback_to_control_sync,
should_finalize_sync_response, should_retry_next_local_candidate_sync,
local_failover_response_text, resolve_core_sync_error_finalize_report_kind,
should_fallback_to_control_sync, should_finalize_sync_response,
should_retry_next_local_candidate_sync, should_stop_local_candidate_failover_sync,
};
use crate::log_ids::short_request_id;
use crate::request_candidate_runtime::{
@@ -85,6 +86,7 @@ pub(crate) async fn execute_execution_runtime_sync(
let plan_request_id = plan.request_id.as_str();
let plan_request_id_for_log = short_request_id(plan_request_id);
let plan_candidate_id = plan.candidate_id.as_deref();
let candidate_started_unix_secs = current_request_candidate_unix_ms();
#[cfg(not(test))]
let result = {
match DirectSyncExecutionRuntime::new()
@@ -140,6 +142,7 @@ pub(crate) async fn execute_execution_runtime_sync(
plan_request_id,
plan_candidate_id,
report_context.as_ref(),
candidate_started_unix_secs,
)
.await?;
match remote_outcome {
@@ -159,8 +162,34 @@ pub(crate) async fn execute_execution_runtime_sync(
.telemetry
.as_ref()
.and_then(|telemetry| telemetry.elapsed_ms);
if should_retry_next_local_candidate_sync(plan_kind, report_context.as_ref(), &result) {
let terminal_unix_secs = current_request_candidate_unix_secs();
let mut headers = result.headers.clone();
let (body_bytes, body_json, body_base64) = decode_execution_result_body(&result, &mut headers)?;
let local_failover_response_text = local_failover_response_text(
body_json.as_ref(),
&body_bytes,
result.error.as_ref().map(|error| error.message.as_str()),
);
let stop_local_failover = should_stop_local_candidate_failover_sync(
state,
&plan,
plan_kind,
report_context.as_ref(),
&result,
local_failover_response_text.as_deref(),
)
.await;
if should_retry_next_local_candidate_sync(
state,
&plan,
plan_kind,
report_context.as_ref(),
&result,
local_failover_response_text.as_deref(),
)
.await
&& !stop_local_failover
{
let terminal_unix_secs = current_request_candidate_unix_ms();
record_local_request_candidate_status(
state,
&plan,
@@ -171,8 +200,8 @@ pub(crate) async fn execute_execution_runtime_sync(
error_type: result_error_type.clone(),
error_message: result_error_message.clone(),
latency_ms: result_latency_ms,
started_at_unix_secs: Some(terminal_unix_secs),
finished_at_unix_secs: Some(terminal_unix_secs),
started_at_unix_ms: Some(candidate_started_unix_secs),
finished_at_unix_ms: Some(terminal_unix_secs),
},
)
.await;
@@ -191,8 +220,6 @@ pub(crate) async fn execute_execution_runtime_sync(
.or(Some(plan_request_id));
let request_id_for_log = short_request_id(request_id.unwrap_or("-"));
let candidate_id = result.candidate_id.as_deref().or(plan_candidate_id);
let mut headers = result.headers.clone();
let (body_bytes, body_json, body_base64) = decode_execution_result_body(&result, &mut headers)?;
let has_body_bytes = body_base64.is_some();
let explicit_finalize = should_finalize_sync_response(report_kind.as_deref());
let mapped_error_finalize_kind =
@@ -220,15 +247,17 @@ pub(crate) async fn execute_execution_runtime_sync(
mapped_error_finalize_kind.clone()
};
if should_fallback_to_control_sync(
plan_kind,
&result,
body_json.as_ref(),
has_body_bytes,
explicit_finalize || implicit_finalize.is_some(),
mapped_error_finalize_kind.is_some(),
) {
let terminal_unix_secs = current_request_candidate_unix_secs();
if !stop_local_failover
&& should_fallback_to_control_sync(
plan_kind,
&result,
body_json.as_ref(),
has_body_bytes,
explicit_finalize || implicit_finalize.is_some(),
mapped_error_finalize_kind.is_some(),
)
{
let terminal_unix_secs = current_request_candidate_unix_ms();
record_local_request_candidate_status(
state,
&plan,
@@ -239,8 +268,8 @@ pub(crate) async fn execute_execution_runtime_sync(
error_type: result_error_type.clone(),
error_message: result_error_message.clone(),
latency_ms: result_latency_ms,
started_at_unix_secs: Some(terminal_unix_secs),
finished_at_unix_secs: Some(terminal_unix_secs),
started_at_unix_ms: Some(candidate_started_unix_secs),
finished_at_unix_ms: Some(terminal_unix_secs),
},
)
.await;
@@ -251,7 +280,7 @@ pub(crate) async fn execute_execution_runtime_sync(
.usage_runtime
.record_pending(state.data.as_ref(), &plan, report_context.as_ref())
.await;
let terminal_unix_secs = current_request_candidate_unix_secs();
let terminal_unix_secs = current_request_candidate_unix_ms();
record_local_request_candidate_status(
state,
&plan,
@@ -266,8 +295,8 @@ pub(crate) async fn execute_execution_runtime_sync(
error_type: result_error_type.clone(),
error_message: result_error_message.clone(),
latency_ms: result_latency_ms,
started_at_unix_secs: Some(terminal_unix_secs),
finished_at_unix_secs: Some(terminal_unix_secs),
started_at_unix_ms: Some(candidate_started_unix_secs),
finished_at_unix_ms: Some(terminal_unix_secs),
},
)
.await;
@@ -579,6 +608,7 @@ async fn execute_sync_via_remote_execution_runtime(
plan_request_id: &str,
plan_candidate_id: Option<&str>,
report_context: Option<&serde_json::Value>,
candidate_started_unix_secs: u64,
) -> Result<RemoteSyncFallbackOutcome, GatewayError> {
let response = match post_sync_plan_to_remote_execution_runtime(
state,
@@ -604,7 +634,7 @@ async fn execute_sync_via_remote_execution_runtime(
};
if response.status() != http::StatusCode::OK {
let terminal_unix_secs = current_request_candidate_unix_secs();
let terminal_unix_secs = current_request_candidate_unix_ms();
record_local_request_candidate_status(
state,
plan,
@@ -618,8 +648,8 @@ async fn execute_sync_via_remote_execution_runtime(
response.status()
)),
latency_ms: None,
started_at_unix_secs: Some(terminal_unix_secs),
finished_at_unix_secs: Some(terminal_unix_secs),
started_at_unix_ms: Some(candidate_started_unix_secs),
finished_at_unix_ms: Some(terminal_unix_secs),
},
)
.await;

154
apps/aether-gateway/src/execution_runtime/tests.rs
View File

@@ -228,6 +228,160 @@ fn build_best_effort_local_core_error_body_converts_claude_cli_error_to_openai_c
);
}
#[test]
fn build_best_effort_local_core_error_body_converts_sync_errors_across_standard_families() {
let cases = vec![
(
"claude chat -> openai chat",
core_finalize_payload(
"openai_chat_sync_finalize",
"openai:chat",
"claude:chat",
429,
json!({
"type": "error",
"error": {
"type": "rate_limit_error",
"message": "slow down"
}
}),
),
json!({
"error": {
"message": "slow down",
"type": "rate_limit_error"
}
}),
),
(
"openai chat -> claude chat",
core_finalize_payload(
"claude_chat_sync_finalize",
"claude:chat",
"openai:chat",
404,
json!({
"error": {
"message": "missing model",
"type": "not_found_error",
"code": "model_missing"
}
}),
),
json!({
"type": "error",
"error": {
"message": "missing model",
"type": "not_found_error",
"code": "model_missing"
}
}),
),
(
"openai chat -> gemini chat",
core_finalize_payload(
"gemini_chat_sync_finalize",
"gemini:chat",
"openai:chat",
401,
json!({
"error": {
"message": "bad auth",
"type": "authentication_error"
}
}),
),
json!({
"error": {
"code": 401,
"message": "bad auth",
"status": "UNAUTHENTICATED"
}
}),
),
(
"gemini cli -> openai cli",
core_finalize_payload(
"openai_cli_sync_finalize",
"openai:cli",
"gemini:cli",
429,
json!({
"error": {
"message": "quota hit",
"status": "RESOURCE_EXHAUSTED",
"code": 429
}
}),
),
json!({
"error": {
"message": "quota hit",
"type": "rate_limit_error",
"code": "429"
}
}),
),
(
"gemini cli -> claude cli",
core_finalize_payload(
"claude_cli_sync_finalize",
"claude:cli",
"gemini:cli",
503,
json!({
"error": {
"message": "backend busy",
"status": "UNAVAILABLE"
}
}),
),
json!({
"type": "error",
"error": {
"message": "backend busy",
"type": "api_error",
"code": "UNAVAILABLE"
}
}),
),
(
"claude cli -> gemini cli",
core_finalize_payload(
"gemini_cli_sync_finalize",
"gemini:cli",
"claude:cli",
404,
json!({
"type": "error",
"error": {
"type": "not_found_error",
"message": "resource missing"
}
}),
),
json!({
"error": {
"code": 404,
"message": "resource missing",
"status": "NOT_FOUND"
}
}),
),
];
for (label, payload, expected) in cases {
let converted = build_best_effort_local_core_error_body(
&payload,
payload.body_json.as_ref().expect("body_json should exist"),
)
.expect("conversion should not error")
.expect("conversion should produce a client error body");
assert_eq!(converted, expected, "unexpected conversion for {label}");
}
}
#[test]
fn resolve_local_core_error_response_body_json_parses_body_base64_json_for_cross_format_cli_error()
{