i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 17:30:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor ai serving modules and crates

Browse Source
This commit is contained in:
fawney19
2026-05-02 13:23:54 +08:00
parent 4fc7cecf30
commit c130d0e2c9
309 changed files with 21549 additions and 14265 deletions
Download Patch File Download Diff File Expand all files Collapse all files

117
apps/aether-gateway/src/orchestration/adaptive.rs
View File

@@ -82,71 +82,46 @@ pub(crate) fn project_local_adaptive_rate_limit(
let last_probe_increase_at_unix_secs = current_key.last_probe_increase_at_unix_secs;
let utilization_samples = Some(Value::Array(Vec::new()));
let is_rpm_observation = matches!(
classification,
LocalFailoverClassification::RetrySemanticRateLimit
) || upstream_limit.is_some();
let last_429_type = if is_rpm_observation { "rpm" } else { "unknown" }.to_string();
let last_429_type = "rpm".to_string();
rpm_429_count = rpm_429_count.saturating_add(1);
record_429_observation(
&mut history,
observed_at_unix_secs,
current_rpm,
upstream_limit,
);
if is_rpm_observation {
rpm_429_count = rpm_429_count.saturating_add(1);
record_429_observation(
&mut history,
observed_at_unix_secs,
current_rpm,
upstream_limit,
let (evaluated_limit, confidence) = evaluate_observations(&history);
if let Some(evaluated_limit) =
evaluated_limit.filter(|_| confidence >= ENFORCEMENT_CONFIDENCE_THRESHOLD)
{
let old_limit = learned_rpm_limit.unwrap_or_default();
let learning_source = if upstream_limit.is_some() {
"header"
} else {
"observation"
};
let mut extra = Map::new();
extra.insert(
"current_rpm".to_string(),
current_rpm.map_or(Value::Null, |value| json!(value)),
);
let (evaluated_limit, confidence) = evaluate_observations(&history);
if let Some(evaluated_limit) =
evaluated_limit.filter(|_| confidence >= ENFORCEMENT_CONFIDENCE_THRESHOLD)
{
let old_limit = learned_rpm_limit.unwrap_or_default();
let learning_source = if upstream_limit.is_some() {
"header"
} else {
"observation"
};
let mut extra = Map::new();
extra.insert(
"current_rpm".to_string(),
current_rpm.map_or(Value::Null, |value| json!(value)),
);
extra.insert(
"upstream_limit".to_string(),
upstream_limit.map_or(Value::Null, |value| json!(value)),
);
extra.insert("confidence".to_string(), json!(round3(confidence)));
extra.insert("learning_source".to_string(), json!(learning_source));
record_adjustment(
&mut history,
observed_at_unix_secs,
old_limit,
evaluated_limit,
"rpm_429",
extra,
);
learned_rpm_limit = Some(evaluated_limit);
last_rpm_peak = upstream_limit.or(current_rpm).or(last_rpm_peak);
}
} else if let Some(old_limit) = learned_rpm_limit {
let new_limit = reduced_limit(old_limit);
extra.insert(
"upstream_limit".to_string(),
upstream_limit.map_or(Value::Null, |value| json!(value)),
);
extra.insert("confidence".to_string(), json!(round3(confidence)));
extra.insert("learning_source".to_string(), json!(learning_source));
record_adjustment(
&mut history,
observed_at_unix_secs,
old_limit,
new_limit,
"unknown_429",
{
let mut extra = Map::new();
extra.insert(
"current_rpm".to_string(),
current_rpm.map_or(Value::Null, |value| json!(value)),
);
extra
},
evaluated_limit,
"rpm_429",
extra,
);
learned_rpm_limit = Some(new_limit);
learned_rpm_limit = Some(evaluated_limit);
last_rpm_peak = upstream_limit.or(current_rpm).or(last_rpm_peak);
}
let status_snapshot = project_local_adaptive_status_snapshot(
@@ -272,14 +247,10 @@ pub(crate) fn project_local_adaptive_success(
}
fn local_candidate_failure_should_record_adaptive_rate_limit(
classification: LocalFailoverClassification,
_classification: LocalFailoverClassification,
status_code: u16,
) -> bool {
status_code == 429
|| matches!(
classification,
LocalFailoverClassification::RetrySemanticRateLimit
)
}
fn project_local_adaptive_status_snapshot(
@@ -713,10 +684,6 @@ fn clamp_limit(value: f64) -> u32 {
.clamp(MIN_RPM_LIMIT as f64, MAX_RPM_LIMIT as f64) as u32
}
fn reduced_limit(value: u32) -> u32 {
((value as f64) * 0.95).floor().max(MIN_RPM_LIMIT as f64) as u32
}
fn record_type(record: &Map<String, Value>) -> Option<&str> {
record.get("type").and_then(Value::as_str)
}
@@ -843,7 +810,7 @@ mod tests {
let projection = project_local_adaptive_rate_limit(
&key,
LocalFailoverClassification::RetrySemanticRateLimit,
LocalFailoverClassification::RetryUpstreamFailure,
429,
Some(19),
None,
@@ -873,7 +840,7 @@ mod tests {
assert!(project_local_adaptive_rate_limit(
&key,
LocalFailoverClassification::RetrySemanticRateLimit,
LocalFailoverClassification::RetryUpstreamFailure,
429,
Some(10),
None,
@@ -913,7 +880,7 @@ mod tests {
let projection = project_local_adaptive_rate_limit(
&key,
LocalFailoverClassification::RetrySemanticRateLimit,
LocalFailoverClassification::RetryUpstreamFailure,
429,
Some(45),
Some(&headers),
@@ -971,7 +938,7 @@ mod tests {
let projection = project_local_adaptive_rate_limit(
&key,
LocalFailoverClassification::RetrySemanticRateLimit,
LocalFailoverClassification::RetryUpstreamFailure,
429,
Some(21),
None,
@@ -992,7 +959,7 @@ mod tests {
}
#[test]
fn unknown_429_reduces_existing_learned_limit() {
fn rate_limit_projection_records_429_as_rpm_observation() {
let mut key = sample_adaptive_key();
key.learned_rpm_limit = Some(100);
@@ -1006,9 +973,9 @@ mod tests {
)
.expect("projection should exist");
assert_eq!(projection.last_429_type, "unknown");
assert_eq!(projection.rpm_429_count, 0);
assert_eq!(projection.learned_rpm_limit, Some(95));
assert_eq!(projection.last_429_type, "rpm");
assert_eq!(projection.rpm_429_count, 1);
assert_eq!(projection.learned_rpm_limit, Some(100));
assert_eq!(
projection
.adjustment_history

405
apps/aether-gateway/src/orchestration/classifier.rs
View File

@@ -3,112 +3,8 @@ use serde_json::Value;
use super::{LocalFailoverPolicy, LocalFailoverRegexRule};
const CLIENT_ERROR_TYPES: &[&str] = &[
"invalid_request_error",
"invalid_argument",
"failed_precondition",
"validation_error",
"bad_request",
];
const CLIENT_ERROR_REASONS: &[&str] = &[
"CONTENT_LENGTH_EXCEEDS_THRESHOLD",
"CONTEXT_LENGTH_EXCEEDED",
"MAX_TOKENS_EXCEEDED",
"INVALID_CONTENT",
"CONTENT_POLICY_VIOLATION",
];
const CLIENT_ERROR_PATTERNS: &[&str] = &[
"could not process image",
"image too large",
"invalid image",
"unsupported image",
"content_policy_violation",
"context_length_exceeded",
"content_length_limit",
"content_length_exceeds",
"invalid_prompt",
"content too long",
"input is too long",
"message is too long",
"prompt is too long",
"image exceeds",
"pdf too large",
"file too large",
"tool_use_id",
"validationexception",
];
const STRICT_CLIENT_ERROR_PATTERNS: &[&str] =
&["unknown parameter", "invalid model for this endpoint"];
const COMPATIBILITY_ERROR_PATTERNS: &[&str] = &[
"unsupported parameter",
"unsupported model",
"unsupported feature",
"not supported with this model",
"model does not support",
"parameter is not supported",
"feature is not supported",
"not available for this model",
];
const THINKING_ERROR_PATTERNS: &[&str] = &[
"invalid `signature` in `thinking` block",
"invalid signature in thinking block",
"thinking.signature: field required",
"thinking.signature:",
"signature verification failed",
"must start with a thinking block",
"expected thinking or redacted_thinking",
"expected `thinking`",
"expected thinking, found",
"expected `thinking`, found",
"expected redacted_thinking, found",
"expected `redacted_thinking`, found",
"thoughtsignature",
"thought_signature",
];
const RETRYABLE_RATE_LIMIT_PATTERNS: &[&str] = &[
"rate_limit",
"rate limited",
"resource_exhausted",
"throttl",
"too many requests",
"quota reached",
"quota exceeded",
"quota hit",
];
const RETRYABLE_ACCOUNT_OR_BILLING_PATTERNS: &[&str] = &[
"organization has been disabled",
"organization_disabled",
"account has been disabled",
"account_disabled",
"account has been deactivated",
"account_deactivated",
"account deactivated",
"account suspended",
"account banned",
"subscription inactive",
"payment_required",
"payment required",
"insufficient_quota",
"insufficient quota",
"quota exhausted",
"credits exhausted",
"credit balance",
"credit limit",
"verify your account",
"account verification",
"verification required",
];
#[derive(Debug, Clone, PartialEq, Eq, Default)]
struct ParsedLocalErrorResponse {
type_name: Option<String>,
message: Option<String>,
reason: Option<String>,
raw: Option<String>,
@@ -136,11 +32,7 @@ pub(crate) enum LocalFailoverClassification {
UseDefault,
StopStatusCode,
StopErrorPattern,
StopSemanticClientError,
RetrySuccessPattern,
RetrySemanticCompatibilityError,
RetrySemanticRateLimit,
RetrySemanticThinkingError,
RetryStatusCode,
RetryUpstreamFailure,
}
@@ -151,11 +43,7 @@ impl LocalFailoverClassification {
Self::UseDefault => "use_default",
Self::StopStatusCode => "stop_status_code",
Self::StopErrorPattern => "stop_error_pattern",
Self::StopSemanticClientError => "stop_semantic_client_error",
Self::RetrySuccessPattern => "retry_success_pattern",
Self::RetrySemanticCompatibilityError => "retry_semantic_compatibility_error",
Self::RetrySemanticRateLimit => "retry_semantic_rate_limit",
Self::RetrySemanticThinkingError => "retry_semantic_thinking_error",
Self::RetryStatusCode => "retry_status_code",
Self::RetryUpstreamFailure => "retry_upstream_failure",
}
@@ -192,32 +80,6 @@ pub(crate) fn classify_local_failover(
return LocalFailoverClassification::RetrySuccessPattern;
}
let parsed_error = parse_local_error_response(input.response_text);
if is_semantic_thinking_error(input.status_code, &parsed_error) {
return LocalFailoverClassification::RetrySemanticThinkingError;
}
if is_strict_semantic_client_error(input.status_code, &parsed_error) {
return LocalFailoverClassification::StopSemanticClientError;
}
if is_semantic_compatibility_error(input.status_code, &parsed_error) {
return LocalFailoverClassification::RetrySemanticCompatibilityError;
}
if is_semantic_rate_limit_error(input.status_code, &parsed_error) {
return LocalFailoverClassification::RetrySemanticRateLimit;
}
if is_semantic_account_or_billing_error(input.status_code, &parsed_error) {
return LocalFailoverClassification::RetryUpstreamFailure;
}
if is_semantic_client_error(input.status_code, &parsed_error) {
return LocalFailoverClassification::StopSemanticClientError;
}
if policy.continue_status_codes.contains(&input.status_code) {
return LocalFailoverClassification::RetryStatusCode;
}
@@ -266,8 +128,6 @@ fn parse_local_error_response(response_text: Option<&str>) -> ParsedLocalErrorRe
.and_then(|object| object.get("error"))
.and_then(Value::as_object);
parsed.type_name = first_non_empty_json_text(error_object, &["type", "__type"])
.or_else(|| first_non_empty_json_text(body_object, &["type", "__type"]));
parsed.message = first_non_empty_json_text(error_object, &["message", "detail", "reason"])
.or_else(|| first_non_empty_json_text(body_object, &["errorMessage"]))
.or_else(|| {
@@ -296,10 +156,6 @@ fn parse_local_error_response(response_text: Option<&str>) -> ParsedLocalErrorRe
let nested_error_object = nested_object
.and_then(|object| object.get("error"))
.and_then(Value::as_object);
parsed.type_name = parsed
.type_name
.or_else(|| first_non_empty_json_text(nested_error_object, &["type", "__type"]))
.or_else(|| first_non_empty_json_text(nested_object, &["type", "__type"]));
parsed.message =
first_non_empty_json_text(nested_error_object, &["message", "detail", "reason"])
.or_else(|| first_non_empty_json_text(nested_object, &["message", "detail", "reason"]))
@@ -330,115 +186,6 @@ fn first_non_empty_json_text(
None
}
fn semantic_search_text(parsed: &ParsedLocalErrorResponse) -> String {
[
parsed.type_name.as_deref(),
parsed.reason.as_deref(),
parsed.message.as_deref(),
parsed.raw.as_deref(),
]
.into_iter()
.flatten()
.map(str::trim)
.filter(|value| !value.is_empty())
.map(str::to_ascii_lowercase)
.collect::<Vec<_>>()
.join(" ")
}
fn is_semantic_client_error(status_code: u16, parsed: &ParsedLocalErrorResponse) -> bool {
if status_code < 400 {
return false;
}
if parsed.type_name.as_deref().is_some_and(|type_name| {
let type_name = type_name.to_ascii_lowercase();
CLIENT_ERROR_TYPES
.iter()
.any(|pattern| type_name.contains(pattern))
}) {
return true;
}
if parsed.reason.as_deref().is_some_and(|reason| {
let reason = reason.to_ascii_uppercase();
CLIENT_ERROR_REASONS
.iter()
.any(|pattern| reason.contains(pattern))
}) {
return true;
}
let search_text = semantic_search_text(parsed);
!search_text.is_empty()
&& CLIENT_ERROR_PATTERNS
.iter()
.any(|pattern| search_text.contains(&pattern.to_ascii_lowercase()))
}
fn is_strict_semantic_client_error(status_code: u16, parsed: &ParsedLocalErrorResponse) -> bool {
if status_code < 400 {
return false;
}
let search_text = semantic_search_text(parsed);
!search_text.is_empty()
&& STRICT_CLIENT_ERROR_PATTERNS
.iter()
.any(|pattern| search_text.contains(&pattern.to_ascii_lowercase()))
}
fn is_semantic_compatibility_error(status_code: u16, parsed: &ParsedLocalErrorResponse) -> bool {
if status_code < 400 {
return false;
}
let search_text = semantic_search_text(parsed);
!search_text.is_empty()
&& COMPATIBILITY_ERROR_PATTERNS
.iter()
.any(|pattern| search_text.contains(&pattern.to_ascii_lowercase()))
}
fn is_semantic_thinking_error(status_code: u16, parsed: &ParsedLocalErrorResponse) -> bool {
if status_code != 400 {
return false;
}
let search_text = semantic_search_text(parsed);
!search_text.is_empty()
&& THINKING_ERROR_PATTERNS
.iter()
.any(|pattern| search_text.contains(&pattern.to_ascii_lowercase()))
}
fn is_semantic_rate_limit_error(status_code: u16, parsed: &ParsedLocalErrorResponse) -> bool {
if status_code < 400 {
return false;
}
let search_text = semantic_search_text(parsed);
!search_text.is_empty()
&& RETRYABLE_RATE_LIMIT_PATTERNS
.iter()
.any(|pattern| search_text.contains(&pattern.to_ascii_lowercase()))
}
fn is_semantic_account_or_billing_error(
status_code: u16,
parsed: &ParsedLocalErrorResponse,
) -> bool {
if status_code < 400 {
return false;
}
let search_text = semantic_search_text(parsed);
!search_text.is_empty()
&& RETRYABLE_ACCOUNT_OR_BILLING_PATTERNS
.iter()
.any(|pattern| search_text.contains(&pattern.to_ascii_lowercase()))
}
fn local_failover_regex_rule_matches(
rule: &LocalFailoverRegexRule,
response_text: &str,
@@ -512,120 +259,68 @@ mod tests {
}
#[test]
fn classifier_stops_semantic_client_errors_without_custom_rule() {
fn classifier_detects_success_continue_status_code() {
let policy = LocalFailoverPolicy {
continue_status_codes: [200].into_iter().collect(),
..LocalFailoverPolicy::default()
};
assert_eq!(
classify_local_failover(
&LocalFailoverPolicy::default(),
LocalFailoverInput::new(
400,
Some(
"{\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long\"}}"
)
)
),
LocalFailoverClassification::StopSemanticClientError
classify_local_failover(&policy, LocalFailoverInput::new(200, None)),
LocalFailoverClassification::RetryStatusCode
);
}
#[test]
fn classifier_retries_semantic_compatibility_errors() {
assert_eq!(
classify_local_failover(
&LocalFailoverPolicy::default(),
LocalFailoverInput::new(
400,
Some("{\"error\":{\"message\":\"Unsupported parameter: max_tokens is not supported with this model\"}}")
)
fn classifier_retries_all_error_statuses_without_custom_rule() {
for (status_code, response_text) in [
(
400,
"{\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long\"}}",
),
LocalFailoverClassification::RetrySemanticCompatibilityError
);
}
#[test]
fn classifier_stops_unknown_parameter_errors_before_compatibility_retry() {
assert_eq!(
classify_local_failover(
&LocalFailoverPolicy::default(),
LocalFailoverInput::new(
400,
Some("{\"error\":{\"message\":\"Unknown parameter: 'tools[0].n'.\"}}")
)
(
400,
"{\"error\":{\"message\":\"Unsupported parameter: max_tokens is not supported with this model\"}}",
),
LocalFailoverClassification::StopSemanticClientError
);
}
#[test]
fn classifier_stops_invalid_model_for_endpoint_errors_before_compatibility_retry() {
assert_eq!(
classify_local_failover(
&LocalFailoverPolicy::default(),
LocalFailoverInput::new(
400,
Some("{\"error\":{\"message\":\"invalid model for this endpoint\"}}")
)
(
400,
"{\"error\":{\"message\":\"Unknown parameter: 'tools[0].n'.\"}}",
),
LocalFailoverClassification::StopSemanticClientError
);
}
#[test]
fn classifier_retries_semantic_thinking_errors() {
assert_eq!(
classify_local_failover(
&LocalFailoverPolicy::default(),
LocalFailoverInput::new(
400,
Some(
"{\"error\":{\"message\":\"invalid `signature` in `thinking` block: signature is for a different request\"}}"
)
)
(
400,
"{\"error\":{\"message\":\"invalid model for this endpoint\"}}",
),
LocalFailoverClassification::RetrySemanticThinkingError
);
}
#[test]
fn classifier_retries_semantic_rate_limit_errors_even_when_status_is_not_429() {
assert_eq!(
classify_local_failover(
&LocalFailoverPolicy::default(),
LocalFailoverInput::new(
400,
Some("{\"error\":{\"message\":\"resource_exhausted: quota reached\"}}")
)
(
400,
"{\"error\":{\"message\":\"invalid `signature` in `thinking` block: signature is for a different request\"}}",
),
LocalFailoverClassification::RetrySemanticRateLimit
);
}
#[test]
fn classifier_retries_account_and_billing_errors_before_client_error_stop() {
assert_eq!(
classify_local_failover(
&LocalFailoverPolicy::default(),
LocalFailoverInput::new(
403,
Some(
"{\"error\":{\"type\":\"invalid_request_error\",\"message\":\"verify your account before continuing\"}}"
)
)
(
400,
"{\"error\":{\"message\":\"resource_exhausted: quota reached\"}}",
),
LocalFailoverClassification::RetryUpstreamFailure
);
assert_eq!(
classify_local_failover(
&LocalFailoverPolicy::default(),
LocalFailoverInput::new(
402,
Some(
"{\"error\":{\"type\":\"invalid_request_error\",\"message\":\"payment required: credit balance exhausted\"}}"
)
)
(
401,
"{\"error\":{\"type\":\"invalid_request_error\",\"message\":\"Your authentication token has been invalidated. Please try signing in again.\"}}",
),
LocalFailoverClassification::RetryUpstreamFailure
);
(
402,
"{\"error\":{\"type\":\"invalid_request_error\",\"message\":\"payment required: credit balance exhausted\"}}",
),
(
403,
"{\"error\":{\"type\":\"invalid_request_error\",\"message\":\"verify your account before continuing\"}}",
),
(429, "{\"error\":{\"message\":\"rate limited\"}}"),
(500, "{\"error\":{\"message\":\"upstream failed\"}}"),
] {
assert_eq!(
classify_local_failover(
&LocalFailoverPolicy::default(),
LocalFailoverInput::new(status_code, Some(response_text))
),
LocalFailoverClassification::RetryUpstreamFailure
);
}
}
#[test]

30
apps/aether-gateway/src/orchestration/effects.rs
View File

@@ -18,7 +18,7 @@ use super::{
project_local_adaptive_success, project_local_failure_health, project_local_success_health,
LocalFailoverClassification,
};
use crate::ai_pipeline::extract_pool_sticky_session_token;
use crate::ai_serving::extract_pool_sticky_session_token;
use crate::clock::current_unix_secs;
use crate::handlers::shared::provider_pool::admin_provider_pool_config_from_config_value;
use crate::handlers::shared::provider_pool::{
@@ -613,16 +613,12 @@ fn local_candidate_failure_should_invalidate_affinity(
match classification {
LocalFailoverClassification::RetrySuccessPattern
| LocalFailoverClassification::RetrySemanticCompatibilityError
| LocalFailoverClassification::RetrySemanticRateLimit
| LocalFailoverClassification::RetrySemanticThinkingError
| LocalFailoverClassification::RetryStatusCode
| LocalFailoverClassification::RetryUpstreamFailure => true,
LocalFailoverClassification::UseDefault | LocalFailoverClassification::StopStatusCode => {
status_code >= 500
}
LocalFailoverClassification::StopErrorPattern
| LocalFailoverClassification::StopSemanticClientError => false,
LocalFailoverClassification::StopErrorPattern => false,
}
}
@@ -1037,7 +1033,7 @@ mod tests {
}
#[tokio::test]
async fn semantic_client_error_keeps_scheduler_affinity_cache() {
async fn configured_stop_pattern_keeps_scheduler_affinity_cache() {
let state = AppState::new().expect("gateway state should build");
let plan = sample_plan();
let report_context = json!({
@@ -1068,7 +1064,7 @@ mod tests {
},
LocalExecutionEffect::AttemptFailure(LocalAttemptFailureEffect {
status_code: 400,
classification: LocalFailoverClassification::StopSemanticClientError,
classification: LocalFailoverClassification::StopErrorPattern,
}),
)
.await;
@@ -1175,13 +1171,13 @@ mod tests {
}
#[test]
fn semantic_client_error_does_not_penalize_pool_feedback() {
fn configured_stop_pattern_does_not_penalize_pool_feedback() {
assert!(!local_candidate_failure_should_record_pool_error(
LocalFailoverClassification::StopSemanticClientError,
LocalFailoverClassification::StopErrorPattern,
400,
));
assert!(local_candidate_failure_should_record_pool_error(
LocalFailoverClassification::RetrySemanticRateLimit,
LocalFailoverClassification::RetryUpstreamFailure,
429,
));
}
@@ -1360,7 +1356,7 @@ mod tests {
},
LocalExecutionEffect::AdaptiveRateLimit(LocalAdaptiveRateLimitEffect {
status_code: 429,
classification: LocalFailoverClassification::RetrySemanticRateLimit,
classification: LocalFailoverClassification::RetryUpstreamFailure,
headers: Some(&BTreeMap::from([(
"x-ratelimit-limit-requests".to_string(),
"42".to_string(),
@@ -1429,7 +1425,7 @@ mod tests {
},
LocalExecutionEffect::AdaptiveRateLimit(LocalAdaptiveRateLimitEffect {
status_code: 429,
classification: LocalFailoverClassification::RetrySemanticRateLimit,
classification: LocalFailoverClassification::RetryUpstreamFailure,
headers: Some(&BTreeMap::from([(
"x-ratelimit-limit-requests".to_string(),
"42".to_string(),
@@ -1451,7 +1447,7 @@ mod tests {
}
#[tokio::test]
async fn adaptive_rate_limit_effect_persists_zero_rpm_count_for_unknown_429() {
async fn adaptive_rate_limit_effect_records_429_as_rpm_observation() {
let mut key = sample_health_key();
key.rpm_limit = None;
key.learned_rpm_limit = Some(20);
@@ -1479,9 +1475,9 @@ mod tests {
.into_iter()
.next()
.expect("stored key should exist");
assert_eq!(stored_key.rpm_429_count, Some(0));
assert_eq!(stored_key.learned_rpm_limit, Some(19));
assert_eq!(stored_key.last_429_type.as_deref(), Some("unknown"));
assert_eq!(stored_key.rpm_429_count, Some(1));
assert_eq!(stored_key.learned_rpm_limit, Some(20));
assert_eq!(stored_key.last_429_type.as_deref(), Some("rpm"));
}
#[tokio::test]

13
apps/aether-gateway/src/orchestration/health.rs
View File

@@ -83,16 +83,12 @@ fn local_candidate_failure_should_project_health(
match classification {
LocalFailoverClassification::RetrySuccessPattern
| LocalFailoverClassification::RetrySemanticCompatibilityError
| LocalFailoverClassification::RetrySemanticRateLimit
| LocalFailoverClassification::RetrySemanticThinkingError
| LocalFailoverClassification::RetryStatusCode
| LocalFailoverClassification::RetryUpstreamFailure => true,
LocalFailoverClassification::UseDefault | LocalFailoverClassification::StopStatusCode => {
status_code >= 500
}
LocalFailoverClassification::StopErrorPattern
| LocalFailoverClassification::StopSemanticClientError => false,
LocalFailoverClassification::StopErrorPattern => false,
}
}
@@ -102,9 +98,6 @@ fn projected_failure_health_score(
consecutive_failures: u64,
) -> f64 {
let base_score = match classification {
LocalFailoverClassification::RetrySemanticRateLimit => 0.7,
LocalFailoverClassification::RetrySemanticCompatibilityError
| LocalFailoverClassification::RetrySemanticThinkingError => 0.8,
LocalFailoverClassification::RetrySuccessPattern => 0.75,
_ if status_code >= 500 => 0.6,
_ => 0.7,
@@ -145,11 +138,11 @@ mod tests {
}
#[test]
fn failure_projection_ignores_semantic_client_error() {
fn failure_projection_ignores_configured_stop_pattern() {
assert!(project_local_failure_health(
None,
"openai:chat",
LocalFailoverClassification::StopSemanticClientError,
LocalFailoverClassification::StopErrorPattern,
400,
1_760_000_000,
)

4
apps/aether-gateway/src/orchestration/mod.rs
View File

@@ -87,9 +87,7 @@ pub(crate) fn build_local_error_flow_metadata(
) -> Value {
let safe_to_expose = matches!(
analysis.classification,
LocalFailoverClassification::StopSemanticClientError
| LocalFailoverClassification::StopStatusCode
| LocalFailoverClassification::StopErrorPattern
LocalFailoverClassification::StopStatusCode | LocalFailoverClassification::StopErrorPattern
);
let propagation = match analysis.decision {
LocalFailoverDecision::RetryNextCandidate => "suppressed",

16
apps/aether-gateway/src/orchestration/recovery.rs
View File

@@ -57,14 +57,8 @@ const fn decision_from_classification(
match classification {
LocalFailoverClassification::UseDefault => LocalFailoverDecision::UseDefault,
LocalFailoverClassification::StopStatusCode
| LocalFailoverClassification::StopErrorPattern
| LocalFailoverClassification::StopSemanticClientError => {
LocalFailoverDecision::StopLocalFailover
}
| LocalFailoverClassification::StopErrorPattern => LocalFailoverDecision::StopLocalFailover,
LocalFailoverClassification::RetrySuccessPattern
| LocalFailoverClassification::RetrySemanticCompatibilityError
| LocalFailoverClassification::RetrySemanticRateLimit
| LocalFailoverClassification::RetrySemanticThinkingError
| LocalFailoverClassification::RetryStatusCode
| LocalFailoverClassification::RetryUpstreamFailure => {
LocalFailoverDecision::RetryNextCandidate
@@ -104,7 +98,7 @@ mod tests {
}
#[test]
fn recovery_maps_semantic_client_error_to_stop_failover() {
fn recovery_retries_default_client_error_without_custom_rule() {
assert_eq!(
recover_local_failover_decision(
&LocalFailoverPolicy::default(),
@@ -113,12 +107,12 @@ mod tests {
Some("{\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long\"}}")
)
),
LocalFailoverDecision::StopLocalFailover
LocalFailoverDecision::RetryNextCandidate
);
}
#[test]
fn recovery_maps_semantic_thinking_error_to_retry_next_candidate() {
fn recovery_retries_any_error_status_without_custom_rule() {
assert_eq!(
recover_local_failover_decision(
&LocalFailoverPolicy::default(),
@@ -144,7 +138,7 @@ mod tests {
assert_eq!(analysis.decision, LocalFailoverDecision::RetryNextCandidate);
assert_eq!(
analysis.classification,
LocalFailoverClassification::RetrySemanticCompatibilityError
LocalFailoverClassification::RetryUpstreamFailure
);
}
}