i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 01:10:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: restore provider key circuit breaker backoff

Browse Source
This commit is contained in:
fawney19
2026-05-19 13:57:36 +08:00
parent 6a1da6a5ff
commit 7e9ca88e00
14 changed files with 525 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
crates/aether-scheduler-core/src/candidate/mod.rs
View File

@@ -441,6 +441,29 @@ mod tests {
);
}
#[test]
fn recent_failures_do_not_skip_without_persisted_circuit() {
let recent_candidates = vec![
stored_candidate("failed", RequestCandidateStatus::Failed, 95_000),
stored_candidate("cancelled", RequestCandidateStatus::Cancelled, 99_000),
];
assert_eq!(
candidate_runtime_skip_reason_with_state(CandidateRuntimeSelectabilityInput {
candidate: &sample_candidate("1", None),
recent_candidates: &recent_candidates,
provider_concurrent_limits: &BTreeMap::new(),
provider_key_rpm_states: &BTreeMap::new(),
now_unix_secs: 100,
provider_quota_blocks_requests: false,
account_quota_exhausted: false,
oauth_invalid: false,
rpm_reset_at: None,
}),
None
);
}
#[test]
fn provider_key_concurrency_limit_preserves_key_circuit_and_rpm_checks() {
let mut circuit_open_key = sample_key_with_concurrent_limit("1", Some(2));

9
crates/aether-scheduler-core/src/candidate/selectability.rs
View File

@@ -62,15 +62,6 @@ pub fn candidate_runtime_skip_reason_with_state(
if oauth_invalid {
return Some("oauth_invalid");
}
if crate::is_candidate_in_recent_failure_cooldown(
recent_candidates,
candidate.provider_id.as_str(),
candidate.endpoint_id.as_str(),
candidate.key_id.as_str(),
now_unix_secs,
) {
return Some("recent_failure_cooldown");
}
if provider_concurrent_limits
.get(&candidate.provider_id)
.is_some_and(|limit| {

17
crates/aether-scheduler-core/src/health.rs
View File

@@ -4,7 +4,7 @@ use aether_data_contracts::repository::candidates::{
use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey;
const FAILURE_COOLDOWN_WINDOW_SECS: u64 = 60;
const FAILURE_COOLDOWN_THRESHOLD: usize = 2;
const FAILURE_COOLDOWN_THRESHOLD: usize = 8;
const ACTIVE_REQUEST_WINDOW_SECS: u64 = 300;
pub const PROVIDER_KEY_RPM_WINDOW_SECS: u64 = 60;
const PROBE_PHASE_REQUESTS: u32 = 100;
@@ -694,11 +694,16 @@ mod tests {
}
#[test]
fn cooldown_triggers_after_two_recent_failures() {
let recent_candidates = vec![
stored_candidate("one", RequestCandidateStatus::Failed, 95),
stored_candidate("two", RequestCandidateStatus::Cancelled, 99),
];
fn cooldown_triggers_after_eight_recent_failures() {
let recent_candidates = (0..8)
.map(|index| {
stored_candidate(
&format!("failed-{index}"),
RequestCandidateStatus::Failed,
92 + index,
)
})
.collect::<Vec<_>>();
assert!(is_candidate_in_recent_failure_cooldown(
&recent_candidates,