i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-10-09 18:59:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

Disable key circuit breaker for pool providers

Browse Source
This commit is contained in:
fawney19
2026-05-29 15:16:06 +08:00
parent 9fb600df1b
commit 1658925f52
12 changed files with 252 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/aether-scheduler-core/src/candidate/mod.rs
+15
View File
@@ -309,6 +309,7 @@ mod tests {
provider_quota_blocks_requests: false,
account_quota_exhausted: false,
oauth_invalid: false,
enforce_key_circuit_breaker: true,
rpm_reset_at: None,
},
));
@@ -333,6 +334,7 @@ mod tests {
provider_quota_blocks_requests: false,
account_quota_exhausted: false,
oauth_invalid: false,
enforce_key_circuit_breaker: true,
rpm_reset_at: None,
}),
None
@@ -358,6 +360,7 @@ mod tests {
provider_quota_blocks_requests: false,
account_quota_exhausted: false,
oauth_invalid: false,
enforce_key_circuit_breaker: true,
rpm_reset_at: None,
}),
Some("provider_key_concurrency_limit_reached")
@@ -386,6 +389,7 @@ mod tests {
provider_quota_blocks_requests: false,
account_quota_exhausted: false,
oauth_invalid: false,
enforce_key_circuit_breaker: true,
rpm_reset_at: None,
}),
Some("provider_key_concurrency_limit_reached")
@@ -415,6 +419,7 @@ mod tests {
provider_quota_blocks_requests: false,
account_quota_exhausted: false,
oauth_invalid: false,
enforce_key_circuit_breaker: true,
rpm_reset_at: None,
}),
None
@@ -435,6 +440,7 @@ mod tests {
provider_quota_blocks_requests: false,
account_quota_exhausted: false,
oauth_invalid: false,
enforce_key_circuit_breaker: true,
rpm_reset_at: None,
}),
None
@@ -458,6 +464,7 @@ mod tests {
provider_quota_blocks_requests: false,
account_quota_exhausted: false,
oauth_invalid: false,
enforce_key_circuit_breaker: true,
rpm_reset_at: None,
}),
None
@@ -481,6 +488,7 @@ mod tests {
provider_quota_blocks_requests: false,
account_quota_exhausted: false,
oauth_invalid: false,
enforce_key_circuit_breaker: true,
rpm_reset_at: None,
}),
Some("key_circuit_open")
@@ -516,6 +524,7 @@ mod tests {
provider_quota_blocks_requests: false,
account_quota_exhausted: false,
oauth_invalid: false,
enforce_key_circuit_breaker: true,
rpm_reset_at: None,
}),
Some("key_rpm_exhausted")
@@ -543,6 +552,7 @@ mod tests {
provider_quota_blocks_requests: false,
account_quota_exhausted: false,
oauth_invalid: false,
enforce_key_circuit_breaker: true,
rpm_reset_at: None,
}),
Some("key_circuit_open")
@@ -557,6 +567,7 @@ mod tests {
provider_quota_blocks_requests: false,
account_quota_exhausted: false,
oauth_invalid: false,
enforce_key_circuit_breaker: true,
rpm_reset_at: None,
}),
None
@@ -577,6 +588,7 @@ mod tests {
provider_quota_blocks_requests: false,
account_quota_exhausted: false,
oauth_invalid: false,
enforce_key_circuit_breaker: true,
rpm_reset_at: None,
},
));
@@ -590,6 +602,7 @@ mod tests {
provider_quota_blocks_requests: true,
account_quota_exhausted: false,
oauth_invalid: false,
enforce_key_circuit_breaker: true,
rpm_reset_at: None,
},
));
@@ -607,6 +620,7 @@ mod tests {
provider_quota_blocks_requests: false,
account_quota_exhausted: true,
oauth_invalid: false,
enforce_key_circuit_breaker: true,
rpm_reset_at: None,
},
));
@@ -624,6 +638,7 @@ mod tests {
provider_quota_blocks_requests: false,
account_quota_exhausted: false,
oauth_invalid: true,
enforce_key_circuit_breaker: true,
rpm_reset_at: None,
},
));
crates/aether-scheduler-core/src/candidate/selectability.rs
+9 -5
View File
@@ -29,6 +29,7 @@ pub struct CandidateRuntimeSelectabilityInput<'a> {
pub provider_quota_blocks_requests: bool,
pub account_quota_exhausted: bool,
pub oauth_invalid: bool,
pub enforce_key_circuit_breaker: bool,
pub rpm_reset_at: Option<u64>,
}
@@ -50,6 +51,7 @@ pub fn candidate_runtime_skip_reason_with_state(
provider_quota_blocks_requests,
account_quota_exhausted,
oauth_invalid,
enforce_key_circuit_breaker,
rpm_reset_at,
} = input;
@@ -94,11 +96,13 @@ pub fn candidate_runtime_skip_reason_with_state(
}
if let Some(provider_key) = provider_key {
if crate::is_provider_key_circuit_open_at(
provider_key,
candidate.endpoint_api_format.as_str(),
now_unix_secs,
) {
if enforce_key_circuit_breaker
&& crate::is_provider_key_circuit_open_at(
provider_key,
candidate.endpoint_api_format.as_str(),
now_unix_secs,
)
{
return Some("key_circuit_open");
}
if crate::provider_key_health_score(provider_key, candidate.endpoint_api_format.as_str())