i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-10-04 16:37:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(routing): move sticky-key retries into routing policy with lazy attempts

Browse Source 
Replace the provider/endpoint max_retries fields as the source of same-key
retries with a routing policy setting, sticky_key_attempts (default 2). Only
the first-ranked candidate is retried on the same key; every failover
candidate gets a single attempt so failover keeps advancing instead of
retrying each fallback key.

Materialize exactly one attempt per candidate and derive same-key retries in
the attempt loop after a candidate-scoped failure, so the retry budget no
longer inflates up-front materialization and needs no upper bound. The budget
travels in the report context; retries reuse the plan with a fresh candidate
id and incremented retry index. Pool groups only retry their first key within
the retry-index stride.

Expose the setting in the routing profile editor and the set_scheduling rule
action, and drop the max_retries input from the provider form.
This commit is contained in:
elky
2026-09-02 20:48:40 +08:00
parent 415b2da81b
commit 7323d41fbe
40 changed files with 851 additions and 570 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/aether-ai/serving/src/attempt_loop.rs
+88 -2
View File
@@ -13,8 +13,23 @@ pub trait AiExecutionAttempt {
fn report_context_ref(&self) -> Option<&serde_json::Value> {
None
}
/// Re-issue this attempt against the same key as a fresh attempt with the
/// given retry index and candidate id. Attempt types that cannot be
/// re-issued return `None`, which disables same-key retries for them.
fn with_same_key_retry(&self, _retry_index: u32, _candidate_id: String) -> Option<Self>
where
Self: Sized,
{
None
}
}
/// Report-context field carrying the routing policy's sticky-key attempt
/// budget for the request, so the attempt loop can derive same-key retries
/// lazily instead of pre-materializing them.
pub const STICKY_KEY_ATTEMPTS_REPORT_FIELD: &str = "sticky_key_attempts";
#[derive(Debug)]
pub enum AiAttemptLoopOutcome<Response, Exhaustion> {
Responded(Response),
@@ -83,6 +98,17 @@ where
Ok(())
}
/// After `attempt` failed with candidate scope, return the next attempt on
/// the same key, or `None` once the sticky-key budget is used up. Retries
/// are derived here on demand so no attempt is materialized before it is
/// actually needed.
async fn next_same_key_retry(
&self,
_attempt: &Attempt,
) -> Result<Option<Attempt>, Self::Error> {
Ok(None)
}
async fn mark_unused_attempts(&self, attempts: Vec<Attempt>) -> Result<(), Self::Error>;
async fn build_exhaustion(
@@ -101,11 +127,15 @@ where
Attempt: AiExecutionAttempt + Send + Sync + 'static,
{
let mut remaining = attempts.into_iter();
let mut pending_same_key_retry: Option<Attempt> = None;
let mut last_attempted = None;
let mut retry_filters: Vec<AiAttemptRetryFilter> = Vec::new();
let mut fallback_response = None;
while let Some(attempt) = remaining.next() {
loop {
let Some(attempt) = pending_same_key_retry.take().or_else(|| remaining.next()) else {
break;
};
if retry_filters.iter().any(|filter| filter.matches(&attempt))
|| port.should_skip_attempt(&attempt).await?
{
@@ -133,7 +163,9 @@ where
if attempt_fallback_response.is_some() {
fallback_response = attempt_fallback_response;
}
if scope != AiAttemptRetryScope::Candidate {
if scope == AiAttemptRetryScope::Candidate {
pending_same_key_retry = port.next_same_key_retry(&attempt).await?;
} else {
retry_filters.push(AiAttemptRetryFilter::new(&attempt, scope));
}
}
@@ -188,6 +220,32 @@ impl AiAttemptRetryFilter {
}
}
/// Clone `plan`/`report_context` for a same-key retry: only the candidate id
/// and retry index change, everything else (url, headers, body) is reused.
fn same_key_retry_parts(
plan: &aether_contracts::ExecutionPlan,
report_context: Option<&serde_json::Value>,
retry_index: u32,
candidate_id: String,
) -> (aether_contracts::ExecutionPlan, Option<serde_json::Value>) {
let mut plan = plan.clone();
plan.candidate_id = Some(candidate_id.clone());
let report_context = report_context.cloned().map(|mut value| {
if let Some(object) = value.as_object_mut() {
object.insert(
"candidate_id".to_string(),
serde_json::Value::String(candidate_id),
);
object.insert(
"retry_index".to_string(),
serde_json::Value::Number(retry_index.into()),
);
}
value
});
(plan, report_context)
}
impl AiExecutionAttempt for crate::dto::AiSyncAttempt {
fn execution_plan(&self) -> &aether_contracts::ExecutionPlan {
&self.plan
@@ -204,6 +262,20 @@ impl AiExecutionAttempt for crate::dto::AiSyncAttempt {
fn report_context_ref(&self) -> Option<&serde_json::Value> {
self.report_context.as_ref()
}
fn with_same_key_retry(&self, retry_index: u32, candidate_id: String) -> Option<Self> {
let (plan, report_context) = same_key_retry_parts(
&self.plan,
self.report_context.as_ref(),
retry_index,
candidate_id,
);
Some(Self {
plan,
report_kind: self.report_kind.clone(),
report_context,
})
}
}
impl AiExecutionAttempt for crate::dto::AiStreamAttempt {
@@ -222,6 +294,20 @@ impl AiExecutionAttempt for crate::dto::AiStreamAttempt {
fn report_context_ref(&self) -> Option<&serde_json::Value> {
self.report_context.as_ref()
}
fn with_same_key_retry(&self, retry_index: u32, candidate_id: String) -> Option<Self> {
let (plan, report_context) = same_key_retry_parts(
&self.plan,
self.report_context.as_ref(),
retry_index,
candidate_id,
);
Some(Self {
plan,
report_kind: self.report_kind.clone(),
report_context,
})
}
}
#[cfg(test)]
crates/aether-ai/serving/src/candidate_persistence.rs
+22 -60
View File
@@ -9,8 +9,6 @@ pub trait AiAvailableCandidatePersistencePort: Send + Sync {
type ExtraData: Clone + Send + Sync;
type Error: Send;
fn attempt_slot_count(&self, candidate: &Self::Candidate) -> u32;
fn build_extra_data(&self, candidate: &Self::Candidate) -> Option<Self::ExtraData>;
fn generate_candidate_id(&self) -> String;
@@ -42,50 +40,28 @@ pub async fn run_ai_available_candidate_persistence<Port>(
where
Port: AiAvailableCandidatePersistencePort,
{
let total_attempts = candidates
.iter()
.map(|candidate| port.attempt_slot_count(candidate) as usize)
.sum();
let mut materialized = Vec::with_capacity(total_attempts);
// One attempt per candidate. Same-key retries are derived lazily by the
// attempt loop (`AiAttemptLoopPort::next_same_key_retry`) after a failure,
// so the sticky-key budget never inflates up-front materialization.
let mut materialized = Vec::with_capacity(candidates.len());
for (candidate_index, candidate) in candidates.into_iter().enumerate() {
let candidate_index = candidate_index as u32;
let attempt_slots = port.attempt_slot_count(&candidate).max(1);
let extra_data = port.build_extra_data(&candidate);
let mut owned_candidate = Some(candidate);
for retry_index in 0..attempt_slots {
let candidate = owned_candidate
.as_ref()
.expect("candidate should remain available until final retry");
let generated_candidate_id = port.generate_candidate_id();
let candidate_id = if port.should_persist_available_candidate(candidate) {
port.persist_available_candidate(
candidate,
candidate_index,
retry_index,
generated_candidate_id.as_str(),
extra_data.clone(),
)
.await?
} else {
generated_candidate_id
};
let candidate = if retry_index + 1 == attempt_slots {
owned_candidate
.take()
.expect("final retry should consume owned candidate")
} else {
candidate.clone()
};
materialized.push(port.build_attempt(
candidate,
let generated_candidate_id = port.generate_candidate_id();
let candidate_id = if port.should_persist_available_candidate(&candidate) {
port.persist_available_candidate(
&candidate,
candidate_index,
retry_index,
candidate_id,
));
}
0,
generated_candidate_id.as_str(),
extra_data,
)
.await?
} else {
generated_candidate_id
};
materialized.push(port.build_attempt(candidate, candidate_index, 0, candidate_id));
}
Ok(materialized)
@@ -177,7 +153,6 @@ mod tests {
#[derive(Debug, Clone, PartialEq, Eq)]
struct TestCandidate {
id: &'static str,
attempt_slots: u32,
persist: bool,
}
@@ -216,10 +191,6 @@ mod tests {
type ExtraData = String;
type Error = std::convert::Infallible;
fn attempt_slot_count(&self, candidate: &Self::Candidate) -> u32 {
candidate.attempt_slots
}
fn build_extra_data(&self, candidate: &Self::Candidate) -> Option<Self::ExtraData> {
Some(format!("extra:{}", candidate.id))
}
@@ -299,7 +270,7 @@ mod tests {
}
#[tokio::test]
async fn available_persistence_expands_candidates_into_retry_attempts() {
async fn available_persistence_materializes_one_attempt_per_candidate() {
let port = TestPort::default();
let attempts = run_ai_available_candidate_persistence(
@@ -307,12 +278,10 @@ mod tests {
vec![
TestCandidate {
id: "a",
attempt_slots: 2,
persist: true,
},
TestCandidate {
id: "b",
attempt_slots: 1,
persist: false,
},
],
@@ -320,6 +289,8 @@ mod tests {
.await
.unwrap();
// Same-key retries are never pre-materialized; the attempt loop
// derives them on demand after a failure.
assert_eq!(
attempts,
[
@@ -329,26 +300,17 @@ mod tests {
retry_index: 0,
candidate_id: "stored-candidate-1".to_string(),
},
TestAttempt {
id: "a",
candidate_index: 0,
retry_index: 1,
candidate_id: "stored-candidate-2".to_string(),
},
TestAttempt {
id: "b",
candidate_index: 1,
retry_index: 0,
candidate_id: "candidate-3".to_string(),
candidate_id: "candidate-2".to_string(),
},
]
);
assert_eq!(
port.calls.lock().unwrap().as_slice(),
[
"available:a:0:0:candidate-1:extra:a",
"available:a:0:1:candidate-2:extra:a",
]
["available:a:0:0:candidate-1:extra:a"]
);
}
crates/aether-ai/serving/src/lib.rs
+1 -1
View File
@@ -54,7 +54,7 @@ pub use aether_pool_core::{
};
pub use attempt_loop::{
run_ai_attempt_loop, AiAttemptExecutionOutcome, AiAttemptLoopOutcome, AiAttemptLoopPort,
AiAttemptRetryScope, AiExecutionAttempt,
AiAttemptRetryScope, AiExecutionAttempt, STICKY_KEY_ATTEMPTS_REPORT_FIELD,
};
pub use attempt_plan::{
build_ai_execution_decision_from_plan, build_ai_execution_plan_from_decision,