i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-10-07 18:07:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(routing): move sticky-key retries into routing policy with lazy attempts

Browse Source 
Replace the provider/endpoint max_retries fields as the source of same-key
retries with a routing policy setting, sticky_key_attempts (default 2). Only
the first-ranked candidate is retried on the same key; every failover
candidate gets a single attempt so failover keeps advancing instead of
retrying each fallback key.

Materialize exactly one attempt per candidate and derive same-key retries in
the attempt loop after a candidate-scoped failure, so the retry budget no
longer inflates up-front materialization and needs no upper bound. The budget
travels in the report context; retries reuse the plan with a fresh candidate
id and incremented retry index. Pool groups only retry their first key within
the retry-index stride.

Expose the setting in the routing profile editor and the set_scheduling rule
action, and drop the max_retries input from the provider form.
This commit is contained in:
elky
2026-09-02 20:48:40 +08:00
parent 415b2da81b
commit 7323d41fbe
40 changed files with 851 additions and 570 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/aether-gateway/src/tests/ai_execute/sync/chat/failover.rs
+54 -45
View File
@@ -112,7 +112,7 @@ async fn gateway_skips_unsupported_local_openai_chat_sync_candidate_before_tryin
false,
false,
None,
Some(2),
Some(1),
None,
Some(20.0),
None,
@@ -134,7 +134,7 @@ async fn gateway_skips_unsupported_local_openai_chat_sync_candidate_before_tryin
"https://api.openai.skip.example".to_string(),
None,
None,
Some(2),
Some(1),
None,
None,
None,
@@ -520,7 +520,7 @@ async fn gateway_surfaces_local_execution_runtime_miss_reason_when_all_openai_ch
false,
false,
None,
Some(2),
Some(1),
None,
Some(20.0),
None,
@@ -542,7 +542,7 @@ async fn gateway_surfaces_local_execution_runtime_miss_reason_when_all_openai_ch
"https://chatgpt.com/backend-api/codex".to_string(),
None,
None,
Some(2),
Some(1),
None,
None,
None,
@@ -802,7 +802,7 @@ async fn gateway_retries_next_local_openai_chat_sync_candidate_after_auth_failur
false,
false,
None,
Some(2),
Some(1),
None,
Some(20.0),
None,
@@ -828,7 +828,7 @@ async fn gateway_retries_next_local_openai_chat_sync_candidate_after_auth_failur
base_url.to_string(),
None,
None,
Some(2),
Some(1),
None,
None,
None,
@@ -970,7 +970,9 @@ async fn gateway_retries_next_local_openai_chat_sync_candidate_after_auth_failur
.to_string(),
});
if attempt == 1 {
// The primary key gets two attempts under the default
// sticky_key_attempts; both must fail to reach the backup.
if attempt <= 2 {
return Json(json!({
"request_id": "trace-openai-chat-local-failover-123",
"status_code": 401,
@@ -1125,56 +1127,63 @@ async fn gateway_retries_next_local_openai_chat_sync_candidate_after_auth_failur
.lock()
.expect("mutex should lock")
.clone();
assert_eq!(seen_execution_runtime_requests.len(), 2);
// Default sticky_key_attempts is 2: the primary key is retried once on
// the same key, then failover moves to the backup with a single attempt.
assert_eq!(seen_execution_runtime_requests.len(), 3);
for primary_request in &seen_execution_runtime_requests[..2] {
assert_eq!(
primary_request.trace_id,
"trace-openai-chat-local-failover-123"
);
assert_eq!(
primary_request.url,
"https://api.openai.primary.example/chat/completions"
);
assert_eq!(
primary_request.authorization,
"Bearer sk-upstream-openai-primary"
);
}
assert_eq!(
seen_execution_runtime_requests[0].trace_id,
"trace-openai-chat-local-failover-123"
);
assert_eq!(
seen_execution_runtime_requests[0].url,
"https://api.openai.primary.example/chat/completions"
);
assert_eq!(
seen_execution_runtime_requests[0].authorization,
"Bearer sk-upstream-openai-primary"
);
assert_eq!(
seen_execution_runtime_requests[1].url,
seen_execution_runtime_requests[2].url,
"https://api.openai.backup.example/chat/completions"
);
assert_eq!(
seen_execution_runtime_requests[1].model,
seen_execution_runtime_requests[2].model,
"gpt-5-upstream-backup"
);
assert_eq!(
seen_execution_runtime_requests[1].authorization,
seen_execution_runtime_requests[2].authorization,
"Bearer sk-upstream-openai-backup"
);
let stored_candidates = request_candidate_repository
.list_by_request_id("trace-openai-chat-local-failover-123")
.await
.expect("request candidate trace should read");
assert_eq!(stored_candidates.len(), 2);
assert_eq!(stored_candidates[0].candidate_index, 0);
assert_eq!(stored_candidates[0].status, RequestCandidateStatus::Failed);
assert_eq!(stored_candidates[0].status_code, Some(401));
assert_eq!(
stored_candidates[0].error_message.as_deref(),
Some("invalid auth token")
);
let failed_upstream_response = stored_candidates[0]
.extra_data
.as_ref()
.and_then(|value| value.get("upstream_response"))
.expect("failed candidate should keep its upstream response");
assert_eq!(failed_upstream_response["status_code"], json!(401));
assert_eq!(
failed_upstream_response["body"]["error"]["message"],
json!("invalid auth token")
);
assert_eq!(stored_candidates[1].candidate_index, 1);
assert_eq!(stored_candidates[1].status, RequestCandidateStatus::Success);
assert_eq!(stored_candidates[1].status_code, Some(200));
assert_eq!(stored_candidates.len(), 3);
for (retry_index, failed_candidate) in stored_candidates[..2].iter().enumerate() {
assert_eq!(failed_candidate.candidate_index, 0);
assert_eq!(failed_candidate.retry_index, retry_index as u32);
assert_eq!(failed_candidate.status, RequestCandidateStatus::Failed);
assert_eq!(failed_candidate.status_code, Some(401));
assert_eq!(
failed_candidate.error_message.as_deref(),
Some("invalid auth token")
);
let failed_upstream_response = failed_candidate
.extra_data
.as_ref()
.and_then(|value| value.get("upstream_response"))
.expect("failed candidate should keep its upstream response");
assert_eq!(failed_upstream_response["status_code"], json!(401));
assert_eq!(
failed_upstream_response["body"]["error"]["message"],
json!("invalid auth token")
);
}
assert_eq!(stored_candidates[2].candidate_index, 1);
assert_eq!(stored_candidates[2].status, RequestCandidateStatus::Success);
assert_eq!(stored_candidates[2].status_code, Some(200));
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
assert!(
@@ -1184,7 +1193,7 @@ async fn gateway_retries_next_local_openai_chat_sync_candidate_after_auth_failur
assert_eq!(
*execution_runtime_hits.lock().expect("mutex should lock"),
2
3
);
assert_eq!(*decision_hits.lock().expect("mutex should lock"), 0);
assert_eq!(*plan_hits.lock().expect("mutex should lock"), 0);