feat(gateway): harden provider request execution

Preserve exact request payloads and model client surface and API operation explicitly.

Add Anthropic compatibility profiles, bounded stream commitment, and scoped OAuth retry behavior across provider transports.
This commit is contained in:
elky
2026-07-27 09:36:31 +08:00
parent 79b70f7b5c
commit 531cf11025
152 changed files with 13984 additions and 2075 deletions
+234 -12
View File
@@ -18,10 +18,45 @@ pub trait AiExecutionAttempt {
#[derive(Debug)]
pub enum AiAttemptLoopOutcome<Response, Exhaustion> {
Responded(Response),
Deferred(Response),
Exhausted(Exhaustion),
NoPath,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum AiAttemptRetryScope {
#[default]
Candidate,
Credential,
Endpoint,
Provider,
}
#[derive(Debug)]
pub enum AiAttemptExecutionOutcome<Response> {
Responded(Response),
Retry {
scope: AiAttemptRetryScope,
fallback_response: Option<Response>,
},
}
impl<Response> AiAttemptExecutionOutcome<Response> {
pub fn retry(scope: AiAttemptRetryScope) -> Self {
Self::Retry {
scope,
fallback_response: None,
}
}
pub fn from_optional_response(response: Option<Response>) -> Self {
match response {
Some(response) => Self::Responded(response),
None => Self::retry(AiAttemptRetryScope::Candidate),
}
}
}
#[async_trait]
pub trait AiAttemptLoopPort<Attempt>: Send + Sync
where
@@ -34,7 +69,7 @@ where
async fn execute_attempt(
&self,
attempt: &Attempt,
) -> Result<Option<Self::Response>, Self::Error>;
) -> Result<AiAttemptExecutionOutcome<Self::Response>, Self::Error>;
async fn should_skip_attempt(&self, _attempt: &Attempt) -> Result<bool, Self::Error> {
Ok(false)
@@ -67,32 +102,52 @@ where
{
let mut remaining = attempts.into_iter();
let mut last_attempted = None;
let mut retry_filters: Vec<AiAttemptRetryFilter> = Vec::new();
let mut fallback_response = None;
while let Some(attempt) = remaining.next() {
if port.should_skip_attempt(&attempt).await? {
if retry_filters.iter().any(|filter| filter.matches(&attempt))
|| port.should_skip_attempt(&attempt).await?
{
port.mark_unused_attempts(vec![attempt]).await?;
continue;
}
port.record_attempt_started(&attempt).await?;
let response = match port.execute_attempt(&attempt).await {
Ok(response) => response,
let execution = match port.execute_attempt(&attempt).await {
Ok(execution) => execution,
Err(err) => {
port.mark_unused_attempts(remaining.collect()).await?;
return Err(err);
}
};
if let Some(response) = response {
port.mark_unused_attempts(remaining.collect()).await?;
return Ok(AiAttemptLoopOutcome::Responded(response));
match execution {
AiAttemptExecutionOutcome::Responded(response) => {
port.mark_unused_attempts(remaining.collect()).await?;
return Ok(AiAttemptLoopOutcome::Responded(response));
}
AiAttemptExecutionOutcome::Retry {
scope,
fallback_response: attempt_fallback_response,
} => {
port.record_attempt_failed(&attempt).await?;
if attempt_fallback_response.is_some() {
fallback_response = attempt_fallback_response;
}
if scope != AiAttemptRetryScope::Candidate {
retry_filters.push(AiAttemptRetryFilter::new(&attempt, scope));
}
}
}
port.record_attempt_failed(&attempt).await?;
// Exhaustion diagnostics are only needed after an attempt fails. Keep
// the common successful path free of a deep plan/report-context clone.
last_attempted = Some((attempt.execution_plan().clone(), attempt.report_context()));
}
if let Some(response) = fallback_response {
return Ok(AiAttemptLoopOutcome::Deferred(response));
}
let Some((last_plan, last_report_context)) = last_attempted else {
return Ok(AiAttemptLoopOutcome::NoPath);
};
@@ -103,6 +158,36 @@ where
))
}
#[derive(Debug)]
struct AiAttemptRetryFilter {
scope: AiAttemptRetryScope,
provider_id: String,
endpoint_id: String,
key_id: String,
}
impl AiAttemptRetryFilter {
fn new<Attempt: AiExecutionAttempt>(attempt: &Attempt, scope: AiAttemptRetryScope) -> Self {
let plan = attempt.execution_plan();
Self {
scope,
provider_id: plan.provider_id.clone(),
endpoint_id: plan.endpoint_id.clone(),
key_id: plan.key_id.clone(),
}
}
fn matches<Attempt: AiExecutionAttempt>(&self, attempt: &Attempt) -> bool {
let plan = attempt.execution_plan();
match self.scope {
AiAttemptRetryScope::Candidate => false,
AiAttemptRetryScope::Credential => plan.key_id == self.key_id,
AiAttemptRetryScope::Endpoint => plan.endpoint_id == self.endpoint_id,
AiAttemptRetryScope::Provider => plan.provider_id == self.provider_id,
}
}
}
impl AiExecutionAttempt for crate::dto::AiSyncAttempt {
fn execution_plan(&self) -> &aether_contracts::ExecutionPlan {
&self.plan
@@ -146,7 +231,10 @@ mod tests {
use async_trait::async_trait;
use super::{run_ai_attempt_loop, AiAttemptLoopPort, AiExecutionAttempt};
use super::{
run_ai_attempt_loop, AiAttemptExecutionOutcome, AiAttemptLoopPort, AiAttemptRetryScope,
AiExecutionAttempt,
};
#[derive(Clone)]
struct TestAttempt {
@@ -173,6 +261,60 @@ mod tests {
unused: Mutex<Vec<&'static str>>,
}
struct ScopedRetryPort {
executed: Mutex<Vec<&'static str>>,
unused: Mutex<Vec<&'static str>>,
}
#[async_trait]
impl AiAttemptLoopPort<TestAttempt> for ScopedRetryPort {
type Response = &'static str;
type Exhaustion = ();
type Error = &'static str;
async fn execute_attempt(
&self,
attempt: &TestAttempt,
) -> Result<AiAttemptExecutionOutcome<Self::Response>, Self::Error> {
self.executed
.lock()
.expect("executed attempts should lock")
.push(attempt.id);
Ok(match attempt.id {
"endpoint-failure" => {
AiAttemptExecutionOutcome::retry(AiAttemptRetryScope::Endpoint)
}
"credential-failure" => {
AiAttemptExecutionOutcome::retry(AiAttemptRetryScope::Credential)
}
"provider-failure" => AiAttemptExecutionOutcome::Retry {
scope: AiAttemptRetryScope::Provider,
fallback_response: Some("provider-error"),
},
_ => AiAttemptExecutionOutcome::Responded(attempt.id),
})
}
async fn mark_unused_attempts(
&self,
attempts: Vec<TestAttempt>,
) -> Result<(), Self::Error> {
self.unused
.lock()
.expect("unused attempts should lock")
.extend(attempts.into_iter().map(|attempt| attempt.id));
Ok(())
}
async fn build_exhaustion(
&self,
_last_plan: aether_contracts::ExecutionPlan,
_last_report_context: Option<serde_json::Value>,
) -> Result<Self::Exhaustion, Self::Error> {
Ok(())
}
}
#[async_trait]
impl AiAttemptLoopPort<TestAttempt> for FailingPort {
type Response = ();
@@ -182,11 +324,13 @@ mod tests {
async fn execute_attempt(
&self,
attempt: &TestAttempt,
) -> Result<Option<Self::Response>, Self::Error> {
) -> Result<AiAttemptExecutionOutcome<Self::Response>, Self::Error> {
if attempt.id == self.fail_on {
Err("attempt failed")
} else {
Ok(None)
Ok(AiAttemptExecutionOutcome::retry(
AiAttemptRetryScope::Candidate,
))
}
}
@@ -237,6 +381,19 @@ mod tests {
}
}
fn routed_attempt(
id: &'static str,
provider_id: &str,
endpoint_id: &str,
key_id: &str,
) -> TestAttempt {
let mut attempt = attempt(id);
attempt.plan.provider_id = provider_id.to_string();
attempt.plan.endpoint_id = endpoint_id.to_string();
attempt.plan.key_id = key_id.to_string();
attempt
}
#[tokio::test]
async fn marks_unattempted_candidates_unused_when_execution_returns_error() {
let port = FailingPort {
@@ -261,4 +418,69 @@ mod tests {
vec!["candidate-3"]
);
}
#[tokio::test]
async fn retry_scopes_skip_matching_static_candidates() {
let port = ScopedRetryPort {
executed: Mutex::new(Vec::new()),
unused: Mutex::new(Vec::new()),
};
let attempts = vec![
routed_attempt("endpoint-failure", "provider-a", "endpoint-a", "key-a"),
routed_attempt("same-endpoint", "provider-a", "endpoint-a", "key-b"),
routed_attempt("credential-failure", "provider-a", "endpoint-b", "key-c"),
routed_attempt("same-credential", "provider-a", "endpoint-c", "key-c"),
routed_attempt("provider-failure", "provider-b", "endpoint-d", "key-d"),
routed_attempt("same-provider", "provider-b", "endpoint-e", "key-e"),
routed_attempt("success", "provider-c", "endpoint-f", "key-f"),
];
let outcome = run_ai_attempt_loop(&port, attempts)
.await
.expect("scoped retry loop should succeed");
assert!(matches!(
outcome,
super::AiAttemptLoopOutcome::Responded("success")
));
assert_eq!(
*port.executed.lock().expect("executed attempts should lock"),
vec![
"endpoint-failure",
"credential-failure",
"provider-failure",
"success"
]
);
assert_eq!(
*port.unused.lock().expect("unused attempts should lock"),
vec!["same-endpoint", "same-credential", "same-provider"]
);
}
#[tokio::test]
async fn returns_preserved_upstream_response_after_candidates_exhaust() {
let port = ScopedRetryPort {
executed: Mutex::new(Vec::new()),
unused: Mutex::new(Vec::new()),
};
let outcome = run_ai_attempt_loop(
&port,
vec![
routed_attempt("provider-failure", "provider-a", "endpoint-a", "key-a"),
routed_attempt("same-provider", "provider-a", "endpoint-b", "key-b"),
],
)
.await
.expect("fallback response loop should succeed");
assert!(matches!(
outcome,
super::AiAttemptLoopOutcome::Deferred("provider-error")
));
assert_eq!(
*port.unused.lock().expect("unused attempts should lock"),
vec!["same-provider"]
);
}
}
+22
View File
@@ -44,6 +44,28 @@ impl ConversionMode {
}
}
/// Request/response adaptation applied after candidate selection.
///
/// This is independent from format conversion: a same-format request may be
/// byte-transparent or may intentionally apply provider compatibility edits.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum AdaptationMode {
NativeTransparent,
SameFormatCompat,
CrossFormat,
}
impl AdaptationMode {
pub const fn as_str(self) -> &'static str {
match self {
Self::NativeTransparent => "native_transparent",
Self::SameFormatCompat => "same_format_compat",
Self::CrossFormat => "cross_format",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
pub struct AiRequestGzipPolicy {
#[serde(default, skip_serializing_if = "Option::is_none")]
+92 -4
View File
@@ -3,6 +3,7 @@ use async_trait::async_trait;
#[derive(Debug)]
pub enum AiServingExecutionOutcome<Response, Exhaustion> {
Responded(Response),
Deferred(Response),
Exhausted(Exhaustion),
NoPath,
}
@@ -98,9 +99,15 @@ where
Port: AiSyncExecutionPathPort,
{
let mut exhausted = None;
let mut deferred = None;
if let Some(response) =
absorb_sync_step(port, AiSyncExecutionStep::VideoTaskFollowUp, &mut exhausted).await?
if let Some(response) = absorb_sync_step(
port,
AiSyncExecutionStep::VideoTaskFollowUp,
&mut deferred,
&mut exhausted,
)
.await?
{
return Ok(response);
}
@@ -116,12 +123,18 @@ where
AiSyncExecutionStep::LocalGeminiFiles,
AiSyncExecutionStep::RemoteDecision,
] {
if let Some(response) = absorb_sync_step(port, step, &mut exhausted).await? {
if let Some(response) =
absorb_sync_step(port, step, &mut deferred, &mut exhausted).await?
{
return Ok(response);
}
}
}
if let Some(response) = deferred {
return Ok(AiServingExecutionOutcome::Deferred(response));
}
if let Some(outcome) = exhausted {
return Ok(AiServingExecutionOutcome::Exhausted(outcome));
}
@@ -135,6 +148,9 @@ where
AiServingExecutionOutcome::Responded(response) => {
Ok(AiServingExecutionOutcome::Responded(response))
}
AiServingExecutionOutcome::Deferred(response) => {
Ok(AiServingExecutionOutcome::Deferred(response))
}
AiServingExecutionOutcome::Exhausted(outcome) => {
Ok(AiServingExecutionOutcome::Exhausted(outcome))
}
@@ -149,17 +165,24 @@ where
Port: AiStreamExecutionPathPort,
{
let mut exhausted = None;
let mut deferred = None;
for step in port.stream_execution_steps() {
if *step != AiStreamExecutionStep::LocalVideoContent && !port.scheduler_decision_supported()
{
continue;
}
if let Some(response) = absorb_stream_step(port, *step, &mut exhausted).await? {
if let Some(response) =
absorb_stream_step(port, *step, &mut deferred, &mut exhausted).await?
{
return Ok(response);
}
}
if let Some(response) = deferred {
return Ok(AiServingExecutionOutcome::Deferred(response));
}
if let Some(outcome) = exhausted {
return Ok(AiServingExecutionOutcome::Exhausted(outcome));
}
@@ -173,6 +196,9 @@ where
AiServingExecutionOutcome::Responded(response) => {
Ok(AiServingExecutionOutcome::Responded(response))
}
AiServingExecutionOutcome::Deferred(response) => {
Ok(AiServingExecutionOutcome::Deferred(response))
}
AiServingExecutionOutcome::Exhausted(outcome) => {
Ok(AiServingExecutionOutcome::Exhausted(outcome))
}
@@ -183,6 +209,7 @@ where
async fn absorb_sync_step<Port>(
port: &Port,
step: AiSyncExecutionStep,
deferred: &mut Option<Port::Response>,
exhausted: &mut Option<Port::Exhaustion>,
) -> Result<Option<AiServingExecutionOutcome<Port::Response, Port::Exhaustion>>, Port::Error>
where
@@ -192,6 +219,10 @@ where
AiServingExecutionOutcome::Responded(response) => {
Ok(Some(AiServingExecutionOutcome::Responded(response)))
}
AiServingExecutionOutcome::Deferred(response) => {
*deferred = Some(response);
Ok(None)
}
AiServingExecutionOutcome::Exhausted(outcome) => {
*exhausted = Some(outcome);
Ok(None)
@@ -203,6 +234,7 @@ where
async fn absorb_stream_step<Port>(
port: &Port,
step: AiStreamExecutionStep,
deferred: &mut Option<Port::Response>,
exhausted: &mut Option<Port::Exhaustion>,
) -> Result<Option<AiServingExecutionOutcome<Port::Response, Port::Exhaustion>>, Port::Error>
where
@@ -212,6 +244,10 @@ where
AiServingExecutionOutcome::Responded(response) => {
Ok(Some(AiServingExecutionOutcome::Responded(response)))
}
AiServingExecutionOutcome::Deferred(response) => {
*deferred = Some(response);
Ok(None)
}
AiServingExecutionOutcome::Exhausted(outcome) => {
*exhausted = Some(outcome);
Ok(None)
@@ -390,6 +426,30 @@ mod tests {
);
}
#[tokio::test]
async fn sync_path_keeps_deferred_error_until_a_later_step_succeeds() {
let port = TestSyncPort {
scheduler_supported: true,
outcomes: Mutex::new(VecDeque::from([
AiServingExecutionOutcome::NoPath,
AiServingExecutionOutcome::Deferred("preserved_upstream_error"),
AiServingExecutionOutcome::Responded("later_success"),
])),
calls: Mutex::default(),
};
let outcome = run_ai_sync_execution_path(&port).await.unwrap();
assert!(matches!(
outcome,
AiServingExecutionOutcome::Responded("later_success")
));
assert_eq!(
port.calls.lock().unwrap().as_slice(),
["VideoTaskFollowUp", "LocalVideo", "LocalImage"]
);
}
#[tokio::test]
async fn stream_path_skips_scheduler_steps_when_unsupported() {
let port = TestStreamPort {
@@ -451,6 +511,34 @@ mod tests {
assert_eq!(port.calls.lock().unwrap().as_slice(), ["LocalOpenAiChat"]);
}
#[tokio::test]
async fn stream_path_returns_deferred_error_only_after_steps_exhaust() {
const TWO_STEPS: &[AiStreamExecutionStep] = &[
AiStreamExecutionStep::LocalOpenAiChat,
AiStreamExecutionStep::LocalSameFormatProvider,
];
let port = TestStreamPort {
scheduler_supported: true,
stream_steps: Some(TWO_STEPS),
outcomes: Mutex::new(VecDeque::from([
AiServingExecutionOutcome::Deferred("preserved_upstream_error"),
AiServingExecutionOutcome::NoPath,
])),
calls: Mutex::default(),
};
let outcome = run_ai_stream_execution_path(&port).await.unwrap();
assert!(matches!(
outcome,
AiServingExecutionOutcome::Deferred("preserved_upstream_error")
));
assert_eq!(
port.calls.lock().unwrap().as_slice(),
["LocalOpenAiChat", "LocalSameFormatProvider"]
);
}
#[tokio::test]
async fn stream_path_returns_last_exhaustion_without_plan_fallback() {
let port = TestStreamPort {
+5 -2
View File
@@ -14,6 +14,7 @@ pub mod decision_payload;
pub mod dto;
pub mod execution_path;
pub mod failure_diagnostic;
pub mod payload_fidelity;
pub mod plan_payload;
pub mod ports;
pub mod ranking_metadata;
@@ -52,7 +53,8 @@ pub use aether_pool_core::{
REQUEST_FAILURE_PENALTY, UNSCHEDULABLE_SCORE_CAP,
};
pub use attempt_loop::{
run_ai_attempt_loop, AiAttemptLoopOutcome, AiAttemptLoopPort, AiExecutionAttempt,
run_ai_attempt_loop, AiAttemptExecutionOutcome, AiAttemptLoopOutcome, AiAttemptLoopPort,
AiAttemptRetryScope, AiExecutionAttempt,
};
pub use attempt_plan::{
build_ai_execution_decision_from_plan, build_ai_execution_plan_from_decision,
@@ -106,7 +108,7 @@ pub use decision_payload::{
AiExecutionDecisionResponseParts,
};
pub use dto::{
augment_sync_report_context, generic_decision_missing_exact_provider_request,
augment_sync_report_context, generic_decision_missing_exact_provider_request, AdaptationMode,
AiExecutionDecision, AiExecutionPlanPayload, AiRequestGzipPolicy, AiStreamAttempt,
AiSyncAttempt, ConversionMode, ExecutionStrategy,
};
@@ -116,6 +118,7 @@ pub use execution_path::{
AiSyncExecutionPathPort, AiSyncExecutionStep, DEFAULT_STREAM_EXECUTION_STEPS,
};
pub use failure_diagnostic::{CandidateFailureDiagnostic, CandidateFailureDiagnosticKind};
pub use payload_fidelity::OriginalRequestPayload;
pub use plan_payload::{
build_ai_stream_execution_plan_payload, build_ai_sync_execution_plan_payload,
};
@@ -0,0 +1,77 @@
use std::sync::Arc;
use base64::Engine as _;
/// The frontdoor-normalized JSON request as both a parsed value and its exact
/// decoded bytes. This is carried only inside the local process through HTTP
/// request extensions; serialized execution contracts continue to use
/// `RequestBody::body_bytes_b64`.
#[derive(Debug, Clone)]
pub struct OriginalRequestPayload {
body_json: Arc<serde_json::Value>,
body_bytes: Arc<[u8]>,
}
impl OriginalRequestPayload {
pub fn from_parsed_json(body_json: serde_json::Value, body_bytes: &[u8]) -> Self {
Self {
body_json: Arc::new(body_json),
body_bytes: Arc::from(body_bytes),
}
}
/// Returns the original body only when the terminal provider JSON is
/// semantically unchanged. Object key order and whitespace are preserved by
/// returning the captured bytes rather than serializing `provider_body`.
pub fn body_bytes_base64_if_unchanged(
&self,
provider_body: &serde_json::Value,
) -> Option<String> {
if self.body_bytes.is_empty() || provider_body != self.body_json.as_ref() {
return None;
}
Some(base64::engine::general_purpose::STANDARD.encode(self.body_bytes.as_ref()))
}
}
#[cfg(test)]
mod tests {
use base64::Engine as _;
use serde_json::json;
use super::OriginalRequestPayload;
#[test]
fn preserves_exact_json_bytes_when_terminal_value_is_unchanged() {
let raw = br#"{ "unknown": true, "model": "claude-sonnet-4" }"#;
let parsed: serde_json::Value = serde_json::from_slice(raw).expect("request should parse");
let payload = OriginalRequestPayload::from_parsed_json(parsed.clone(), raw);
let encoded = payload
.body_bytes_base64_if_unchanged(&parsed)
.expect("unchanged body should preserve bytes");
assert_eq!(
base64::engine::general_purpose::STANDARD
.decode(encoded)
.expect("body should decode"),
raw
);
}
#[test]
fn rejects_original_bytes_when_terminal_value_changed() {
let raw = br#"{"model":"claude-sonnet-4","messages":[]}"#;
let parsed: serde_json::Value = serde_json::from_slice(raw).expect("request should parse");
let payload = OriginalRequestPayload::from_parsed_json(parsed, raw);
assert_eq!(
payload.body_bytes_base64_if_unchanged(&json!({
"model": "claude-sonnet-4-5",
"messages": []
})),
None
);
}
}