Improve gateway transport and usage runtime

2026-10-09 18:59:50 +08:00 · 2026-06-25 22:36:27 +08:00
parent d336d1a7fa
commit 6f00e9fc67
112 changed files with 12456 additions and 1387 deletions
@@ -6,6 +6,7 @@ mod request;
 mod support;

 pub(super) use self::payload::maybe_build_local_openai_chat_decision_payload_for_candidate;
+pub(super) use self::request::LocalOpenAiChatRequestPreparation;
 pub(super) use self::support::{
    build_lazy_local_openai_chat_candidate_attempt_source,
    build_local_openai_chat_candidate_attempt_source,
@@ -18,7 +18,9 @@ use crate::{
    AiExecutionDecision, AppState, GatewayError,
 };

-use super::request::resolve_local_openai_chat_candidate_payload_parts;
+use super::request::{
+    resolve_local_openai_chat_candidate_payload_parts, LocalOpenAiChatRequestPreparation,
+};
 use super::support::{LocalOpenAiChatCandidateAttempt, LocalOpenAiChatDecisionInput};

 #[allow(clippy::too_many_arguments)]
@@ -28,6 +30,7 @@ pub(crate) async fn maybe_build_local_openai_chat_decision_payload_for_candidate
    trace_id: &str,
    body_json: &serde_json::Value,
    input: &LocalOpenAiChatDecisionInput,
+    preparation: Option<&mut LocalOpenAiChatRequestPreparation>,
    attempt: LocalOpenAiChatCandidateAttempt,
    decision_kind: &str,
    report_kind: &str,
@@ -48,6 +51,7 @@ pub(crate) async fn maybe_build_local_openai_chat_decision_payload_for_candidate
        trace_id,
        body_json,
        input,
+        preparation,
        &eligible,
        candidate_index,
        &candidate_id,
@@ -82,6 +82,39 @@ pub(crate) struct LocalOpenAiChatCandidatePayloadParts {
    pub(super) image_request_summary: Option<Value>,
 }

+#[derive(Default)]
+pub(crate) struct LocalOpenAiChatRequestPreparation {
+    model_directives_enabled: BTreeMap<(String, String), bool>,
+}
+
+impl LocalOpenAiChatRequestPreparation {
+    async fn model_directives_enabled(
+        &mut self,
+        state: &AppState,
+        provider_api_format: &str,
+        requested_model: &str,
+    ) -> bool {
+        let key = (
+            provider_api_format.trim().to_ascii_lowercase(),
+            requested_model.trim().to_string(),
+        );
+        if let Some(enabled) = self.model_directives_enabled.get(&key) {
+            crate::stage_metrics::record_openai_chat_model_directive_cache_hit();
+            return *enabled;
+        }
+        crate::stage_metrics::record_openai_chat_model_directive_cache_miss();
+        let enabled =
+            crate::system_features::reasoning_model_directive_enabled_for_api_format_and_model(
+                state,
+                provider_api_format,
+                Some(requested_model),
+            )
+            .await;
+        self.model_directives_enabled.insert(key, enabled);
+        enabled
+    }
+}
+
 fn is_grok_text_provider_api_format(provider_api_format: &str) -> bool {
    matches!(
        crate::ai_serving::normalize_api_format_alias(provider_api_format).as_str(),
@@ -96,6 +129,7 @@ pub(crate) async fn resolve_local_openai_chat_candidate_payload_parts(
    trace_id: &str,
    body_json: &serde_json::Value,
    input: &LocalOpenAiChatDecisionInput,
+    mut preparation: Option<&mut LocalOpenAiChatRequestPreparation>,
    eligible: &EligibleLocalExecutionCandidate,
    candidate_index: u32,
    candidate_id: &str,
@@ -112,13 +146,18 @@ pub(crate) async fn resolve_local_openai_chat_candidate_payload_parts(
    let force_body_stream_field =
        endpoint_config_forces_body_stream_field(transport.endpoint.config.as_ref());
    let model_directives_started_at = std::time::Instant::now();
-    let enable_model_directives =
+    let enable_model_directives = if let Some(preparation) = preparation {
+        preparation
+            .model_directives_enabled(state, provider_api_format, &input.requested_model)
+            .await
+    } else {
        crate::system_features::reasoning_model_directive_enabled_for_api_format_and_model(
            state,
            provider_api_format,
            Some(&input.requested_model),
        )
-        .await;
+        .await
+    };
    observe_gateway_stage_ms(
        "openai_chat_payload_model_directives",
        model_directives_started_at.elapsed().as_millis() as u64,
@@ -1907,6 +1946,7 @@ mod tests {
            "trace-openai-chat-gemini-cli",
            &body_json,
            &sample_input(),
+            None,
            &sample_gemini_cli_eligible(),
            0,
            "candidate-0",
@@ -13,6 +13,7 @@ use self::decision::{
    build_lazy_local_openai_chat_candidate_attempt_source,
    maybe_build_local_openai_chat_decision_payload_for_candidate, LocalOpenAiChatCandidateAttempt,
    LocalOpenAiChatCandidateAttemptSource, LocalOpenAiChatDecisionInput,
+    LocalOpenAiChatRequestPreparation,
 };
 use self::plans::{
    build_local_openai_chat_stream_attempt_source, build_local_openai_chat_stream_plan_and_reports,
@@ -159,6 +160,7 @@ pub(crate) async fn maybe_build_sync_local_decision_payload(
            trace_id,
            body_json,
            &input,
+            None,
            attempt,
            OPENAI_CHAT_SYNC_PLAN_KIND,
            "openai_chat_sync_success",
@@ -211,6 +213,7 @@ pub(crate) async fn maybe_build_stream_local_decision_payload(
            trace_id,
            body_json,
            &input,
+            None,
            attempt,
            OPENAI_CHAT_STREAM_PLAN_KIND,
            "openai_chat_stream_success",
@@ -9,6 +9,7 @@ use crate::ai_serving::planner::decision_input::{
 };
 use crate::ai_serving::resolve_local_decision_execution_runtime_auth_context;
 use crate::client_session_affinity::client_session_affinity_from_parts;
+use crate::stage_metrics::observe_gateway_stage_ms;
 use crate::{AppState, GatewayError};

 pub(crate) async fn resolve_local_openai_chat_decision_input(
@@ -59,6 +60,7 @@ pub(crate) async fn resolve_local_openai_chat_decision_input(
        return Ok(None);
    };

+    let auth_started_at = std::time::Instant::now();
    let resolved_input = match resolve_local_authenticated_decision_input(
        state,
        auth_context.clone(),
@@ -106,10 +108,20 @@ pub(crate) async fn resolve_local_openai_chat_decision_input(
            return Err(err);
        }
    };
+    observe_gateway_stage_ms(
+        "openai_chat_decision_input_auth",
+        auth_started_at.elapsed().as_millis() as u64,
+    );

    let mut input = build_local_requested_model_decision_input(resolved_input, requested_model);
    input.request_auth_channel = decision.request_auth_channel.clone();
+    let affinity_started_at = std::time::Instant::now();
    input.client_session_affinity = client_session_affinity_from_parts(parts, Some(body_json));
+    observe_gateway_stage_ms(
+        "openai_chat_decision_input_affinity",
+        affinity_started_at.elapsed().as_millis() as u64,
+    );
+    let routing_started_at = std::time::Instant::now();
    if let Err(err) = attach_routing_policy_to_local_requested_model_input(
        state,
        parts,
@@ -126,5 +138,9 @@ pub(crate) async fn resolve_local_openai_chat_decision_input(
        );
        return Err(err);
    }
+    observe_gateway_stage_ms(
+        "openai_chat_decision_input_routing",
+        routing_started_at.elapsed().as_millis() as u64,
+    );
    Ok(Some(input))
 }
@@ -1,11 +1,12 @@
 use async_trait::async_trait;
+use std::collections::VecDeque;
 use tracing::warn;

 use super::super::{
    build_lazy_local_openai_chat_candidate_attempt_source,
    maybe_build_local_openai_chat_decision_payload_for_candidate, AppState, GatewayControlDecision,
    GatewayError, LocalOpenAiChatCandidateAttempt, LocalOpenAiChatCandidateAttemptSource,
-    LocalOpenAiChatDecisionInput,
+    LocalOpenAiChatDecisionInput, LocalOpenAiChatRequestPreparation,
 };
 use super::diagnostic::{
    set_local_openai_chat_candidate_evaluation_diagnostic, set_local_openai_chat_miss_diagnostic,
@@ -18,7 +19,23 @@ use crate::ai_serving::planner::plan_builders::{
    build_openai_chat_stream_plan_from_decision, AiStreamAttempt,
 };
 use crate::ai_serving::planner::runtime_miss::apply_local_runtime_candidate_terminal_reason;
-use crate::stage_metrics::observe_gateway_stage_ms;
+use crate::ai_serving::planner::standard::build_local_openai_chat_upstream_url;
+use crate::ai_serving::transport::{
+    is_windsurf_provider_transport, local_openai_chat_transport_unsupported_reason,
+};
+use crate::clock::request_distribution_seed;
+use crate::stage_metrics::{
+    observe_gateway_stage_ms, record_openai_chat_stream_payload_build_prefetch_avoided,
+    record_openai_chat_stream_payload_build_selected,
+    record_openai_chat_stream_raw_candidates_scanned,
+    record_openai_chat_stream_target_select_selected_rank,
+};
+use crate::upstream_admission::upstream_target_key_from_url;
+
+const OPENAI_CHAT_STREAM_TARGET_SELECT_WINDOW_ENV: &str =
+    "AETHER_GATEWAY_OPENAI_CHAT_STREAM_TARGET_SELECT_WINDOW";
+const DEFAULT_OPENAI_CHAT_STREAM_TARGET_SELECT_WINDOW: usize = 2;
+const MAX_OPENAI_CHAT_STREAM_TARGET_SELECT_WINDOW: usize = 8;

 pub(crate) struct LocalOpenAiChatStreamAttemptSource<'a> {
    state: &'a AppState,
@@ -27,6 +44,8 @@ pub(crate) struct LocalOpenAiChatStreamAttemptSource<'a> {
    body_json: serde_json::Value,
    input: LocalOpenAiChatDecisionInput,
    candidates: LocalOpenAiChatCandidateAttemptSource<'a>,
+    prefetched_attempts: VecDeque<LocalOpenAiChatCandidateAttempt>,
+    request_preparation: LocalOpenAiChatRequestPreparation,
 }

 pub(crate) async fn build_local_openai_chat_stream_attempt_source<'a>(
@@ -41,6 +60,7 @@ pub(crate) async fn build_local_openai_chat_stream_attempt_source<'a>(
        return Ok(None);
    }

+    let attempt_source_started_at = std::time::Instant::now();
    let Some(input) = resolve_local_openai_chat_decision_input(
        state, parts, trace_id, decision, body_json, plan_kind, true,
    )
@@ -77,6 +97,10 @@ pub(crate) async fn build_local_openai_chat_stream_attempt_source<'a>(
        Some(input.requested_model.as_str()),
        candidate_count,
    );
+    observe_gateway_stage_ms(
+        "openai_chat_attempt_source_build",
+        attempt_source_started_at.elapsed().as_millis() as u64,
+    );

    Ok(Some((
        LocalOpenAiChatStreamAttemptSource {
@@ -86,6 +110,8 @@ pub(crate) async fn build_local_openai_chat_stream_attempt_source<'a>(
            body_json: effective_body_json,
            input,
            candidates,
+            prefetched_attempts: VecDeque::new(),
+            request_preparation: LocalOpenAiChatRequestPreparation::default(),
        },
        candidate_count,
    )))
@@ -94,21 +120,42 @@ pub(crate) async fn build_local_openai_chat_stream_attempt_source<'a>(
 #[async_trait]
 impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalOpenAiChatStreamAttemptSource<'_> {
    async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
+        let select_started_at = std::time::Instant::now();
+        let selected = self.next_execution_attempt_with_target_select().await?;
+        observe_gateway_stage_ms(
+            "openai_chat_stream_target_select",
+            select_started_at.elapsed().as_millis() as u64,
+        );
+        Ok(selected)
+    }
+
+    async fn drain_execution_attempts(&mut self) -> Result<Vec<AiStreamAttempt>, GatewayError> {
+        let mut drained = Vec::new();
+        for attempt in self.candidates.drain_static_attempts() {
+            if let Some(attempt) = self.build_stream_attempt(attempt).await? {
+                drained.push(attempt);
+            }
+        }
+        Ok(drained)
+    }
+}
+
+impl LocalOpenAiChatStreamAttemptSource<'_> {
+    async fn next_execution_attempt_with_target_select(
+        &mut self,
+    ) -> Result<Option<AiStreamAttempt>, GatewayError> {
        loop {
-            let source_started_at = std::time::Instant::now();
-            let Some(attempt) = self.candidates.next_attempt().await? else {
-                observe_gateway_stage_ms(
-                    "stream_candidate_source_next",
-                    source_started_at.elapsed().as_millis() as u64,
+            let Some(attempt) = self.next_raw_attempt_with_target_select().await? else {
+                apply_local_runtime_candidate_terminal_reason(
+                    self.state,
+                    self.trace_id,
+                    "no_local_stream_plans",
                );
-                break;
+                return Ok(None);
            };
-            observe_gateway_stage_ms(
-                "stream_candidate_source_next",
-                source_started_at.elapsed().as_millis() as u64,
-            );

            let plan_started_at = std::time::Instant::now();
+            record_openai_chat_stream_payload_build_selected();
            match self.build_stream_attempt(attempt).await? {
                Some(attempt) => {
                    observe_gateway_stage_ms(
@@ -126,28 +173,122 @@ impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalOpenAiChatStreamAttem
                }
            }
        }
-        apply_local_runtime_candidate_terminal_reason(
-            self.state,
-            self.trace_id,
-            "no_local_stream_plans",
-        );
-        Ok(None)
    }

-    async fn drain_execution_attempts(&mut self) -> Result<Vec<AiStreamAttempt>, GatewayError> {
-        let mut drained = Vec::new();
-        for attempt in self.candidates.drain_static_attempts() {
-            if let Some(attempt) = self.build_stream_attempt(attempt).await? {
-                drained.push(attempt);
+    async fn next_raw_attempt_with_target_select(
+        &mut self,
+    ) -> Result<Option<LocalOpenAiChatCandidateAttempt>, GatewayError> {
+        let select_window = openai_chat_stream_target_select_window();
+        if select_window <= 1 {
+            return self.next_raw_attempt_linear().await;
+        }
+        let mut attempts = Vec::with_capacity(select_window);
+        for _ in 0..select_window {
+            match self.next_raw_attempt_linear().await? {
+                Some(attempt) => attempts.push(attempt),
+                None => break,
            }
        }
-        Ok(drained)
+        if attempts.is_empty() {
+            return Ok(None);
+        }
+        record_openai_chat_stream_raw_candidates_scanned(attempts.len());
+        let seed = request_distribution_seed();
+        let target_keys = attempts
+            .iter()
+            .map(|attempt| self.lightweight_target_key_for_attempt(attempt))
+            .collect::<Vec<_>>();
+        for target_key in target_keys.iter().flatten() {
+            self.state
+                .upstream_target_admission
+                .record_raw_seen_for_target_key(target_key);
+        }
+        let selected_index = if target_keys.iter().all(Option::is_some) {
+            let choices = attempts
+                .iter()
+                .zip(target_keys.iter())
+                .map(|(attempt, target_key)| {
+                    let target_key = target_key.as_deref().unwrap_or("-");
+                    let snapshot = self
+                        .state
+                        .upstream_target_admission
+                        .snapshot_for_target_key(target_key);
+                    TargetSelectChoice {
+                        target_key,
+                        identity: target_select_candidate_identity(attempt),
+                        in_flight: snapshot
+                            .as_ref()
+                            .map(|snapshot| snapshot.in_flight)
+                            .unwrap_or(0),
+                        selection_pressure_total: snapshot
+                            .as_ref()
+                            .map(|snapshot| snapshot.selection_pressure_total)
+                            .unwrap_or(0),
+                    }
+                })
+                .collect::<Vec<_>>();
+            select_target_index(seed, &choices)
+        } else {
+            0
+        };
+        record_openai_chat_stream_target_select_selected_rank(selected_index);
+        record_openai_chat_stream_payload_build_prefetch_avoided(attempts.len().saturating_sub(1));
+        if let Some(Some(target_key)) = target_keys.get(selected_index) {
+            self.state
+                .upstream_target_admission
+                .record_preselect_for_target_key(target_key);
+        }
+        let selected = attempts.remove(selected_index);
+        self.prefetched_attempts.extend(attempts);
+        Ok(Some(selected))
    }
-}

-impl LocalOpenAiChatStreamAttemptSource<'_> {
-    async fn build_stream_attempt(
+    async fn next_raw_attempt_linear(
+        &mut self,
+    ) -> Result<Option<LocalOpenAiChatCandidateAttempt>, GatewayError> {
+        if let Some(attempt) = self.prefetched_attempts.pop_front() {
+            return Ok(Some(attempt));
+        }
+        let source_started_at = std::time::Instant::now();
+        let attempt = self.candidates.next_attempt().await?;
+        observe_gateway_stage_ms(
+            "stream_candidate_source_next",
+            source_started_at.elapsed().as_millis() as u64,
+        );
+        Ok(attempt)
+    }
+
+    fn lightweight_target_key_for_attempt(
        &self,
+        attempt: &LocalOpenAiChatCandidateAttempt,
+    ) -> Option<String> {
+        let provider_api_format = attempt.eligible.provider_api_format.trim();
+        if !provider_api_format.eq_ignore_ascii_case("openai:chat") {
+            return None;
+        }
+        let transport = &attempt.eligible.transport;
+        if transport
+            .provider
+            .provider_type
+            .trim()
+            .eq_ignore_ascii_case("grok")
+            || is_windsurf_provider_transport(transport)
+            || local_openai_chat_transport_unsupported_reason(transport).is_some()
+        {
+            return None;
+        }
+        if transport.provider.proxy.is_some()
+            || transport.endpoint.proxy.is_some()
+            || transport.key.proxy.is_some()
+        {
+            return None;
+        }
+        let upstream_url = build_local_openai_chat_upstream_url(self.parts, transport)?;
+        upstream_target_key_from_url(upstream_url.as_str(), None)
+    }
+
+    async fn build_stream_attempt(
+        &mut self,
        attempt: LocalOpenAiChatCandidateAttempt,
    ) -> Result<Option<AiStreamAttempt>, GatewayError> {
        let upstream_is_stream = openai_chat_upstream_is_stream_for_candidate(
@@ -161,6 +302,7 @@ impl LocalOpenAiChatStreamAttemptSource<'_> {
            self.trace_id,
            &self.body_json,
            &self.input,
+            Some(&mut self.request_preparation),
            attempt,
            OPENAI_CHAT_STREAM_PLAN_KIND,
            "openai_chat_stream_success",
@@ -185,6 +327,97 @@ impl LocalOpenAiChatStreamAttemptSource<'_> {
    }
 }

+fn openai_chat_stream_target_select_window() -> usize {
+    std::env::var(OPENAI_CHAT_STREAM_TARGET_SELECT_WINDOW_ENV)
+        .ok()
+        .and_then(|value| value.trim().parse::<usize>().ok())
+        .filter(|value| *value > 0)
+        .unwrap_or(DEFAULT_OPENAI_CHAT_STREAM_TARGET_SELECT_WINDOW)
+        .clamp(1, MAX_OPENAI_CHAT_STREAM_TARGET_SELECT_WINDOW)
+}
+
+#[derive(Clone, Copy)]
+struct TargetSelectCandidateIdentity<'a> {
+    provider_id: &'a str,
+    endpoint_id: &'a str,
+    key_id: &'a str,
+    candidate_id: &'a str,
+}
+
+#[derive(Clone, Copy)]
+struct TargetSelectChoice<'a> {
+    target_key: &'a str,
+    identity: TargetSelectCandidateIdentity<'a>,
+    in_flight: usize,
+    selection_pressure_total: u64,
+}
+
+fn select_target_index(seed: u64, choices: &[TargetSelectChoice<'_>]) -> usize {
+    choices
+        .iter()
+        .enumerate()
+        .min_by_key(|(index, choice)| {
+            target_select_score(
+                seed,
+                choice.target_key,
+                &choice.identity,
+                *index,
+                choice.in_flight,
+                choice.selection_pressure_total,
+            )
+        })
+        .map(|(index, _)| index)
+        .unwrap_or(0)
+}
+
+fn target_select_candidate_identity(
+    attempt: &LocalOpenAiChatCandidateAttempt,
+) -> TargetSelectCandidateIdentity<'_> {
+    TargetSelectCandidateIdentity {
+        provider_id: &attempt.eligible.candidate.provider_id,
+        endpoint_id: &attempt.eligible.candidate.endpoint_id,
+        key_id: &attempt.eligible.candidate.key_id,
+        candidate_id: &attempt.candidate_id,
+    }
+}
+
+fn target_select_tie_break(
+    seed: u64,
+    target_key: &str,
+    identity: &TargetSelectCandidateIdentity<'_>,
+    index: usize,
+) -> u64 {
+    let mut hash = seed ^ ((index as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15));
+    hash = hash_string(hash, target_key);
+    hash = hash_string(hash, identity.provider_id);
+    hash = hash_string(hash, identity.endpoint_id);
+    hash = hash_string(hash, identity.key_id);
+    hash_string(hash, identity.candidate_id)
+}
+
+fn hash_string(mut hash: u64, value: &str) -> u64 {
+    for byte in value.as_bytes() {
+        hash ^= u64::from(*byte);
+        hash = hash.wrapping_mul(0x100_0000_01B3);
+    }
+    hash
+}
+
+fn target_select_score(
+    seed: u64,
+    target_key: &str,
+    identity: &TargetSelectCandidateIdentity<'_>,
+    index: usize,
+    in_flight: usize,
+    selected_total: u64,
+) -> (usize, u64, u64) {
+    (
+        in_flight,
+        selected_total,
+        target_select_tie_break(seed, target_key, identity, index),
+    )
+}
+
 pub(crate) async fn build_local_openai_chat_stream_plan_and_reports(
    state: &AppState,
    parts: &http::request::Parts,
@@ -234,3 +467,82 @@ pub(crate) async fn build_local_openai_chat_stream_plan_and_reports(

    Ok(plans)
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn identity<'a>(
+        endpoint_id: &'a str,
+        candidate_id: &'a str,
+    ) -> TargetSelectCandidateIdentity<'a> {
+        TargetSelectCandidateIdentity {
+            provider_id: "provider",
+            endpoint_id,
+            key_id: "key",
+            candidate_id,
+        }
+    }
+
+    #[test]
+    fn target_select_score_prefers_lower_in_flight() {
+        let busy = identity("endpoint-a", "candidate-a");
+        let idle = identity("endpoint-b", "candidate-b");
+
+        assert!(
+            target_select_score(7, "http://127.0.0.1:18182|proxy=-", &idle, 1, 0, 10)
+                < target_select_score(7, "http://127.0.0.1:18181|proxy=-", &busy, 0, 5, 0)
+        );
+    }
+
+    #[test]
+    fn target_select_tie_break_distinguishes_equivalent_targets() {
+        let left = identity("endpoint-a", "candidate-a");
+        let right = identity("endpoint-b", "candidate-b");
+
+        assert_ne!(
+            target_select_tie_break(11, "http://127.0.0.1:18181|proxy=-", &left, 0),
+            target_select_tie_break(11, "http://127.0.0.1:18182|proxy=-", &right, 1)
+        );
+    }
+
+    #[test]
+    fn select_target_index_prefers_lower_in_flight_target() {
+        let choices = [
+            TargetSelectChoice {
+                target_key: "http://127.0.0.1:18181|proxy=-",
+                identity: identity("endpoint-a", "candidate-a"),
+                in_flight: 8,
+                selection_pressure_total: 0,
+            },
+            TargetSelectChoice {
+                target_key: "http://127.0.0.1:18182|proxy=-",
+                identity: identity("endpoint-b", "candidate-b"),
+                in_flight: 1,
+                selection_pressure_total: 100,
+            },
+        ];
+
+        assert_eq!(select_target_index(17, &choices), 1);
+    }
+
+    #[test]
+    fn select_target_index_uses_selection_pressure_before_tie_break() {
+        let choices = [
+            TargetSelectChoice {
+                target_key: "http://127.0.0.1:18181|proxy=-",
+                identity: identity("endpoint-a", "candidate-a"),
+                in_flight: 0,
+                selection_pressure_total: 20,
+            },
+            TargetSelectChoice {
+                target_key: "http://127.0.0.1:18182|proxy=-",
+                identity: identity("endpoint-b", "candidate-b"),
+                in_flight: 0,
+                selection_pressure_total: 1,
+            },
+        ];
+
+        assert_eq!(select_target_index(19, &choices), 1);
+    }
+}
@@ -134,6 +134,7 @@ impl LocalOpenAiChatSyncAttemptSource<'_> {
            self.trace_id,
            &self.body_json,
            &self.input,
+            None,
            attempt,
            OPENAI_CHAT_SYNC_PLAN_KIND,
            "openai_chat_sync_success",