use std::collections::HashMap; use std::future::Future; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex, OnceLock}; use std::time::Duration; use aether_cache::ExpiringMap; use aether_data::DataLayerError; use aether_data_contracts::repository::routing_profiles::{ RoutingGroupBindingQuery, RoutingGroupBindingSubject, RoutingGroupLookupKey, RoutingGroupReadRepository, StoredRoutingGroup, StoredRoutingGroupBinding, StoredRoutingGroupVersion, }; use async_trait::async_trait; use tokio::sync::Notify; // Routing profile writes clear this local cache. Keep the fallback TTL bounded // so a missed cross-node invalidation cannot route traffic stale for minutes. const ROUTING_GROUP_CACHE_STALE_TTL: Duration = Duration::from_secs(60); const ROUTING_GROUP_CACHE_MAX_ENTRIES: usize = 4_096; const ROUTING_GROUP_CACHE_MAX_INFLIGHT: usize = 4_096; const ROUTING_GROUP_CACHE_LOAD_TIMEOUT: Duration = Duration::from_secs(10); pub(super) struct CachedRoutingGroupReadRepository { inner: Arc, entries: ExpiringMap, inflight: Mutex>>, admission: Arc, generation: AtomicU64, mutation: Mutex<()>, } impl CachedRoutingGroupReadRepository { pub(super) fn new(inner: Arc) -> Self { Self { inner, entries: ExpiringMap::new(), inflight: Mutex::new(HashMap::new()), admission: Arc::new(tokio::sync::Semaphore::new( ROUTING_GROUP_CACHE_MAX_INFLIGHT, )), generation: AtomicU64::new(0), mutation: Mutex::new(()), } } fn clear(&self) { let _mutation = self .mutation .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); self.generation.fetch_add(1, Ordering::AcqRel); self.entries.clear(); let states = self .inflight .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .drain() .map(|(_, state)| { let _ = state .completion .set(RoutingGroupInflightCompletion::Invalidated); state }) .collect::>(); drop(_mutation); for state in states { state.notify.notify_waiters(); } } async fn get_or_load( &self, key: RoutingGroupCacheKey, mut load: F, ) -> Result where F: FnMut() -> Fut, Fut: Future>, { loop { if let Some(value) = self.cached_value(&key) { return Ok(value); } match self.register_inflight(&key) { RoutingGroupInflightRegistration::Saturated => { return Err(DataLayerError::TimedOut(format!( "routing group cache admission saturated for {key:?}" ))); } RoutingGroupInflightRegistration::Leader(mut guard) => { // A previous leader can publish between the optimistic // cache read and this registration. if let Some(value) = self.cached_value(&key) { guard.finish(RoutingGroupInflightCompletion::Loaded(value.clone())); return Ok(value); } let result = match tokio::time::timeout( ROUTING_GROUP_CACHE_LOAD_TIMEOUT, load(), ) .await { Ok(result) => result, Err(_) => Err(DataLayerError::TimedOut(format!( "routing group cache load exceeded {}ms for {key:?}", ROUTING_GROUP_CACHE_LOAD_TIMEOUT.as_millis() ))), }; match result { Ok(value) => { self.insert_if_generation(key.clone(), value.clone(), guard.generation); guard.finish(RoutingGroupInflightCompletion::Loaded(value.clone())); return Ok(value); } Err(error) => { guard.finish(RoutingGroupInflightCompletion::Failed( SharedDataLayerError::from(&error), )); return Err(error); } } } RoutingGroupInflightRegistration::Follower(state) => { state.wait().await; match self.follower_completion(&state) { Some(RoutingGroupInflightCompletion::Loaded(value)) => return Ok(value), Some(RoutingGroupInflightCompletion::Failed(error)) => { return Err(error.into_data_layer_error()); } Some( RoutingGroupInflightCompletion::Cancelled | RoutingGroupInflightCompletion::Invalidated, ) | None => continue, } } } } } fn cached_value(&self, key: &RoutingGroupCacheKey) -> Option { self.entries .get_with_age(key, ROUTING_GROUP_CACHE_STALE_TTL) .map(|(value, _age)| value) } fn follower_completion( &self, state: &RoutingGroupInflightState, ) -> Option { // Double-check the generation around the completion read. This keeps // the hot follower path lock-free while ensuring clear() cannot race // between an old-generation check and returning a loaded value. let before = self.generation.load(Ordering::Acquire); let completion = state.completion.get().cloned(); let after = self.generation.load(Ordering::Acquire); (before == state.generation && before == after) .then_some(completion) .flatten() } fn insert_if_generation( &self, key: RoutingGroupCacheKey, value: RoutingGroupCacheValue, generation: u64, ) { let _mutation = self .mutation .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); if self.generation.load(Ordering::Acquire) != generation { return; } self.entries.insert( key, value, ROUTING_GROUP_CACHE_STALE_TTL, ROUTING_GROUP_CACHE_MAX_ENTRIES, ); } fn register_inflight( &self, key: &RoutingGroupCacheKey, ) -> RoutingGroupInflightRegistration<'_> { // Existing followers only touch the per-key map. Avoid taking the // mutation lock on the 20k-request hot path; that lock is reserved // for leader insertion and invalidation ordering. { let inflight = self .inflight .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); if let Some(state) = inflight.get(key) { return RoutingGroupInflightRegistration::Follower(Arc::clone(state)); } } let _mutation = self .mutation .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); let mut inflight = self .inflight .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); if let Some(state) = inflight.get(key) { return RoutingGroupInflightRegistration::Follower(Arc::clone(state)); } if inflight.len() >= ROUTING_GROUP_CACHE_MAX_INFLIGHT { return RoutingGroupInflightRegistration::Saturated; } let Ok(admission) = Arc::clone(&self.admission).try_acquire_owned() else { return RoutingGroupInflightRegistration::Saturated; }; let state = Arc::new(RoutingGroupInflightState { notify: Arc::new(Notify::new()), completion: OnceLock::new(), generation: self.generation.load(Ordering::Acquire), }); let generation = state.generation; inflight.insert(key.clone(), Arc::clone(&state)); RoutingGroupInflightRegistration::Leader(RoutingGroupInflightGuard { cache: self, key: Some(key.clone()), state, generation, admission: Some(admission), }) } fn finish_inflight( &self, key: &RoutingGroupCacheKey, state: &Arc, admission: tokio::sync::OwnedSemaphorePermit, completion: RoutingGroupInflightCompletion, ) { let _mutation = self .mutation .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); let removed = { let mut inflight = self .inflight .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); drop(admission); debug_assert!(self.admission.available_permits() > 0); if inflight .get(key) .is_some_and(|current| Arc::ptr_eq(current, state)) { let _ = state.completion.set(completion); inflight.remove(key); true } else { false } }; drop(_mutation); if removed { state.notify.notify_waiters(); } } } #[derive(Debug, Clone, PartialEq, Eq, Hash)] enum RoutingGroupCacheKey { ListGroups, HasAnyBinding, FindById(String), FindByName(String), FindSystemDefault, Bindings { group_id: Option, subject_type: Option<&'static str>, subject_id: Option, }, Versions(String), } #[derive(Debug, Clone)] enum RoutingGroupCacheValue { Groups(Vec), Bool(bool), Group(Option), Bindings(Vec), Versions(Vec), } struct RoutingGroupInflightState { notify: Arc, completion: OnceLock, generation: u64, } impl RoutingGroupInflightState { async fn wait(&self) { loop { if self.completion.get().is_some() { return; } // Register before checking completion a second time. This closes // the completion/notify race even if the follower is not polled // until after the leader has broadcast with notify_waiters(). let mut notified = Box::pin(self.notify.notified()); notified.as_mut().enable(); if self.completion.get().is_some() { return; } notified.await; } } } #[derive(Clone)] enum RoutingGroupInflightCompletion { Loaded(RoutingGroupCacheValue), Failed(SharedDataLayerError), Cancelled, Invalidated, } enum RoutingGroupInflightRegistration<'a> { Leader(RoutingGroupInflightGuard<'a>), Follower(Arc), Saturated, } struct RoutingGroupInflightGuard<'a> { cache: &'a CachedRoutingGroupReadRepository, key: Option, state: Arc, generation: u64, admission: Option, } impl RoutingGroupInflightGuard<'_> { fn finish(&mut self, completion: RoutingGroupInflightCompletion) { if let Some(key) = self.key.take() { let admission = self .admission .take() .expect("active routing group leader must own admission"); self.cache .finish_inflight(&key, &self.state, admission, completion); } } } impl Drop for RoutingGroupInflightGuard<'_> { fn drop(&mut self) { self.finish(RoutingGroupInflightCompletion::Cancelled); } } #[derive(Clone)] enum SharedDataLayerError { InvalidConfiguration(String), InvalidInput(String), Postgres(String), Redis(String), Sql(String), TimedOut(String), UnexpectedValue(String), } impl From<&DataLayerError> for SharedDataLayerError { fn from(error: &DataLayerError) -> Self { match error { DataLayerError::InvalidConfiguration(message) => { Self::InvalidConfiguration(message.clone()) } DataLayerError::InvalidInput(message) => Self::InvalidInput(message.clone()), DataLayerError::Postgres(message) => Self::Postgres(message.clone()), DataLayerError::Redis(message) => Self::Redis(message.clone()), DataLayerError::Sql(message) => Self::Sql(message.clone()), DataLayerError::TimedOut(message) => Self::TimedOut(message.clone()), DataLayerError::UnexpectedValue(message) => Self::UnexpectedValue(message.clone()), } } } impl SharedDataLayerError { fn into_data_layer_error(self) -> DataLayerError { match self { Self::InvalidConfiguration(message) => DataLayerError::InvalidConfiguration(message), Self::InvalidInput(message) => DataLayerError::InvalidInput(message), Self::Postgres(message) => DataLayerError::Postgres(message), Self::Redis(message) => DataLayerError::Redis(message), Self::Sql(message) => DataLayerError::Sql(message), Self::TimedOut(message) => DataLayerError::TimedOut(message), Self::UnexpectedValue(message) => DataLayerError::UnexpectedValue(message), } } } fn lookup_cache_key(lookup: &RoutingGroupLookupKey<'_>) -> RoutingGroupCacheKey { match lookup { RoutingGroupLookupKey::Id(id) => RoutingGroupCacheKey::FindById((*id).to_string()), RoutingGroupLookupKey::Name(name) => RoutingGroupCacheKey::FindByName((*name).to_string()), RoutingGroupLookupKey::SystemDefault => RoutingGroupCacheKey::FindSystemDefault, } } fn subject_cache_key(subject: Option) -> Option<&'static str> { match subject { Some(RoutingGroupBindingSubject::User) => Some("user"), Some(RoutingGroupBindingSubject::ApiKey) => Some("api_key"), Some(RoutingGroupBindingSubject::UserGroup) => Some("user_group"), None => None, } } #[async_trait] impl RoutingGroupReadRepository for CachedRoutingGroupReadRepository { fn clear_local_cache(&self) { self.clear(); } async fn list_routing_groups(&self) -> Result, DataLayerError> { match self .get_or_load(RoutingGroupCacheKey::ListGroups, || async { self.inner .list_routing_groups() .await .map(RoutingGroupCacheValue::Groups) }) .await? { RoutingGroupCacheValue::Groups(groups) => Ok(groups), _ => Ok(Vec::new()), } } async fn find_routing_group( &self, lookup: RoutingGroupLookupKey<'_>, ) -> Result, DataLayerError> { let key = lookup_cache_key(&lookup); let lookup_for_load = lookup.clone(); match self .get_or_load(key, move || { let lookup = lookup_for_load.clone(); async move { self.inner .find_routing_group(lookup) .await .map(RoutingGroupCacheValue::Group) } }) .await? { RoutingGroupCacheValue::Group(group) => Ok(group), _ => Ok(None), } } async fn list_routing_group_bindings( &self, query: &RoutingGroupBindingQuery, ) -> Result, DataLayerError> { let key = RoutingGroupCacheKey::Bindings { group_id: query.group_id.clone(), subject_type: subject_cache_key(query.subject_type), subject_id: query.subject_id.clone(), }; match self .get_or_load(key, || async { self.inner .list_routing_group_bindings(query) .await .map(RoutingGroupCacheValue::Bindings) }) .await? { RoutingGroupCacheValue::Bindings(bindings) => Ok(bindings), _ => Ok(Vec::new()), } } async fn has_any_routing_group_binding(&self) -> Result { match self .get_or_load(RoutingGroupCacheKey::HasAnyBinding, || async { self.inner .has_any_routing_group_binding() .await .map(RoutingGroupCacheValue::Bool) }) .await? { RoutingGroupCacheValue::Bool(value) => Ok(value), _ => Ok(false), } } async fn list_routing_group_versions( &self, group_id: &str, ) -> Result, DataLayerError> { let key = RoutingGroupCacheKey::Versions(group_id.to_string()); match self .get_or_load(key, || async { self.inner .list_routing_group_versions(group_id) .await .map(RoutingGroupCacheValue::Versions) }) .await? { RoutingGroupCacheValue::Versions(versions) => Ok(versions), _ => Ok(Vec::new()), } } } #[cfg(test)] mod tests { use std::sync::atomic::{AtomicUsize, Ordering}; use tokio::sync::{Barrier, Semaphore}; use super::*; #[derive(Default)] struct CountingRoutingGroupReadRepository { list_calls: AtomicUsize, has_any_binding_calls: AtomicUsize, } #[async_trait] impl RoutingGroupReadRepository for CountingRoutingGroupReadRepository { async fn list_routing_groups(&self) -> Result, DataLayerError> { self.list_calls.fetch_add(1, Ordering::AcqRel); Ok(Vec::new()) } async fn find_routing_group( &self, _lookup: RoutingGroupLookupKey<'_>, ) -> Result, DataLayerError> { Ok(None) } async fn list_routing_group_bindings( &self, _query: &RoutingGroupBindingQuery, ) -> Result, DataLayerError> { Ok(Vec::new()) } async fn has_any_routing_group_binding(&self) -> Result { self.has_any_binding_calls.fetch_add(1, Ordering::AcqRel); Ok(false) } async fn list_routing_group_versions( &self, _group_id: &str, ) -> Result, DataLayerError> { Ok(Vec::new()) } } #[derive(Clone, Copy)] enum ControlledHasAnyBindingBehavior { WaitThenSuccess(bool), WaitThenError, FirstWaitFalseThenTrue, FirstPendingThenTrue, } struct ControlledRoutingGroupReadRepository { behavior: ControlledHasAnyBindingBehavior, has_any_binding_calls: AtomicUsize, started: Semaphore, release: Semaphore, } impl ControlledRoutingGroupReadRepository { fn new(behavior: ControlledHasAnyBindingBehavior) -> Self { Self { behavior, has_any_binding_calls: AtomicUsize::new(0), started: Semaphore::new(0), release: Semaphore::new(0), } } fn calls(&self) -> usize { self.has_any_binding_calls.load(Ordering::Acquire) } async fn wait_until_started(&self) { self.started .acquire() .await .expect("started semaphore should remain open") .forget(); } async fn wait_for_release(&self) { self.release .acquire() .await .expect("release semaphore should remain open") .forget(); } } #[async_trait] impl RoutingGroupReadRepository for ControlledRoutingGroupReadRepository { async fn list_routing_groups(&self) -> Result, DataLayerError> { Ok(Vec::new()) } async fn find_routing_group( &self, _lookup: RoutingGroupLookupKey<'_>, ) -> Result, DataLayerError> { Ok(None) } async fn list_routing_group_bindings( &self, _query: &RoutingGroupBindingQuery, ) -> Result, DataLayerError> { Ok(Vec::new()) } async fn has_any_routing_group_binding(&self) -> Result { let call = self.has_any_binding_calls.fetch_add(1, Ordering::AcqRel) + 1; self.started.add_permits(1); match self.behavior { ControlledHasAnyBindingBehavior::WaitThenSuccess(value) => { self.wait_for_release().await; Ok(value) } ControlledHasAnyBindingBehavior::WaitThenError => { self.wait_for_release().await; Err(DataLayerError::TimedOut( "shared routing load error".to_string(), )) } ControlledHasAnyBindingBehavior::FirstWaitFalseThenTrue if call == 1 => { self.wait_for_release().await; Ok(false) } ControlledHasAnyBindingBehavior::FirstPendingThenTrue if call == 1 => { std::future::pending().await } ControlledHasAnyBindingBehavior::FirstWaitFalseThenTrue | ControlledHasAnyBindingBehavior::FirstPendingThenTrue => Ok(true), } } async fn list_routing_group_versions( &self, _group_id: &str, ) -> Result, DataLayerError> { Ok(Vec::new()) } } async fn wait_for_same_key_inflight_participants( repository: &CachedRoutingGroupReadRepository, participants: usize, ) { tokio::time::timeout(Duration::from_secs(1), async { loop { let participant_count = repository .inflight .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .get(&RoutingGroupCacheKey::HasAnyBinding) .map(Arc::strong_count) .unwrap_or_default(); // One Arc is retained by the map and every active request // owns one through its leader guard or follower state. if participant_count > participants { break; } tokio::task::yield_now().await; } }) .await .expect("all same-key requests should join one inflight load"); } #[tokio::test] async fn clear_local_cache_forces_next_load() { let inner = Arc::new(CountingRoutingGroupReadRepository::default()); let repository = CachedRoutingGroupReadRepository::new(inner.clone()); repository .list_routing_groups() .await .expect("initial list should load"); repository .list_routing_groups() .await .expect("cached list should load"); assert_eq!(inner.list_calls.load(Ordering::Acquire), 1); assert!(!repository .has_any_routing_group_binding() .await .expect("initial binding existence should load")); assert!(!repository .has_any_routing_group_binding() .await .expect("cached binding existence should load")); assert_eq!(inner.has_any_binding_calls.load(Ordering::Acquire), 1); repository.clear_local_cache(); repository .list_routing_groups() .await .expect("cleared list should reload"); assert_eq!(inner.list_calls.load(Ordering::Acquire), 2); assert!(!repository .has_any_routing_group_binding() .await .expect("cleared binding existence should reload")); assert_eq!(inner.has_any_binding_calls.load(Ordering::Acquire), 2); } #[tokio::test] async fn follower_does_not_miss_completion_before_first_poll() { let inner = Arc::new(CountingRoutingGroupReadRepository::default()); let repository = CachedRoutingGroupReadRepository::new(inner); let key = RoutingGroupCacheKey::HasAnyBinding; let mut leader = match repository.register_inflight(&key) { RoutingGroupInflightRegistration::Leader(leader) => leader, _ => panic!("first registration should lead"), }; let follower = match repository.register_inflight(&key) { RoutingGroupInflightRegistration::Follower(state) => state, _ => panic!("second registration should follow"), }; // Complete before the follower wait future is created or polled. A // bare notify_waiters().await sequence would sleep forever here. leader.finish(RoutingGroupInflightCompletion::Loaded( RoutingGroupCacheValue::Bool(true), )); tokio::time::timeout(Duration::from_millis(100), follower.wait()) .await .expect("completed follower should not miss the broadcast"); assert!(matches!( repository.follower_completion(&follower), Some(RoutingGroupInflightCompletion::Loaded( RoutingGroupCacheValue::Bool(true) )) )); } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn concurrent_same_key_requests_share_one_completed_load() { const TASKS: usize = 64; let inner = Arc::new(ControlledRoutingGroupReadRepository::new( ControlledHasAnyBindingBehavior::WaitThenSuccess(true), )); let repository = Arc::new(CachedRoutingGroupReadRepository::new(inner.clone())); let barrier = Arc::new(Barrier::new(TASKS + 1)); let mut tasks = Vec::with_capacity(TASKS); for _ in 0..TASKS { let repository = Arc::clone(&repository); let barrier = Arc::clone(&barrier); tasks.push(tokio::spawn(async move { barrier.wait().await; repository.has_any_routing_group_binding().await })); } barrier.wait().await; inner.wait_until_started().await; wait_for_same_key_inflight_participants(&repository, TASKS).await; inner.release.add_permits(TASKS); for task in tasks { assert!(task .await .expect("request task should finish") .expect("shared load should succeed")); } assert_eq!(inner.calls(), 1); assert!(repository .inflight .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .is_empty()); } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn concurrent_same_key_followers_share_leader_error() { const TASKS: usize = 32; let inner = Arc::new(ControlledRoutingGroupReadRepository::new( ControlledHasAnyBindingBehavior::WaitThenError, )); let repository = Arc::new(CachedRoutingGroupReadRepository::new(inner.clone())); let barrier = Arc::new(Barrier::new(TASKS + 1)); let mut tasks = Vec::with_capacity(TASKS); for _ in 0..TASKS { let repository = Arc::clone(&repository); let barrier = Arc::clone(&barrier); tasks.push(tokio::spawn(async move { barrier.wait().await; repository.has_any_routing_group_binding().await })); } barrier.wait().await; inner.wait_until_started().await; wait_for_same_key_inflight_participants(&repository, TASKS).await; inner.release.add_permits(TASKS); for task in tasks { let error = task .await .expect("request task should finish") .expect_err("shared load should fail"); assert!(matches!( error, DataLayerError::TimedOut(message) if message == "shared routing load error" )); } assert_eq!(inner.calls(), 1); } #[tokio::test] async fn cancelled_leader_wakes_follower_for_retry() { let inner = Arc::new(ControlledRoutingGroupReadRepository::new( ControlledHasAnyBindingBehavior::FirstPendingThenTrue, )); let repository = Arc::new(CachedRoutingGroupReadRepository::new(inner.clone())); let leader_repository = Arc::clone(&repository); let leader = tokio::spawn(async move { leader_repository.has_any_routing_group_binding().await }); inner.wait_until_started().await; let follower_repository = Arc::clone(&repository); let follower = tokio::spawn(async move { follower_repository.has_any_routing_group_binding().await }); wait_for_same_key_inflight_participants(&repository, 2).await; leader.abort(); let _ = leader.await; assert!(tokio::time::timeout(Duration::from_secs(1), follower) .await .expect("follower should not remain stuck after leader cancellation") .expect("follower task should finish") .expect("retried load should succeed")); assert_eq!(inner.calls(), 2); } #[tokio::test] async fn clear_wakes_followers_and_rejects_old_generation_publication() { let inner = Arc::new(ControlledRoutingGroupReadRepository::new( ControlledHasAnyBindingBehavior::FirstWaitFalseThenTrue, )); let repository = Arc::new(CachedRoutingGroupReadRepository::new(inner.clone())); let leader_repository = Arc::clone(&repository); let leader = tokio::spawn(async move { leader_repository.has_any_routing_group_binding().await }); inner.wait_until_started().await; let follower_repository = Arc::clone(&repository); let follower = tokio::spawn(async move { follower_repository.has_any_routing_group_binding().await }); wait_for_same_key_inflight_participants(&repository, 2).await; repository.clear_local_cache(); assert!(tokio::time::timeout(Duration::from_secs(1), follower) .await .expect("clear should wake the follower") .expect("follower task should finish") .expect("new-generation load should succeed")); assert_eq!(inner.calls(), 2); inner.release.add_permits(1); assert!(!leader .await .expect("old leader task should finish") .expect("old leader load should succeed")); assert!(repository .has_any_routing_group_binding() .await .expect("new-generation value should remain cached")); assert_eq!(inner.calls(), 2); } #[test] fn capacity_full_cancelled_follower_can_retry_after_repeated_clear() { let inner = Arc::new(CountingRoutingGroupReadRepository::default()); let repository = CachedRoutingGroupReadRepository::new(inner); let key = RoutingGroupCacheKey::HasAnyBinding; let mut active = Vec::with_capacity(ROUTING_GROUP_CACHE_MAX_INFLIGHT); for _ in 0..ROUTING_GROUP_CACHE_MAX_INFLIGHT - 1 { let leader = match repository.register_inflight(&key) { RoutingGroupInflightRegistration::Leader(guard) => guard, _ => panic!("each available permit should admit one leader"), }; active.push(leader); repository.clear(); } let current = match repository.register_inflight(&key) { RoutingGroupInflightRegistration::Leader(guard) => guard, _ => panic!("the final available permit should admit a leader"), }; let follower = match repository.register_inflight(&key) { RoutingGroupInflightRegistration::Follower(state) => state, _ => panic!("the same-key request should follow at full capacity"), }; assert_eq!(repository.admission.available_permits(), 0); assert!(matches!( repository.register_inflight(&RoutingGroupCacheKey::ListGroups), RoutingGroupInflightRegistration::Saturated )); drop(current); assert!(matches!( repository.follower_completion(&follower), Some(RoutingGroupInflightCompletion::Cancelled) )); let mut replacement = match repository.register_inflight(&key) { RoutingGroupInflightRegistration::Leader(guard) => guard, _ => panic!("cancelled follower retry should use the released permit"), }; replacement.finish(RoutingGroupInflightCompletion::Cancelled); assert_eq!(repository.admission.available_permits(), 1); } }