refactor(workspace): enforce layered crate boundaries

2026-09-12 22:20:19 +08:00 · 2026-07-15 23:47:19 +08:00
parent a728c090a9
commit 8616fe6ee2
969 changed files with 40187 additions and 27240 deletions
@@ -0,0 +1,292 @@
+use async_stream::stream;
+use axum::body::Body;
+use axum::http::Response;
+use futures_util::StreamExt;
+use std::time::Duration;
+
+use crate::concurrency::ConcurrencyPermit;
+
+const ADMISSION_HEALTH_POLL_INTERVAL: Duration = Duration::from_secs(1);
+
+pub trait AdmissionPermitHealth: Send + Sync {
+    fn is_healthy(&self) -> bool;
+}
+
+impl AdmissionPermitHealth for ConcurrencyPermit {
+    fn is_healthy(&self) -> bool {
+        true
+    }
+}
+
+pub struct AdmissionPermit {
+    _local: Option<ConcurrencyPermit>,
+    _distributed: Option<Box<dyn AdmissionPermitHealth>>,
+}
+
+impl std::fmt::Debug for AdmissionPermit {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("AdmissionPermit")
+            .field("has_local", &self._local.is_some())
+            .field("has_distributed", &self._distributed.is_some())
+            .finish()
+    }
+}
+
+impl AdmissionPermit {
+    pub fn from_parts<D: AdmissionPermitHealth + 'static>(
+        local: Option<ConcurrencyPermit>,
+        distributed: Option<D>,
+    ) -> Option<Self> {
+        if local.is_none() && distributed.is_none() {
+            None
+        } else {
+            Some(Self {
+                _local: local,
+                _distributed: distributed
+                    .map(|permit| Box::new(permit) as Box<dyn AdmissionPermitHealth>),
+            })
+        }
+    }
+
+    pub fn is_healthy(&self) -> bool {
+        self._distributed
+            .as_ref()
+            .map(|permit| permit.is_healthy())
+            .unwrap_or(true)
+    }
+
+    fn requires_health_poll(&self) -> bool {
+        self._distributed.is_some()
+    }
+}
+
+impl From<ConcurrencyPermit> for AdmissionPermit {
+    fn from(value: ConcurrencyPermit) -> Self {
+        Self {
+            _local: Some(value),
+            _distributed: None,
+        }
+    }
+}
+
+pub fn maybe_hold_axum_response_permit(
+    response: Response<Body>,
+    permit: Option<AdmissionPermit>,
+) -> Response<Body> {
+    match permit {
+        Some(permit) => hold_axum_response_permit(response, permit),
+        None => response,
+    }
+}
+
+pub async fn hold_admission_permit_until<F>(permit: Option<AdmissionPermit>, future: F)
+where
+    F: std::future::Future<Output = ()>,
+{
+    hold_admission_permit_until_with_interval(permit, future, ADMISSION_HEALTH_POLL_INTERVAL).await;
+}
+
+fn hold_axum_response_permit(response: Response<Body>, permit: AdmissionPermit) -> Response<Body> {
+    hold_axum_response_permit_with_interval(response, permit, ADMISSION_HEALTH_POLL_INTERVAL)
+}
+
+async fn hold_admission_permit_until_with_interval<F>(
+    permit: Option<AdmissionPermit>,
+    future: F,
+    health_poll_interval: Duration,
+) where
+    F: std::future::Future<Output = ()>,
+{
+    let Some(permit) = permit else {
+        future.await;
+        return;
+    };
+    if !permit.requires_health_poll() {
+        let _permit = permit;
+        future.await;
+        return;
+    }
+
+    tokio::pin!(future);
+    let mut health = tokio::time::interval(health_poll_interval);
+    health.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
+    health.tick().await;
+    loop {
+        if !permit.is_healthy() {
+            break;
+        }
+        tokio::select! {
+            _ = &mut future => break,
+            _ = health.tick() => {
+                if !permit.is_healthy() {
+                    break;
+                }
+            }
+        }
+    }
+}
+
+fn hold_axum_response_permit_with_interval(
+    response: Response<Body>,
+    permit: AdmissionPermit,
+    health_poll_interval: Duration,
+) -> Response<Body> {
+    if !permit.requires_health_poll() {
+        let (parts, body) = response.into_parts();
+        let stream = stream! {
+            let _permit = permit;
+            let mut body_stream = body.into_data_stream();
+            while let Some(item) = body_stream.next().await {
+                yield item;
+            }
+        };
+        return Response::from_parts(parts, Body::from_stream(stream));
+    }
+
+    let (parts, body) = response.into_parts();
+    let stream = stream! {
+        let _permit = permit;
+        let mut body_stream = body.into_data_stream();
+        let mut health = tokio::time::interval(health_poll_interval);
+        health.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
+        health.tick().await;
+        loop {
+            if !_permit.is_healthy() {
+                break;
+            }
+            tokio::select! {
+                item = body_stream.next() => match item {
+                    Some(item) if _permit.is_healthy() => yield item,
+                    Some(_) | None => break,
+                },
+                _ = health.tick() => {
+                    if !_permit.is_healthy() {
+                        break;
+                    }
+                }
+            }
+        }
+    };
+    Response::from_parts(parts, Body::from_stream(stream))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{
+        hold_admission_permit_until, hold_admission_permit_until_with_interval,
+        hold_axum_response_permit_with_interval, maybe_hold_axum_response_permit, AdmissionPermit,
+        AdmissionPermitHealth,
+    };
+    use crate::ConcurrencyGate;
+    use axum::body::{to_bytes, Body};
+    use axum::http::Response;
+    use std::sync::atomic::{AtomicBool, Ordering};
+    use std::sync::Arc;
+
+    struct TestPermitHealth(Arc<AtomicBool>);
+
+    impl AdmissionPermitHealth for TestPermitHealth {
+        fn is_healthy(&self) -> bool {
+            self.0.load(Ordering::Acquire)
+        }
+    }
+
+    #[tokio::test]
+    async fn holds_permit_until_response_body_is_consumed() {
+        let gate = ConcurrencyGate::new("test", 1);
+        let permit = gate.try_acquire().expect("first permit");
+        let response = Response::new(Body::from_stream(
+            async_stream::stream! { yield Ok::<_, std::convert::Infallible>(axum::body::Bytes::from_static(b"ok")); },
+        ));
+
+        let wrapped = maybe_hold_axum_response_permit(response, Some(permit.into()));
+        assert_eq!(gate.snapshot().in_flight, 1);
+        assert!(gate.try_acquire().is_err(), "permit should still be held");
+
+        let body = to_bytes(wrapped.into_body(), usize::MAX)
+            .await
+            .expect("body should drain");
+        assert_eq!(body.as_ref(), b"ok");
+        assert_eq!(gate.snapshot().in_flight, 0);
+    }
+
+    #[tokio::test]
+    async fn holds_combined_local_and_distributed_permit_until_future_finishes() {
+        let local_gate = ConcurrencyGate::new("local", 1);
+        let local = local_gate.try_acquire().expect("local permit");
+        let distributed_gate = ConcurrencyGate::new("distributed", 1);
+        let distributed = distributed_gate.try_acquire().expect("distributed permit");
+
+        let task = tokio::spawn(hold_admission_permit_until(
+            AdmissionPermit::from_parts(Some(local), Some(distributed)),
+            async {
+                tokio::time::sleep(std::time::Duration::from_millis(50)).await;
+            },
+        ));
+
+        tokio::time::sleep(std::time::Duration::from_millis(5)).await;
+        assert!(
+            local_gate.try_acquire().is_err(),
+            "local permit should still be held"
+        );
+        assert!(
+            distributed_gate.try_acquire().is_err(),
+            "distributed permit should still be held"
+        );
+
+        task.await.expect("task should complete");
+        assert_eq!(local_gate.snapshot().in_flight, 0);
+        assert_eq!(distributed_gate.snapshot().in_flight, 0);
+    }
+
+    #[tokio::test]
+    async fn unhealthy_distributed_permit_cancels_held_future() {
+        let local_gate = ConcurrencyGate::new("local", 1);
+        let local = local_gate.try_acquire().expect("local permit");
+        let healthy = Arc::new(AtomicBool::new(true));
+        let permit =
+            AdmissionPermit::from_parts(Some(local), Some(TestPermitHealth(Arc::clone(&healthy))));
+
+        let task = tokio::spawn(hold_admission_permit_until_with_interval(
+            permit,
+            std::future::pending(),
+            std::time::Duration::from_millis(5),
+        ));
+        healthy.store(false, Ordering::Release);
+
+        tokio::time::timeout(std::time::Duration::from_millis(100), task)
+            .await
+            .expect("unhealthy permit should cancel the held future")
+            .expect("held future task should not panic");
+        assert_eq!(local_gate.snapshot().in_flight, 0);
+    }
+
+    #[tokio::test]
+    async fn unhealthy_distributed_permit_ends_idle_response_body() {
+        let local_gate = ConcurrencyGate::new("local", 1);
+        let local = local_gate.try_acquire().expect("local permit");
+        let healthy = Arc::new(AtomicBool::new(true));
+        let permit =
+            AdmissionPermit::from_parts(Some(local), Some(TestPermitHealth(Arc::clone(&healthy))))
+                .expect("combined permit");
+        let response = Response::new(Body::from_stream(futures_util::stream::pending::<
+            Result<axum::body::Bytes, std::convert::Infallible>,
+        >()));
+        let wrapped = hold_axum_response_permit_with_interval(
+            response,
+            permit,
+            std::time::Duration::from_millis(5),
+        );
+
+        healthy.store(false, Ordering::Release);
+        let body = tokio::time::timeout(
+            std::time::Duration::from_millis(100),
+            to_bytes(wrapped.into_body(), usize::MAX),
+        )
+        .await
+        .expect("unhealthy permit should end an idle body")
+        .expect("body collection should succeed");
+        assert!(body.is_empty());
+        assert_eq!(local_gate.snapshot().in_flight, 0);
+    }
+}
@@ -0,0 +1,8 @@
+use crate::config::ServiceRuntimeConfig;
+use crate::error::RuntimeBootstrapError;
+
+pub fn init_service_runtime(config: ServiceRuntimeConfig) -> Result<(), RuntimeBootstrapError> {
+    crate::tracing::init_tracing(config.clone())?;
+    crate::metrics::init_metrics(config);
+    Ok(())
+}
@@ -0,0 +1,200 @@
+use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
+use std::sync::Arc;
+
+use tokio::sync::{OwnedSemaphorePermit, Semaphore};
+
+use crate::metrics::{MetricKind, MetricLabel, MetricSample};
+
+#[derive(Debug, thiserror::Error, PartialEq, Eq)]
+pub enum ConcurrencyError {
+    #[error("concurrency gate {gate} is saturated at {limit}")]
+    Saturated { gate: &'static str, limit: usize },
+    #[error("concurrency gate {gate} is closed")]
+    Closed { gate: &'static str },
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct ConcurrencySnapshot {
+    pub limit: usize,
+    pub in_flight: usize,
+    pub available_permits: usize,
+    pub high_watermark: usize,
+    pub rejected: u64,
+}
+
+impl ConcurrencySnapshot {
+    pub fn to_metric_samples(&self, gate: &'static str) -> Vec<MetricSample> {
+        let labels = vec![MetricLabel::new("gate", gate)];
+        vec![
+            MetricSample::new(
+                "concurrency_in_flight",
+                "Current number of in-flight operations guarded by the concurrency gate.",
+                MetricKind::Gauge,
+                self.in_flight as u64,
+            )
+            .with_labels(labels.clone()),
+            MetricSample::new(
+                "concurrency_available_permits",
+                "Currently available permits for the concurrency gate.",
+                MetricKind::Gauge,
+                self.available_permits as u64,
+            )
+            .with_labels(labels.clone()),
+            MetricSample::new(
+                "concurrency_high_watermark",
+                "Highest observed in-flight count for the concurrency gate.",
+                MetricKind::Gauge,
+                self.high_watermark as u64,
+            )
+            .with_labels(labels.clone()),
+            MetricSample::new(
+                "concurrency_rejected_total",
+                "Number of operations rejected by the concurrency gate.",
+                MetricKind::Counter,
+                self.rejected,
+            )
+            .with_labels(labels),
+        ]
+    }
+}
+
+#[derive(Debug)]
+struct ConcurrencyState {
+    gate: &'static str,
+    limit: usize,
+    semaphore: Arc<Semaphore>,
+    in_flight: AtomicUsize,
+    high_watermark: AtomicUsize,
+    rejected: AtomicU64,
+}
+
+#[derive(Debug, Clone)]
+pub struct ConcurrencyGate {
+    state: Arc<ConcurrencyState>,
+}
+
+impl ConcurrencyGate {
+    pub fn new(gate: &'static str, limit: usize) -> Self {
+        assert!(limit > 0, "concurrency gate limit must be positive");
+        Self {
+            state: Arc::new(ConcurrencyState {
+                gate,
+                limit,
+                semaphore: Arc::new(Semaphore::new(limit)),
+                in_flight: AtomicUsize::new(0),
+                high_watermark: AtomicUsize::new(0),
+                rejected: AtomicU64::new(0),
+            }),
+        }
+    }
+
+    pub async fn acquire(&self) -> Result<ConcurrencyPermit, ConcurrencyError> {
+        let permit = self
+            .state
+            .semaphore
+            .clone()
+            .acquire_owned()
+            .await
+            .map_err(|_| ConcurrencyError::Closed {
+                gate: self.state.gate,
+            })?;
+        Ok(ConcurrencyPermit::new(self.state.clone(), permit))
+    }
+
+    pub fn try_acquire(&self) -> Result<ConcurrencyPermit, ConcurrencyError> {
+        match self.state.semaphore.clone().try_acquire_owned() {
+            Ok(permit) => Ok(ConcurrencyPermit::new(self.state.clone(), permit)),
+            Err(tokio::sync::TryAcquireError::NoPermits) => {
+                self.state.rejected.fetch_add(1, Ordering::Relaxed);
+                Err(ConcurrencyError::Saturated {
+                    gate: self.state.gate,
+                    limit: self.state.limit,
+                })
+            }
+            Err(tokio::sync::TryAcquireError::Closed) => Err(ConcurrencyError::Closed {
+                gate: self.state.gate,
+            }),
+        }
+    }
+
+    pub fn snapshot(&self) -> ConcurrencySnapshot {
+        ConcurrencySnapshot {
+            limit: self.state.limit,
+            in_flight: self.state.in_flight.load(Ordering::Relaxed),
+            available_permits: self.state.semaphore.available_permits(),
+            high_watermark: self.state.high_watermark.load(Ordering::Relaxed),
+            rejected: self.state.rejected.load(Ordering::Relaxed),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct ConcurrencyPermit {
+    state: Arc<ConcurrencyState>,
+    _permit: OwnedSemaphorePermit,
+}
+
+impl ConcurrencyPermit {
+    fn new(state: Arc<ConcurrencyState>, permit: OwnedSemaphorePermit) -> Self {
+        let in_flight = state.in_flight.fetch_add(1, Ordering::AcqRel) + 1;
+        let mut observed = state.high_watermark.load(Ordering::Acquire);
+        while in_flight > observed {
+            match state.high_watermark.compare_exchange_weak(
+                observed,
+                in_flight,
+                Ordering::AcqRel,
+                Ordering::Acquire,
+            ) {
+                Ok(_) => break,
+                Err(next) => observed = next,
+            }
+        }
+        Self {
+            state,
+            _permit: permit,
+        }
+    }
+}
+
+impl Drop for ConcurrencyPermit {
+    fn drop(&mut self) {
+        self.state.in_flight.fetch_sub(1, Ordering::AcqRel);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{ConcurrencyError, ConcurrencyGate};
+
+    #[tokio::test]
+    async fn tracks_in_flight_and_high_watermark() {
+        let gate = ConcurrencyGate::new("test", 2);
+
+        let permit_a = gate.acquire().await.expect("permit a");
+        let permit_b = gate.acquire().await.expect("permit b");
+        let snapshot = gate.snapshot();
+
+        assert_eq!(snapshot.in_flight, 2);
+        assert_eq!(snapshot.high_watermark, 2);
+        assert_eq!(snapshot.available_permits, 0);
+
+        drop((permit_a, permit_b));
+        assert_eq!(gate.snapshot().in_flight, 0);
+    }
+
+    #[test]
+    fn rejects_when_saturated() {
+        let gate = ConcurrencyGate::new("test", 1);
+        let _permit = gate.try_acquire().expect("first permit");
+
+        let error = gate.try_acquire().expect_err("second permit should fail");
+        assert_eq!(
+            error,
+            ConcurrencyError::Saturated {
+                gate: "test",
+                limit: 1,
+            }
+        );
+        assert_eq!(gate.snapshot().rejected, 1);
+    }
+}
@@ -0,0 +1,48 @@
+use crate::observability::{FileLoggingConfig, LogDestination, ServiceObservabilityConfig};
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ServiceRuntimeConfig {
+    pub service_name: &'static str,
+    pub default_log_filter: &'static str,
+    pub observability: ServiceObservabilityConfig,
+}
+
+impl ServiceRuntimeConfig {
+    pub const fn new(service_name: &'static str, default_log_filter: &'static str) -> Self {
+        Self {
+            service_name,
+            default_log_filter,
+            observability: ServiceObservabilityConfig::new(crate::LogFormat::Pretty, service_name),
+        }
+    }
+
+    pub const fn with_log_format(mut self, log_format: crate::LogFormat) -> Self {
+        self.observability.log_format = log_format;
+        self
+    }
+
+    pub const fn with_log_destination(mut self, log_destination: LogDestination) -> Self {
+        self.observability.log_destination = log_destination;
+        self
+    }
+
+    pub fn with_file_logging(mut self, file_logging: FileLoggingConfig) -> Self {
+        self.observability.file_logging = Some(file_logging);
+        self
+    }
+
+    pub fn with_node_role(mut self, node_role: impl Into<String>) -> Self {
+        self.observability.node_role = Some(node_role.into());
+        self
+    }
+
+    pub fn with_instance_id(mut self, instance_id: impl Into<String>) -> Self {
+        self.observability.instance_id = Some(instance_id.into());
+        self
+    }
+
+    pub const fn with_metrics_namespace(mut self, metrics_namespace: &'static str) -> Self {
+        self.observability.metrics_namespace = metrics_namespace;
+        self
+    }
+}
@@ -0,0 +1,171 @@
+use std::sync::Arc;
+
+use crate::concurrency::{ConcurrencyGate, ConcurrencyPermit};
+use crate::metrics::{MetricKind, MetricLabel, MetricSample};
+
+#[derive(Debug, thiserror::Error, PartialEq, Eq)]
+pub enum DistributedConcurrencyError {
+    #[error("distributed concurrency gate {gate} is saturated at {limit}")]
+    Saturated { gate: &'static str, limit: usize },
+    #[error("distributed concurrency gate {gate} is unavailable: {message}")]
+    Unavailable {
+        gate: &'static str,
+        limit: usize,
+        message: String,
+    },
+    #[error("{0}")]
+    InvalidConfiguration(String),
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct DistributedConcurrencySnapshot {
+    pub limit: usize,
+    pub in_flight: usize,
+    pub available_permits: usize,
+    pub high_watermark: usize,
+    pub rejected: u64,
+}
+
+impl DistributedConcurrencySnapshot {
+    pub fn to_metric_samples(&self, gate: &'static str) -> Vec<MetricSample> {
+        let labels = vec![MetricLabel::new("gate", gate)];
+        vec![
+            MetricSample::new(
+                "concurrency_in_flight",
+                "Current number of in-flight operations guarded by the concurrency gate.",
+                MetricKind::Gauge,
+                self.in_flight as u64,
+            )
+            .with_labels(labels.clone()),
+            MetricSample::new(
+                "concurrency_available_permits",
+                "Currently available permits for the concurrency gate.",
+                MetricKind::Gauge,
+                self.available_permits as u64,
+            )
+            .with_labels(labels.clone()),
+            MetricSample::new(
+                "concurrency_high_watermark",
+                "Highest observed in-flight count for the concurrency gate.",
+                MetricKind::Gauge,
+                self.high_watermark as u64,
+            )
+            .with_labels(labels.clone()),
+            MetricSample::new(
+                "concurrency_rejected_total",
+                "Number of operations rejected by the concurrency gate.",
+                MetricKind::Counter,
+                self.rejected,
+            )
+            .with_labels(labels),
+        ]
+    }
+}
+
+#[derive(Debug)]
+struct DistributedConcurrencyState {
+    gate: &'static str,
+    limit: usize,
+    gate_impl: Arc<ConcurrencyGate>,
+}
+
+#[derive(Debug, Clone)]
+pub struct DistributedConcurrencyGate {
+    state: Arc<DistributedConcurrencyState>,
+}
+
+impl DistributedConcurrencyGate {
+    pub fn new_in_memory(gate: &'static str, limit: usize) -> Self {
+        assert!(
+            limit > 0,
+            "distributed concurrency gate limit must be positive"
+        );
+        Self {
+            state: Arc::new(DistributedConcurrencyState {
+                gate,
+                limit,
+                gate_impl: Arc::new(ConcurrencyGate::new(gate, limit)),
+            }),
+        }
+    }
+
+    pub fn gate(&self) -> &'static str {
+        self.state.gate
+    }
+
+    pub fn limit(&self) -> usize {
+        self.state.limit
+    }
+
+    pub async fn try_acquire(
+        &self,
+    ) -> Result<DistributedConcurrencyPermit, DistributedConcurrencyError> {
+        self.state
+            .gate_impl
+            .try_acquire()
+            .map(|permit| DistributedConcurrencyPermit { _permit: permit })
+            .map_err(|err| match err {
+                crate::ConcurrencyError::Saturated { gate, limit } => {
+                    DistributedConcurrencyError::Saturated { gate, limit }
+                }
+                crate::ConcurrencyError::Closed { gate } => {
+                    DistributedConcurrencyError::Unavailable {
+                        gate,
+                        limit: self.state.limit,
+                        message: "in-memory distributed concurrency gate is closed".to_string(),
+                    }
+                }
+            })
+    }
+
+    pub async fn snapshot(
+        &self,
+    ) -> Result<DistributedConcurrencySnapshot, DistributedConcurrencyError> {
+        let snapshot = self.state.gate_impl.snapshot();
+        Ok(DistributedConcurrencySnapshot {
+            limit: snapshot.limit,
+            in_flight: snapshot.in_flight,
+            available_permits: snapshot.available_permits,
+            high_watermark: snapshot.high_watermark,
+            rejected: snapshot.rejected,
+        })
+    }
+}
+
+#[derive(Debug)]
+pub struct DistributedConcurrencyPermit {
+    _permit: ConcurrencyPermit,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{DistributedConcurrencyError, DistributedConcurrencyGate};
+
+    #[tokio::test]
+    async fn shared_in_memory_gate_rejects_second_acquire() {
+        let gate = DistributedConcurrencyGate::new_in_memory("shared", 1);
+        let permit = gate.try_acquire().await.expect("first permit");
+
+        let error = gate
+            .try_acquire()
+            .await
+            .expect_err("second permit should fail");
+        assert_eq!(
+            error,
+            DistributedConcurrencyError::Saturated {
+                gate: "shared",
+                limit: 1,
+            }
+        );
+
+        let snapshot = gate.snapshot().await.expect("snapshot should build");
+        assert_eq!(snapshot.in_flight, 1);
+        assert_eq!(snapshot.available_permits, 0);
+        assert_eq!(snapshot.high_watermark, 1);
+        assert_eq!(snapshot.rejected, 1);
+
+        drop(permit);
+        let snapshot = gate.snapshot().await.expect("snapshot should build");
+        assert_eq!(snapshot.in_flight, 0);
+    }
+}
@@ -0,0 +1,5 @@
+#[derive(Debug, thiserror::Error)]
+pub enum RuntimeBootstrapError {
+    #[error("failed to initialize tracing: {0}")]
+    Tracing(String),
+}
@@ -0,0 +1,38 @@
+pub mod admission;
+mod bootstrap;
+pub mod concurrency;
+mod config;
+pub mod distributed;
+mod error;
+pub mod metrics;
+mod observability;
+pub mod queue;
+pub mod redaction;
+pub mod shutdown;
+pub mod task;
+mod tracing;
+
+pub use admission::{
+    hold_admission_permit_until, maybe_hold_axum_response_permit, AdmissionPermit,
+    AdmissionPermitHealth,
+};
+pub use bootstrap::init_service_runtime;
+pub use concurrency::{ConcurrencyError, ConcurrencyGate, ConcurrencyPermit, ConcurrencySnapshot};
+pub use config::ServiceRuntimeConfig;
+pub use distributed::{
+    DistributedConcurrencyError, DistributedConcurrencyGate, DistributedConcurrencyPermit,
+    DistributedConcurrencySnapshot,
+};
+pub use error::RuntimeBootstrapError;
+pub use metrics::{prometheus_response, service_up_sample, MetricKind, MetricLabel, MetricSample};
+pub use observability::{
+    FileLoggingConfig, LogDestination, LogRotation, ServiceObservabilityConfig,
+};
+pub use queue::{
+    bounded_queue, BoundedQueueReceiver, BoundedQueueSender, QueueSendError, QueueSnapshot,
+};
+pub use redaction::{summarize_text_payload, TextPayloadSummary};
+pub use shutdown::wait_for_shutdown_signal;
+pub use tracing::{
+    init_reloadable_service_tracing, init_reloadable_tracing, LogFormat, LogReloader,
+};
@@ -0,0 +1,186 @@
+use crate::config::ServiceRuntimeConfig;
+use axum::body::Body;
+use axum::http::header::{HeaderValue, CONTENT_TYPE};
+use axum::http::Response;
+
+static METRICS_NAMESPACE: std::sync::OnceLock<&'static str> = std::sync::OnceLock::new();
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum MetricKind {
+    Counter,
+    Gauge,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct MetricLabel {
+    pub key: &'static str,
+    pub value: String,
+}
+
+impl MetricLabel {
+    pub fn new(key: &'static str, value: impl Into<String>) -> Self {
+        Self {
+            key,
+            value: value.into(),
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct MetricSample {
+    pub name: &'static str,
+    pub help: &'static str,
+    pub kind: MetricKind,
+    pub value: u64,
+    pub labels: Vec<MetricLabel>,
+}
+
+impl MetricSample {
+    pub fn new(name: &'static str, help: &'static str, kind: MetricKind, value: u64) -> Self {
+        Self {
+            name,
+            help,
+            kind,
+            value,
+            labels: Vec::new(),
+        }
+    }
+
+    pub fn with_labels(mut self, labels: Vec<MetricLabel>) -> Self {
+        self.labels = labels;
+        self
+    }
+}
+
+pub fn init_metrics(config: ServiceRuntimeConfig) {
+    let _ = METRICS_NAMESPACE.set(config.observability.metrics_namespace);
+}
+
+pub fn metrics_namespace() -> Option<&'static str> {
+    METRICS_NAMESPACE.get().copied()
+}
+
+pub fn render_prometheus_text(samples: &[MetricSample]) -> String {
+    let mut body = String::new();
+    let namespace = metrics_namespace();
+
+    for sample in samples {
+        let metric_name = format_metric_name(namespace, sample.name);
+        body.push_str(&format!("# HELP {} {}\n", metric_name, sample.help));
+        body.push_str(&format!(
+            "# TYPE {} {}\n",
+            metric_name,
+            match sample.kind {
+                MetricKind::Counter => "counter",
+                MetricKind::Gauge => "gauge",
+            }
+        ));
+        body.push_str(&metric_name);
+        if !sample.labels.is_empty() {
+            body.push('{');
+            for (index, label) in sample.labels.iter().enumerate() {
+                if index > 0 {
+                    body.push(',');
+                }
+                body.push_str(label.key);
+                body.push_str("=\"");
+                body.push_str(&escape_prometheus_label(&label.value));
+                body.push('"');
+            }
+            body.push('}');
+        }
+        body.push(' ');
+        body.push_str(&sample.value.to_string());
+        body.push('\n');
+    }
+
+    body
+}
+
+pub fn prometheus_response(samples: &[MetricSample]) -> Response<Body> {
+    let mut response = Response::new(Body::from(render_prometheus_text(samples)));
+    response.headers_mut().insert(
+        CONTENT_TYPE,
+        HeaderValue::from_static("text/plain; version=0.0.4; charset=utf-8"),
+    );
+    response
+}
+
+pub fn service_up_sample(service: &'static str) -> MetricSample {
+    MetricSample::new(
+        "service_up",
+        "Whether the service process is currently up.",
+        MetricKind::Gauge,
+        1,
+    )
+    .with_labels(vec![MetricLabel::new("service", service)])
+}
+
+fn format_metric_name(namespace: Option<&str>, name: &str) -> String {
+    match namespace {
+        Some(namespace) if !namespace.is_empty() => format!("{}_{}", namespace, name),
+        _ => name.to_string(),
+    }
+}
+
+fn escape_prometheus_label(value: &str) -> String {
+    value
+        .replace('\\', "\\\\")
+        .replace('\n', "\\n")
+        .replace('"', "\\\"")
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{
+        prometheus_response, render_prometheus_text, service_up_sample, MetricKind, MetricLabel,
+        MetricSample,
+    };
+    use axum::body::to_bytes;
+
+    #[test]
+    fn renders_prometheus_samples_with_labels() {
+        let text = render_prometheus_text(&[MetricSample::new(
+            "queue_depth",
+            "Current queue depth",
+            MetricKind::Gauge,
+            3,
+        )
+        .with_labels(vec![MetricLabel::new("queue", "proxy_writer")])]);
+
+        assert!(text.contains("# HELP queue_depth Current queue depth"));
+        assert!(text.contains("# TYPE queue_depth gauge"));
+        assert!(text.contains("queue_depth{queue=\"proxy_writer\"} 3"));
+    }
+
+    #[test]
+    fn escapes_prometheus_labels() {
+        let text = render_prometheus_text(&[MetricSample::new(
+            "errors_total",
+            "Errors",
+            MetricKind::Counter,
+            1,
+        )
+        .with_labels(vec![MetricLabel::new("message", "bad\"line\nx")])]);
+
+        assert!(text.contains("message=\"bad\\\"line\\nx\""));
+    }
+
+    #[tokio::test]
+    async fn builds_prometheus_http_response() {
+        let response = prometheus_response(&[service_up_sample("gateway")]);
+        let content_type = response
+            .headers()
+            .get(axum::http::header::CONTENT_TYPE)
+            .and_then(|value| value.to_str().ok());
+        assert_eq!(
+            content_type,
+            Some("text/plain; version=0.0.4; charset=utf-8")
+        );
+        let body = to_bytes(response.into_body(), usize::MAX)
+            .await
+            .expect("body should read");
+        let text = String::from_utf8(body.to_vec()).expect("body should be utf8");
+        assert!(text.contains("service_up{service=\"gateway\"} 1"));
+    }
+}
@@ -0,0 +1,88 @@
+use crate::tracing::LogFormat;
+use std::path::PathBuf;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum LogDestination {
+    Stdout,
+    File,
+    Both,
+}
+
+impl LogDestination {
+    pub const fn needs_file_sink(self) -> bool {
+        matches!(self, Self::File | Self::Both)
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum LogRotation {
+    Hourly,
+    Daily,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct FileLoggingConfig {
+    pub dir: PathBuf,
+    pub rotation: LogRotation,
+    pub retention_days: u64,
+    pub max_files: usize,
+}
+
+impl FileLoggingConfig {
+    pub fn new(
+        dir: impl Into<PathBuf>,
+        rotation: LogRotation,
+        retention_days: u64,
+        max_files: usize,
+    ) -> Self {
+        Self {
+            dir: dir.into(),
+            rotation,
+            retention_days,
+            max_files,
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ServiceObservabilityConfig {
+    pub log_format: LogFormat,
+    pub metrics_namespace: &'static str,
+    pub log_destination: LogDestination,
+    pub file_logging: Option<FileLoggingConfig>,
+    pub node_role: Option<String>,
+    pub instance_id: Option<String>,
+}
+
+impl ServiceObservabilityConfig {
+    pub const fn new(log_format: LogFormat, metrics_namespace: &'static str) -> Self {
+        Self {
+            log_format,
+            metrics_namespace,
+            log_destination: LogDestination::Stdout,
+            file_logging: None,
+            node_role: None,
+            instance_id: None,
+        }
+    }
+
+    pub const fn with_log_destination(mut self, log_destination: LogDestination) -> Self {
+        self.log_destination = log_destination;
+        self
+    }
+
+    pub fn with_file_logging(mut self, file_logging: FileLoggingConfig) -> Self {
+        self.file_logging = Some(file_logging);
+        self
+    }
+
+    pub fn with_node_role(mut self, node_role: impl Into<String>) -> Self {
+        self.node_role = Some(node_role.into());
+        self
+    }
+
+    pub fn with_instance_id(mut self, instance_id: impl Into<String>) -> Self {
+        self.instance_id = Some(instance_id.into());
+        self
+    }
+}
@@ -0,0 +1,240 @@
+use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
+use std::sync::Arc;
+
+use tokio::sync::mpsc;
+
+use crate::metrics::{MetricKind, MetricLabel, MetricSample};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct QueueSnapshot {
+    pub capacity: usize,
+    pub depth: usize,
+    pub high_watermark: usize,
+    pub enqueued_total: u64,
+    pub rejected_full_total: u64,
+    pub rejected_closed_total: u64,
+}
+
+impl QueueSnapshot {
+    pub fn to_metric_samples(&self, queue: &'static str) -> Vec<MetricSample> {
+        let labels = vec![MetricLabel::new("queue", queue)];
+        vec![
+            MetricSample::new(
+                "queue_depth",
+                "Current number of items buffered in the queue.",
+                MetricKind::Gauge,
+                self.depth as u64,
+            )
+            .with_labels(labels.clone()),
+            MetricSample::new(
+                "queue_high_watermark",
+                "Highest observed queue depth.",
+                MetricKind::Gauge,
+                self.high_watermark as u64,
+            )
+            .with_labels(labels.clone()),
+            MetricSample::new(
+                "queue_enqueued_total",
+                "Total number of items successfully enqueued.",
+                MetricKind::Counter,
+                self.enqueued_total,
+            )
+            .with_labels(labels.clone()),
+            MetricSample::new(
+                "queue_rejected_full_total",
+                "Total number of items rejected because the queue was full.",
+                MetricKind::Counter,
+                self.rejected_full_total,
+            )
+            .with_labels(labels.clone()),
+            MetricSample::new(
+                "queue_rejected_closed_total",
+                "Total number of items rejected because the queue was closed.",
+                MetricKind::Counter,
+                self.rejected_closed_total,
+            )
+            .with_labels(labels),
+        ]
+    }
+}
+
+#[derive(Debug)]
+struct QueueState {
+    capacity: usize,
+    depth: AtomicUsize,
+    high_watermark: AtomicUsize,
+    enqueued_total: AtomicU64,
+    rejected_full_total: AtomicU64,
+    rejected_closed_total: AtomicU64,
+}
+
+#[derive(Debug)]
+pub enum QueueSendError<T> {
+    Full(T),
+    Closed(T),
+}
+
+#[derive(Debug, Clone)]
+pub struct BoundedQueueSender<T> {
+    inner: mpsc::Sender<T>,
+    state: Arc<QueueState>,
+}
+
+#[derive(Debug)]
+pub struct BoundedQueueReceiver<T> {
+    inner: mpsc::Receiver<T>,
+    state: Arc<QueueState>,
+}
+
+pub fn bounded_queue<T>(capacity: usize) -> (BoundedQueueSender<T>, BoundedQueueReceiver<T>) {
+    assert!(capacity > 0, "bounded queue capacity must be positive");
+    let (tx, rx) = mpsc::channel(capacity);
+    let state = Arc::new(QueueState {
+        capacity,
+        depth: AtomicUsize::new(0),
+        high_watermark: AtomicUsize::new(0),
+        enqueued_total: AtomicU64::new(0),
+        rejected_full_total: AtomicU64::new(0),
+        rejected_closed_total: AtomicU64::new(0),
+    });
+    (
+        BoundedQueueSender {
+            inner: tx,
+            state: state.clone(),
+        },
+        BoundedQueueReceiver { inner: rx, state },
+    )
+}
+
+impl<T> BoundedQueueSender<T> {
+    pub async fn send(&self, value: T) -> Result<(), QueueSendError<T>> {
+        let permit = match self.inner.reserve().await {
+            Ok(permit) => permit,
+            Err(_) => {
+                self.state
+                    .rejected_closed_total
+                    .fetch_add(1, Ordering::Relaxed);
+                return Err(QueueSendError::Closed(value));
+            }
+        };
+        self.record_enqueue();
+        permit.send(value);
+        Ok(())
+    }
+
+    pub fn try_send(&self, value: T) -> Result<(), QueueSendError<T>> {
+        let permit = match self.inner.try_reserve() {
+            Ok(permit) => permit,
+            Err(mpsc::error::TrySendError::Full(_)) => {
+                self.state
+                    .rejected_full_total
+                    .fetch_add(1, Ordering::Relaxed);
+                return Err(QueueSendError::Full(value));
+            }
+            Err(mpsc::error::TrySendError::Closed(_)) => {
+                self.state
+                    .rejected_closed_total
+                    .fetch_add(1, Ordering::Relaxed);
+                return Err(QueueSendError::Closed(value));
+            }
+        };
+        self.record_enqueue();
+        permit.send(value);
+        Ok(())
+    }
+
+    pub fn snapshot(&self) -> QueueSnapshot {
+        QueueSnapshot {
+            capacity: self.state.capacity,
+            depth: self.state.depth.load(Ordering::Relaxed),
+            high_watermark: self.state.high_watermark.load(Ordering::Relaxed),
+            enqueued_total: self.state.enqueued_total.load(Ordering::Relaxed),
+            rejected_full_total: self.state.rejected_full_total.load(Ordering::Relaxed),
+            rejected_closed_total: self.state.rejected_closed_total.load(Ordering::Relaxed),
+        }
+    }
+
+    pub fn capacity(&self) -> usize {
+        self.state.capacity
+    }
+
+    fn record_enqueue(&self) {
+        let depth = self.state.depth.fetch_add(1, Ordering::AcqRel) + 1;
+        self.state.enqueued_total.fetch_add(1, Ordering::Relaxed);
+        let mut observed = self.state.high_watermark.load(Ordering::Acquire);
+        while depth > observed {
+            match self.state.high_watermark.compare_exchange_weak(
+                observed,
+                depth,
+                Ordering::AcqRel,
+                Ordering::Acquire,
+            ) {
+                Ok(_) => break,
+                Err(next) => observed = next,
+            }
+        }
+    }
+}
+
+impl<T> BoundedQueueReceiver<T> {
+    pub async fn recv(&mut self) -> Option<T> {
+        let value = self.inner.recv().await?;
+        self.state.depth.fetch_sub(1, Ordering::AcqRel);
+        Some(value)
+    }
+
+    pub fn try_recv(&mut self) -> Result<T, mpsc::error::TryRecvError> {
+        let value = self.inner.try_recv()?;
+        self.state.depth.fetch_sub(1, Ordering::AcqRel);
+        Ok(value)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{bounded_queue, QueueSendError};
+
+    #[tokio::test]
+    async fn tracks_queue_depth_and_high_watermark() {
+        let (tx, mut rx) = bounded_queue::<u32>(2);
+        tx.send(1).await.expect("enqueue 1");
+        tx.send(2).await.expect("enqueue 2");
+
+        let snapshot = tx.snapshot();
+        assert_eq!(snapshot.depth, 2);
+        assert_eq!(snapshot.high_watermark, 2);
+        assert_eq!(snapshot.enqueued_total, 2);
+
+        assert_eq!(rx.recv().await, Some(1));
+        assert_eq!(tx.snapshot().depth, 1);
+    }
+
+    #[test]
+    fn counts_full_rejections() {
+        let (tx, _rx) = bounded_queue::<u32>(1);
+        tx.try_send(1).expect("first send should work");
+        let error = tx.try_send(2).expect_err("second send should fail");
+        assert!(matches!(error, QueueSendError::Full(2)));
+        assert_eq!(tx.snapshot().rejected_full_total, 1);
+    }
+
+    #[tokio::test]
+    async fn send_does_not_underflow_depth_when_receiver_races() {
+        let (tx, mut rx) = bounded_queue::<u32>(1);
+        let receiver = tokio::spawn(async move {
+            for _ in 0..256 {
+                assert!(rx.recv().await.is_some());
+            }
+        });
+
+        for value in 0..256 {
+            tx.send(value).await.expect("send should succeed");
+        }
+
+        receiver.await.expect("receiver task should join");
+        let snapshot = tx.snapshot();
+        assert_eq!(snapshot.depth, 0);
+        assert!(snapshot.high_watermark <= 1);
+        assert_eq!(snapshot.enqueued_total, 256);
+    }
+}
@@ -0,0 +1,36 @@
+use std::fmt::Write as _;
+
+use sha2::{Digest, Sha256};
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct TextPayloadSummary {
+    pub bytes: usize,
+    pub sha256: String,
+}
+
+pub fn summarize_text_payload(text: &str) -> TextPayloadSummary {
+    let digest = Sha256::digest(text.as_bytes());
+    let mut sha256 = String::with_capacity(digest.len() * 2);
+    for byte in digest {
+        write!(&mut sha256, "{byte:02x}").expect("writing to string should not fail");
+    }
+    TextPayloadSummary {
+        bytes: text.len(),
+        sha256,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::summarize_text_payload;
+
+    #[test]
+    fn summarizes_text_payload_without_exposing_content() {
+        let summary = summarize_text_payload("secret-body");
+        assert_eq!(summary.bytes, 11);
+        assert_eq!(
+            summary.sha256,
+            "7c3029502007b2beae470d090221f0a8f7708be361be4662f4b649426e5767b3"
+        );
+    }
+}
@@ -0,0 +1,15 @@
+#[cfg(unix)]
+pub async fn wait_for_shutdown_signal() -> Result<(), std::io::Error> {
+    use tokio::signal::unix::{signal, SignalKind};
+
+    let mut terminate = signal(SignalKind::terminate())?;
+    tokio::select! {
+        _ = tokio::signal::ctrl_c() => Ok(()),
+        _ = terminate.recv() => Ok(()),
+    }
+}
+
+#[cfg(not(unix))]
+pub async fn wait_for_shutdown_signal() -> Result<(), std::io::Error> {
+    tokio::signal::ctrl_c().await
+}
@@ -0,0 +1,12 @@
+use std::future::Future;
+
+pub fn spawn_named<F>(task_name: &'static str, future: F) -> tokio::task::JoinHandle<F::Output>
+where
+    F: Future + Send + 'static,
+    F::Output: Send + 'static,
+{
+    tokio::spawn(async move {
+        tracing::debug!(task = task_name, "spawned runtime task");
+        future.await
+    })
+}