refactor: 拆分 gateway 单体为独立 crate，新增 systemd 部署方案

将 gateway 内部的 model-fetch、provider-transport、scheduler-core、
usage-runtime、video-tasks-core 模块提取为独立 crate；重构 gateway
内部模块结构（state/router/cache/data/query 等）；移除大量遗留模块
文件；新增 systemd 二进制部署骨架及相关文档；更新前端 usage 相关
API 和组件。

crates/aether-scheduler-core/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "aether-scheduler-core"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+description = "Pure scheduler health and quota logic extracted from aether-gateway"
+
+[dependencies]
+aether-contracts.workspace = true
+aether-data.workspace = true
+aether-wallet.workspace = true
+regex.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+sha2.workspace = true

crates/aether-scheduler-core/src/affinity.rs

@@ -0,0 +1,173 @@
+use sha2::{Digest, Sha256};
+
+use crate::SchedulerMinimalCandidateSelectionCandidate;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct SchedulerAffinityTarget {
+    pub provider_id: String,
+    pub endpoint_id: String,
+    pub key_id: String,
+}
+
+pub fn build_scheduler_affinity_cache_key_for_api_key_id(
+    api_key_id: &str,
+    api_format: &str,
+    global_model_name: &str,
+) -> Option<String> {
+    let api_key_id = api_key_id.trim();
+    if api_key_id.is_empty() {
+        return None;
+    }
+    let api_format = crate::normalize_api_format(api_format);
+    let global_model_name = global_model_name.trim();
+    if api_format.is_empty() || global_model_name.is_empty() {
+        return None;
+    }
+
+    Some(format!(
+        "scheduler_affinity:{api_key_id}:{api_format}:{global_model_name}"
+    ))
+}
+
+pub fn compare_affinity_order(
+    left: &SchedulerMinimalCandidateSelectionCandidate,
+    right: &SchedulerMinimalCandidateSelectionCandidate,
+    affinity_key: Option<&str>,
+) -> std::cmp::Ordering {
+    let Some(affinity_key) = affinity_key else {
+        return std::cmp::Ordering::Equal;
+    };
+
+    candidate_affinity_hash(affinity_key, left).cmp(&candidate_affinity_hash(affinity_key, right))
+}
+
+pub fn candidate_affinity_hash(
+    affinity_key: &str,
+    candidate: &SchedulerMinimalCandidateSelectionCandidate,
+) -> u64 {
+    let mut hasher = Sha256::new();
+    hasher.update(affinity_key.as_bytes());
+    hasher.update(b":");
+    hasher.update(candidate.provider_id.as_bytes());
+    hasher.update(b":");
+    hasher.update(candidate.endpoint_id.as_bytes());
+    hasher.update(b":");
+    hasher.update(candidate.key_id.as_bytes());
+    let digest = hasher.finalize();
+    u64::from_be_bytes([
+        digest[0], digest[1], digest[2], digest[3], digest[4], digest[5], digest[6], digest[7],
+    ])
+}
+
+pub fn matches_affinity_target(
+    candidate: &SchedulerMinimalCandidateSelectionCandidate,
+    target: &SchedulerAffinityTarget,
+) -> bool {
+    candidate.provider_id == target.provider_id
+        && candidate.endpoint_id == target.endpoint_id
+        && candidate.key_id == target.key_id
+}
+
+pub fn candidate_key(
+    candidate: &SchedulerMinimalCandidateSelectionCandidate,
+) -> (String, String, String) {
+    (
+        candidate.provider_id.clone(),
+        candidate.endpoint_id.clone(),
+        candidate.key_id.clone(),
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{
+        build_scheduler_affinity_cache_key_for_api_key_id, candidate_affinity_hash, candidate_key,
+        compare_affinity_order, matches_affinity_target, SchedulerAffinityTarget,
+    };
+    use crate::SchedulerMinimalCandidateSelectionCandidate;
+
+    fn sample_candidate(id: &str) -> SchedulerMinimalCandidateSelectionCandidate {
+        SchedulerMinimalCandidateSelectionCandidate {
+            provider_id: format!("provider-{id}"),
+            provider_name: format!("Provider {id}"),
+            provider_type: "custom".to_string(),
+            provider_priority: 1,
+            endpoint_id: format!("endpoint-{id}"),
+            endpoint_api_format: "openai:chat".to_string(),
+            key_id: format!("key-{id}"),
+            key_name: format!("Key {id}"),
+            key_auth_type: "api_key".to_string(),
+            key_internal_priority: 1,
+            key_global_priority_for_format: Some(1),
+            key_capabilities: None,
+            model_id: format!("model-{id}"),
+            global_model_id: format!("global-model-{id}"),
+            global_model_name: "gpt-5".to_string(),
+            selected_provider_model_name: "gpt-5".to_string(),
+            mapping_matched_model: None,
+        }
+    }
+
+    #[test]
+    fn builds_normalized_scheduler_affinity_cache_key() {
+        assert_eq!(
+            build_scheduler_affinity_cache_key_for_api_key_id("api-key-1", "OPENAI:CHAT", "gpt-5"),
+            Some("scheduler_affinity:api-key-1:openai:chat:gpt-5".to_string())
+        );
+    }
+
+    #[test]
+    fn rejects_blank_affinity_key_components() {
+        assert_eq!(
+            build_scheduler_affinity_cache_key_for_api_key_id("", "openai:chat", "gpt-5"),
+            None
+        );
+        assert_eq!(
+            build_scheduler_affinity_cache_key_for_api_key_id("api-key-1", "", "gpt-5"),
+            None
+        );
+        assert_eq!(
+            build_scheduler_affinity_cache_key_for_api_key_id("api-key-1", "openai:chat", ""),
+            None
+        );
+    }
+
+    #[test]
+    fn affinity_hash_and_order_are_candidate_specific() {
+        let left = sample_candidate("1");
+        let right = sample_candidate("2");
+
+        assert_ne!(
+            candidate_affinity_hash("api-key-1", &left),
+            candidate_affinity_hash("api-key-1", &right)
+        );
+        assert_ne!(
+            compare_affinity_order(&left, &right, Some("api-key-1")),
+            std::cmp::Ordering::Equal
+        );
+        assert_eq!(
+            compare_affinity_order(&left, &right, None),
+            std::cmp::Ordering::Equal
+        );
+    }
+
+    #[test]
+    fn affinity_target_and_candidate_key_reuse_candidate_identity() {
+        let candidate = sample_candidate("1");
+        let target = SchedulerAffinityTarget {
+            provider_id: candidate.provider_id.clone(),
+            endpoint_id: candidate.endpoint_id.clone(),
+            key_id: candidate.key_id.clone(),
+        };
+
+        assert!(matches_affinity_target(&candidate, &target));
+        assert_eq!(
+            candidate_key(&candidate),
+            (
+                "provider-1".to_string(),
+                "endpoint-1".to_string(),
+                "key-1".to_string()
+            )
+        );
+    }
+}

crates/aether-scheduler-core/src/auth.rs

@@ -0,0 +1,108 @@
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct SchedulerAuthConstraints {
+    pub allowed_providers: Option<Vec<String>>,
+    pub allowed_api_formats: Option<Vec<String>>,
+    pub allowed_models: Option<Vec<String>>,
+}
+
+pub fn auth_constraints_allow_provider(
+    constraints: Option<&SchedulerAuthConstraints>,
+    provider_id: &str,
+    provider_name: &str,
+) -> bool {
+    let Some(allowed) =
+        constraints.and_then(|constraints| constraints.allowed_providers.as_deref())
+    else {
+        return true;
+    };
+
+    allowed.iter().any(|value| {
+        value.trim().eq_ignore_ascii_case(provider_id.trim())
+            || value.trim().eq_ignore_ascii_case(provider_name.trim())
+    })
+}
+
+pub fn auth_constraints_allow_api_format(
+    constraints: Option<&SchedulerAuthConstraints>,
+    api_format: &str,
+) -> bool {
+    let Some(allowed) =
+        constraints.and_then(|constraints| constraints.allowed_api_formats.as_deref())
+    else {
+        return true;
+    };
+
+    allowed
+        .iter()
+        .any(|value| crate::normalize_api_format(value) == api_format)
+}
+
+pub fn auth_constraints_allow_model(
+    constraints: Option<&SchedulerAuthConstraints>,
+    requested_model_name: &str,
+    resolved_global_model_name: &str,
+) -> bool {
+    let Some(allowed) = constraints.and_then(|constraints| constraints.allowed_models.as_deref())
+    else {
+        return true;
+    };
+
+    allowed
+        .iter()
+        .any(|value| value == requested_model_name || value == resolved_global_model_name)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{
+        auth_constraints_allow_api_format, auth_constraints_allow_model,
+        auth_constraints_allow_provider, SchedulerAuthConstraints,
+    };
+
+    fn sample_constraints() -> SchedulerAuthConstraints {
+        SchedulerAuthConstraints {
+            allowed_providers: Some(vec!["provider-1".to_string(), "OpenAI".to_string()]),
+            allowed_api_formats: Some(vec!["OPENAI:CHAT".to_string()]),
+            allowed_models: Some(vec!["gpt-5".to_string()]),
+        }
+    }
+
+    #[test]
+    fn constraints_allow_matching_provider_identifier_or_name() {
+        let constraints = sample_constraints();
+        assert!(auth_constraints_allow_provider(
+            Some(&constraints),
+            "provider-1",
+            "other"
+        ));
+        assert!(auth_constraints_allow_provider(
+            Some(&constraints),
+            "other",
+            "openai"
+        ));
+        assert!(!auth_constraints_allow_provider(
+            Some(&constraints),
+            "other",
+            "other"
+        ));
+    }
+
+    #[test]
+    fn constraints_normalize_api_formats_and_models() {
+        let constraints = sample_constraints();
+        assert!(auth_constraints_allow_api_format(
+            Some(&constraints),
+            "openai:chat"
+        ));
+        assert!(auth_constraints_allow_model(
+            Some(&constraints),
+            "gpt-5",
+            "gpt-5"
+        ));
+        assert!(!auth_constraints_allow_model(
+            Some(&constraints),
+            "gpt-4.1",
+            "gpt-4.1"
+        ));
+    }
+}

crates/aether-scheduler-core/src/candidate.rs

@@ -0,0 +1,708 @@
+use std::collections::{BTreeMap, BTreeSet};
+
+use aether_data::repository::candidate_selection::StoredMinimalCandidateSelectionRow;
+use aether_data::repository::candidates::StoredRequestCandidate;
+use aether_data::repository::provider_catalog::StoredProviderCatalogKey;
+use aether_data::DataLayerError;
+
+#[derive(Debug, Clone, PartialEq, serde::Serialize)]
+pub struct SchedulerMinimalCandidateSelectionCandidate {
+    pub provider_id: String,
+    pub provider_name: String,
+    pub provider_type: String,
+    pub provider_priority: i32,
+    pub endpoint_id: String,
+    pub endpoint_api_format: String,
+    pub key_id: String,
+    pub key_name: String,
+    pub key_auth_type: String,
+    pub key_internal_priority: i32,
+    pub key_global_priority_for_format: Option<i32>,
+    pub key_capabilities: Option<serde_json::Value>,
+    pub model_id: String,
+    pub global_model_id: String,
+    pub global_model_name: String,
+    pub selected_provider_model_name: String,
+    pub mapping_matched_model: Option<String>,
+}
+
+pub fn candidate_supports_required_capability(
+    candidate: &SchedulerMinimalCandidateSelectionCandidate,
+    required_capability: &str,
+) -> bool {
+    let required_capability = required_capability.trim();
+    if required_capability.is_empty() {
+        return true;
+    }
+    let Some(capabilities) = candidate.key_capabilities.as_ref() else {
+        return false;
+    };
+
+    if let Some(object) = capabilities.as_object() {
+        return object.iter().any(|(key, value)| {
+            key.eq_ignore_ascii_case(required_capability)
+                && match value {
+                    serde_json::Value::Bool(value) => *value,
+                    serde_json::Value::String(value) => value.eq_ignore_ascii_case("true"),
+                    serde_json::Value::Number(value) => {
+                        value.as_i64().is_some_and(|value| value > 0)
+                    }
+                    _ => false,
+                }
+        });
+    }
+
+    if let Some(items) = capabilities.as_array() {
+        return items.iter().any(|value| {
+            value
+                .as_str()
+                .is_some_and(|value| value.eq_ignore_ascii_case(required_capability))
+        });
+    }
+
+    false
+}
+
+pub fn auth_api_key_concurrency_limit_reached(
+    recent_candidates: &[StoredRequestCandidate],
+    now_unix_secs: u64,
+    api_key_id: &str,
+    concurrent_limit: usize,
+) -> bool {
+    if api_key_id.trim().is_empty() || concurrent_limit == 0 {
+        return false;
+    }
+
+    crate::count_recent_active_requests_for_api_key(recent_candidates, api_key_id, now_unix_secs)
+        >= concurrent_limit
+}
+
+pub fn build_minimal_candidate_selection(
+    rows: Vec<StoredMinimalCandidateSelectionRow>,
+    normalized_api_format: &str,
+    requested_model_name: &str,
+    resolved_global_model_name: &str,
+    require_streaming: bool,
+    auth_constraints: Option<&crate::SchedulerAuthConstraints>,
+    affinity_key: Option<&str>,
+) -> Result<Vec<SchedulerMinimalCandidateSelectionCandidate>, DataLayerError> {
+    if normalized_api_format.is_empty() {
+        return Ok(Vec::new());
+    }
+    if !crate::auth_constraints_allow_api_format(auth_constraints, normalized_api_format) {
+        return Ok(Vec::new());
+    }
+    if !crate::auth_constraints_allow_model(
+        auth_constraints,
+        requested_model_name,
+        resolved_global_model_name,
+    ) {
+        return Ok(Vec::new());
+    }
+
+    let mut candidates = Vec::new();
+    for row in rows {
+        if !crate::auth_constraints_allow_provider(
+            auth_constraints,
+            &row.provider_id,
+            &row.provider_name,
+        ) {
+            continue;
+        }
+        if require_streaming && !row.supports_streaming() {
+            continue;
+        }
+        let Some((selected_provider_model_name, mapping_matched_model)) =
+            crate::resolve_provider_model_name(&row, requested_model_name, normalized_api_format)
+        else {
+            continue;
+        };
+
+        candidates.push(SchedulerMinimalCandidateSelectionCandidate {
+            provider_id: row.provider_id,
+            provider_name: row.provider_name,
+            provider_type: row.provider_type,
+            provider_priority: row.provider_priority,
+            endpoint_id: row.endpoint_id,
+            endpoint_api_format: row.endpoint_api_format,
+            key_id: row.key_id,
+            key_name: row.key_name,
+            key_auth_type: row.key_auth_type,
+            key_internal_priority: row.key_internal_priority,
+            key_global_priority_for_format: crate::extract_global_priority_for_format(
+                row.key_global_priority_by_format.as_ref(),
+                normalized_api_format,
+            )?,
+            key_capabilities: row.key_capabilities,
+            model_id: row.model_id,
+            global_model_id: row.global_model_id,
+            global_model_name: row.global_model_name,
+            selected_provider_model_name,
+            mapping_matched_model,
+        });
+    }
+
+    candidates.sort_by(|left, right| {
+        left.key_global_priority_for_format
+            .unwrap_or(i32::MAX)
+            .cmp(&right.key_global_priority_for_format.unwrap_or(i32::MAX))
+            .then_with(|| crate::compare_affinity_order(left, right, affinity_key))
+            .then(left.provider_priority.cmp(&right.provider_priority))
+            .then(left.key_internal_priority.cmp(&right.key_internal_priority))
+            .then(left.provider_id.cmp(&right.provider_id))
+            .then(left.endpoint_id.cmp(&right.endpoint_id))
+            .then(left.key_id.cmp(&right.key_id))
+            .then(
+                left.selected_provider_model_name
+                    .cmp(&right.selected_provider_model_name),
+            )
+    });
+
+    Ok(candidates)
+}
+
+pub fn collect_global_model_names_for_required_capability(
+    rows: Vec<StoredMinimalCandidateSelectionRow>,
+    normalized_api_format: &str,
+    required_capability: &str,
+    require_streaming: bool,
+    auth_constraints: Option<&crate::SchedulerAuthConstraints>,
+) -> Vec<String> {
+    if normalized_api_format.is_empty() || required_capability.trim().is_empty() {
+        return Vec::new();
+    }
+    if !crate::auth_constraints_allow_api_format(auth_constraints, normalized_api_format) {
+        return Vec::new();
+    }
+
+    let mut model_names = BTreeSet::new();
+    for row in rows {
+        if !crate::auth_constraints_allow_provider(
+            auth_constraints,
+            &row.provider_id,
+            &row.provider_name,
+        ) {
+            continue;
+        }
+        if !crate::row_supports_required_capability(&row, required_capability) {
+            continue;
+        }
+        if require_streaming && !row.supports_streaming() {
+            continue;
+        }
+        if !crate::auth_constraints_allow_model(
+            auth_constraints,
+            &row.global_model_name,
+            &row.global_model_name,
+        ) {
+            continue;
+        }
+        model_names.insert(row.global_model_name);
+    }
+
+    model_names.into_iter().collect()
+}
+
+pub fn collect_selectable_candidates_from_keys(
+    candidates: Vec<SchedulerMinimalCandidateSelectionCandidate>,
+    selectable_keys: &BTreeSet<(String, String, String)>,
+    cached_affinity_target: Option<&crate::SchedulerAffinityTarget>,
+) -> Vec<SchedulerMinimalCandidateSelectionCandidate> {
+    let mut selected = Vec::new();
+    let mut emitted_keys = BTreeSet::new();
+
+    if let Some(target) = cached_affinity_target {
+        if let Some(candidate) = candidates
+            .iter()
+            .find(|candidate| crate::matches_affinity_target(candidate, target))
+            .cloned()
+        {
+            let key = crate::candidate_key(&candidate);
+            if selectable_keys.contains(&key) && emitted_keys.insert(key) {
+                selected.push(candidate);
+            }
+        }
+    }
+
+    for candidate in candidates {
+        let key = crate::candidate_key(&candidate);
+        if !selectable_keys.contains(&key) || !emitted_keys.insert(key) {
+            continue;
+        }
+        selected.push(candidate);
+    }
+
+    selected
+}
+
+pub fn reorder_candidates_by_scheduler_health(
+    candidates: &mut [SchedulerMinimalCandidateSelectionCandidate],
+    provider_key_rpm_states: &BTreeMap<String, StoredProviderCatalogKey>,
+    affinity_key: Option<&str>,
+) {
+    candidates.sort_by(|left, right| {
+        left.key_global_priority_for_format
+            .unwrap_or(i32::MAX)
+            .cmp(&right.key_global_priority_for_format.unwrap_or(i32::MAX))
+            .then_with(|| compare_provider_key_health_order(left, right, provider_key_rpm_states))
+            .then_with(|| crate::compare_affinity_order(left, right, affinity_key))
+            .then(left.provider_priority.cmp(&right.provider_priority))
+            .then(left.key_internal_priority.cmp(&right.key_internal_priority))
+            .then(left.provider_id.cmp(&right.provider_id))
+            .then(left.endpoint_id.cmp(&right.endpoint_id))
+            .then(left.key_id.cmp(&right.key_id))
+            .then(
+                left.selected_provider_model_name
+                    .cmp(&right.selected_provider_model_name),
+            )
+    });
+}
+
+pub fn candidate_is_selectable_with_runtime_state(
+    candidate: &SchedulerMinimalCandidateSelectionCandidate,
+    recent_candidates: &[StoredRequestCandidate],
+    provider_concurrent_limits: &BTreeMap<String, usize>,
+    provider_key_rpm_states: &BTreeMap<String, StoredProviderCatalogKey>,
+    now_unix_secs: u64,
+    cached_affinity_target: Option<&crate::SchedulerAffinityTarget>,
+    provider_quota_blocks_requests: bool,
+    rpm_reset_at: Option<u64>,
+) -> bool {
+    if provider_quota_blocks_requests {
+        return false;
+    }
+    if crate::is_candidate_in_recent_failure_cooldown(
+        recent_candidates,
+        candidate.provider_id.as_str(),
+        candidate.endpoint_id.as_str(),
+        candidate.key_id.as_str(),
+        now_unix_secs,
+    ) {
+        return false;
+    }
+    if provider_concurrent_limits
+        .get(&candidate.provider_id)
+        .is_some_and(|limit| {
+            crate::count_recent_active_requests_for_provider(
+                recent_candidates,
+                candidate.provider_id.as_str(),
+                now_unix_secs,
+            ) >= *limit
+        })
+    {
+        return false;
+    }
+
+    let is_cached_user = cached_affinity_target
+        .is_some_and(|target| crate::matches_affinity_target(candidate, target));
+    if let Some(provider_key) = provider_key_rpm_states.get(&candidate.key_id) {
+        if crate::is_provider_key_circuit_open(provider_key, candidate.endpoint_api_format.as_str())
+        {
+            return false;
+        }
+        if crate::provider_key_health_score(provider_key, candidate.endpoint_api_format.as_str())
+            .is_some_and(|score| score <= 0.0)
+        {
+            return false;
+        }
+        if !crate::provider_key_rpm_allows_request_since(
+            provider_key,
+            recent_candidates,
+            now_unix_secs,
+            is_cached_user,
+            rpm_reset_at,
+        ) {
+            return false;
+        }
+    }
+
+    true
+}
+
+fn compare_provider_key_health_order(
+    left: &SchedulerMinimalCandidateSelectionCandidate,
+    right: &SchedulerMinimalCandidateSelectionCandidate,
+    provider_key_rpm_states: &BTreeMap<String, StoredProviderCatalogKey>,
+) -> std::cmp::Ordering {
+    let left_bucket = candidate_provider_key_health_bucket(left, provider_key_rpm_states);
+    let right_bucket = candidate_provider_key_health_bucket(right, provider_key_rpm_states);
+    right_bucket.cmp(&left_bucket).then_with(|| {
+        let left_score = candidate_provider_key_health_score(left, provider_key_rpm_states);
+        let right_score = candidate_provider_key_health_score(right, provider_key_rpm_states);
+        right_score
+            .partial_cmp(&left_score)
+            .unwrap_or(std::cmp::Ordering::Equal)
+    })
+}
+
+fn candidate_provider_key_health_bucket(
+    candidate: &SchedulerMinimalCandidateSelectionCandidate,
+    provider_key_rpm_states: &BTreeMap<String, StoredProviderCatalogKey>,
+) -> Option<crate::ProviderKeyHealthBucket> {
+    provider_key_rpm_states
+        .get(&candidate.key_id)
+        .and_then(|key| {
+            crate::provider_key_health_bucket(key, candidate.endpoint_api_format.as_str())
+        })
+}
+
+fn candidate_provider_key_health_score(
+    candidate: &SchedulerMinimalCandidateSelectionCandidate,
+    provider_key_rpm_states: &BTreeMap<String, StoredProviderCatalogKey>,
+) -> f64 {
+    provider_key_rpm_states
+        .get(&candidate.key_id)
+        .and_then(|key| {
+            crate::effective_provider_key_health_score(key, candidate.endpoint_api_format.as_str())
+        })
+        .unwrap_or(1.0)
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::{BTreeMap, BTreeSet};
+
+    use aether_data::repository::candidate_selection::{
+        StoredMinimalCandidateSelectionRow, StoredProviderModelMapping,
+    };
+    use aether_data::repository::candidates::{RequestCandidateStatus, StoredRequestCandidate};
+    use aether_data::repository::provider_catalog::StoredProviderCatalogKey;
+
+    use super::{
+        auth_api_key_concurrency_limit_reached, build_minimal_candidate_selection,
+        candidate_is_selectable_with_runtime_state, candidate_supports_required_capability,
+        collect_global_model_names_for_required_capability,
+        collect_selectable_candidates_from_keys, reorder_candidates_by_scheduler_health,
+        SchedulerMinimalCandidateSelectionCandidate,
+    };
+    use crate::SchedulerAuthConstraints;
+
+    fn sample_row(id: &str) -> StoredMinimalCandidateSelectionRow {
+        StoredMinimalCandidateSelectionRow {
+            provider_id: format!("provider-{id}"),
+            provider_name: format!("Provider {id}"),
+            provider_type: "custom".to_string(),
+            provider_priority: 10,
+            provider_is_active: true,
+            endpoint_id: format!("endpoint-{id}"),
+            endpoint_api_format: "openai:chat".to_string(),
+            endpoint_api_family: Some("openai".to_string()),
+            endpoint_kind: Some("chat".to_string()),
+            endpoint_is_active: true,
+            key_id: format!("key-{id}"),
+            key_name: format!("prod-{id}"),
+            key_auth_type: "api_key".to_string(),
+            key_is_active: true,
+            key_api_formats: Some(vec!["openai:chat".to_string()]),
+            key_allowed_models: None,
+            key_capabilities: Some(serde_json::json!({"cache_1h": true})),
+            key_internal_priority: 50,
+            key_global_priority_by_format: Some(serde_json::json!({"openai:chat": 2})),
+            model_id: format!("model-{id}"),
+            global_model_id: format!("global-model-{id}"),
+            global_model_name: "gpt-5".to_string(),
+            global_model_mappings: Some(vec!["gpt-5(?:\\.\\d+)?".to_string()]),
+            global_model_supports_streaming: Some(true),
+            model_provider_model_name: format!("gpt-5-upstream-{id}"),
+            model_provider_model_mappings: Some(vec![StoredProviderModelMapping {
+                name: format!("gpt-5-canary-{id}"),
+                priority: 1,
+                api_formats: Some(vec!["openai:chat".to_string()]),
+            }]),
+            model_supports_streaming: None,
+            model_is_active: true,
+            model_is_available: true,
+        }
+    }
+    fn sample_candidate(
+        id: &str,
+        capabilities: Option<serde_json::Value>,
+    ) -> SchedulerMinimalCandidateSelectionCandidate {
+        SchedulerMinimalCandidateSelectionCandidate {
+            provider_id: format!("provider-{id}"),
+            provider_name: format!("Provider {id}"),
+            provider_type: "openai".to_string(),
+            provider_priority: 0,
+            endpoint_id: format!("endpoint-{id}"),
+            endpoint_api_format: "openai:chat".to_string(),
+            key_id: format!("key-{id}"),
+            key_name: format!("key-{id}"),
+            key_auth_type: "bearer".to_string(),
+            key_internal_priority: 0,
+            key_global_priority_for_format: None,
+            key_capabilities: capabilities,
+            model_id: format!("model-{id}"),
+            global_model_id: format!("global-model-{id}"),
+            global_model_name: "gpt-5".to_string(),
+            selected_provider_model_name: "gpt-5".to_string(),
+            mapping_matched_model: None,
+        }
+    }
+
+    fn sample_key(id: &str, health_score: f64) -> StoredProviderCatalogKey {
+        let mut key = StoredProviderCatalogKey::new(
+            format!("key-{id}"),
+            format!("provider-{id}"),
+            format!("key-{id}"),
+            "api_key".to_string(),
+            None,
+            true,
+        )
+        .expect("provider key should build");
+        key.health_by_format = Some(serde_json::json!({
+            "openai:chat": {
+                "health_score": health_score
+            }
+        }));
+        key
+    }
+
+    fn stored_candidate(
+        id: &str,
+        status: RequestCandidateStatus,
+        created_at_unix_secs: i64,
+    ) -> StoredRequestCandidate {
+        let finished_at_unix_secs = match status {
+            RequestCandidateStatus::Pending | RequestCandidateStatus::Streaming => None,
+            _ => Some(created_at_unix_secs),
+        };
+        StoredRequestCandidate::new(
+            id.to_string(),
+            format!("req-{id}"),
+            None,
+            None,
+            None,
+            None,
+            0,
+            0,
+            Some("provider-1".to_string()),
+            Some("endpoint-1".to_string()),
+            Some("key-1".to_string()),
+            status,
+            None,
+            false,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            created_at_unix_secs,
+            Some(created_at_unix_secs),
+            finished_at_unix_secs,
+        )
+        .expect("candidate should build")
+    }
+
+    #[test]
+    fn reads_required_capability_from_object_and_array_forms() {
+        assert!(candidate_supports_required_capability(
+            &sample_candidate("1", Some(serde_json::json!({"vision": true}))),
+            "vision"
+        ));
+        assert!(candidate_supports_required_capability(
+            &sample_candidate("1", Some(serde_json::json!(["vision", "tools"]))),
+            "tools"
+        ));
+        assert!(!candidate_supports_required_capability(
+            &sample_candidate("1", Some(serde_json::json!({"vision": false}))),
+            "vision"
+        ));
+    }
+
+    #[test]
+    fn builds_minimal_candidate_selection_with_auth_constraints() {
+        let mut disallowed = sample_row("2");
+        disallowed.provider_id = "provider-blocked".to_string();
+        disallowed.provider_name = "Blocked".to_string();
+
+        let constraints = SchedulerAuthConstraints {
+            allowed_providers: Some(vec!["provider-1".to_string()]),
+            allowed_api_formats: Some(vec!["OPENAI:CHAT".to_string()]),
+            allowed_models: Some(vec!["gpt-5".to_string()]),
+        };
+        let candidates = build_minimal_candidate_selection(
+            vec![sample_row("1"), disallowed],
+            "openai:chat",
+            "gpt-5",
+            "gpt-5",
+            false,
+            Some(&constraints),
+            None,
+        )
+        .expect("candidate selection should build");
+
+        assert_eq!(candidates.len(), 1);
+        assert_eq!(candidates[0].provider_id, "provider-1");
+        assert_eq!(candidates[0].selected_provider_model_name, "gpt-5-canary-1");
+    }
+
+    #[test]
+    fn collects_global_model_names_for_required_capability_with_auth_constraints() {
+        let mut disallowed = sample_row("2");
+        disallowed.global_model_name = "gpt-4.1".to_string();
+        disallowed.provider_id = "provider-blocked".to_string();
+        disallowed.provider_name = "Blocked".to_string();
+
+        let constraints = SchedulerAuthConstraints {
+            allowed_providers: Some(vec!["provider-1".to_string()]),
+            allowed_api_formats: Some(vec!["openai:chat".to_string()]),
+            allowed_models: Some(vec!["gpt-5".to_string()]),
+        };
+        let model_names = collect_global_model_names_for_required_capability(
+            vec![sample_row("1"), disallowed],
+            "openai:chat",
+            "cache_1h",
+            false,
+            Some(&constraints),
+        );
+
+        assert_eq!(model_names, vec!["gpt-5".to_string()]);
+    }
+
+    #[test]
+    fn reorders_candidates_by_health_before_affinity_tiebreak() {
+        let mut candidates = vec![
+            sample_candidate("1", None),
+            sample_candidate("2", None),
+            sample_candidate("3", None),
+        ];
+        let provider_key_rpm_states = BTreeMap::from([
+            ("key-1".to_string(), sample_key("1", 0.95)),
+            ("key-2".to_string(), sample_key("2", 0.40)),
+            ("key-3".to_string(), sample_key("3", 0.95)),
+        ]);
+
+        reorder_candidates_by_scheduler_health(
+            &mut candidates,
+            &provider_key_rpm_states,
+            Some("api-key-1"),
+        );
+
+        assert_ne!(candidates[0].key_id, "key-2");
+        assert_ne!(candidates[1].key_id, "key-2");
+        assert_eq!(candidates[2].key_id, "key-2");
+    }
+
+    #[test]
+    fn collects_selectable_candidates_with_affinity_priority_and_dedup() {
+        let candidates = vec![
+            sample_candidate("1", None),
+            sample_candidate("2", None),
+            sample_candidate("1", None),
+        ];
+        let selectable_keys = BTreeSet::from([
+            (
+                "provider-1".to_string(),
+                "endpoint-1".to_string(),
+                "key-1".to_string(),
+            ),
+            (
+                "provider-2".to_string(),
+                "endpoint-2".to_string(),
+                "key-2".to_string(),
+            ),
+        ]);
+        let selected = collect_selectable_candidates_from_keys(
+            candidates,
+            &selectable_keys,
+            Some(&crate::SchedulerAffinityTarget {
+                provider_id: "provider-2".to_string(),
+                endpoint_id: "endpoint-2".to_string(),
+                key_id: "key-2".to_string(),
+            }),
+        );
+
+        assert_eq!(selected.len(), 2);
+        assert_eq!(selected[0].key_id, "key-2");
+        assert_eq!(selected[1].key_id, "key-1");
+    }
+
+    #[test]
+    fn candidate_selectability_respects_provider_concurrency_limit() {
+        let recent_candidates = vec![stored_candidate("one", RequestCandidateStatus::Pending, 95)];
+        let provider_concurrent_limits = BTreeMap::from([("provider-1".to_string(), 1)]);
+
+        assert!(!candidate_is_selectable_with_runtime_state(
+            &sample_candidate("1", None),
+            &recent_candidates,
+            &provider_concurrent_limits,
+            &BTreeMap::new(),
+            100,
+            None,
+            false,
+            None,
+        ));
+    }
+
+    #[test]
+    fn candidate_selectability_rejects_quota_or_zero_health() {
+        let provider_key_rpm_states = BTreeMap::from([("key-1".to_string(), sample_key("1", 0.0))]);
+
+        assert!(!candidate_is_selectable_with_runtime_state(
+            &sample_candidate("1", None),
+            &[],
+            &BTreeMap::new(),
+            &provider_key_rpm_states,
+            100,
+            None,
+            false,
+            None,
+        ));
+        assert!(!candidate_is_selectable_with_runtime_state(
+            &sample_candidate("1", None),
+            &[],
+            &BTreeMap::new(),
+            &BTreeMap::new(),
+            100,
+            None,
+            true,
+            None,
+        ));
+    }
+
+    #[test]
+    fn detects_auth_api_key_concurrency_limit_from_recent_active_requests() {
+        let recent_candidates = vec![StoredRequestCandidate::new(
+            "one".to_string(),
+            "req-one".to_string(),
+            None,
+            Some("api-key-1".to_string()),
+            None,
+            None,
+            0,
+            0,
+            Some("provider-1".to_string()),
+            Some("endpoint-1".to_string()),
+            Some("key-1".to_string()),
+            RequestCandidateStatus::Pending,
+            None,
+            false,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            95,
+            Some(95),
+            None,
+        )
+        .expect("candidate should build")];
+
+        assert!(auth_api_key_concurrency_limit_reached(
+            &recent_candidates,
+            100,
+            "api-key-1",
+            1,
+        ));
+        assert!(!auth_api_key_concurrency_limit_reached(
+            &recent_candidates,
+            100,
+            "api-key-1",
+            2,
+        ));
+    }
+}

crates/aether-scheduler-core/src/health.rs

@@ -0,0 +1,948 @@
+use aether_data::repository::candidates::{RequestCandidateStatus, StoredRequestCandidate};
+use aether_data::repository::provider_catalog::StoredProviderCatalogKey;
+
+const FAILURE_COOLDOWN_WINDOW_SECS: u64 = 60;
+const FAILURE_COOLDOWN_THRESHOLD: usize = 2;
+const ACTIVE_REQUEST_WINDOW_SECS: u64 = 300;
+pub const PROVIDER_KEY_RPM_WINDOW_SECS: u64 = 60;
+const PROBE_PHASE_REQUESTS: u32 = 100;
+const PROBE_RESERVATION_RATIO: f64 = 0.1;
+const STABLE_MIN_RESERVATION_RATIO: f64 = 0.1;
+const STABLE_MAX_RESERVATION_RATIO: f64 = 0.35;
+const SUCCESS_COUNT_FOR_FULL_CONFIDENCE: u32 = 50;
+const COOLDOWN_HOURS_FOR_FULL_CONFIDENCE: f64 = 24.0;
+const LOW_LOAD_THRESHOLD: f64 = 0.5;
+const HIGH_LOAD_THRESHOLD: f64 = 0.8;
+const ENFORCEMENT_CONFIDENCE_THRESHOLD: f64 = 0.6;
+const HEALTH_DEGRADED_THRESHOLD: f64 = 0.8;
+const HEALTH_LOW_THRESHOLD: f64 = 0.5;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub enum ProviderKeyHealthBucket {
+    Low,
+    Degraded,
+    Healthy,
+}
+
+impl ProviderKeyHealthBucket {
+    fn from_score(score: f64) -> Self {
+        let score = score.clamp(0.0, 1.0);
+        if score < HEALTH_LOW_THRESHOLD {
+            return Self::Low;
+        }
+        if score < HEALTH_DEGRADED_THRESHOLD {
+            return Self::Degraded;
+        }
+        Self::Healthy
+    }
+}
+
+pub fn is_candidate_in_recent_failure_cooldown(
+    recent_candidates: &[StoredRequestCandidate],
+    provider_id: &str,
+    endpoint_id: &str,
+    key_id: &str,
+    now_unix_secs: u64,
+) -> bool {
+    let mut recent_failures = 0usize;
+
+    for candidate in recent_candidates {
+        if candidate.provider_id.as_deref() != Some(provider_id)
+            || candidate.endpoint_id.as_deref() != Some(endpoint_id)
+            || candidate.key_id.as_deref() != Some(key_id)
+        {
+            continue;
+        }
+
+        let observed_at_unix_secs = candidate
+            .finished_at_unix_secs
+            .or(candidate.started_at_unix_secs)
+            .unwrap_or(candidate.created_at_unix_secs);
+        if now_unix_secs.saturating_sub(observed_at_unix_secs) > FAILURE_COOLDOWN_WINDOW_SECS {
+            continue;
+        }
+
+        match candidate.status {
+            RequestCandidateStatus::Success => return false,
+            RequestCandidateStatus::Failed | RequestCandidateStatus::Cancelled => {
+                recent_failures += 1;
+                if recent_failures >= FAILURE_COOLDOWN_THRESHOLD {
+                    return true;
+                }
+            }
+            RequestCandidateStatus::Available
+            | RequestCandidateStatus::Unused
+            | RequestCandidateStatus::Pending
+            | RequestCandidateStatus::Streaming
+            | RequestCandidateStatus::Skipped => {}
+        }
+    }
+
+    false
+}
+
+pub fn count_recent_active_requests_for_provider(
+    recent_candidates: &[StoredRequestCandidate],
+    provider_id: &str,
+    now_unix_secs: u64,
+) -> usize {
+    recent_candidates
+        .iter()
+        .filter(|candidate| candidate.provider_id.as_deref() == Some(provider_id))
+        .filter(|candidate| is_recently_active(candidate, now_unix_secs))
+        .count()
+}
+
+pub fn count_recent_active_requests_for_api_key(
+    recent_candidates: &[StoredRequestCandidate],
+    api_key_id: &str,
+    now_unix_secs: u64,
+) -> usize {
+    recent_candidates
+        .iter()
+        .filter(|candidate| candidate.api_key_id.as_deref() == Some(api_key_id))
+        .filter(|candidate| is_recently_active(candidate, now_unix_secs))
+        .count()
+}
+
+pub fn effective_provider_key_rpm_limit(
+    key: &StoredProviderCatalogKey,
+    now_unix_secs: u64,
+) -> Option<usize> {
+    if let Some(limit) = key.rpm_limit.filter(|limit| *limit > 0) {
+        return usize::try_from(limit).ok();
+    }
+
+    let learned_limit = key
+        .learned_rpm_limit
+        .filter(|limit| *limit > 0)
+        .and_then(|limit| usize::try_from(limit).ok())?;
+    if provider_key_reservation_confidence(key, now_unix_secs) < ENFORCEMENT_CONFIDENCE_THRESHOLD {
+        return None;
+    }
+
+    Some(learned_limit)
+}
+
+pub fn count_recent_rpm_requests_for_provider_key(
+    recent_candidates: &[StoredRequestCandidate],
+    key_id: &str,
+    now_unix_secs: u64,
+) -> usize {
+    count_recent_rpm_requests_for_provider_key_since(recent_candidates, key_id, now_unix_secs, None)
+}
+
+pub fn count_recent_rpm_requests_for_provider_key_since(
+    recent_candidates: &[StoredRequestCandidate],
+    key_id: &str,
+    now_unix_secs: u64,
+    reset_after_unix_secs: Option<u64>,
+) -> usize {
+    let mut attempted_count = 0usize;
+    let mut max_observed = 0usize;
+
+    for candidate in recent_candidates {
+        if candidate.key_id.as_deref() != Some(key_id) {
+            continue;
+        }
+        if !is_recent_rpm_observation(candidate, now_unix_secs) {
+            continue;
+        }
+        let observed_at_unix_secs = candidate
+            .started_at_unix_secs
+            .unwrap_or(candidate.created_at_unix_secs);
+        if reset_after_unix_secs.is_some_and(|reset_after| observed_at_unix_secs <= reset_after) {
+            continue;
+        }
+        attempted_count += 1;
+        max_observed = max_observed.max(candidate.concurrent_requests.unwrap_or_default() as usize);
+    }
+
+    max_observed.max(attempted_count)
+}
+
+pub fn provider_key_rpm_allows_request(
+    key: &StoredProviderCatalogKey,
+    recent_candidates: &[StoredRequestCandidate],
+    now_unix_secs: u64,
+    is_cached_user: bool,
+) -> bool {
+    provider_key_rpm_allows_request_since(
+        key,
+        recent_candidates,
+        now_unix_secs,
+        is_cached_user,
+        None,
+    )
+}
+
+pub fn provider_key_rpm_allows_request_since(
+    key: &StoredProviderCatalogKey,
+    recent_candidates: &[StoredRequestCandidate],
+    now_unix_secs: u64,
+    is_cached_user: bool,
+    reset_after_unix_secs: Option<u64>,
+) -> bool {
+    let Some(effective_limit) = effective_provider_key_rpm_limit(key, now_unix_secs) else {
+        return true;
+    };
+    if effective_limit == 0 {
+        return false;
+    }
+
+    let current_usage = count_recent_rpm_requests_for_provider_key_since(
+        recent_candidates,
+        key.id.as_str(),
+        now_unix_secs,
+        reset_after_unix_secs,
+    );
+    if is_cached_user {
+        return current_usage < effective_limit;
+    }
+
+    let available_for_new = available_provider_key_rpm_slots_for_new_user(
+        key,
+        current_usage,
+        effective_limit,
+        now_unix_secs,
+    );
+    current_usage < available_for_new
+}
+
+pub fn provider_key_health_score(key: &StoredProviderCatalogKey, api_format: &str) -> Option<f64> {
+    let score = key
+        .health_by_format
+        .as_ref()
+        .and_then(serde_json::Value::as_object)
+        .and_then(|values| values.get(api_format))
+        .and_then(serde_json::Value::as_object)
+        .and_then(|payload| payload.get("health_score"))
+        .and_then(json_value_as_f64)?;
+    Some(score.clamp(0.0, 1.0))
+}
+
+pub fn aggregate_provider_key_health_score(key: &StoredProviderCatalogKey) -> Option<f64> {
+    let health_by_format = key.health_by_format.as_ref()?.as_object()?;
+    let mut scores = Vec::new();
+    for payload in health_by_format.values() {
+        let Some(score) = payload
+            .as_object()
+            .and_then(|payload| payload.get("health_score"))
+            .and_then(json_value_as_f64)
+        else {
+            continue;
+        };
+        scores.push(score.clamp(0.0, 1.0));
+    }
+    scores.into_iter().reduce(f64::min)
+}
+
+pub fn effective_provider_key_health_score(
+    key: &StoredProviderCatalogKey,
+    api_format: &str,
+) -> Option<f64> {
+    provider_key_health_score(key, api_format).or_else(|| aggregate_provider_key_health_score(key))
+}
+
+pub fn provider_key_health_bucket(
+    key: &StoredProviderCatalogKey,
+    api_format: &str,
+) -> Option<ProviderKeyHealthBucket> {
+    effective_provider_key_health_score(key, api_format).map(ProviderKeyHealthBucket::from_score)
+}
+
+pub fn is_provider_key_circuit_open(key: &StoredProviderCatalogKey, api_format: &str) -> bool {
+    key.circuit_breaker_by_format
+        .as_ref()
+        .and_then(serde_json::Value::as_object)
+        .and_then(|values| values.get(api_format))
+        .and_then(serde_json::Value::as_object)
+        .and_then(|payload| payload.get("open"))
+        .and_then(serde_json::Value::as_bool)
+        .unwrap_or(false)
+}
+
+fn available_provider_key_rpm_slots_for_new_user(
+    key: &StoredProviderCatalogKey,
+    current_usage: usize,
+    effective_limit: usize,
+    now_unix_secs: u64,
+) -> usize {
+    let reservation_ratio =
+        provider_key_dynamic_reservation_ratio(key, current_usage, effective_limit, now_unix_secs);
+    usize::max(
+        1,
+        (effective_limit as f64 * (1.0 - reservation_ratio)).floor() as usize,
+    )
+}
+
+fn provider_key_dynamic_reservation_ratio(
+    key: &StoredProviderCatalogKey,
+    current_usage: usize,
+    effective_limit: usize,
+    now_unix_secs: u64,
+) -> f64 {
+    let total_requests = provider_key_total_requests(key);
+    if total_requests < PROBE_PHASE_REQUESTS {
+        return PROBE_RESERVATION_RATIO;
+    }
+
+    let confidence = provider_key_reservation_confidence(key, now_unix_secs);
+    let load_ratio = provider_key_load_ratio(current_usage, effective_limit);
+    if load_ratio < LOW_LOAD_THRESHOLD {
+        return STABLE_MIN_RESERVATION_RATIO;
+    }
+    if load_ratio < HIGH_LOAD_THRESHOLD {
+        let load_factor =
+            (load_ratio - LOW_LOAD_THRESHOLD) / (HIGH_LOAD_THRESHOLD - LOW_LOAD_THRESHOLD);
+        return STABLE_MIN_RESERVATION_RATIO
+            + confidence
+                * load_factor
+                * (STABLE_MAX_RESERVATION_RATIO - STABLE_MIN_RESERVATION_RATIO);
+    }
+
+    STABLE_MIN_RESERVATION_RATIO
+        + confidence * (STABLE_MAX_RESERVATION_RATIO - STABLE_MIN_RESERVATION_RATIO)
+}
+
+fn is_recently_active(candidate: &StoredRequestCandidate, now_unix_secs: u64) -> bool {
+    if candidate.finished_at_unix_secs.is_some() {
+        return false;
+    }
+
+    if !matches!(
+        candidate.status,
+        RequestCandidateStatus::Pending | RequestCandidateStatus::Streaming
+    ) {
+        return false;
+    }
+
+    let observed_at_unix_secs = candidate
+        .started_at_unix_secs
+        .unwrap_or(candidate.created_at_unix_secs);
+    now_unix_secs.saturating_sub(observed_at_unix_secs) <= ACTIVE_REQUEST_WINDOW_SECS
+}
+
+fn is_recent_rpm_observation(candidate: &StoredRequestCandidate, now_unix_secs: u64) -> bool {
+    if !candidate
+        .status
+        .is_attempted(candidate.started_at_unix_secs)
+    {
+        return false;
+    }
+
+    let observed_at_unix_secs = candidate
+        .started_at_unix_secs
+        .unwrap_or(candidate.created_at_unix_secs);
+    now_unix_secs.saturating_sub(observed_at_unix_secs) <= PROVIDER_KEY_RPM_WINDOW_SECS
+}
+
+fn provider_key_total_requests(key: &StoredProviderCatalogKey) -> u32 {
+    let request_count = key.request_count.unwrap_or_default();
+    if request_count > 0 {
+        return request_count;
+    }
+
+    let history_count = key
+        .adjustment_history
+        .as_ref()
+        .and_then(serde_json::Value::as_array)
+        .map(|values| values.len() as u32 * 10)
+        .unwrap_or_default();
+    key.concurrent_429_count.unwrap_or_default()
+        + key.rpm_429_count.unwrap_or_default()
+        + key.success_count.unwrap_or_default()
+        + history_count
+}
+
+fn provider_key_load_ratio(current_usage: usize, effective_limit: usize) -> f64 {
+    if effective_limit == 0 {
+        return 0.0;
+    }
+
+    (current_usage as f64 / effective_limit as f64).min(1.0)
+}
+
+fn provider_key_reservation_confidence(key: &StoredProviderCatalogKey, now_unix_secs: u64) -> f64 {
+    let request_count = key.request_count.unwrap_or_default() as f64;
+    let success_count = key.success_count.unwrap_or_default() as f64;
+
+    let success_score = if request_count >= SUCCESS_COUNT_FOR_FULL_CONFIDENCE as f64 {
+        let success_rate = if request_count > 0.0 {
+            success_count / request_count
+        } else {
+            0.0
+        };
+        success_rate * 0.4
+    } else if request_count > 0.0 {
+        let success_rate = success_count / request_count;
+        let progress_ratio = request_count / SUCCESS_COUNT_FOR_FULL_CONFIDENCE as f64;
+        success_rate * progress_ratio * 0.4
+    } else {
+        0.0
+    };
+
+    let cooldown_score = match key.last_429_at_unix_secs {
+        Some(last_429_at_unix_secs) => {
+            let hours_since_429 =
+                now_unix_secs.saturating_sub(last_429_at_unix_secs) as f64 / 3600.0;
+            (hours_since_429 / COOLDOWN_HOURS_FOR_FULL_CONFIDENCE).min(1.0) * 0.3
+        }
+        None => 0.3,
+    };
+
+    let stability_score = provider_key_stability_score(key);
+    (success_score + cooldown_score + stability_score).min(1.0)
+}
+
+fn provider_key_stability_score(key: &StoredProviderCatalogKey) -> f64 {
+    let Some(history) = key
+        .adjustment_history
+        .as_ref()
+        .and_then(serde_json::Value::as_array)
+    else {
+        return 0.15;
+    };
+    if history.len() < 3 {
+        return 0.15;
+    }
+
+    let recent = if history.len() > 5 {
+        &history[history.len() - 5..]
+    } else {
+        history.as_slice()
+    };
+    let limits = recent
+        .iter()
+        .filter_map(|entry| entry.get("new_limit"))
+        .filter_map(json_value_as_f64)
+        .collect::<Vec<_>>();
+    if limits.len() < 2 {
+        return 0.15;
+    }
+
+    let mean = limits.iter().sum::<f64>() / limits.len() as f64;
+    let variance = limits
+        .iter()
+        .map(|limit| {
+            let delta = *limit - mean;
+            delta * delta
+        })
+        .sum::<f64>()
+        / (limits.len() as f64 - 1.0);
+    let stability_ratio = (1.0 - variance / 10.0).max(0.0);
+    stability_ratio * 0.3
+}
+
+fn json_value_as_f64(value: &serde_json::Value) -> Option<f64> {
+    value
+        .as_f64()
+        .or_else(|| value.as_i64().map(|raw| raw as f64))
+        .or_else(|| value.as_u64().map(|raw| raw as f64))
+}
+
+#[cfg(test)]
+mod tests {
+    use aether_data::repository::candidates::{RequestCandidateStatus, StoredRequestCandidate};
+    use aether_data::repository::provider_catalog::StoredProviderCatalogKey;
+
+    use super::{
+        aggregate_provider_key_health_score, count_recent_active_requests_for_api_key,
+        count_recent_active_requests_for_provider, count_recent_rpm_requests_for_provider_key,
+        count_recent_rpm_requests_for_provider_key_since, effective_provider_key_health_score,
+        effective_provider_key_rpm_limit, is_candidate_in_recent_failure_cooldown,
+        is_provider_key_circuit_open, provider_key_health_bucket, provider_key_health_score,
+        provider_key_rpm_allows_request, provider_key_rpm_allows_request_since,
+        ProviderKeyHealthBucket,
+    };
+
+    fn stored_candidate(
+        id: &str,
+        status: RequestCandidateStatus,
+        created_at_unix_secs: i64,
+    ) -> StoredRequestCandidate {
+        StoredRequestCandidate::new(
+            id.to_string(),
+            format!("req-{id}"),
+            None,
+            None,
+            None,
+            None,
+            0,
+            0,
+            Some("provider-a".to_string()),
+            Some("endpoint-a".to_string()),
+            Some("key-a".to_string()),
+            status,
+            None,
+            false,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            created_at_unix_secs,
+            Some(created_at_unix_secs),
+            Some(created_at_unix_secs),
+        )
+        .expect("candidate should build")
+    }
+
+    fn provider_catalog_key(id: &str) -> StoredProviderCatalogKey {
+        StoredProviderCatalogKey::new(
+            id.to_string(),
+            "provider-a".to_string(),
+            "primary".to_string(),
+            "api_key".to_string(),
+            None,
+            true,
+        )
+        .expect("provider key should build")
+    }
+
+    #[test]
+    fn cooldown_triggers_after_two_recent_failures() {
+        let recent_candidates = vec![
+            stored_candidate("one", RequestCandidateStatus::Failed, 95),
+            stored_candidate("two", RequestCandidateStatus::Cancelled, 99),
+        ];
+
+        assert!(is_candidate_in_recent_failure_cooldown(
+            &recent_candidates,
+            "provider-a",
+            "endpoint-a",
+            "key-a",
+            100,
+        ));
+    }
+
+    #[test]
+    fn recent_success_clears_cooldown() {
+        let recent_candidates = vec![
+            stored_candidate("one", RequestCandidateStatus::Failed, 95),
+            stored_candidate("two", RequestCandidateStatus::Success, 99),
+            stored_candidate("three", RequestCandidateStatus::Cancelled, 98),
+        ];
+
+        assert!(!is_candidate_in_recent_failure_cooldown(
+            &recent_candidates,
+            "provider-a",
+            "endpoint-a",
+            "key-a",
+            100,
+        ));
+    }
+
+    #[test]
+    fn counts_only_recently_active_provider_requests() {
+        let recent_candidates = vec![
+            StoredRequestCandidate::new(
+                "one".to_string(),
+                "req-one".to_string(),
+                None,
+                Some("api-key-1".to_string()),
+                None,
+                None,
+                0,
+                0,
+                Some("provider-a".to_string()),
+                Some("endpoint-a".to_string()),
+                Some("key-a".to_string()),
+                RequestCandidateStatus::Pending,
+                None,
+                false,
+                None,
+                None,
+                None,
+                None,
+                None,
+                None,
+                None,
+                95,
+                Some(95),
+                None,
+            )
+            .expect("candidate should build"),
+            StoredRequestCandidate::new(
+                "two".to_string(),
+                "req-two".to_string(),
+                None,
+                Some("api-key-1".to_string()),
+                None,
+                None,
+                0,
+                0,
+                Some("provider-a".to_string()),
+                Some("endpoint-a".to_string()),
+                Some("key-a".to_string()),
+                RequestCandidateStatus::Streaming,
+                None,
+                false,
+                None,
+                None,
+                None,
+                None,
+                None,
+                None,
+                None,
+                96,
+                Some(96),
+                None,
+            )
+            .expect("candidate should build"),
+            StoredRequestCandidate::new(
+                "three".to_string(),
+                "req-three".to_string(),
+                None,
+                Some("api-key-1".to_string()),
+                None,
+                None,
+                0,
+                0,
+                Some("provider-a".to_string()),
+                Some("endpoint-a".to_string()),
+                Some("key-a".to_string()),
+                RequestCandidateStatus::Success,
+                None,
+                false,
+                None,
+                None,
+                None,
+                None,
+                None,
+                None,
+                None,
+                97,
+                Some(97),
+                Some(98),
+            )
+            .expect("candidate should build"),
+        ];
+
+        assert_eq!(
+            count_recent_active_requests_for_provider(&recent_candidates, "provider-a", 100),
+            2
+        );
+        assert_eq!(
+            count_recent_active_requests_for_api_key(&recent_candidates, "api-key-1", 100),
+            2
+        );
+    }
+
+    #[test]
+    fn fixed_provider_key_rpm_limit_takes_precedence() {
+        let key = provider_catalog_key("key-a").with_rate_limit_fields(
+            Some(120),
+            Some(80),
+            None,
+            None,
+            None,
+            None,
+            Some(10),
+            Some(10),
+        );
+
+        assert_eq!(effective_provider_key_rpm_limit(&key, 100), Some(120));
+    }
+
+    #[test]
+    fn learned_provider_key_rpm_limit_requires_confidence() {
+        let low_confidence = provider_catalog_key("key-a").with_rate_limit_fields(
+            None,
+            Some(80),
+            Some(0),
+            Some(0),
+            Some(99),
+            None,
+            Some(5),
+            Some(1),
+        );
+        assert_eq!(effective_provider_key_rpm_limit(&low_confidence, 100), None);
+
+        let high_confidence = provider_catalog_key("key-a").with_rate_limit_fields(
+            None,
+            Some(80),
+            Some(0),
+            Some(0),
+            None,
+            Some(serde_json::json!([
+                {"new_limit": 80},
+                {"new_limit": 81},
+                {"new_limit": 80},
+            ])),
+            Some(120),
+            Some(118),
+        );
+        assert_eq!(
+            effective_provider_key_rpm_limit(&high_confidence, 100),
+            Some(80)
+        );
+    }
+
+    #[test]
+    fn counts_recent_provider_key_rpm_from_snapshot_or_recent_attempts() {
+        let recent_candidates = vec![
+            StoredRequestCandidate::new(
+                "one".to_string(),
+                "req-one".to_string(),
+                None,
+                None,
+                None,
+                None,
+                0,
+                0,
+                Some("provider-a".to_string()),
+                Some("endpoint-a".to_string()),
+                Some("key-a".to_string()),
+                RequestCandidateStatus::Success,
+                None,
+                false,
+                Some(200),
+                None,
+                None,
+                Some(10),
+                Some(7),
+                None,
+                None,
+                95,
+                Some(95),
+                Some(96),
+            )
+            .expect("candidate should build"),
+            StoredRequestCandidate::new(
+                "two".to_string(),
+                "req-two".to_string(),
+                None,
+                None,
+                None,
+                None,
+                0,
+                0,
+                Some("provider-a".to_string()),
+                Some("endpoint-a".to_string()),
+                Some("key-a".to_string()),
+                RequestCandidateStatus::Failed,
+                None,
+                false,
+                Some(502),
+                None,
+                None,
+                Some(10),
+                None,
+                None,
+                None,
+                98,
+                Some(98),
+                Some(99),
+            )
+            .expect("candidate should build"),
+        ];
+
+        assert_eq!(
+            count_recent_rpm_requests_for_provider_key(&recent_candidates, "key-a", 100),
+            7
+        );
+    }
+
+    #[test]
+    fn ignores_rpm_observations_before_reset_watermark() {
+        let recent_candidates = vec![
+            StoredRequestCandidate::new(
+                "one".to_string(),
+                "req-one".to_string(),
+                None,
+                None,
+                None,
+                None,
+                0,
+                0,
+                Some("provider-a".to_string()),
+                Some("endpoint-a".to_string()),
+                Some("key-a".to_string()),
+                RequestCandidateStatus::Success,
+                None,
+                false,
+                Some(200),
+                None,
+                None,
+                Some(10),
+                Some(7),
+                None,
+                None,
+                95,
+                Some(95),
+                Some(96),
+            )
+            .expect("candidate should build"),
+            StoredRequestCandidate::new(
+                "two".to_string(),
+                "req-two".to_string(),
+                None,
+                None,
+                None,
+                None,
+                0,
+                0,
+                Some("provider-a".to_string()),
+                Some("endpoint-a".to_string()),
+                Some("key-a".to_string()),
+                RequestCandidateStatus::Success,
+                None,
+                false,
+                Some(200),
+                None,
+                None,
+                Some(10),
+                Some(2),
+                None,
+                None,
+                99,
+                Some(99),
+                Some(100),
+            )
+            .expect("candidate should build"),
+        ];
+
+        assert_eq!(
+            count_recent_rpm_requests_for_provider_key_since(
+                &recent_candidates,
+                "key-a",
+                100,
+                Some(98),
+            ),
+            2
+        );
+    }
+
+    #[test]
+    fn provider_key_rpm_reserves_capacity_for_new_users() {
+        let key = provider_catalog_key("key-a").with_rate_limit_fields(
+            Some(10),
+            None,
+            None,
+            None,
+            None,
+            None,
+            Some(5),
+            Some(5),
+        );
+        let recent_candidates = vec![StoredRequestCandidate::new(
+            "one".to_string(),
+            "req-one".to_string(),
+            None,
+            None,
+            None,
+            None,
+            0,
+            0,
+            Some("provider-a".to_string()),
+            Some("endpoint-a".to_string()),
+            Some("key-a".to_string()),
+            RequestCandidateStatus::Success,
+            None,
+            false,
+            Some(200),
+            None,
+            None,
+            Some(10),
+            Some(9),
+            None,
+            None,
+            95,
+            Some(95),
+            Some(96),
+        )
+        .expect("candidate should build")];
+
+        assert!(!provider_key_rpm_allows_request(
+            &key,
+            &recent_candidates,
+            100,
+            false,
+        ));
+        assert!(provider_key_rpm_allows_request(
+            &key,
+            &recent_candidates,
+            100,
+            true,
+        ));
+        assert!(provider_key_rpm_allows_request_since(
+            &key,
+            &recent_candidates,
+            100,
+            false,
+            Some(97),
+        ));
+    }
+
+    #[test]
+    fn reads_provider_key_health_and_circuit_status_for_api_format() {
+        let key = provider_catalog_key("key-a").with_health_fields(
+            Some(serde_json::json!({
+                "openai:chat": {"health_score": 0.25},
+                "openai:responses": {"health_score": 0.75}
+            })),
+            Some(serde_json::json!({
+                "openai:chat": {"open": true},
+                "openai:responses": {"open": false}
+            })),
+        );
+
+        assert_eq!(provider_key_health_score(&key, "openai:chat"), Some(0.25));
+        assert_eq!(
+            provider_key_health_score(&key, "openai:responses"),
+            Some(0.75)
+        );
+        assert!(is_provider_key_circuit_open(&key, "openai:chat"));
+        assert!(!is_provider_key_circuit_open(&key, "openai:responses"));
+    }
+
+    #[test]
+    fn aggregates_provider_key_health_score_with_lower_bound_strategy() {
+        let key = provider_catalog_key("key-a").with_health_fields(
+            Some(serde_json::json!({
+                "openai:chat": {"health_score": 0.85},
+                "openai:responses": {"health_score": 0.45},
+                "claude:chat": {"health_score": 0.70}
+            })),
+            None,
+        );
+
+        assert_eq!(aggregate_provider_key_health_score(&key), Some(0.45));
+        assert_eq!(
+            effective_provider_key_health_score(&key, "gemini:chat"),
+            Some(0.45)
+        );
+    }
+
+    #[test]
+    fn classifies_provider_key_health_bucket_from_effective_score() {
+        let low = provider_catalog_key("key-low").with_health_fields(
+            Some(serde_json::json!({"openai:chat": {"health_score": 0.30}})),
+            None,
+        );
+        let degraded = provider_catalog_key("key-degraded").with_health_fields(
+            Some(serde_json::json!({"openai:chat": {"health_score": 0.65}})),
+            None,
+        );
+        let healthy = provider_catalog_key("key-healthy").with_health_fields(
+            Some(serde_json::json!({"openai:chat": {"health_score": 0.92}})),
+            None,
+        );
+
+        assert_eq!(
+            provider_key_health_bucket(&low, "openai:chat"),
+            Some(ProviderKeyHealthBucket::Low)
+        );
+        assert_eq!(
+            provider_key_health_bucket(&degraded, "openai:chat"),
+            Some(ProviderKeyHealthBucket::Degraded)
+        );
+        assert_eq!(
+            provider_key_health_bucket(&healthy, "openai:chat"),
+            Some(ProviderKeyHealthBucket::Healthy)
+        );
+    }
+}

crates/aether-scheduler-core/src/lib.rs

@@ -0,0 +1,45 @@
+mod affinity;
+mod auth;
+mod candidate;
+mod health;
+mod model;
+mod provider;
+mod request_candidate;
+
+pub use affinity::{
+    build_scheduler_affinity_cache_key_for_api_key_id, candidate_affinity_hash, candidate_key,
+    compare_affinity_order, matches_affinity_target, SchedulerAffinityTarget,
+};
+pub use auth::{
+    auth_constraints_allow_api_format, auth_constraints_allow_model,
+    auth_constraints_allow_provider, SchedulerAuthConstraints,
+};
+pub use candidate::{
+    auth_api_key_concurrency_limit_reached, build_minimal_candidate_selection,
+    candidate_is_selectable_with_runtime_state, candidate_supports_required_capability,
+    collect_global_model_names_for_required_capability, collect_selectable_candidates_from_keys,
+    reorder_candidates_by_scheduler_health, SchedulerMinimalCandidateSelectionCandidate,
+};
+pub use health::{
+    aggregate_provider_key_health_score, count_recent_active_requests_for_api_key,
+    count_recent_active_requests_for_provider, count_recent_rpm_requests_for_provider_key,
+    count_recent_rpm_requests_for_provider_key_since, effective_provider_key_health_score,
+    effective_provider_key_rpm_limit, is_candidate_in_recent_failure_cooldown,
+    is_provider_key_circuit_open, provider_key_health_bucket, provider_key_health_score,
+    provider_key_rpm_allows_request, provider_key_rpm_allows_request_since,
+    ProviderKeyHealthBucket, PROVIDER_KEY_RPM_WINDOW_SECS,
+};
+pub use model::{
+    candidate_model_names, extract_global_priority_for_format, matches_model_mapping,
+    normalize_api_format, resolve_provider_model_name, resolve_requested_global_model_name,
+    row_supports_required_capability, select_provider_model_name,
+};
+pub use provider::{build_provider_concurrent_limit_map, should_skip_provider_quota};
+pub use request_candidate::{
+    build_execution_request_candidate_seed, build_local_request_candidate_status_record,
+    build_report_request_candidate_status_record, execution_error_details,
+    finalize_execution_request_candidate_report_context, is_terminal_candidate_status,
+    parse_request_candidate_report_context, resolve_report_request_candidate_slot,
+    SchedulerExecutionRequestCandidateSeed, SchedulerRequestCandidateReportContext,
+    SchedulerResolvedReportRequestCandidateSlot,
+};

crates/aether-scheduler-core/src/model.rs

@@ -0,0 +1,257 @@
+use std::collections::BTreeSet;
+
+use aether_data::repository::candidate_selection::{
+    StoredMinimalCandidateSelectionRow, StoredProviderModelMapping,
+};
+use aether_data::DataLayerError;
+use regex::Regex;
+
+pub fn resolve_requested_global_model_name(
+    rows: &[StoredMinimalCandidateSelectionRow],
+    requested_model_name: &str,
+    api_format: &str,
+) -> Option<String> {
+    resolve_global_model_name_by(rows, |row| {
+        row.model_provider_model_name == requested_model_name
+    })
+    .or_else(|| {
+        resolve_global_model_name_by(rows, |row| {
+            row.model_provider_model_mappings
+                .as_ref()
+                .is_some_and(|mappings| {
+                    mappings.iter().any(|mapping| {
+                        mapping_scope_matches(mapping, api_format)
+                            && mapping.name == requested_model_name
+                    })
+                })
+        })
+    })
+    .or_else(|| {
+        resolve_global_model_name_by(rows, |row| {
+            row.global_model_mappings.as_ref().is_some_and(|patterns| {
+                patterns
+                    .iter()
+                    .any(|pattern| matches_model_mapping(pattern, requested_model_name))
+            })
+        })
+    })
+}
+
+fn resolve_global_model_name_by<F>(
+    rows: &[StoredMinimalCandidateSelectionRow],
+    matches: F,
+) -> Option<String>
+where
+    F: Fn(&StoredMinimalCandidateSelectionRow) -> bool,
+{
+    let mut matches = rows
+        .iter()
+        .filter(|row| matches(row))
+        .map(|row| row.global_model_name.trim())
+        .filter(|value| !value.is_empty())
+        .map(ToOwned::to_owned)
+        .collect::<BTreeSet<_>>()
+        .into_iter();
+    matches.next()
+}
+
+pub fn resolve_provider_model_name(
+    row: &StoredMinimalCandidateSelectionRow,
+    requested_model_name: &str,
+    api_format: &str,
+) -> Option<(String, Option<String>)> {
+    let selected_provider_model_name = select_provider_model_name(row, api_format);
+    let Some(key_allowed_models) = row.key_allowed_models.as_ref() else {
+        return Some((selected_provider_model_name, None));
+    };
+    if key_allowed_models.is_empty() {
+        return None;
+    }
+
+    if key_allowed_models
+        .iter()
+        .any(|value| value == requested_model_name)
+    {
+        return Some((selected_provider_model_name, None));
+    }
+
+    let candidate_models = candidate_model_names(row, api_format);
+    let mut sorted_allowed_models = key_allowed_models
+        .iter()
+        .map(|value| value.trim())
+        .filter(|value| !value.is_empty())
+        .map(ToOwned::to_owned)
+        .collect::<Vec<_>>();
+    sorted_allowed_models.sort();
+
+    for allowed_model in &sorted_allowed_models {
+        if candidate_models.contains(allowed_model.as_str()) {
+            return Some((allowed_model.clone(), Some(allowed_model.clone())));
+        }
+    }
+
+    let Some(global_model_mappings) = row.global_model_mappings.as_ref() else {
+        return None;
+    };
+    for allowed_model in sorted_allowed_models {
+        for pattern in global_model_mappings {
+            if matches_model_mapping(pattern, &allowed_model) {
+                return Some((allowed_model.clone(), Some(allowed_model)));
+            }
+        }
+    }
+
+    None
+}
+
+pub fn select_provider_model_name(
+    row: &StoredMinimalCandidateSelectionRow,
+    api_format: &str,
+) -> String {
+    let Some(mappings) = row.model_provider_model_mappings.as_ref() else {
+        return row.model_provider_model_name.clone();
+    };
+
+    let mut scoped = mappings
+        .iter()
+        .filter(|mapping| mapping_scope_matches(mapping, api_format))
+        .collect::<Vec<_>>();
+    if scoped.is_empty() {
+        return row.model_provider_model_name.clone();
+    }
+
+    scoped.sort_by(|left, right| {
+        left.priority
+            .cmp(&right.priority)
+            .then(left.name.cmp(&right.name))
+    });
+    let top_priority = scoped[0].priority;
+    scoped
+        .into_iter()
+        .find(|mapping| mapping.priority == top_priority)
+        .map(|mapping| mapping.name.clone())
+        .unwrap_or_else(|| row.model_provider_model_name.clone())
+}
+
+pub fn candidate_model_names(
+    row: &StoredMinimalCandidateSelectionRow,
+    api_format: &str,
+) -> BTreeSet<String> {
+    let mut names = BTreeSet::from([row.model_provider_model_name.clone()]);
+    if let Some(mappings) = row.model_provider_model_mappings.as_ref() {
+        for mapping in mappings {
+            if mapping_scope_matches(mapping, api_format) {
+                names.insert(mapping.name.clone());
+            }
+        }
+    }
+    names
+}
+
+fn mapping_scope_matches(mapping: &StoredProviderModelMapping, api_format: &str) -> bool {
+    let Some(api_formats) = mapping.api_formats.as_ref() else {
+        return true;
+    };
+
+    api_formats
+        .iter()
+        .any(|value| normalize_api_format(value) == api_format)
+}
+
+pub fn row_supports_required_capability(
+    row: &StoredMinimalCandidateSelectionRow,
+    required_capability: &str,
+) -> bool {
+    capabilities_support_required_capability(row.key_capabilities.as_ref(), required_capability)
+}
+
+fn capabilities_support_required_capability(
+    capabilities: Option<&serde_json::Value>,
+    required_capability: &str,
+) -> bool {
+    let required_capability = required_capability.trim();
+    if required_capability.is_empty() {
+        return true;
+    }
+    let Some(capabilities) = capabilities else {
+        return false;
+    };
+
+    if let Some(object) = capabilities.as_object() {
+        return object.iter().any(|(key, value)| {
+            key.eq_ignore_ascii_case(required_capability)
+                && match value {
+                    serde_json::Value::Bool(value) => *value,
+                    serde_json::Value::String(value) => value.eq_ignore_ascii_case("true"),
+                    serde_json::Value::Number(value) => {
+                        value.as_i64().is_some_and(|value| value > 0)
+                    }
+                    _ => false,
+                }
+        });
+    }
+
+    if let Some(items) = capabilities.as_array() {
+        return items.iter().any(|value| {
+            value
+                .as_str()
+                .is_some_and(|value| value.eq_ignore_ascii_case(required_capability))
+        });
+    }
+
+    false
+}
+
+pub fn matches_model_mapping(pattern: &str, model_name: &str) -> bool {
+    let Ok(compiled) = Regex::new(&format!("^(?:{pattern})$")) else {
+        return false;
+    };
+    compiled.is_match(model_name)
+}
+
+pub fn extract_global_priority_for_format(
+    raw: Option<&serde_json::Value>,
+    api_format: &str,
+) -> Result<Option<i32>, DataLayerError> {
+    let Some(raw) = raw else {
+        return Ok(None);
+    };
+    let Some(object) = raw.as_object() else {
+        return Err(DataLayerError::UnexpectedValue(
+            "provider_api_keys.global_priority_by_format is not a JSON object".to_string(),
+        ));
+    };
+
+    let Some(value) = object
+        .iter()
+        .find(|(key, _)| normalize_api_format(key) == api_format)
+        .map(|(_, value)| value)
+    else {
+        return Ok(None);
+    };
+
+    if let Some(value) = value.as_i64() {
+        return i32::try_from(value).map(Some).map_err(|_| {
+            DataLayerError::UnexpectedValue(format!(
+                "invalid provider_api_keys.global_priority_by_format value: {value}"
+            ))
+        });
+    }
+
+    if let Some(value) = value.as_str() {
+        let value = value.trim().parse::<i32>().map_err(|_| {
+            DataLayerError::UnexpectedValue(format!(
+                "invalid provider_api_keys.global_priority_by_format value: {value}"
+            ))
+        })?;
+        return Ok(Some(value));
+    }
+
+    Err(DataLayerError::UnexpectedValue(
+        "provider_api_keys.global_priority_by_format contains a non-integer value".to_string(),
+    ))
+}
+
+pub fn normalize_api_format(value: &str) -> String {
+    value.trim().to_ascii_lowercase()
+}

crates/aether-scheduler-core/src/provider.rs

@@ -0,0 +1,127 @@
+use std::collections::BTreeMap;
+
+use aether_data::repository::provider_catalog::StoredProviderCatalogProvider;
+use aether_data::repository::quota::StoredProviderQuotaSnapshot;
+use aether_wallet::{ProviderBillingType, ProviderQuotaSnapshot};
+
+pub fn should_skip_provider_quota(quota: &StoredProviderQuotaSnapshot, now_unix_secs: u64) -> bool {
+    let snapshot = ProviderQuotaSnapshot {
+        provider_id: quota.provider_id.clone(),
+        billing_type: ProviderBillingType::parse(&quota.billing_type),
+        monthly_quota_usd: quota.monthly_quota_usd,
+        monthly_used_usd: quota.monthly_used_usd,
+        quota_reset_day: quota.quota_reset_day,
+        quota_last_reset_at_unix_secs: quota.quota_last_reset_at_unix_secs,
+        quota_expires_at_unix_secs: quota.quota_expires_at_unix_secs,
+        is_active: quota.is_active,
+    };
+
+    if !snapshot.is_active || snapshot.is_expired(now_unix_secs) {
+        return true;
+    }
+
+    match snapshot.billing_type {
+        ProviderBillingType::MonthlyQuota | ProviderBillingType::FreeTier => snapshot
+            .remaining_quota_usd()
+            .is_some_and(|remaining| remaining <= 0.0),
+        ProviderBillingType::PayAsYouGo | ProviderBillingType::Unknown => false,
+    }
+}
+
+pub fn build_provider_concurrent_limit_map(
+    providers: Vec<StoredProviderCatalogProvider>,
+) -> BTreeMap<String, usize> {
+    providers
+        .into_iter()
+        .filter_map(|provider| {
+            provider
+                .concurrent_limit
+                .and_then(|limit| usize::try_from(limit).ok())
+                .filter(|limit| *limit > 0)
+                .map(|limit| (provider.id, limit))
+        })
+        .collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{build_provider_concurrent_limit_map, should_skip_provider_quota};
+    use aether_data::repository::provider_catalog::StoredProviderCatalogProvider;
+    use aether_data::repository::quota::StoredProviderQuotaSnapshot;
+
+    fn sample_provider(id: &str, concurrent_limit: Option<i32>) -> StoredProviderCatalogProvider {
+        StoredProviderCatalogProvider::new(
+            id.to_string(),
+            format!("provider-{id}"),
+            Some("https://example.com".to_string()),
+            "custom".to_string(),
+        )
+        .expect("provider should build")
+        .with_transport_fields(
+            true,
+            false,
+            false,
+            concurrent_limit,
+            None,
+            None,
+            None,
+            None,
+            None,
+        )
+    }
+
+    #[test]
+    fn skips_inactive_or_exhausted_monthly_quota_provider() {
+        let inactive = StoredProviderQuotaSnapshot::new(
+            "provider-1".to_string(),
+            "monthly_quota".to_string(),
+            Some(10.0),
+            1.0,
+            Some(30),
+            Some(1_000),
+            None,
+            false,
+        )
+        .expect("quota should build");
+        assert!(should_skip_provider_quota(&inactive, 2_000));
+
+        let exhausted = StoredProviderQuotaSnapshot::new(
+            "provider-1".to_string(),
+            "monthly_quota".to_string(),
+            Some(10.0),
+            10.0,
+            Some(30),
+            Some(1_000),
+            None,
+            true,
+        )
+        .expect("quota should build");
+        assert!(should_skip_provider_quota(&exhausted, 2_000));
+
+        let payg = StoredProviderQuotaSnapshot::new(
+            "provider-1".to_string(),
+            "pay_as_you_go".to_string(),
+            None,
+            10.0,
+            None,
+            None,
+            None,
+            true,
+        )
+        .expect("quota should build");
+        assert!(!should_skip_provider_quota(&payg, 2_000));
+    }
+
+    #[test]
+    fn builds_provider_concurrent_limit_map_for_positive_limits_only() {
+        let limits = build_provider_concurrent_limit_map(vec![
+            sample_provider("provider-a", Some(10)),
+            sample_provider("provider-b", Some(0)),
+            sample_provider("provider-c", None),
+        ]);
+
+        assert_eq!(limits.get("provider-a"), Some(&10));
+        assert!(!limits.contains_key("provider-b"));
+        assert!(!limits.contains_key("provider-c"));
+    }
+}

crates/aether-scheduler-core/src/request_candidate.rs

@@ -0,0 +1,664 @@
+use aether_contracts::{ExecutionError, ExecutionPlan};
+use aether_data::repository::candidates::{
+    RequestCandidateStatus, StoredRequestCandidate, UpsertRequestCandidateRecord,
+};
+use serde_json::Value;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct SchedulerRequestCandidateReportContext {
+    pub request_id: Option<String>,
+    pub candidate_id: Option<String>,
+    pub user_id: Option<String>,
+    pub api_key_id: Option<String>,
+    pub candidate_index: Option<u32>,
+    pub retry_index: u32,
+    pub provider_id: Option<String>,
+    pub endpoint_id: Option<String>,
+    pub key_id: Option<String>,
+    pub client_api_format: Option<String>,
+    pub provider_api_format: Option<String>,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct SchedulerResolvedReportRequestCandidateSlot {
+    pub id: String,
+    pub request_id: String,
+    pub user_id: Option<String>,
+    pub api_key_id: Option<String>,
+    pub candidate_index: u32,
+    pub retry_index: u32,
+    pub provider_id: Option<String>,
+    pub endpoint_id: Option<String>,
+    pub key_id: Option<String>,
+    pub extra_data: Option<Value>,
+    pub created_at_unix_secs: u64,
+    pub started_at_unix_secs: Option<u64>,
+    pub finished_at_unix_secs: Option<u64>,
+}
+
+pub struct SchedulerExecutionRequestCandidateSeed {
+    pub upsert_record: UpsertRequestCandidateRecord,
+    pub report_context: Value,
+}
+
+pub fn execution_error_details(
+    error: Option<&ExecutionError>,
+    body_json: Option<&Value>,
+) -> (Option<String>, Option<String>) {
+    match error {
+        Some(error) => (
+            Some(format!("{:?}", error.kind)),
+            Some(error.message.trim().to_string()).filter(|value| !value.is_empty()),
+        ),
+        None => (
+            None,
+            body_json
+                .and_then(extract_error_message)
+                .map(str::trim)
+                .filter(|value| !value.is_empty())
+                .map(ToOwned::to_owned),
+        ),
+    }
+}
+
+pub fn parse_request_candidate_report_context(
+    report_context: Option<&Value>,
+) -> Option<SchedulerRequestCandidateReportContext> {
+    let report_context = report_context?;
+    let retry_index = report_context
+        .get("retry_index")
+        .and_then(Value::as_u64)
+        .unwrap_or_default();
+    Some(SchedulerRequestCandidateReportContext {
+        request_id: string_field(report_context, "request_id"),
+        candidate_id: string_field(report_context, "candidate_id"),
+        user_id: string_field(report_context, "user_id"),
+        api_key_id: string_field(report_context, "api_key_id"),
+        candidate_index: report_context
+            .get("candidate_index")
+            .and_then(Value::as_u64)
+            .and_then(|value| u32::try_from(value).ok()),
+        retry_index: u32::try_from(retry_index).unwrap_or(u32::MAX),
+        provider_id: string_field(report_context, "provider_id"),
+        endpoint_id: string_field(report_context, "endpoint_id"),
+        key_id: string_field(report_context, "key_id"),
+        client_api_format: string_field(report_context, "client_api_format"),
+        provider_api_format: string_field(report_context, "provider_api_format"),
+    })
+}
+
+pub fn resolve_report_request_candidate_slot(
+    existing_candidates: &[StoredRequestCandidate],
+    metadata: SchedulerRequestCandidateReportContext,
+    now_unix_secs: u64,
+    generated_candidate_id: String,
+) -> Option<SchedulerResolvedReportRequestCandidateSlot> {
+    let request_id = metadata.request_id.clone()?;
+    let matched_candidate = match_existing_report_candidate(existing_candidates, &metadata);
+    let synthesized_extra_data = build_report_candidate_extra_data(&metadata);
+    let created_at_unix_secs = matched_candidate
+        .as_ref()
+        .map(|candidate| candidate.created_at_unix_secs)
+        .unwrap_or(now_unix_secs);
+    let candidate_index = matched_candidate
+        .as_ref()
+        .map(|candidate| candidate.candidate_index)
+        .or(metadata.candidate_index)
+        .unwrap_or_else(|| next_candidate_index(existing_candidates));
+    let retry_index = matched_candidate
+        .as_ref()
+        .map(|candidate| candidate.retry_index)
+        .unwrap_or(metadata.retry_index);
+
+    Some(SchedulerResolvedReportRequestCandidateSlot {
+        id: matched_candidate
+            .as_ref()
+            .map(|candidate| candidate.id.clone())
+            .or(metadata.candidate_id)
+            .unwrap_or(generated_candidate_id),
+        request_id,
+        user_id: matched_candidate
+            .as_ref()
+            .and_then(|candidate| candidate.user_id.clone())
+            .or(metadata.user_id),
+        api_key_id: matched_candidate
+            .as_ref()
+            .and_then(|candidate| candidate.api_key_id.clone())
+            .or(metadata.api_key_id),
+        candidate_index,
+        retry_index,
+        provider_id: matched_candidate
+            .as_ref()
+            .and_then(|candidate| candidate.provider_id.clone())
+            .or(metadata.provider_id),
+        endpoint_id: matched_candidate
+            .as_ref()
+            .and_then(|candidate| candidate.endpoint_id.clone())
+            .or(metadata.endpoint_id),
+        key_id: matched_candidate
+            .as_ref()
+            .and_then(|candidate| candidate.key_id.clone())
+            .or(metadata.key_id),
+        extra_data: matched_candidate
+            .as_ref()
+            .and_then(|candidate| candidate.extra_data.clone())
+            .or(synthesized_extra_data),
+        created_at_unix_secs,
+        started_at_unix_secs: matched_candidate
+            .as_ref()
+            .and_then(|candidate| candidate.started_at_unix_secs),
+        finished_at_unix_secs: matched_candidate
+            .as_ref()
+            .and_then(|candidate| candidate.finished_at_unix_secs),
+    })
+}
+
+pub fn build_execution_request_candidate_seed(
+    plan: &ExecutionPlan,
+    report_context: Option<&Value>,
+    started_at_unix_secs: u64,
+    generated_candidate_id: String,
+) -> SchedulerExecutionRequestCandidateSeed {
+    let mut context = report_context
+        .and_then(Value::as_object)
+        .cloned()
+        .unwrap_or_default();
+    let request_id = string_field(&Value::Object(context.clone()), "request_id")
+        .unwrap_or_else(|| plan.request_id.clone());
+    let candidate_index = context
+        .get("candidate_index")
+        .and_then(Value::as_u64)
+        .and_then(|value| u32::try_from(value).ok())
+        .unwrap_or(0);
+    let retry_index = context
+        .get("retry_index")
+        .and_then(Value::as_u64)
+        .and_then(|value| u32::try_from(value).ok())
+        .unwrap_or(0);
+    let candidate_id = string_field(&Value::Object(context.clone()), "candidate_id")
+        .unwrap_or(generated_candidate_id);
+    let user_id = string_field(&Value::Object(context.clone()), "user_id");
+    let api_key_id = string_field(&Value::Object(context.clone()), "api_key_id");
+
+    context.insert("request_id".to_string(), Value::String(request_id.clone()));
+    context.insert(
+        "candidate_id".to_string(),
+        Value::String(candidate_id.clone()),
+    );
+    context.insert(
+        "candidate_index".to_string(),
+        Value::Number(candidate_index.into()),
+    );
+    context.insert(
+        "provider_id".to_string(),
+        Value::String(plan.provider_id.clone()),
+    );
+    context.insert(
+        "endpoint_id".to_string(),
+        Value::String(plan.endpoint_id.clone()),
+    );
+    context.insert("key_id".to_string(), Value::String(plan.key_id.clone()));
+
+    SchedulerExecutionRequestCandidateSeed {
+        upsert_record: UpsertRequestCandidateRecord {
+            id: candidate_id,
+            request_id,
+            user_id,
+            api_key_id,
+            username: None,
+            api_key_name: None,
+            candidate_index,
+            retry_index,
+            provider_id: Some(plan.provider_id.clone()),
+            endpoint_id: Some(plan.endpoint_id.clone()),
+            key_id: Some(plan.key_id.clone()),
+            status: RequestCandidateStatus::Pending,
+            skip_reason: None,
+            is_cached: Some(false),
+            status_code: None,
+            error_type: None,
+            error_message: None,
+            latency_ms: None,
+            concurrent_requests: None,
+            extra_data: None,
+            required_capabilities: None,
+            created_at_unix_secs: Some(started_at_unix_secs),
+            started_at_unix_secs: Some(started_at_unix_secs),
+            finished_at_unix_secs: None,
+        },
+        report_context: Value::Object(context),
+    }
+}
+
+pub fn build_local_request_candidate_status_record(
+    plan: &ExecutionPlan,
+    report_context: Option<&Value>,
+    status: RequestCandidateStatus,
+    status_code: Option<u16>,
+    error_type: Option<String>,
+    error_message: Option<String>,
+    latency_ms: Option<u64>,
+    started_at_unix_secs: Option<u64>,
+    finished_at_unix_secs: Option<u64>,
+) -> Option<UpsertRequestCandidateRecord> {
+    let candidate_id = plan
+        .candidate_id
+        .as_deref()
+        .map(str::trim)
+        .filter(|value| !value.is_empty())?;
+    let metadata = parse_request_candidate_report_context(report_context)?;
+    let candidate_index = metadata.candidate_index?;
+
+    Some(UpsertRequestCandidateRecord {
+        id: candidate_id.to_string(),
+        request_id: plan.request_id.clone(),
+        user_id: metadata.user_id,
+        api_key_id: metadata.api_key_id,
+        username: None,
+        api_key_name: None,
+        candidate_index,
+        retry_index: metadata.retry_index,
+        provider_id: Some(plan.provider_id.clone()),
+        endpoint_id: Some(plan.endpoint_id.clone()),
+        key_id: Some(plan.key_id.clone()),
+        status,
+        skip_reason: None,
+        is_cached: None,
+        status_code,
+        error_type,
+        error_message,
+        latency_ms,
+        concurrent_requests: None,
+        extra_data: None,
+        required_capabilities: None,
+        created_at_unix_secs: None,
+        started_at_unix_secs,
+        finished_at_unix_secs,
+    })
+}
+
+pub fn build_report_request_candidate_status_record(
+    slot: SchedulerResolvedReportRequestCandidateSlot,
+    status: RequestCandidateStatus,
+    status_code: Option<u16>,
+    error_type: Option<String>,
+    error_message: Option<String>,
+    latency_ms: Option<u64>,
+    started_at_unix_secs: Option<u64>,
+    finished_at_unix_secs: Option<u64>,
+    now_unix_secs: u64,
+) -> UpsertRequestCandidateRecord {
+    let terminal_unix_secs = finished_at_unix_secs.unwrap_or(now_unix_secs);
+    let started_at_unix_secs = started_at_unix_secs
+        .or(slot.started_at_unix_secs)
+        .or_else(|| status.is_attempted(None).then_some(terminal_unix_secs));
+    let finished_at_unix_secs = finished_at_unix_secs
+        .or(slot.finished_at_unix_secs)
+        .or_else(|| is_terminal_candidate_status(status).then_some(terminal_unix_secs));
+
+    UpsertRequestCandidateRecord {
+        id: slot.id,
+        request_id: slot.request_id,
+        user_id: slot.user_id,
+        api_key_id: slot.api_key_id,
+        username: None,
+        api_key_name: None,
+        candidate_index: slot.candidate_index,
+        retry_index: slot.retry_index,
+        provider_id: slot.provider_id,
+        endpoint_id: slot.endpoint_id,
+        key_id: slot.key_id,
+        status,
+        skip_reason: None,
+        is_cached: None,
+        status_code,
+        error_type,
+        error_message,
+        latency_ms,
+        concurrent_requests: None,
+        extra_data: slot.extra_data,
+        required_capabilities: None,
+        created_at_unix_secs: Some(slot.created_at_unix_secs),
+        started_at_unix_secs,
+        finished_at_unix_secs,
+    }
+}
+
+pub fn finalize_execution_request_candidate_report_context(
+    report_context: Value,
+    candidate_id: &str,
+) -> Value {
+    let mut context = report_context.as_object().cloned().unwrap_or_default();
+    let candidate_id = candidate_id.trim();
+    if !candidate_id.is_empty() {
+        context.insert(
+            "candidate_id".to_string(),
+            Value::String(candidate_id.to_string()),
+        );
+    }
+    Value::Object(context)
+}
+
+pub fn is_terminal_candidate_status(status: RequestCandidateStatus) -> bool {
+    matches!(
+        status,
+        RequestCandidateStatus::Unused
+            | RequestCandidateStatus::Success
+            | RequestCandidateStatus::Failed
+            | RequestCandidateStatus::Cancelled
+            | RequestCandidateStatus::Skipped
+    )
+}
+
+fn extract_error_message(body_json: &Value) -> Option<&str> {
+    body_json
+        .get("error")
+        .and_then(|error| {
+            error
+                .get("message")
+                .and_then(Value::as_str)
+                .or_else(|| error.as_str())
+        })
+        .or_else(|| body_json.get("message").and_then(Value::as_str))
+}
+
+fn string_field(value: &Value, key: &str) -> Option<String> {
+    value
+        .get(key)
+        .and_then(Value::as_str)
+        .map(str::trim)
+        .filter(|value| !value.is_empty())
+        .map(ToOwned::to_owned)
+}
+
+fn match_existing_report_candidate<'a>(
+    candidates: &'a [StoredRequestCandidate],
+    metadata: &SchedulerRequestCandidateReportContext,
+) -> Option<&'a StoredRequestCandidate> {
+    if let Some(candidate_id) = metadata.candidate_id.as_deref() {
+        if let Some(candidate) = candidates
+            .iter()
+            .find(|candidate| candidate.id == candidate_id)
+        {
+            return Some(candidate);
+        }
+    }
+
+    if let Some(candidate_index) = metadata.candidate_index {
+        if let Some(candidate) = candidates.iter().find(|candidate| {
+            candidate.candidate_index == candidate_index
+                && candidate.retry_index == metadata.retry_index
+        }) {
+            return Some(candidate);
+        }
+    }
+
+    candidates
+        .iter()
+        .filter(|candidate| {
+            candidate.provider_id.as_deref() == metadata.provider_id.as_deref()
+                && candidate.endpoint_id.as_deref() == metadata.endpoint_id.as_deref()
+                && candidate.key_id.as_deref() == metadata.key_id.as_deref()
+        })
+        .max_by_key(|candidate| {
+            (
+                candidate.retry_index,
+                candidate.candidate_index,
+                candidate.created_at_unix_secs,
+            )
+        })
+}
+
+fn next_candidate_index(candidates: &[StoredRequestCandidate]) -> u32 {
+    candidates
+        .iter()
+        .map(|candidate| candidate.candidate_index)
+        .max()
+        .map(|value| value.saturating_add(1))
+        .unwrap_or_default()
+}
+
+fn build_report_candidate_extra_data(
+    metadata: &SchedulerRequestCandidateReportContext,
+) -> Option<Value> {
+    let mut extra_data = serde_json::Map::new();
+    extra_data.insert("gateway_execution_runtime".to_string(), Value::Bool(true));
+    extra_data.insert("phase".to_string(), Value::String("3c_trial".to_string()));
+    if let Some(client_api_format) = metadata.client_api_format.clone() {
+        extra_data.insert(
+            "client_api_format".to_string(),
+            Value::String(client_api_format),
+        );
+    }
+    if let Some(provider_api_format) = metadata.provider_api_format.clone() {
+        extra_data.insert(
+            "provider_api_format".to_string(),
+            Value::String(provider_api_format),
+        );
+    }
+    (!extra_data.is_empty()).then_some(Value::Object(extra_data))
+}
+
+#[cfg(test)]
+mod tests {
+    use aether_contracts::{
+        ExecutionError, ExecutionErrorKind, ExecutionPhase, ExecutionPlan, RequestBody,
+    };
+    use aether_data::repository::candidates::{RequestCandidateStatus, StoredRequestCandidate};
+    use serde_json::{json, Value};
+
+    use super::{
+        build_execution_request_candidate_seed, build_local_request_candidate_status_record,
+        build_report_request_candidate_status_record, execution_error_details,
+        finalize_execution_request_candidate_report_context,
+        parse_request_candidate_report_context, resolve_report_request_candidate_slot,
+        SchedulerResolvedReportRequestCandidateSlot,
+    };
+
+    fn sample_candidate(
+        id: &str,
+        candidate_index: u32,
+        retry_index: u32,
+    ) -> StoredRequestCandidate {
+        StoredRequestCandidate::new(
+            id.to_string(),
+            "req-1".to_string(),
+            Some("user-1".to_string()),
+            Some("key-1".to_string()),
+            None,
+            None,
+            candidate_index as i32,
+            retry_index as i32,
+            Some("provider-1".to_string()),
+            Some("endpoint-1".to_string()),
+            Some("catalog-key-1".to_string()),
+            RequestCandidateStatus::Pending,
+            None,
+            false,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            100,
+            Some(110),
+            None,
+        )
+        .expect("candidate should build")
+    }
+
+    fn sample_plan() -> ExecutionPlan {
+        ExecutionPlan {
+            request_id: "req-1".to_string(),
+            candidate_id: None,
+            provider_name: Some("openai".to_string()),
+            provider_id: "provider-1".to_string(),
+            endpoint_id: "endpoint-1".to_string(),
+            key_id: "key-1".to_string(),
+            method: "POST".to_string(),
+            url: "https://example.com/v1/chat/completions".to_string(),
+            headers: Default::default(),
+            content_type: Some("application/json".to_string()),
+            content_encoding: None,
+            body: RequestBody::from_json(json!({"model": "gpt-5"})),
+            stream: false,
+            client_api_format: "openai:chat".to_string(),
+            provider_api_format: "openai:chat".to_string(),
+            model_name: Some("gpt-5".to_string()),
+            proxy: None,
+            tls_profile: None,
+            timeouts: None,
+        }
+    }
+
+    #[test]
+    fn parses_report_context_and_resolves_existing_candidate_slot() {
+        let metadata = parse_request_candidate_report_context(Some(&json!({
+            "request_id": "req-1",
+            "candidate_index": 1,
+            "retry_index": 2,
+            "provider_id": "provider-1",
+            "endpoint_id": "endpoint-1",
+            "key_id": "catalog-key-1",
+            "client_api_format": "openai:chat"
+        })))
+        .expect("metadata");
+        let slot = resolve_report_request_candidate_slot(
+            &[sample_candidate("cand-1", 1, 2)],
+            metadata,
+            123,
+            "generated-1".to_string(),
+        )
+        .expect("slot");
+
+        assert_eq!(slot.id, "cand-1");
+        assert_eq!(slot.candidate_index, 1);
+        assert_eq!(slot.retry_index, 2);
+        assert_eq!(slot.request_id, "req-1");
+    }
+
+    #[test]
+    fn resolves_error_details_from_execution_error_or_body_json() {
+        let error = ExecutionError {
+            kind: ExecutionErrorKind::Upstream5xx,
+            phase: ExecutionPhase::FirstByte,
+            message: " upstream failed ".to_string(),
+            upstream_status: Some(502),
+            retryable: true,
+            failover_recommended: true,
+        };
+        assert_eq!(
+            execution_error_details(Some(&error), None),
+            (
+                Some("Upstream5xx".to_string()),
+                Some("upstream failed".to_string())
+            )
+        );
+        assert_eq!(
+            execution_error_details(None, Some(&json!({"error": {"message": "bad request"}}))),
+            (None, Some("bad request".to_string()))
+        );
+    }
+
+    #[test]
+    fn builds_execution_request_candidate_seed_and_finalizes_report_context() {
+        let seed = build_execution_request_candidate_seed(
+            &sample_plan(),
+            Some(&json!({
+                "request_id": "req-override",
+                "candidate_index": 3,
+                "retry_index": 2,
+                "user_id": "user-1",
+                "api_key_id": "api-key-1",
+                "client_api_format": "openai:chat"
+            })),
+            123,
+            "generated-1".to_string(),
+        );
+
+        assert_eq!(seed.upsert_record.id, "generated-1");
+        assert_eq!(seed.upsert_record.request_id, "req-override");
+        assert_eq!(seed.upsert_record.candidate_index, 3);
+        assert_eq!(seed.upsert_record.retry_index, 2);
+        assert_eq!(seed.upsert_record.user_id.as_deref(), Some("user-1"));
+        assert_eq!(
+            seed.report_context
+                .get("provider_id")
+                .and_then(Value::as_str),
+            Some("provider-1")
+        );
+
+        let finalized =
+            finalize_execution_request_candidate_report_context(seed.report_context, "cand-final");
+        assert_eq!(
+            finalized.get("candidate_id").and_then(Value::as_str),
+            Some("cand-final")
+        );
+    }
+
+    #[test]
+    fn builds_local_request_candidate_status_record() {
+        let mut plan = sample_plan();
+        plan.candidate_id = Some("cand-1".to_string());
+
+        let record = build_local_request_candidate_status_record(
+            &plan,
+            Some(&json!({
+                "candidate_index": 1,
+                "retry_index": 2,
+                "user_id": "user-1",
+                "api_key_id": "api-key-1"
+            })),
+            RequestCandidateStatus::Failed,
+            Some(500),
+            Some("Upstream5xx".to_string()),
+            Some("boom".to_string()),
+            Some(42),
+            Some(100),
+            Some(101),
+        )
+        .expect("record should build");
+
+        assert_eq!(record.id, "cand-1");
+        assert_eq!(record.candidate_index, 1);
+        assert_eq!(record.retry_index, 2);
+        assert_eq!(record.user_id.as_deref(), Some("user-1"));
+        assert_eq!(record.status, RequestCandidateStatus::Failed);
+    }
+
+    #[test]
+    fn builds_report_request_candidate_status_record_with_terminal_timestamps() {
+        let record = build_report_request_candidate_status_record(
+            SchedulerResolvedReportRequestCandidateSlot {
+                id: "cand-1".to_string(),
+                request_id: "req-1".to_string(),
+                user_id: Some("user-1".to_string()),
+                api_key_id: Some("api-key-1".to_string()),
+                candidate_index: 1,
+                retry_index: 0,
+                provider_id: Some("provider-1".to_string()),
+                endpoint_id: Some("endpoint-1".to_string()),
+                key_id: Some("key-1".to_string()),
+                extra_data: None,
+                created_at_unix_secs: 10,
+                started_at_unix_secs: None,
+                finished_at_unix_secs: None,
+            },
+            RequestCandidateStatus::Success,
+            Some(200),
+            None,
+            None,
+            Some(12),
+            None,
+            None,
+            123,
+        );
+
+        assert_eq!(record.started_at_unix_secs, Some(123));
+        assert_eq!(record.finished_at_unix_secs, Some(123));
+        assert_eq!(record.created_at_unix_secs, Some(10));
+        assert_eq!(record.status, RequestCandidateStatus::Success);
+    }
+}