Implement independent provider pool scheduling runtime

This commit is contained in:
fawney19
2026-04-17 23:05:49 +08:00
parent dd4641d618
commit 3321bb3ccc
38 changed files with 3222 additions and 83 deletions
@@ -10,21 +10,47 @@ pub(crate) const ADMIN_PROVIDER_POOL_SCAN_BATCH: u64 = 200;
pub(crate) const ADMIN_PROVIDER_OAUTH_DATA_UNAVAILABLE_DETAIL: &str =
"Admin provider OAuth data unavailable";
#[derive(Debug, Clone, Copy)]
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct AdminProviderPoolSchedulingPreset {
pub(crate) preset: String,
pub(crate) enabled: bool,
pub(crate) mode: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct AdminProviderPoolUnschedulableRule {
pub(crate) keyword: String,
pub(crate) duration_minutes: u64,
}
#[derive(Debug, Clone)]
pub(crate) struct AdminProviderPoolConfig {
pub(crate) scheduling_presets: Vec<AdminProviderPoolSchedulingPreset>,
pub(crate) unschedulable_rules: Vec<AdminProviderPoolUnschedulableRule>,
pub(crate) lru_enabled: bool,
pub(crate) skip_exhausted_accounts: bool,
pub(crate) sticky_session_ttl_seconds: u64,
pub(crate) latency_window_seconds: u64,
pub(crate) latency_sample_limit: u64,
pub(crate) cost_window_seconds: u64,
pub(crate) cost_limit_per_key_tokens: Option<u64>,
pub(crate) rate_limit_cooldown_seconds: u64,
pub(crate) overload_cooldown_seconds: u64,
pub(crate) health_policy_enabled: bool,
pub(crate) stream_timeout_threshold: u64,
pub(crate) stream_timeout_window_seconds: u64,
pub(crate) stream_timeout_cooldown_seconds: u64,
}
#[derive(Debug, Default)]
pub(crate) struct AdminProviderPoolRuntimeState {
pub(crate) total_sticky_sessions: usize,
pub(crate) sticky_sessions_by_key: BTreeMap<String, usize>,
pub(crate) sticky_bound_key_id: Option<String>,
pub(crate) cooldown_reason_by_key: BTreeMap<String, String>,
pub(crate) cooldown_ttl_by_key: BTreeMap<String, u64>,
pub(crate) cost_window_usage_by_key: BTreeMap<String, u64>,
pub(crate) latency_avg_ms_by_key: BTreeMap<String, f64>,
pub(crate) lru_score_by_key: BTreeMap<String, f64>,
}