use std::time::{SystemTime, UNIX_EPOCH}; use aether_data_contracts::repository::provider_catalog::StoredProviderCatalogKey; use serde_json::{json, Map, Value}; use crate::provider::ProviderPoolMemberInput; use crate::service::ProviderPoolService; pub fn provider_pool_key_account_quota_exhausted( key: &StoredProviderCatalogKey, provider_type: &str, ) -> bool { let adapter = ProviderPoolService::with_builtin_adapters().adapter(provider_type); adapter.quota_exhausted(&ProviderPoolMemberInput { provider_type, key, auth_config: None, provider_model_name: None, }) } pub fn provider_pool_key_quota_hard_blocked( key: &StoredProviderCatalogKey, provider_type: &str, ) -> bool { let adapter = ProviderPoolService::with_builtin_adapters().adapter(provider_type); adapter.quota_hard_blocked(&ProviderPoolMemberInput { provider_type, key, auth_config: None, provider_model_name: None, }) } /// Model-aware hard-block lookup for pre-scheduler candidate filtering. Some /// providers expose permanent account flags alongside independent model /// buckets; adapters can suppress the account flag when the selected model /// has its own usable quota. pub fn provider_pool_key_model_quota_hard_blocked( key: &StoredProviderCatalogKey, provider_type: &str, provider_model_name: &str, ) -> bool { let adapter = ProviderPoolService::with_builtin_adapters().adapter(provider_type); adapter.quota_hard_blocked(&ProviderPoolMemberInput { provider_type, key, auth_config: None, provider_model_name: Some(provider_model_name), }) } pub fn provider_pool_member_quota_snapshot<'a>( key: &'a StoredProviderCatalogKey, provider_type: &str, ) -> Option<&'a Map> { let quota_snapshot = key .status_snapshot .as_ref() .and_then(Value::as_object) .and_then(|snapshot| snapshot.get("quota")) .and_then(Value::as_object)?; provider_pool_quota_snapshot_matches_provider(quota_snapshot, provider_type) .then_some(quota_snapshot) } /// Resolve exhaustion for the quota bucket applicable to one provider model. /// /// Providers are free to expose quota windows in different shapes. Newer /// snapshots should put an explicit `model`/`models` (or `quota_group`) on a /// window; legacy Codex snapshots use a family prefix in `code` (for example /// `spark_5h`). We deliberately do not name any product or model here: the /// resolver compares the metadata supplied by the provider with the selected /// model and only falls back to account-level exhaustion when no model bucket /// can be identified. pub(crate) fn provider_pool_model_quota_exhausted( key: &StoredProviderCatalogKey, provider_type: &str, provider_model_name: &str, ) -> Option { provider_pool_quota_reaches_reserve(key, provider_type, Some(provider_model_name), 0.0) } fn provider_pool_quota_reaches_reserve( key: &StoredProviderCatalogKey, provider_type: &str, provider_model_name: Option<&str>, reserve_ratio: f64, ) -> Option { let is_codex = provider_type.trim().eq_ignore_ascii_case("codex"); let requested = provider_model_name.map(provider_pool_identifier_tokens); if reserve_ratio <= 0.0 && requested.as_ref().is_some_and(|tokens| tokens.is_empty()) { return None; } // Prefer the materialized status snapshot, but also inspect the raw // provider metadata. A quota refresh and a request can race, leaving the // latter newer than the snapshot; resolving both avoids falling back to an // account-wide signal and incorrectly blocking an unrelated model bucket. let sources = [ provider_pool_member_quota_snapshot(key, provider_type), provider_pool_metadata_bucket(key.upstream_metadata.as_ref(), provider_type), ]; let mut resolved = None::<(Option, bool)>; let mut resolved_specific_bucket = false; for source in sources.into_iter().flatten() { let observed_at = provider_pool_timestamp_unix_secs(source.get("observed_at")) .or_else(|| provider_pool_timestamp_unix_secs(source.get("updated_at"))); let account_signal = (is_codex && reserve_ratio <= 0.0) .then(|| provider_pool_codex_account_quota_signal(source, observed_at)) .flatten(); let mut windows = provider_pool_collect_quota_windows(source); if is_codex { // Raw refresh metadata can be newer than the materialized windows. // Include all four legacy slots before selecting the model bucket. for prefix in ["primary", "secondary", "spark_primary", "spark_secondary"] { let Some(used_percent) = provider_pool_json_f64(source.get(&format!("{prefix}_used_percent"))) else { continue; }; if provider_pool_json_f64(source.get(&format!("{prefix}_window_minutes"))) == Some(0.0) { continue; } let mut window = Map::from_iter([ ("code".to_string(), json!(prefix)), ("used_percent".to_string(), json!(used_percent)), ]); for field in [ "reset_at", "next_reset_at", "reset_seconds", "reset_after_seconds", ] { if let Some(value) = source.get(&format!("{prefix}_{field}")) { window.insert(field.to_string(), value.clone()); } } windows.push(window); } windows.retain(provider_pool_quota_window_has_observation); if let Some(exhausted) = account_signal { if !windows.iter().any(provider_pool_window_is_generic) { // A flags-only quota refresh is still a newer account // observation, but must not override a model-specific bucket. windows.push(Map::from_iter([ ("code".to_string(), json!("account")), ("is_exhausted".to_string(), json!(exhausted)), ])); } } } if windows.is_empty() { continue; } let model_matches = windows .iter() .filter(|window| { provider_model_name.is_some_and(|model| { provider_pool_window_explicitly_matches_model(window, model) }) }) .collect::>(); if !model_matches.is_empty() { let exhausted = if reserve_ratio > 0.0 { // Reserving quota protects every applicable rate-limit window, // including short and weekly limits for an explicit model. provider_pool_any_window_exhausted(model_matches, observed_at, reserve_ratio) } else { provider_pool_explicit_model_windows_exhausted(model_matches, observed_at) }; if resolved.is_none() || (is_codex && !resolved_specific_bucket) || provider_pool_should_replace_model_quota_resolution( resolved.as_ref().and_then(|(observed_at, _)| *observed_at), observed_at, ) { resolved = Some((observed_at, exhausted)); resolved_specific_bucket = true; } continue; } // Legacy snapshots may not carry a model field. Match an opaque // family token (the prefix before `_`/`:` in `code`, or an explicit // family key) against the model's tokens. This keeps independent // windows isolated without baking in names such as "spark". let family_matches = windows .iter() .filter(|window| { requested .as_ref() .is_some_and(|tokens| provider_pool_window_family_matches_model(window, tokens)) }) .collect::>(); if !family_matches.is_empty() { let exhausted = provider_pool_any_window_exhausted(family_matches, observed_at, reserve_ratio); if resolved.is_none() || (is_codex && !resolved_specific_bucket) || provider_pool_should_replace_model_quota_resolution( resolved.as_ref().and_then(|(observed_at, _)| *observed_at), observed_at, ) { resolved = Some((observed_at, exhausted)); resolved_specific_bucket = true; } continue; } // A newer account observation does not update an independent model // bucket. Only compare freshness between applicable model sources. if is_codex && resolved_specific_bucket { continue; } // Account-scoped windows (for example the ordinary weekly and short // windows emitted by Codex) apply to every model that has no more // specific family. Restrict this fallback to well-known structural // window names; opaque family names such as `alpha_weekly` must not // accidentally make an unrelated model look schedulable. let generic_matches = windows .iter() .filter(|window| provider_pool_window_is_generic(window)) .collect::>(); if !generic_matches.is_empty() { let exhausted = account_signal.unwrap_or_else(|| { provider_pool_any_window_exhausted(generic_matches, observed_at, reserve_ratio) }); if resolved.is_none() || provider_pool_should_replace_model_quota_resolution( resolved.as_ref().and_then(|(observed_at, _)| *observed_at), observed_at, ) { resolved = Some((observed_at, exhausted)); } } } resolved.map(|(_, exhausted)| exhausted) } fn provider_pool_codex_account_quota_signal( source: &Map, observed_at: Option, ) -> Option { let allowed = provider_pool_json_bool(source.get("allowed")); let limit_reached = provider_pool_json_bool(source.get("limit_reached")); if allowed == Some(false) || limit_reached == Some(true) { let reset_elapsed = provider_pool_current_unix_secs().is_some_and(|now| { provider_pool_reset_deadline_elapsed(source, observed_at, now) || ["primary", "secondary"].into_iter().any(|prefix| { let window = [ "reset_at", "next_reset_at", "reset_seconds", "reset_after_seconds", ] .into_iter() .filter_map(|field| { source .get(&format!("{prefix}_{field}")) .map(|value| (field.to_string(), value.clone())) }) .collect::>(); provider_pool_reset_deadline_elapsed(&window, observed_at, now) }) }); return (!reset_elapsed).then_some(true); } (allowed == Some(true) || limit_reached == Some(false)).then_some(false) } fn provider_pool_should_replace_model_quota_resolution( previous_observed_at: Option, next_observed_at: Option, ) -> bool { match (previous_observed_at, next_observed_at) { (Some(previous), Some(next)) => next >= previous, (None, Some(_)) => true, (Some(_), None) => false, // Preserve source order when neither side carries freshness metadata; // the materialized status snapshot is preferred over raw metadata. (None, None) => false, } } /// Public adapter-independent model quota lookup used by schedulers that need /// to prefilter candidates before constructing provider-pool signals. pub fn provider_pool_key_model_quota_exhausted( key: &StoredProviderCatalogKey, provider_type: &str, provider_model_name: &str, ) -> Option { provider_pool_model_quota_exhausted(key, provider_type, provider_model_name) } fn provider_pool_explicit_model_windows_exhausted( windows: Vec<&Map>, snapshot_observed_at: Option, ) -> bool { let now_unix_secs = provider_pool_current_unix_secs(); // Explicit model buckets represent independent windows for one model. The // model remains usable while at least one of those windows still has // capacity, so exhaustion is reported only when all are exhausted. windows.iter().all(|window| { provider_pool_quota_window_is_exhausted(window) && !now_unix_secs.is_some_and(|now| { provider_pool_reset_deadline_elapsed(window, snapshot_observed_at, now) }) }) } fn provider_pool_any_window_exhausted( windows: Vec<&Map>, snapshot_observed_at: Option, reserve_ratio: f64, ) -> bool { let now_unix_secs = provider_pool_current_unix_secs(); windows.iter().any(|window| { provider_pool_quota_window_reaches_reserve(window, reserve_ratio) && !now_unix_secs.is_some_and(|now| { provider_pool_reset_deadline_elapsed(window, snapshot_observed_at, now) }) }) } fn provider_pool_quota_window_has_observation(window: &Map) -> bool { ["is_exhausted", "exhausted"] .into_iter() .any(|field| provider_pool_json_bool(window.get(field)).is_some()) || [ "used_ratio", "usage_ratio", "used_percent", "remaining_ratio", "remaining_fraction", "remaining_percent", ] .into_iter() .any(|field| provider_pool_json_f64(window.get(field)).is_some()) || (provider_pool_json_f64( window .get("remaining") .or_else(|| window.get("remaining_value")), ) .is_some() && provider_pool_json_f64( window .get("limit") .or_else(|| window.get("limit_value")) .or_else(|| window.get("total")), ) .is_some_and(|limit| limit > 0.0)) } pub(crate) fn provider_pool_source_account_quota_exhausted( source: &Map, ) -> Option { let windows = provider_pool_collect_quota_windows(source); let account_windows = windows .iter() .filter(|window| provider_pool_window_is_generic(window)) .filter(|window| provider_pool_quota_window_has_observation(window)) .collect::>(); if account_windows.is_empty() { return None; } let observed_at = provider_pool_timestamp_unix_secs(source.get("observed_at")) .or_else(|| provider_pool_timestamp_unix_secs(source.get("updated_at"))); Some(provider_pool_any_window_exhausted( account_windows, observed_at, 0.0, )) } /// Whether Codex metadata contains an account quota observation rather than an /// identity-only update or an independent model quota bucket. pub fn provider_pool_codex_metadata_has_account_quota(source: &Map) -> bool { ["primary_used_percent", "secondary_used_percent"] .into_iter() .any(|field| provider_pool_json_f64(source.get(field)).is_some()) || provider_pool_source_account_quota_exhausted(source).is_some() || ["allowed", "limit_reached", "has_credits"] .into_iter() .any(|field| provider_pool_json_bool(source.get(field)).is_some()) || provider_pool_json_bool(source.get("credits_unlimited")) == Some(true) } /// Whether an applicable Codex window has at most 1% remaining. Missing quota /// data and windows whose reset has elapsed do not trigger this opt-in guard. pub fn provider_pool_key_minimum_quota_reached( key: &StoredProviderCatalogKey, provider_type: &str, provider_model_name: Option<&str>, ) -> bool { if !provider_type.trim().eq_ignore_ascii_case("codex") { return false; } provider_pool_quota_reaches_reserve(key, provider_type, provider_model_name, 0.01) .unwrap_or(false) } fn provider_pool_quota_window_reaches_reserve( window: &Map, reserve_ratio: f64, ) -> bool { if provider_pool_quota_window_is_exhausted(window) { return true; } if reserve_ratio <= 0.0 { return false; } let used_ratio = provider_pool_json_f64(window.get("used_ratio")) .or_else(|| provider_pool_json_f64(window.get("usage_ratio"))) .or_else(|| provider_pool_json_f64(window.get("used_percent")).map(|value| value / 100.0)) .or_else(|| { provider_pool_json_f64(window.get("remaining_ratio")) .or_else(|| provider_pool_json_f64(window.get("remaining_fraction"))) .map(|value| 1.0 - value) }) .or_else(|| { provider_pool_json_f64(window.get("remaining_percent")).map(|value| 1.0 - value / 100.0) }) .or_else(|| { let remaining = provider_pool_json_f64( window .get("remaining") .or_else(|| window.get("remaining_value")), )?; let limit = provider_pool_json_f64( window .get("limit") .or_else(|| window.get("limit_value")) .or_else(|| window.get("total")), )?; (limit > 0.0).then_some(1.0 - remaining / limit) }); used_ratio.is_some_and(|used_ratio| used_ratio >= 1.0 - reserve_ratio) } fn provider_pool_window_is_generic(window: &Map) -> bool { if provider_pool_window_has_explicit_model(window) || window .get("scope") .and_then(Value::as_str) .is_some_and(|scope| scope.trim().eq_ignore_ascii_case("model")) { return false; } let code = window .get("code") .and_then(Value::as_str) .unwrap_or_default(); let family = code .split_once(['_', ':', '/']) .map(|(prefix, _)| prefix) .unwrap_or(code) .trim() .to_ascii_lowercase(); if family.is_empty() { return window .get("scope") .and_then(Value::as_str) .is_some_and(|scope| scope.trim().eq_ignore_ascii_case("account")); } [ "weekly", "5h", "daily", "monthly", "primary", "secondary", "account", "quota", "window", "rate", "reset", ] .contains(&family.as_str()) } fn provider_pool_window_explicitly_matches_model( window: &Map, requested_model: &str, ) -> bool { let requested = provider_pool_normalize_identifier(requested_model); let requested_tokens = provider_pool_identifier_tokens(requested_model); if requested.is_empty() { return false; } [ "model", "model_name", "model_id", "quota_model", "quota_model_name", "target_model", "limit_name", ] .iter() .filter_map(|key| window.get(*key)) .any(|value| match value { Value::String(value) => { provider_pool_identifiers_match(&requested, value, &requested_tokens) } Value::Array(values) => values .iter() .filter_map(Value::as_str) .any(|value| provider_pool_identifiers_match(&requested, value, &requested_tokens)), _ => false, }) || ["models", "model_ids"] .iter() .filter_map(|key| window.get(*key).and_then(Value::as_array)) .flatten() .filter_map(Value::as_str) .any(|value| provider_pool_identifiers_match(&requested, value, &requested_tokens)) } fn provider_pool_window_family_matches_model( window: &Map, requested_tokens: &std::collections::BTreeSet, ) -> bool { let explicit_scope = window .get("scope") .and_then(Value::as_str) .map(|scope| scope.trim().to_ascii_lowercase()); // A model-scoped window without an explicit model must not accidentally // match a token from its opaque code. if explicit_scope.as_deref() == Some("model") || provider_pool_window_has_explicit_model(window) { return false; } let mut families = Vec::new(); for key in ["quota_group", "quota_family", "family", "bucket"] { if let Some(value) = window.get(key).and_then(Value::as_str) { families.push(value.to_string()); } } if let Some(code) = window.get("code").and_then(Value::as_str) { let code = code.trim(); if let Some((prefix, _)) = code.split_once(['_', ':', '/']) { families.push(prefix.to_string()); } } families.into_iter().any(|family| { let normalized = provider_pool_normalize_identifier(&family); if normalized.is_empty() || [ "account", "quota", "window", "primary", "secondary", "rate", "reset", ] .iter() .any(|generic| normalized == *generic) { return false; } requested_tokens.iter().any(|token| { token.len() >= 3 && (token == &normalized || token.contains(&normalized) || normalized.contains(token)) }) }) } fn provider_pool_identifiers_match( requested: &str, candidate: &str, requested_tokens: &std::collections::BTreeSet, ) -> bool { let candidate_tokens = provider_pool_identifier_tokens(candidate); let candidate = provider_pool_normalize_identifier(candidate); if candidate.is_empty() { return false; } requested == candidate || candidate_tokens .iter() .any(|token| { requested_tokens.contains(token) && provider_pool_is_specific_model_token(token) }) // Handle compact upstream identifiers such as `spark` embedded in a // provider model name (`vendor-codex-spark`) while avoiding accidental // one/two-character matches. || (candidate.len() >= 4 && requested.len() >= 6 && (requested.contains(&candidate) || candidate.contains(requested))) } fn provider_pool_is_specific_model_token(token: &str) -> bool { token.len() >= 4 && !token.chars().all(|character| character.is_ascii_digit()) && ![ "auto", "base", "claude", "codex", "default", "fast", "flash", "free", "gemini", "gpt", "latest", "mini", "model", "plus", "pro", "reasoning", "team", "think", "thinking", "tiered", "vendor", ] .contains(&token) } fn provider_pool_window_has_explicit_model(window: &Map) -> bool { [ "model", "model_name", "model_id", "quota_model", "quota_model_name", "target_model", "models", "model_ids", ] .iter() .any(|key| match window.get(*key) { Some(Value::String(value)) => !value.trim().is_empty(), Some(Value::Array(values)) => values .iter() .any(|value| value.as_str().is_some_and(|value| !value.trim().is_empty())), _ => false, }) } fn provider_pool_normalize_identifier(value: &str) -> String { value .trim() .to_ascii_lowercase() .chars() .filter(|character| character.is_ascii_alphanumeric()) .collect() } fn provider_pool_identifier_tokens(value: &str) -> std::collections::BTreeSet { value .split(|character: char| !character.is_ascii_alphanumeric()) .map(|token| token.trim().to_ascii_lowercase()) .filter(|token| token.len() >= 3) .collect() } /// Materialize quota windows from the small set of shapes emitted by current /// and legacy adapters. Model maps (`quota_by_model`/`models`) are converted /// to the same window representation used by status snapshots, with the map /// key retained as the model identity. Keeping this normalization here means /// provider adapters do not need to grow model-specific quota code whenever an /// upstream introduces another independent bucket. fn provider_pool_collect_quota_windows(source: &Map) -> Vec> { let mut windows = Vec::new(); for key in ["windows", "additional_quota_windows"] { if let Some(values) = source.get(key).and_then(Value::as_array) { windows.extend(values.iter().filter_map(Value::as_object).cloned()); } } for key in ["quota_by_model", "models", "model_quotas"] { let Some(models) = source.get(key).and_then(Value::as_object) else { continue; }; for (model_name, item) in models { let Some(item) = item.as_object() else { continue; }; let mut window = item.clone(); window .entry("model".to_string()) .or_insert_with(|| json!(model_name)); window .entry("scope".to_string()) .or_insert_with(|| json!("model")); window .entry("code".to_string()) .or_insert_with(|| json!(format!("model:{model_name}"))); windows.push(window); } } windows } pub fn provider_pool_quota_snapshot_updated_at( key: &StoredProviderCatalogKey, provider_type: &str, ) -> Option { let quota_snapshot = provider_pool_member_quota_snapshot(key, provider_type)?; provider_pool_timestamp_unix_secs(quota_snapshot.get("updated_at")) } pub fn provider_pool_quota_metadata_updated_at( upstream_metadata: Option<&Value>, provider_type: &str, ) -> Option { let bucket = provider_pool_metadata_bucket(upstream_metadata, provider_type)?; provider_pool_timestamp_unix_secs(bucket.get("updated_at")) } pub fn provider_pool_quota_metadata_provider_type(metadata_update: &Value) -> Option { let object = metadata_update.as_object()?; let service = ProviderPoolService::with_builtin_adapters(); let known_provider_type = service .provider_types() .find(|provider_type| object.contains_key(*provider_type)) .map(ToOwned::to_owned); known_provider_type.or_else(|| { object .iter() .find(|(_, value)| value.is_object()) .map(|(provider_type, _)| provider_type.clone()) }) } pub fn provider_pool_key_scheduling_label( is_active: bool, cooldown_reason: Option<&str>, cooldown_ttl_seconds: Option, ) -> (String, String, String, Vec) { if !is_active { return ( "blocked".to_string(), "inactive".to_string(), "已禁用".to_string(), vec![json!({ "code": "inactive", "label": "已禁用", "blocking": true, "source": "manual", "ttl_seconds": Value::Null, "detail": Value::Null, })], ); } if let Some(reason) = cooldown_reason { return ( "degraded".to_string(), "cooldown".to_string(), "冷却中".to_string(), vec![json!({ "code": "cooldown", "label": "冷却中", "blocking": true, "source": "pool", "ttl_seconds": cooldown_ttl_seconds, "detail": reason, })], ); } ( "available".to_string(), "available".to_string(), "可用".to_string(), Vec::new(), ) } pub(crate) fn provider_pool_metadata_bucket<'a>( upstream_metadata: Option<&'a Value>, provider_type: &str, ) -> Option<&'a Map> { upstream_metadata .and_then(Value::as_object) .and_then(|metadata| metadata.get(&provider_type.trim().to_ascii_lowercase())) .and_then(Value::as_object) } pub(crate) fn provider_pool_json_bool(value: Option<&Value>) -> Option { match value { Some(Value::Bool(value)) => Some(*value), Some(Value::String(value)) => match value.trim().to_ascii_lowercase().as_str() { "true" | "1" => Some(true), "false" | "0" => Some(false), _ => None, }, _ => None, } } pub(crate) fn provider_pool_json_f64(value: Option<&Value>) -> Option { match value { Some(Value::Number(number)) => number.as_f64(), Some(Value::String(value)) => value.trim().parse::().ok(), _ => None, } .filter(|value| value.is_finite()) } pub(crate) fn provider_pool_timestamp_unix_secs(value: Option<&Value>) -> Option { let mut timestamp = match provider_pool_json_f64(value) { Some(timestamp) => timestamp, None => { return chrono::DateTime::parse_from_rfc3339(value?.as_str()?.trim()) .ok()? .timestamp() .try_into() .ok(); } }; if timestamp <= 0.0 { return None; } if timestamp > 1_000_000_000_000.0 { timestamp /= 1000.0; } Some(timestamp as u64) } pub(crate) fn provider_pool_current_unix_secs() -> Option { SystemTime::now() .duration_since(UNIX_EPOCH) .ok() .map(|duration| duration.as_secs()) } fn provider_pool_reset_deadline_unix_secs( item: &Map, fallback_observed_at: Option, ) -> Option { provider_pool_timestamp_unix_secs(item.get("reset_at")) .or_else(|| provider_pool_timestamp_unix_secs(item.get("next_reset_at"))) .or_else(|| { let reset_seconds = provider_pool_json_f64(item.get("reset_seconds")) .or_else(|| provider_pool_json_f64(item.get("reset_after_seconds")))?; if reset_seconds < 0.0 { return None; } let base = provider_pool_timestamp_unix_secs(item.get("observed_at")) .or_else(|| provider_pool_timestamp_unix_secs(item.get("updated_at"))) .or(fallback_observed_at)?; Some(base.saturating_add(reset_seconds.ceil() as u64)) }) } /// Whether the quota window's reset deadline has already passed. /// /// The deadline is taken from `reset_at`/`next_reset_at` when present, otherwise /// derived from `reset_seconds`/`reset_after_seconds` anchored at the window (or /// fallback) observation time. Scheduling ignores exhausted windows once this is /// true; read paths can reuse the same predicate so the displayed quota matches /// the scheduling decision after a reset. pub fn provider_pool_reset_deadline_elapsed( item: &Map, fallback_observed_at: Option, now_unix_secs: u64, ) -> bool { provider_pool_reset_deadline_unix_secs(item, fallback_observed_at) .is_some_and(|reset_at| reset_at <= now_unix_secs) } fn provider_pool_quota_window_is_exhausted(window: &Map) -> bool { provider_pool_json_bool(window.get("is_exhausted")) .or_else(|| provider_pool_json_bool(window.get("exhausted"))) .or_else(|| { provider_pool_json_f64( window .get("used_ratio") .or_else(|| window.get("usage_ratio")), ) .map(|value| value >= 1.0 - 1e-6) }) .or_else(|| { provider_pool_json_f64(window.get("used_percent")).map(|value| value >= 100.0 - 1e-6) }) .or_else(|| { provider_pool_json_f64( window .get("remaining_ratio") .or_else(|| window.get("remaining_fraction")), ) .map(|value| value <= 1e-6) }) .or_else(|| { provider_pool_json_f64(window.get("remaining_percent")).map(|value| value <= 1e-6) }) .or_else(|| { let remaining = provider_pool_json_f64( window .get("remaining") .or_else(|| window.get("remaining_value")), )?; let limit = provider_pool_json_f64( window .get("limit") .or_else(|| window.get("limit_value")) .or_else(|| window.get("total")), )?; (limit > 0.0).then_some(remaining <= 0.0) }) .unwrap_or(false) } fn provider_pool_window_is_model_scoped(window: &Map) -> bool { window .get("scope") .and_then(Value::as_str) .is_some_and(|scope| scope.trim().eq_ignore_ascii_case("model")) || provider_pool_window_has_explicit_model(window) } fn provider_pool_quota_snapshot_matches_provider( quota_snapshot: &Map, provider_type: &str, ) -> bool { let normalized_provider_type = provider_type.trim().to_ascii_lowercase(); match quota_snapshot .get("provider_type") .and_then(Value::as_str) .map(str::trim) .filter(|value| !value.is_empty()) { Some(provider_type) => provider_type.eq_ignore_ascii_case(&normalized_provider_type), None => { provider_pool_json_bool(quota_snapshot.get("exhausted")) == Some(true) || quota_snapshot .get("code") .and_then(Value::as_str) .is_some_and(|code| !code.trim().eq_ignore_ascii_case("unknown")) || quota_snapshot .get("updated_at") .is_some_and(|value| !value.is_null()) || quota_snapshot .get("observed_at") .is_some_and(|value| !value.is_null()) || quota_snapshot .get("usage_ratio") .is_some_and(|value| !value.is_null()) || quota_snapshot .get("reset_seconds") .is_some_and(|value| !value.is_null()) || quota_snapshot .get("windows") .and_then(Value::as_array) .is_some_and(|windows| !windows.is_empty()) || quota_snapshot .get("additional_quota_windows") .and_then(Value::as_array) .is_some_and(|windows| !windows.is_empty()) || ["quota_by_model", "models", "model_quotas"] .iter() .any(|key| { quota_snapshot .get(*key) .and_then(Value::as_object) .is_some_and(|models| !models.is_empty()) }) || quota_snapshot .get("credits") .and_then(Value::as_object) .is_some_and(|credits| !credits.is_empty()) } } } pub(crate) fn provider_pool_quota_snapshot_exhausted_decision( key: &StoredProviderCatalogKey, provider_type: &str, ) -> Option { let quota_snapshot = key .status_snapshot .as_ref() .and_then(Value::as_object) .and_then(|snapshot| snapshot.get("quota")) .and_then(Value::as_object)?; if !provider_pool_quota_snapshot_matches_provider(quota_snapshot, provider_type) { return None; } let exhausted = provider_pool_json_bool(quota_snapshot.get("exhausted"))?; if exhausted { let now_unix_secs = provider_pool_current_unix_secs(); let snapshot_observed_at = provider_pool_timestamp_unix_secs(quota_snapshot.get("observed_at")) .or_else(|| provider_pool_timestamp_unix_secs(quota_snapshot.get("updated_at"))); let materialized_windows = provider_pool_collect_quota_windows(quota_snapshot); if !materialized_windows.is_empty() { // Model-scoped windows are evaluated only when the request model // is known. They must not turn the account-level fallback into // an exhausted state for unrelated models. let account_scoped_windows = materialized_windows .iter() .filter(|window| !provider_pool_window_is_model_scoped(window)) .collect::>(); // A snapshot containing only model-scoped buckets has no // account-wide signal to apply when the caller did not provide a // model name (for example, an admin status listing). Do not let a // single exhausted model poison every sibling bucket. if account_scoped_windows.is_empty() { return Some(false); } let windows = account_scoped_windows; let mut saw_exhausted_window = false; let mut saw_active_exhausted_window = false; let mut windows_max_ratio = None::; for window in windows.iter() { if let Some(ratio) = provider_pool_json_f64(window.get("used_ratio")) { windows_max_ratio = Some(windows_max_ratio.map_or(ratio, |current| current.max(ratio))); } if provider_pool_quota_window_is_exhausted(window) { saw_exhausted_window = true; let reset_elapsed = now_unix_secs.is_some_and(|now| { provider_pool_reset_deadline_elapsed(window, snapshot_observed_at, now) }); if !reset_elapsed { saw_active_exhausted_window = true; } } } if saw_exhausted_window { return Some(saw_active_exhausted_window); } if windows_max_ratio.is_some_and(|ratio| ratio < 1.0 - 1e-6) { return Some(false); } } else if now_unix_secs.is_some_and(|now| { provider_pool_reset_deadline_elapsed(quota_snapshot, snapshot_observed_at, now) }) { return Some(false); } } Some(exhausted) } pub(crate) fn provider_pool_quota_usage_ratio(key: &StoredProviderCatalogKey) -> Option { key.status_snapshot .as_ref() .and_then(Value::as_object) .and_then(|snapshot| snapshot.get("quota")) .and_then(Value::as_object) .and_then(|quota| provider_pool_json_f64(quota.get("usage_ratio"))) } pub(crate) fn provider_pool_quota_reset_seconds(key: &StoredProviderCatalogKey) -> Option { key.status_snapshot .as_ref() .and_then(Value::as_object) .and_then(|snapshot| snapshot.get("quota")) .and_then(Value::as_object) .and_then(|quota| provider_pool_json_f64(quota.get("reset_seconds"))) } pub(crate) fn provider_pool_account_blocked(key: &StoredProviderCatalogKey) -> bool { key.oauth_invalid_reason.as_deref().is_some_and(|reason| { let normalized = reason.trim().to_ascii_lowercase(); !normalized.is_empty() && [ "banned", "forbidden", "blocked", "suspend", "deactivated", "disabled", "verification", "workspace", "受限", "封", "禁", ] .iter() .any(|hint| normalized.contains(hint)) }) }