i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-03 01:40:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: 移除独立 hub/proxy/executor/gateway crate，统一为 gateway tunnel 架构

Browse Source 
- 删除 aether-hub、aether-proxy 独立项目及其 Dockerfile/配置
- 删除 crates/aether-executor 和 crates/aether-gateway 全部模块
- 新增 apps/ 目录作为应用入口
- 将 hub 概念重构为 gateway tunnel transport
- 将 executor 重构为 execution runtime
- 新增 tunnel.rs 合约定义和 testkit tunnel/execution_runtime 模块
- 更新 Python 服务层和测试适配新架构命名
This commit is contained in:
fawney19
2026-04-03 14:59:58 +08:00
parent ddf18fed9a
commit 8f26e1a31f
983 changed files with 103098 additions and 105837 deletions
Download Patch File Download Diff File Expand all files Collapse all files

103
apps/aether-gateway/src/scheduler/candidate/affinity.rs Normal file
View File

@@ -0,0 +1,103 @@
use super::*;
pub(super) fn build_scheduler_affinity_cache_key(
auth_snapshot: Option<&StoredGatewayAuthApiKeySnapshot>,
api_format: &str,
global_model_name: &str,
) -> Option<String> {
let api_key_id = auth_snapshot
.map(|snapshot| snapshot.api_key_id.trim())
.filter(|value| !value.is_empty())?;
build_scheduler_affinity_cache_key_for_api_key_id(api_key_id, api_format, global_model_name)
}
pub(super) fn build_scheduler_affinity_cache_key_for_api_key_id(
api_key_id: &str,
api_format: &str,
global_model_name: &str,
) -> Option<String> {
let api_key_id = api_key_id.trim();
if api_key_id.is_empty() {
return None;
}
let api_format = normalize_api_format(api_format);
let global_model_name = global_model_name.trim();
if api_format.is_empty() || global_model_name.is_empty() {
return None;
}
Some(format!(
"scheduler_affinity:{api_key_id}:{api_format}:{global_model_name}"
))
}
pub(super) fn compare_affinity_order(
left: &GatewayMinimalCandidateSelectionCandidate,
right: &GatewayMinimalCandidateSelectionCandidate,
affinity_key: Option<&str>,
) -> std::cmp::Ordering {
let Some(affinity_key) = affinity_key else {
return std::cmp::Ordering::Equal;
};
candidate_affinity_hash(affinity_key, left).cmp(&candidate_affinity_hash(affinity_key, right))
}
pub(super) fn candidate_affinity_hash(
affinity_key: &str,
candidate: &GatewayMinimalCandidateSelectionCandidate,
) -> u64 {
let mut hasher = Sha256::new();
hasher.update(affinity_key.as_bytes());
hasher.update(b":");
hasher.update(candidate.provider_id.as_bytes());
hasher.update(b":");
hasher.update(candidate.endpoint_id.as_bytes());
hasher.update(b":");
hasher.update(candidate.key_id.as_bytes());
let digest = hasher.finalize();
u64::from_be_bytes([
digest[0], digest[1], digest[2], digest[3], digest[4], digest[5], digest[6], digest[7],
])
}
pub(super) fn matches_affinity_target(
candidate: &GatewayMinimalCandidateSelectionCandidate,
target: &SchedulerAffinityTarget,
) -> bool {
candidate.provider_id == target.provider_id
&& candidate.endpoint_id == target.endpoint_id
&& candidate.key_id == target.key_id
}
pub(super) fn candidate_key(
candidate: &GatewayMinimalCandidateSelectionCandidate,
) -> (String, String, String) {
(
candidate.provider_id.clone(),
candidate.endpoint_id.clone(),
candidate.key_id.clone(),
)
}
#[cfg_attr(not(test), allow(dead_code))]
pub(super) fn remember_scheduler_affinity(
affinity_cache_key: Option<&str>,
state: &AppState,
candidate: &GatewayMinimalCandidateSelectionCandidate,
) {
let Some(cache_key) = affinity_cache_key else {
return;
};
state.scheduler_affinity_cache.insert(
cache_key.to_string(),
SchedulerAffinityTarget {
provider_id: candidate.provider_id.clone(),
endpoint_id: candidate.endpoint_id.clone(),
key_id: candidate.key_id.clone(),
},
SCHEDULER_AFFINITY_TTL,
SCHEDULER_AFFINITY_MAX_ENTRIES,
);
}

395
apps/aether-gateway/src/scheduler/candidate/mod.rs Normal file
View File

@@ -0,0 +1,395 @@
use self::affinity::{
build_scheduler_affinity_cache_key, build_scheduler_affinity_cache_key_for_api_key_id,
candidate_affinity_hash, candidate_key, compare_affinity_order, matches_affinity_target,
remember_scheduler_affinity,
};
use self::model::{
auth_snapshot_allows_api_format, auth_snapshot_allows_model, auth_snapshot_allows_provider,
candidate_model_names, candidate_supports_required_capability,
extract_global_priority_for_format, matches_model_mapping, normalize_api_format,
read_requested_model_rows, resolve_provider_model_name, resolve_requested_global_model_name,
row_supports_required_capability, select_provider_model_name,
};
use self::selection::{
is_candidate_selectable, read_provider_concurrent_limits, read_provider_key_rpm_states,
reorder_candidates_by_scheduler_health, should_skip_provider_quota,
};
mod affinity;
mod model;
mod selection;
#[cfg(test)]
mod tests;
use std::collections::{BTreeMap, BTreeSet};
use std::time::Duration;
use aether_data::repository::candidate_selection::{
StoredMinimalCandidateSelectionRow, StoredProviderModelMapping,
};
use aether_data::repository::candidates::StoredRequestCandidate;
use aether_data::repository::provider_catalog::{
StoredProviderCatalogKey, StoredProviderCatalogProvider,
};
use aether_data::repository::quota::StoredProviderQuotaSnapshot;
use aether_data::DataLayerError;
use aether_wallet::{ProviderBillingType, ProviderQuotaSnapshot};
use regex::Regex;
use sha2::{Digest, Sha256};
use crate::gateway::gateway_cache::SchedulerAffinityTarget;
use crate::gateway::gateway_data::{GatewayDataState, StoredGatewayAuthApiKeySnapshot};
use crate::gateway::{AppState, GatewayError};
use super::health::{
count_recent_active_requests_for_api_key, count_recent_active_requests_for_provider,
effective_provider_key_health_score, is_candidate_in_recent_failure_cooldown,
is_provider_key_circuit_open, provider_key_health_bucket, provider_key_health_score,
provider_key_rpm_allows_request_since,
};
const SCHEDULER_AFFINITY_TTL: Duration = Duration::from_secs(300);
#[cfg_attr(not(test), allow(dead_code))]
const SCHEDULER_AFFINITY_MAX_ENTRIES: usize = 10_000;
#[allow(dead_code)]
#[derive(Debug, Clone, PartialEq, serde::Serialize)]
pub(crate) struct GatewayMinimalCandidateSelectionCandidate {
pub(crate) provider_id: String,
pub(crate) provider_name: String,
pub(crate) provider_type: String,
pub(crate) provider_priority: i32,
pub(crate) endpoint_id: String,
pub(crate) endpoint_api_format: String,
pub(crate) key_id: String,
pub(crate) key_name: String,
pub(crate) key_auth_type: String,
pub(crate) key_internal_priority: i32,
pub(crate) key_global_priority_for_format: Option<i32>,
pub(crate) key_capabilities: Option<serde_json::Value>,
pub(crate) model_id: String,
pub(crate) global_model_id: String,
pub(crate) global_model_name: String,
pub(crate) selected_provider_model_name: String,
pub(crate) mapping_matched_model: Option<String>,
}
#[allow(dead_code)]
pub(crate) async fn read_minimal_candidate_selection(
state: &GatewayDataState,
api_format: &str,
requested_model_name: &str,
require_streaming: bool,
auth_snapshot: Option<&StoredGatewayAuthApiKeySnapshot>,
) -> Result<Vec<GatewayMinimalCandidateSelectionCandidate>, DataLayerError> {
let normalized_api_format = normalize_api_format(api_format);
if normalized_api_format.is_empty() {
return Ok(Vec::new());
}
if !auth_snapshot_allows_api_format(auth_snapshot, &normalized_api_format) {
return Ok(Vec::new());
}
let Some((resolved_global_model_name, rows)) =
read_requested_model_rows(state, &normalized_api_format, requested_model_name).await?
else {
return Ok(Vec::new());
};
if !auth_snapshot_allows_model(
auth_snapshot,
requested_model_name,
resolved_global_model_name.as_str(),
) {
return Ok(Vec::new());
}
let mut candidates = Vec::new();
for row in rows {
if !auth_snapshot_allows_provider(auth_snapshot, &row.provider_id, &row.provider_name) {
continue;
}
if require_streaming && !row.supports_streaming() {
continue;
}
let Some((selected_provider_model_name, mapping_matched_model)) =
resolve_provider_model_name(&row, requested_model_name, &normalized_api_format)
else {
continue;
};
candidates.push(GatewayMinimalCandidateSelectionCandidate {
provider_id: row.provider_id,
provider_name: row.provider_name,
provider_type: row.provider_type,
provider_priority: row.provider_priority,
endpoint_id: row.endpoint_id,
endpoint_api_format: row.endpoint_api_format,
key_id: row.key_id,
key_name: row.key_name,
key_auth_type: row.key_auth_type,
key_internal_priority: row.key_internal_priority,
key_global_priority_for_format: extract_global_priority_for_format(
row.key_global_priority_by_format.as_ref(),
&normalized_api_format,
)?,
key_capabilities: row.key_capabilities,
model_id: row.model_id,
global_model_id: row.global_model_id,
global_model_name: row.global_model_name,
selected_provider_model_name,
mapping_matched_model,
});
}
let affinity_key = auth_snapshot
.map(|snapshot| snapshot.api_key_id.trim())
.filter(|value| !value.is_empty());
candidates.sort_by(|left, right| {
left.key_global_priority_for_format
.unwrap_or(i32::MAX)
.cmp(&right.key_global_priority_for_format.unwrap_or(i32::MAX))
.then_with(|| compare_affinity_order(left, right, affinity_key))
.then(left.provider_priority.cmp(&right.provider_priority))
.then(left.key_internal_priority.cmp(&right.key_internal_priority))
.then(left.provider_id.cmp(&right.provider_id))
.then(left.endpoint_id.cmp(&right.endpoint_id))
.then(left.key_id.cmp(&right.key_id))
.then(
left.selected_provider_model_name
.cmp(&right.selected_provider_model_name),
)
});
Ok(candidates)
}
#[cfg_attr(not(test), allow(dead_code))]
pub(crate) async fn select_minimal_candidate(
state: &AppState,
api_format: &str,
global_model_name: &str,
require_streaming: bool,
auth_snapshot: Option<&StoredGatewayAuthApiKeySnapshot>,
now_unix_secs: u64,
) -> Result<Option<GatewayMinimalCandidateSelectionCandidate>, GatewayError> {
let affinity_cache_key =
build_scheduler_affinity_cache_key(auth_snapshot, api_format, global_model_name);
let selected = collect_selectable_candidates(
state,
api_format,
global_model_name,
require_streaming,
auth_snapshot,
now_unix_secs,
)
.await?
.into_iter()
.next();
if let Some(candidate) = selected.as_ref() {
remember_scheduler_affinity(affinity_cache_key.as_deref(), state, candidate);
}
Ok(selected)
}
pub(crate) async fn list_selectable_candidates(
state: &AppState,
api_format: &str,
global_model_name: &str,
require_streaming: bool,
auth_snapshot: Option<&StoredGatewayAuthApiKeySnapshot>,
now_unix_secs: u64,
) -> Result<Vec<GatewayMinimalCandidateSelectionCandidate>, GatewayError> {
collect_selectable_candidates(
state,
api_format,
global_model_name,
require_streaming,
auth_snapshot,
now_unix_secs,
)
.await
}
pub(crate) async fn list_selectable_candidates_for_required_capability_without_requested_model(
state: &AppState,
candidate_api_format: &str,
required_capability: &str,
require_streaming: bool,
auth_snapshot: Option<&StoredGatewayAuthApiKeySnapshot>,
now_unix_secs: u64,
) -> Result<Vec<GatewayMinimalCandidateSelectionCandidate>, GatewayError> {
let normalized_api_format = normalize_api_format(candidate_api_format);
let required_capability = required_capability.trim();
if normalized_api_format.is_empty() || required_capability.is_empty() {
return Ok(Vec::new());
}
if !auth_snapshot_allows_api_format(auth_snapshot, &normalized_api_format) {
return Ok(Vec::new());
}
let rows = state
.list_minimal_candidate_selection_rows_for_api_format(&normalized_api_format)
.await?;
let mut model_names = BTreeSet::new();
for row in rows {
if !auth_snapshot_allows_provider(auth_snapshot, &row.provider_id, &row.provider_name) {
continue;
}
if !row_supports_required_capability(&row, required_capability) {
continue;
}
if require_streaming && !row.supports_streaming() {
continue;
}
if !auth_snapshot_allows_model(
auth_snapshot,
&row.global_model_name,
&row.global_model_name,
) {
continue;
}
model_names.insert(row.global_model_name);
}
for global_model_name in model_names {
let candidates = list_selectable_candidates(
state,
&normalized_api_format,
&global_model_name,
require_streaming,
auth_snapshot,
now_unix_secs,
)
.await?;
let filtered = candidates
.into_iter()
.filter(|candidate| {
candidate_supports_required_capability(candidate, required_capability)
})
.collect::<Vec<_>>();
if !filtered.is_empty() {
return Ok(filtered);
}
}
Ok(Vec::new())
}
pub(crate) fn read_cached_scheduler_affinity_target(
state: &AppState,
api_key_id: &str,
api_format: &str,
global_model_name: &str,
) -> Option<SchedulerAffinityTarget> {
let cache_key = build_scheduler_affinity_cache_key_for_api_key_id(
api_key_id,
api_format,
global_model_name,
)?;
state
.scheduler_affinity_cache
.get_fresh(&cache_key, SCHEDULER_AFFINITY_TTL)
}
async fn collect_selectable_candidates(
state: &AppState,
api_format: &str,
global_model_name: &str,
require_streaming: bool,
auth_snapshot: Option<&StoredGatewayAuthApiKeySnapshot>,
now_unix_secs: u64,
) -> Result<Vec<GatewayMinimalCandidateSelectionCandidate>, GatewayError> {
let mut candidates = state
.read_minimal_candidate_selection(
api_format,
global_model_name,
require_streaming,
auth_snapshot,
)
.await?;
let recent_candidates = state.read_recent_request_candidates(128).await?;
let provider_concurrent_limits = read_provider_concurrent_limits(state, &candidates).await?;
let provider_key_rpm_states = read_provider_key_rpm_states(state, &candidates).await?;
reorder_candidates_by_scheduler_health(
&mut candidates,
&provider_key_rpm_states,
auth_snapshot,
);
let affinity_cache_key =
build_scheduler_affinity_cache_key(auth_snapshot, api_format, global_model_name);
let cached_affinity_target = affinity_cache_key.as_deref().and_then(|cache_key| {
state
.scheduler_affinity_cache
.get_fresh(cache_key, SCHEDULER_AFFINITY_TTL)
});
if let Some((api_key_id, limit)) = auth_snapshot.and_then(|snapshot| {
usize::try_from(snapshot.api_key_concurrent_limit?)
.ok()
.and_then(|limit| {
if limit == 0 {
return None;
}
Some((snapshot.api_key_id.as_str(), limit))
})
}) {
let active_requests =
count_recent_active_requests_for_api_key(&recent_candidates, api_key_id, now_unix_secs);
if active_requests >= limit {
return Ok(Vec::new());
}
}
let mut selected = Vec::new();
let mut selected_keys = BTreeSet::new();
if let Some(target) = cached_affinity_target.as_ref() {
if let Some(candidate) = candidates
.iter()
.find(|candidate| matches_affinity_target(candidate, target))
.cloned()
{
if is_candidate_selectable(
&candidate,
&recent_candidates,
&provider_concurrent_limits,
&provider_key_rpm_states,
now_unix_secs,
cached_affinity_target.as_ref(),
state,
)
.await?
{
selected_keys.insert(candidate_key(&candidate));
selected.push(candidate);
}
}
}
for candidate in candidates {
if selected_keys.contains(&candidate_key(&candidate)) {
continue;
}
if !is_candidate_selectable(
&candidate,
&recent_candidates,
&provider_concurrent_limits,
&provider_key_rpm_states,
now_unix_secs,
cached_affinity_target.as_ref(),
state,
)
.await?
{
continue;
}
selected_keys.insert(candidate_key(&candidate));
selected.push(candidate);
}
Ok(selected)
}

338
apps/aether-gateway/src/scheduler/candidate/model.rs Normal file
View File

@@ -0,0 +1,338 @@
use super::*;
pub(super) fn auth_snapshot_allows_provider(
auth_snapshot: Option<&StoredGatewayAuthApiKeySnapshot>,
provider_id: &str,
provider_name: &str,
) -> bool {
let Some(allowed) =
auth_snapshot.and_then(StoredGatewayAuthApiKeySnapshot::effective_allowed_providers)
else {
return true;
};
allowed.iter().any(|value| {
value.trim().eq_ignore_ascii_case(provider_id.trim())
|| value.trim().eq_ignore_ascii_case(provider_name.trim())
})
}
pub(super) fn auth_snapshot_allows_api_format(
auth_snapshot: Option<&StoredGatewayAuthApiKeySnapshot>,
api_format: &str,
) -> bool {
let Some(allowed) =
auth_snapshot.and_then(StoredGatewayAuthApiKeySnapshot::effective_allowed_api_formats)
else {
return true;
};
allowed
.iter()
.any(|value| normalize_api_format(value) == api_format)
}
pub(super) fn auth_snapshot_allows_model(
auth_snapshot: Option<&StoredGatewayAuthApiKeySnapshot>,
requested_model_name: &str,
resolved_global_model_name: &str,
) -> bool {
let Some(allowed) =
auth_snapshot.and_then(StoredGatewayAuthApiKeySnapshot::effective_allowed_models)
else {
return true;
};
allowed
.iter()
.any(|value| value == requested_model_name || value == resolved_global_model_name)
}
pub(super) async fn read_requested_model_rows(
state: &GatewayDataState,
api_format: &str,
requested_model_name: &str,
) -> Result<Option<(String, Vec<StoredMinimalCandidateSelectionRow>)>, DataLayerError> {
let exact_rows = state
.list_minimal_candidate_selection_rows(api_format, requested_model_name)
.await?;
if !exact_rows.is_empty() {
return Ok(Some((requested_model_name.to_string(), exact_rows)));
}
let rows = state
.list_minimal_candidate_selection_rows_for_api_format(api_format)
.await?;
let Some(resolved_global_model_name) =
resolve_requested_global_model_name(&rows, requested_model_name, api_format)
else {
return Ok(None);
};
Ok(Some((
resolved_global_model_name.clone(),
rows.into_iter()
.filter(|row| row.global_model_name == resolved_global_model_name)
.collect(),
)))
}
pub(super) fn resolve_requested_global_model_name(
rows: &[StoredMinimalCandidateSelectionRow],
requested_model_name: &str,
api_format: &str,
) -> Option<String> {
resolve_global_model_name_by(rows, |row| {
row.model_provider_model_name == requested_model_name
})
.or_else(|| {
resolve_global_model_name_by(rows, |row| {
row.model_provider_model_mappings
.as_ref()
.is_some_and(|mappings| {
mappings.iter().any(|mapping| {
mapping_scope_matches(mapping, api_format)
&& mapping.name == requested_model_name
})
})
})
})
.or_else(|| {
resolve_global_model_name_by(rows, |row| {
row.global_model_mappings.as_ref().is_some_and(|patterns| {
patterns
.iter()
.any(|pattern| matches_model_mapping(pattern, requested_model_name))
})
})
})
}
fn resolve_global_model_name_by<F>(
rows: &[StoredMinimalCandidateSelectionRow],
matches: F,
) -> Option<String>
where
F: Fn(&StoredMinimalCandidateSelectionRow) -> bool,
{
let mut matches = rows
.iter()
.filter(|row| matches(row))
.map(|row| row.global_model_name.trim())
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
.collect::<BTreeSet<_>>()
.into_iter();
matches.next()
}
pub(super) fn resolve_provider_model_name(
row: &StoredMinimalCandidateSelectionRow,
requested_model_name: &str,
api_format: &str,
) -> Option<(String, Option<String>)> {
let selected_provider_model_name = select_provider_model_name(row, api_format);
let Some(key_allowed_models) = row.key_allowed_models.as_ref() else {
return Some((selected_provider_model_name, None));
};
if key_allowed_models.is_empty() {
return None;
}
if key_allowed_models
.iter()
.any(|value| value == requested_model_name)
{
return Some((selected_provider_model_name, None));
}
let candidate_models = candidate_model_names(row, api_format);
let mut sorted_allowed_models = key_allowed_models
.iter()
.map(|value| value.trim())
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
.collect::<Vec<_>>();
sorted_allowed_models.sort();
for allowed_model in &sorted_allowed_models {
if candidate_models.contains(allowed_model.as_str()) {
return Some((allowed_model.clone(), Some(allowed_model.clone())));
}
}
let Some(global_model_mappings) = row.global_model_mappings.as_ref() else {
return None;
};
for allowed_model in sorted_allowed_models {
for pattern in global_model_mappings {
if matches_model_mapping(pattern, &allowed_model) {
return Some((allowed_model.clone(), Some(allowed_model)));
}
}
}
None
}
pub(super) fn select_provider_model_name(
row: &StoredMinimalCandidateSelectionRow,
api_format: &str,
) -> String {
let Some(mappings) = row.model_provider_model_mappings.as_ref() else {
return row.model_provider_model_name.clone();
};
let mut scoped = mappings
.iter()
.filter(|mapping| mapping_scope_matches(mapping, api_format))
.collect::<Vec<_>>();
if scoped.is_empty() {
return row.model_provider_model_name.clone();
}
scoped.sort_by(|left, right| {
left.priority
.cmp(&right.priority)
.then(left.name.cmp(&right.name))
});
let top_priority = scoped[0].priority;
scoped
.into_iter()
.find(|mapping| mapping.priority == top_priority)
.map(|mapping| mapping.name.clone())
.unwrap_or_else(|| row.model_provider_model_name.clone())
}
pub(super) fn candidate_model_names(
row: &StoredMinimalCandidateSelectionRow,
api_format: &str,
) -> BTreeSet<String> {
let mut names = BTreeSet::from([row.model_provider_model_name.clone()]);
if let Some(mappings) = row.model_provider_model_mappings.as_ref() {
for mapping in mappings {
if mapping_scope_matches(mapping, api_format) {
names.insert(mapping.name.clone());
}
}
}
names
}
fn mapping_scope_matches(mapping: &StoredProviderModelMapping, api_format: &str) -> bool {
let Some(api_formats) = mapping.api_formats.as_ref() else {
return true;
};
api_formats
.iter()
.any(|value| normalize_api_format(value) == api_format)
}
pub(super) fn row_supports_required_capability(
row: &StoredMinimalCandidateSelectionRow,
required_capability: &str,
) -> bool {
capabilities_support_required_capability(row.key_capabilities.as_ref(), required_capability)
}
pub(super) fn candidate_supports_required_capability(
candidate: &GatewayMinimalCandidateSelectionCandidate,
required_capability: &str,
) -> bool {
capabilities_support_required_capability(
candidate.key_capabilities.as_ref(),
required_capability,
)
}
fn capabilities_support_required_capability(
capabilities: Option<&serde_json::Value>,
required_capability: &str,
) -> bool {
let required_capability = required_capability.trim();
if required_capability.is_empty() {
return true;
}
let Some(capabilities) = capabilities else {
return false;
};
if let Some(object) = capabilities.as_object() {
return object.iter().any(|(key, value)| {
key.eq_ignore_ascii_case(required_capability)
&& match value {
serde_json::Value::Bool(value) => *value,
serde_json::Value::String(value) => value.eq_ignore_ascii_case("true"),
serde_json::Value::Number(value) => {
value.as_i64().is_some_and(|value| value > 0)
}
_ => false,
}
});
}
if let Some(items) = capabilities.as_array() {
return items.iter().any(|value| {
value
.as_str()
.is_some_and(|value| value.eq_ignore_ascii_case(required_capability))
});
}
false
}
pub(super) fn matches_model_mapping(pattern: &str, model_name: &str) -> bool {
let Ok(compiled) = Regex::new(&format!("^(?:{pattern})$")) else {
return false;
};
compiled.is_match(model_name)
}
pub(super) fn extract_global_priority_for_format(
raw: Option<&serde_json::Value>,
api_format: &str,
) -> Result<Option<i32>, DataLayerError> {
let Some(raw) = raw else {
return Ok(None);
};
let Some(object) = raw.as_object() else {
return Err(DataLayerError::UnexpectedValue(
"provider_api_keys.global_priority_by_format is not a JSON object".to_string(),
));
};
let Some(value) = object
.iter()
.find(|(key, _)| normalize_api_format(key) == api_format)
.map(|(_, value)| value)
else {
return Ok(None);
};
if let Some(value) = value.as_i64() {
return i32::try_from(value).map(Some).map_err(|_| {
DataLayerError::UnexpectedValue(format!(
"invalid provider_api_keys.global_priority_by_format value: {value}"
))
});
}
if let Some(value) = value.as_str() {
let value = value.trim().parse::<i32>().map_err(|_| {
DataLayerError::UnexpectedValue(format!(
"invalid provider_api_keys.global_priority_by_format value: {value}"
))
})?;
return Ok(Some(value));
}
Err(DataLayerError::UnexpectedValue(
"provider_api_keys.global_priority_by_format contains a non-integer value".to_string(),
))
}
pub(super) fn normalize_api_format(value: &str) -> String {
value.trim().to_ascii_lowercase()
}

221
apps/aether-gateway/src/scheduler/candidate/selection.rs Normal file
View File

@@ -0,0 +1,221 @@
use super::*;
pub(super) fn reorder_candidates_by_scheduler_health(
candidates: &mut [GatewayMinimalCandidateSelectionCandidate],
provider_key_rpm_states: &BTreeMap<String, StoredProviderCatalogKey>,
auth_snapshot: Option<&StoredGatewayAuthApiKeySnapshot>,
) {
let affinity_key = auth_snapshot
.map(|snapshot| snapshot.api_key_id.trim())
.filter(|value| !value.is_empty());
candidates.sort_by(|left, right| {
left.key_global_priority_for_format
.unwrap_or(i32::MAX)
.cmp(&right.key_global_priority_for_format.unwrap_or(i32::MAX))
.then_with(|| compare_provider_key_health_order(left, right, provider_key_rpm_states))
.then_with(|| compare_affinity_order(left, right, affinity_key))
.then(left.provider_priority.cmp(&right.provider_priority))
.then(left.key_internal_priority.cmp(&right.key_internal_priority))
.then(left.provider_id.cmp(&right.provider_id))
.then(left.endpoint_id.cmp(&right.endpoint_id))
.then(left.key_id.cmp(&right.key_id))
.then(
left.selected_provider_model_name
.cmp(&right.selected_provider_model_name),
)
});
}
fn compare_provider_key_health_order(
left: &GatewayMinimalCandidateSelectionCandidate,
right: &GatewayMinimalCandidateSelectionCandidate,
provider_key_rpm_states: &BTreeMap<String, StoredProviderCatalogKey>,
) -> std::cmp::Ordering {
let left_bucket = candidate_provider_key_health_bucket(left, provider_key_rpm_states);
let right_bucket = candidate_provider_key_health_bucket(right, provider_key_rpm_states);
right_bucket.cmp(&left_bucket).then_with(|| {
let left_score = candidate_provider_key_health_score(left, provider_key_rpm_states);
let right_score = candidate_provider_key_health_score(right, provider_key_rpm_states);
right_score
.partial_cmp(&left_score)
.unwrap_or(std::cmp::Ordering::Equal)
})
}
fn candidate_provider_key_health_bucket(
candidate: &GatewayMinimalCandidateSelectionCandidate,
provider_key_rpm_states: &BTreeMap<String, StoredProviderCatalogKey>,
) -> Option<super::super::health::ProviderKeyHealthBucket> {
provider_key_rpm_states
.get(&candidate.key_id)
.and_then(|key| provider_key_health_bucket(key, candidate.endpoint_api_format.as_str()))
}
fn candidate_provider_key_health_score(
candidate: &GatewayMinimalCandidateSelectionCandidate,
provider_key_rpm_states: &BTreeMap<String, StoredProviderCatalogKey>,
) -> f64 {
provider_key_rpm_states
.get(&candidate.key_id)
.and_then(|key| {
effective_provider_key_health_score(key, candidate.endpoint_api_format.as_str())
})
.unwrap_or(1.0)
}
pub(super) fn should_skip_provider_quota(
quota: &StoredProviderQuotaSnapshot,
now_unix_secs: u64,
) -> bool {
let snapshot = ProviderQuotaSnapshot {
provider_id: quota.provider_id.clone(),
billing_type: ProviderBillingType::parse(&quota.billing_type),
monthly_quota_usd: quota.monthly_quota_usd,
monthly_used_usd: quota.monthly_used_usd,
quota_reset_day: quota.quota_reset_day,
quota_last_reset_at_unix_secs: quota.quota_last_reset_at_unix_secs,
quota_expires_at_unix_secs: quota.quota_expires_at_unix_secs,
is_active: quota.is_active,
};
if !snapshot.is_active || snapshot.is_expired(now_unix_secs) {
return true;
}
match snapshot.billing_type {
ProviderBillingType::MonthlyQuota | ProviderBillingType::FreeTier => snapshot
.remaining_quota_usd()
.is_some_and(|remaining| remaining <= 0.0),
ProviderBillingType::PayAsYouGo | ProviderBillingType::Unknown => false,
}
}
fn is_candidate_cooled_down(
candidate: &GatewayMinimalCandidateSelectionCandidate,
recent_candidates: &[StoredRequestCandidate],
now_unix_secs: u64,
) -> bool {
is_candidate_in_recent_failure_cooldown(
recent_candidates,
candidate.provider_id.as_str(),
candidate.endpoint_id.as_str(),
candidate.key_id.as_str(),
now_unix_secs,
)
}
pub(super) async fn is_candidate_selectable(
candidate: &GatewayMinimalCandidateSelectionCandidate,
recent_candidates: &[StoredRequestCandidate],
provider_concurrent_limits: &BTreeMap<String, usize>,
provider_key_rpm_states: &BTreeMap<String, StoredProviderCatalogKey>,
now_unix_secs: u64,
cached_affinity_target: Option<&SchedulerAffinityTarget>,
state: &AppState,
) -> Result<bool, GatewayError> {
let quota = state
.read_provider_quota_snapshot(&candidate.provider_id)
.await?;
if quota
.as_ref()
.is_some_and(|quota| should_skip_provider_quota(quota, now_unix_secs))
{
return Ok(false);
}
if is_candidate_cooled_down(candidate, recent_candidates, now_unix_secs) {
return Ok(false);
}
if provider_concurrent_limits
.get(&candidate.provider_id)
.is_some_and(|limit| {
count_recent_active_requests_for_provider(
recent_candidates,
candidate.provider_id.as_str(),
now_unix_secs,
) >= *limit
})
{
return Ok(false);
}
let is_cached_user =
cached_affinity_target.is_some_and(|target| matches_affinity_target(candidate, target));
if let Some(provider_key) = provider_key_rpm_states.get(&candidate.key_id) {
if is_provider_key_circuit_open(provider_key, candidate.endpoint_api_format.as_str()) {
return Ok(false);
}
if provider_key_health_score(provider_key, candidate.endpoint_api_format.as_str())
.is_some_and(|score| score <= 0.0)
{
return Ok(false);
}
let rpm_reset_at =
state.provider_key_rpm_reset_at(candidate.key_id.as_str(), now_unix_secs);
if !provider_key_rpm_allows_request_since(
provider_key,
recent_candidates,
now_unix_secs,
is_cached_user,
rpm_reset_at,
) {
return Ok(false);
}
}
Ok(true)
}
pub(super) async fn read_provider_concurrent_limits(
state: &AppState,
candidates: &[GatewayMinimalCandidateSelectionCandidate],
) -> Result<BTreeMap<String, usize>, GatewayError> {
let provider_ids = candidates
.iter()
.map(|candidate| candidate.provider_id.clone())
.collect::<BTreeSet<_>>()
.into_iter()
.collect::<Vec<_>>();
if provider_ids.is_empty() {
return Ok(BTreeMap::new());
}
let providers = state
.read_provider_catalog_providers_by_ids(&provider_ids)
.await?;
Ok(build_provider_concurrent_limit_map(providers))
}
fn build_provider_concurrent_limit_map(
providers: Vec<StoredProviderCatalogProvider>,
) -> BTreeMap<String, usize> {
providers
.into_iter()
.filter_map(|provider| {
provider
.concurrent_limit
.and_then(|limit| usize::try_from(limit).ok())
.filter(|limit| *limit > 0)
.map(|limit| (provider.id, limit))
})
.collect()
}
pub(super) async fn read_provider_key_rpm_states(
state: &AppState,
candidates: &[GatewayMinimalCandidateSelectionCandidate],
) -> Result<BTreeMap<String, StoredProviderCatalogKey>, GatewayError> {
let key_ids = candidates
.iter()
.map(|candidate| candidate.key_id.clone())
.collect::<BTreeSet<_>>()
.into_iter()
.collect::<Vec<_>>();
if key_ids.is_empty() {
return Ok(BTreeMap::new());
}
let keys = state.read_provider_catalog_keys_by_ids(&key_ids).await?;
Ok(keys
.into_iter()
.map(|key| (key.id.clone(), key))
.collect::<BTreeMap<_, _>>())
}

1267
apps/aether-gateway/src/scheduler/candidate/tests.rs Normal file
View File

File diff suppressed because it is too large Load Diff