i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 17:30:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: 移除独立 hub/proxy/executor/gateway crate，统一为 gateway tunnel 架构

Browse Source 
- 删除 aether-hub、aether-proxy 独立项目及其 Dockerfile/配置
- 删除 crates/aether-executor 和 crates/aether-gateway 全部模块
- 新增 apps/ 目录作为应用入口
- 将 hub 概念重构为 gateway tunnel transport
- 将 executor 重构为 execution runtime
- 新增 tunnel.rs 合约定义和 testkit tunnel/execution_runtime 模块
- 更新 Python 服务层和测试适配新架构命名
This commit is contained in:
fawney19
2026-04-03 14:59:58 +08:00
parent ddf18fed9a
commit 8f26e1a31f
983 changed files with 103098 additions and 105837 deletions
Download Patch File Download Diff File Expand all files Collapse all files

103
apps/aether-gateway/src/scheduler/candidate/affinity.rs Normal file
View File

@@ -0,0 +1,103 @@
use super::*;
pub(super) fn build_scheduler_affinity_cache_key(
auth_snapshot: Option<&StoredGatewayAuthApiKeySnapshot>,
api_format: &str,
global_model_name: &str,
) -> Option<String> {
let api_key_id = auth_snapshot
.map(|snapshot| snapshot.api_key_id.trim())
.filter(|value| !value.is_empty())?;
build_scheduler_affinity_cache_key_for_api_key_id(api_key_id, api_format, global_model_name)
}
pub(super) fn build_scheduler_affinity_cache_key_for_api_key_id(
api_key_id: &str,
api_format: &str,
global_model_name: &str,
) -> Option<String> {
let api_key_id = api_key_id.trim();
if api_key_id.is_empty() {
return None;
}
let api_format = normalize_api_format(api_format);
let global_model_name = global_model_name.trim();
if api_format.is_empty() || global_model_name.is_empty() {
return None;
}
Some(format!(
"scheduler_affinity:{api_key_id}:{api_format}:{global_model_name}"
))
}
pub(super) fn compare_affinity_order(
left: &GatewayMinimalCandidateSelectionCandidate,
right: &GatewayMinimalCandidateSelectionCandidate,
affinity_key: Option<&str>,
) -> std::cmp::Ordering {
let Some(affinity_key) = affinity_key else {
return std::cmp::Ordering::Equal;
};
candidate_affinity_hash(affinity_key, left).cmp(&candidate_affinity_hash(affinity_key, right))
}
pub(super) fn candidate_affinity_hash(
affinity_key: &str,
candidate: &GatewayMinimalCandidateSelectionCandidate,
) -> u64 {
let mut hasher = Sha256::new();
hasher.update(affinity_key.as_bytes());
hasher.update(b":");
hasher.update(candidate.provider_id.as_bytes());
hasher.update(b":");
hasher.update(candidate.endpoint_id.as_bytes());
hasher.update(b":");
hasher.update(candidate.key_id.as_bytes());
let digest = hasher.finalize();
u64::from_be_bytes([
digest[0], digest[1], digest[2], digest[3], digest[4], digest[5], digest[6], digest[7],
])
}
pub(super) fn matches_affinity_target(
candidate: &GatewayMinimalCandidateSelectionCandidate,
target: &SchedulerAffinityTarget,
) -> bool {
candidate.provider_id == target.provider_id
&& candidate.endpoint_id == target.endpoint_id
&& candidate.key_id == target.key_id
}
pub(super) fn candidate_key(
candidate: &GatewayMinimalCandidateSelectionCandidate,
) -> (String, String, String) {
(
candidate.provider_id.clone(),
candidate.endpoint_id.clone(),
candidate.key_id.clone(),
)
}
#[cfg_attr(not(test), allow(dead_code))]
pub(super) fn remember_scheduler_affinity(
affinity_cache_key: Option<&str>,
state: &AppState,
candidate: &GatewayMinimalCandidateSelectionCandidate,
) {
let Some(cache_key) = affinity_cache_key else {
return;
};
state.scheduler_affinity_cache.insert(
cache_key.to_string(),
SchedulerAffinityTarget {
provider_id: candidate.provider_id.clone(),
endpoint_id: candidate.endpoint_id.clone(),
key_id: candidate.key_id.clone(),
},
SCHEDULER_AFFINITY_TTL,
SCHEDULER_AFFINITY_MAX_ENTRIES,
);
}

395
apps/aether-gateway/src/scheduler/candidate/mod.rs Normal file
View File

@@ -0,0 +1,395 @@
use self::affinity::{
build_scheduler_affinity_cache_key, build_scheduler_affinity_cache_key_for_api_key_id,
candidate_affinity_hash, candidate_key, compare_affinity_order, matches_affinity_target,
remember_scheduler_affinity,
};
use self::model::{
auth_snapshot_allows_api_format, auth_snapshot_allows_model, auth_snapshot_allows_provider,
candidate_model_names, candidate_supports_required_capability,
extract_global_priority_for_format, matches_model_mapping, normalize_api_format,
read_requested_model_rows, resolve_provider_model_name, resolve_requested_global_model_name,
row_supports_required_capability, select_provider_model_name,
};
use self::selection::{
is_candidate_selectable, read_provider_concurrent_limits, read_provider_key_rpm_states,
reorder_candidates_by_scheduler_health, should_skip_provider_quota,
};
mod affinity;
mod model;
mod selection;
#[cfg(test)]
mod tests;
use std::collections::{BTreeMap, BTreeSet};
use std::time::Duration;
use aether_data::repository::candidate_selection::{
StoredMinimalCandidateSelectionRow, StoredProviderModelMapping,
};
use aether_data::repository::candidates::StoredRequestCandidate;
use aether_data::repository::provider_catalog::{
StoredProviderCatalogKey, StoredProviderCatalogProvider,
};
use aether_data::repository::quota::StoredProviderQuotaSnapshot;
use aether_data::DataLayerError;
use aether_wallet::{ProviderBillingType, ProviderQuotaSnapshot};
use regex::Regex;
use sha2::{Digest, Sha256};
use crate::gateway::gateway_cache::SchedulerAffinityTarget;
use crate::gateway::gateway_data::{GatewayDataState, StoredGatewayAuthApiKeySnapshot};
use crate::gateway::{AppState, GatewayError};
use super::health::{
count_recent_active_requests_for_api_key, count_recent_active_requests_for_provider,
effective_provider_key_health_score, is_candidate_in_recent_failure_cooldown,
is_provider_key_circuit_open, provider_key_health_bucket, provider_key_health_score,
provider_key_rpm_allows_request_since,
};
const SCHEDULER_AFFINITY_TTL: Duration = Duration::from_secs(300);
#[cfg_attr(not(test), allow(dead_code))]
const SCHEDULER_AFFINITY_MAX_ENTRIES: usize = 10_000;
#[allow(dead_code)]
#[derive(Debug, Clone, PartialEq, serde::Serialize)]
pub(crate) struct GatewayMinimalCandidateSelectionCandidate {
pub(crate) provider_id: String,
pub(crate) provider_name: String,
pub(crate) provider_type: String,
pub(crate) provider_priority: i32,
pub(crate) endpoint_id: String,
pub(crate) endpoint_api_format: String,
pub(crate) key_id: String,
pub(crate) key_name: String,
pub(crate) key_auth_type: String,
pub(crate) key_internal_priority: i32,
pub(crate) key_global_priority_for_format: Option<i32>,
pub(crate) key_capabilities: Option<serde_json::Value>,
pub(crate) model_id: String,
pub(crate) global_model_id: String,
pub(crate) global_model_name: String,
pub(crate) selected_provider_model_name: String,
pub(crate) mapping_matched_model: Option<String>,
}
#[allow(dead_code)]
pub(crate) async fn read_minimal_candidate_selection(
state: &GatewayDataState,
api_format: &str,
requested_model_name: &str,
require_streaming: bool,
auth_snapshot: Option<&StoredGatewayAuthApiKeySnapshot>,
) -> Result<Vec<GatewayMinimalCandidateSelectionCandidate>, DataLayerError> {
let normalized_api_format = normalize_api_format(api_format);
if normalized_api_format.is_empty() {
return Ok(Vec::new());
}
if !auth_snapshot_allows_api_format(auth_snapshot, &normalized_api_format) {
return Ok(Vec::new());
}
let Some((resolved_global_model_name, rows)) =
read_requested_model_rows(state, &normalized_api_format, requested_model_name).await?
else {
return Ok(Vec::new());
};
if !auth_snapshot_allows_model(
auth_snapshot,
requested_model_name,
resolved_global_model_name.as_str(),
) {
return Ok(Vec::new());
}
let mut candidates = Vec::new();
for row in rows {
if !auth_snapshot_allows_provider(auth_snapshot, &row.provider_id, &row.provider_name) {
continue;
}
if require_streaming && !row.supports_streaming() {
continue;
}
let Some((selected_provider_model_name, mapping_matched_model)) =
resolve_provider_model_name(&row, requested_model_name, &normalized_api_format)
else {
continue;
};
candidates.push(GatewayMinimalCandidateSelectionCandidate {
provider_id: row.provider_id,
provider_name: row.provider_name,
provider_type: row.provider_type,
provider_priority: row.provider_priority,
endpoint_id: row.endpoint_id,
endpoint_api_format: row.endpoint_api_format,
key_id: row.key_id,
key_name: row.key_name,
key_auth_type: row.key_auth_type,
key_internal_priority: row.key_internal_priority,
key_global_priority_for_format: extract_global_priority_for_format(
row.key_global_priority_by_format.as_ref(),
&normalized_api_format,
)?,
key_capabilities: row.key_capabilities,
model_id: row.model_id,
global_model_id: row.global_model_id,
global_model_name: row.global_model_name,
selected_provider_model_name,
mapping_matched_model,
});
}
let affinity_key = auth_snapshot
.map(|snapshot| snapshot.api_key_id.trim())
.filter(|value| !value.is_empty());
candidates.sort_by(|left, right| {
left.key_global_priority_for_format
.unwrap_or(i32::MAX)
.cmp(&right.key_global_priority_for_format.unwrap_or(i32::MAX))
.then_with(|| compare_affinity_order(left, right, affinity_key))
.then(left.provider_priority.cmp(&right.provider_priority))
.then(left.key_internal_priority.cmp(&right.key_internal_priority))
.then(left.provider_id.cmp(&right.provider_id))
.then(left.endpoint_id.cmp(&right.endpoint_id))
.then(left.key_id.cmp(&right.key_id))
.then(
left.selected_provider_model_name
.cmp(&right.selected_provider_model_name),
)
});
Ok(candidates)
}
#[cfg_attr(not(test), allow(dead_code))]
pub(crate) async fn select_minimal_candidate(
state: &AppState,
api_format: &str,
global_model_name: &str,
require_streaming: bool,
auth_snapshot: Option<&StoredGatewayAuthApiKeySnapshot>,
now_unix_secs: u64,
) -> Result<Option<GatewayMinimalCandidateSelectionCandidate>, GatewayError> {
let affinity_cache_key =
build_scheduler_affinity_cache_key(auth_snapshot, api_format, global_model_name);
let selected = collect_selectable_candidates(
state,
api_format,
global_model_name,
require_streaming,
auth_snapshot,
now_unix_secs,
)
.await?
.into_iter()
.next();
if let Some(candidate) = selected.as_ref() {
remember_scheduler_affinity(affinity_cache_key.as_deref(), state, candidate);
}
Ok(selected)
}
pub(crate) async fn list_selectable_candidates(
state: &AppState,
api_format: &str,
global_model_name: &str,
require_streaming: bool,
auth_snapshot: Option<&StoredGatewayAuthApiKeySnapshot>,
now_unix_secs: u64,
) -> Result<Vec<GatewayMinimalCandidateSelectionCandidate>, GatewayError> {
collect_selectable_candidates(
state,
api_format,
global_model_name,
require_streaming,
auth_snapshot,
now_unix_secs,
)
.await
}
pub(crate) async fn list_selectable_candidates_for_required_capability_without_requested_model(
state: &AppState,
candidate_api_format: &str,
required_capability: &str,
require_streaming: bool,
auth_snapshot: Option<&StoredGatewayAuthApiKeySnapshot>,
now_unix_secs: u64,
) -> Result<Vec<GatewayMinimalCandidateSelectionCandidate>, GatewayError> {
let normalized_api_format = normalize_api_format(candidate_api_format);
let required_capability = required_capability.trim();
if normalized_api_format.is_empty() || required_capability.is_empty() {
return Ok(Vec::new());
}
if !auth_snapshot_allows_api_format(auth_snapshot, &normalized_api_format) {
return Ok(Vec::new());
}
let rows = state
.list_minimal_candidate_selection_rows_for_api_format(&normalized_api_format)
.await?;
let mut model_names = BTreeSet::new();
for row in rows {
if !auth_snapshot_allows_provider(auth_snapshot, &row.provider_id, &row.provider_name) {
continue;
}
if !row_supports_required_capability(&row, required_capability) {
continue;
}
if require_streaming && !row.supports_streaming() {
continue;
}
if !auth_snapshot_allows_model(
auth_snapshot,
&row.global_model_name,
&row.global_model_name,
) {
continue;
}
model_names.insert(row.global_model_name);
}
for global_model_name in model_names {
let candidates = list_selectable_candidates(
state,
&normalized_api_format,
&global_model_name,
require_streaming,
auth_snapshot,
now_unix_secs,
)
.await?;
let filtered = candidates
.into_iter()
.filter(|candidate| {
candidate_supports_required_capability(candidate, required_capability)
})
.collect::<Vec<_>>();
if !filtered.is_empty() {
return Ok(filtered);
}
}
Ok(Vec::new())
}
pub(crate) fn read_cached_scheduler_affinity_target(
state: &AppState,
api_key_id: &str,
api_format: &str,
global_model_name: &str,
) -> Option<SchedulerAffinityTarget> {
let cache_key = build_scheduler_affinity_cache_key_for_api_key_id(
api_key_id,
api_format,
global_model_name,
)?;
state
.scheduler_affinity_cache
.get_fresh(&cache_key, SCHEDULER_AFFINITY_TTL)
}
async fn collect_selectable_candidates(
state: &AppState,
api_format: &str,
global_model_name: &str,
require_streaming: bool,
auth_snapshot: Option<&StoredGatewayAuthApiKeySnapshot>,
now_unix_secs: u64,
) -> Result<Vec<GatewayMinimalCandidateSelectionCandidate>, GatewayError> {
let mut candidates = state
.read_minimal_candidate_selection(
api_format,
global_model_name,
require_streaming,
auth_snapshot,
)
.await?;
let recent_candidates = state.read_recent_request_candidates(128).await?;
let provider_concurrent_limits = read_provider_concurrent_limits(state, &candidates).await?;
let provider_key_rpm_states = read_provider_key_rpm_states(state, &candidates).await?;
reorder_candidates_by_scheduler_health(
&mut candidates,
&provider_key_rpm_states,
auth_snapshot,
);
let affinity_cache_key =
build_scheduler_affinity_cache_key(auth_snapshot, api_format, global_model_name);
let cached_affinity_target = affinity_cache_key.as_deref().and_then(|cache_key| {
state
.scheduler_affinity_cache
.get_fresh(cache_key, SCHEDULER_AFFINITY_TTL)
});
if let Some((api_key_id, limit)) = auth_snapshot.and_then(|snapshot| {
usize::try_from(snapshot.api_key_concurrent_limit?)
.ok()
.and_then(|limit| {
if limit == 0 {
return None;
}
Some((snapshot.api_key_id.as_str(), limit))
})
}) {
let active_requests =
count_recent_active_requests_for_api_key(&recent_candidates, api_key_id, now_unix_secs);
if active_requests >= limit {
return Ok(Vec::new());
}
}
let mut selected = Vec::new();
let mut selected_keys = BTreeSet::new();
if let Some(target) = cached_affinity_target.as_ref() {
if let Some(candidate) = candidates
.iter()
.find(|candidate| matches_affinity_target(candidate, target))
.cloned()
{
if is_candidate_selectable(
&candidate,
&recent_candidates,
&provider_concurrent_limits,
&provider_key_rpm_states,
now_unix_secs,
cached_affinity_target.as_ref(),
state,
)
.await?
{
selected_keys.insert(candidate_key(&candidate));
selected.push(candidate);
}
}
}
for candidate in candidates {
if selected_keys.contains(&candidate_key(&candidate)) {
continue;
}
if !is_candidate_selectable(
&candidate,
&recent_candidates,
&provider_concurrent_limits,
&provider_key_rpm_states,
now_unix_secs,
cached_affinity_target.as_ref(),
state,
)
.await?
{
continue;
}
selected_keys.insert(candidate_key(&candidate));
selected.push(candidate);
}
Ok(selected)
}

338
apps/aether-gateway/src/scheduler/candidate/model.rs Normal file
View File

@@ -0,0 +1,338 @@
use super::*;
pub(super) fn auth_snapshot_allows_provider(
auth_snapshot: Option<&StoredGatewayAuthApiKeySnapshot>,
provider_id: &str,
provider_name: &str,
) -> bool {
let Some(allowed) =
auth_snapshot.and_then(StoredGatewayAuthApiKeySnapshot::effective_allowed_providers)
else {
return true;
};
allowed.iter().any(|value| {
value.trim().eq_ignore_ascii_case(provider_id.trim())
|| value.trim().eq_ignore_ascii_case(provider_name.trim())
})
}
pub(super) fn auth_snapshot_allows_api_format(
auth_snapshot: Option<&StoredGatewayAuthApiKeySnapshot>,
api_format: &str,
) -> bool {
let Some(allowed) =
auth_snapshot.and_then(StoredGatewayAuthApiKeySnapshot::effective_allowed_api_formats)
else {
return true;
};
allowed
.iter()
.any(|value| normalize_api_format(value) == api_format)
}
pub(super) fn auth_snapshot_allows_model(
auth_snapshot: Option<&StoredGatewayAuthApiKeySnapshot>,
requested_model_name: &str,
resolved_global_model_name: &str,
) -> bool {
let Some(allowed) =
auth_snapshot.and_then(StoredGatewayAuthApiKeySnapshot::effective_allowed_models)
else {
return true;
};
allowed
.iter()
.any(|value| value == requested_model_name || value == resolved_global_model_name)
}
pub(super) async fn read_requested_model_rows(
state: &GatewayDataState,
api_format: &str,
requested_model_name: &str,
) -> Result<Option<(String, Vec<StoredMinimalCandidateSelectionRow>)>, DataLayerError> {
let exact_rows = state
.list_minimal_candidate_selection_rows(api_format, requested_model_name)
.await?;
if !exact_rows.is_empty() {
return Ok(Some((requested_model_name.to_string(), exact_rows)));
}
let rows = state
.list_minimal_candidate_selection_rows_for_api_format(api_format)
.await?;
let Some(resolved_global_model_name) =
resolve_requested_global_model_name(&rows, requested_model_name, api_format)
else {
return Ok(None);
};
Ok(Some((
resolved_global_model_name.clone(),
rows.into_iter()
.filter(|row| row.global_model_name == resolved_global_model_name)
.collect(),
)))
}
pub(super) fn resolve_requested_global_model_name(
rows: &[StoredMinimalCandidateSelectionRow],
requested_model_name: &str,
api_format: &str,
) -> Option<String> {
resolve_global_model_name_by(rows, |row| {
row.model_provider_model_name == requested_model_name
})
.or_else(|| {
resolve_global_model_name_by(rows, |row| {
row.model_provider_model_mappings
.as_ref()
.is_some_and(|mappings| {
mappings.iter().any(|mapping| {
mapping_scope_matches(mapping, api_format)
&& mapping.name == requested_model_name
})
})
})
})
.or_else(|| {
resolve_global_model_name_by(rows, |row| {
row.global_model_mappings.as_ref().is_some_and(|patterns| {
patterns
.iter()
.any(|pattern| matches_model_mapping(pattern, requested_model_name))
})
})
})
}
fn resolve_global_model_name_by<F>(
rows: &[StoredMinimalCandidateSelectionRow],
matches: F,
) -> Option<String>
where
F: Fn(&StoredMinimalCandidateSelectionRow) -> bool,
{
let mut matches = rows
.iter()
.filter(|row| matches(row))
.map(|row| row.global_model_name.trim())
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
.collect::<BTreeSet<_>>()
.into_iter();
matches.next()
}
pub(super) fn resolve_provider_model_name(
row: &StoredMinimalCandidateSelectionRow,
requested_model_name: &str,
api_format: &str,
) -> Option<(String, Option<String>)> {
let selected_provider_model_name = select_provider_model_name(row, api_format);
let Some(key_allowed_models) = row.key_allowed_models.as_ref() else {
return Some((selected_provider_model_name, None));
};
if key_allowed_models.is_empty() {
return None;
}
if key_allowed_models
.iter()
.any(|value| value == requested_model_name)
{
return Some((selected_provider_model_name, None));
}
let candidate_models = candidate_model_names(row, api_format);
let mut sorted_allowed_models = key_allowed_models
.iter()
.map(|value| value.trim())
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
.collect::<Vec<_>>();
sorted_allowed_models.sort();
for allowed_model in &sorted_allowed_models {
if candidate_models.contains(allowed_model.as_str()) {
return Some((allowed_model.clone(), Some(allowed_model.clone())));
}
}
let Some(global_model_mappings) = row.global_model_mappings.as_ref() else {
return None;
};
for allowed_model in sorted_allowed_models {
for pattern in global_model_mappings {
if matches_model_mapping(pattern, &allowed_model) {
return Some((allowed_model.clone(), Some(allowed_model)));
}
}
}
None
}
pub(super) fn select_provider_model_name(
row: &StoredMinimalCandidateSelectionRow,
api_format: &str,
) -> String {
let Some(mappings) = row.model_provider_model_mappings.as_ref() else {
return row.model_provider_model_name.clone();
};
let mut scoped = mappings
.iter()
.filter(|mapping| mapping_scope_matches(mapping, api_format))
.collect::<Vec<_>>();
if scoped.is_empty() {
return row.model_provider_model_name.clone();
}
scoped.sort_by(|left, right| {
left.priority
.cmp(&right.priority)
.then(left.name.cmp(&right.name))
});
let top_priority = scoped[0].priority;
scoped
.into_iter()
.find(|mapping| mapping.priority == top_priority)
.map(|mapping| mapping.name.clone())
.unwrap_or_else(|| row.model_provider_model_name.clone())
}
pub(super) fn candidate_model_names(
row: &StoredMinimalCandidateSelectionRow,
api_format: &str,
) -> BTreeSet<String> {
let mut names = BTreeSet::from([row.model_provider_model_name.clone()]);
if let Some(mappings) = row.model_provider_model_mappings.as_ref() {
for mapping in mappings {
if mapping_scope_matches(mapping, api_format) {
names.insert(mapping.name.clone());
}
}
}
names
}
fn mapping_scope_matches(mapping: &StoredProviderModelMapping, api_format: &str) -> bool {
let Some(api_formats) = mapping.api_formats.as_ref() else {
return true;
};
api_formats
.iter()
.any(|value| normalize_api_format(value) == api_format)
}
pub(super) fn row_supports_required_capability(
row: &StoredMinimalCandidateSelectionRow,
required_capability: &str,
) -> bool {
capabilities_support_required_capability(row.key_capabilities.as_ref(), required_capability)
}
pub(super) fn candidate_supports_required_capability(
candidate: &GatewayMinimalCandidateSelectionCandidate,
required_capability: &str,
) -> bool {
capabilities_support_required_capability(
candidate.key_capabilities.as_ref(),
required_capability,
)
}
fn capabilities_support_required_capability(
capabilities: Option<&serde_json::Value>,
required_capability: &str,
) -> bool {
let required_capability = required_capability.trim();
if required_capability.is_empty() {
return true;
}
let Some(capabilities) = capabilities else {
return false;
};
if let Some(object) = capabilities.as_object() {
return object.iter().any(|(key, value)| {
key.eq_ignore_ascii_case(required_capability)
&& match value {
serde_json::Value::Bool(value) => *value,
serde_json::Value::String(value) => value.eq_ignore_ascii_case("true"),
serde_json::Value::Number(value) => {
value.as_i64().is_some_and(|value| value > 0)
}
_ => false,
}
});
}
if let Some(items) = capabilities.as_array() {
return items.iter().any(|value| {
value
.as_str()
.is_some_and(|value| value.eq_ignore_ascii_case(required_capability))
});
}
false
}
pub(super) fn matches_model_mapping(pattern: &str, model_name: &str) -> bool {
let Ok(compiled) = Regex::new(&format!("^(?:{pattern})$")) else {
return false;
};
compiled.is_match(model_name)
}
pub(super) fn extract_global_priority_for_format(
raw: Option<&serde_json::Value>,
api_format: &str,
) -> Result<Option<i32>, DataLayerError> {
let Some(raw) = raw else {
return Ok(None);
};
let Some(object) = raw.as_object() else {
return Err(DataLayerError::UnexpectedValue(
"provider_api_keys.global_priority_by_format is not a JSON object".to_string(),
));
};
let Some(value) = object
.iter()
.find(|(key, _)| normalize_api_format(key) == api_format)
.map(|(_, value)| value)
else {
return Ok(None);
};
if let Some(value) = value.as_i64() {
return i32::try_from(value).map(Some).map_err(|_| {
DataLayerError::UnexpectedValue(format!(
"invalid provider_api_keys.global_priority_by_format value: {value}"
))
});
}
if let Some(value) = value.as_str() {
let value = value.trim().parse::<i32>().map_err(|_| {
DataLayerError::UnexpectedValue(format!(
"invalid provider_api_keys.global_priority_by_format value: {value}"
))
})?;
return Ok(Some(value));
}
Err(DataLayerError::UnexpectedValue(
"provider_api_keys.global_priority_by_format contains a non-integer value".to_string(),
))
}
pub(super) fn normalize_api_format(value: &str) -> String {
value.trim().to_ascii_lowercase()
}

221
apps/aether-gateway/src/scheduler/candidate/selection.rs Normal file
View File

@@ -0,0 +1,221 @@
use super::*;
pub(super) fn reorder_candidates_by_scheduler_health(
candidates: &mut [GatewayMinimalCandidateSelectionCandidate],
provider_key_rpm_states: &BTreeMap<String, StoredProviderCatalogKey>,
auth_snapshot: Option<&StoredGatewayAuthApiKeySnapshot>,
) {
let affinity_key = auth_snapshot
.map(|snapshot| snapshot.api_key_id.trim())
.filter(|value| !value.is_empty());
candidates.sort_by(|left, right| {
left.key_global_priority_for_format
.unwrap_or(i32::MAX)
.cmp(&right.key_global_priority_for_format.unwrap_or(i32::MAX))
.then_with(|| compare_provider_key_health_order(left, right, provider_key_rpm_states))
.then_with(|| compare_affinity_order(left, right, affinity_key))
.then(left.provider_priority.cmp(&right.provider_priority))
.then(left.key_internal_priority.cmp(&right.key_internal_priority))
.then(left.provider_id.cmp(&right.provider_id))
.then(left.endpoint_id.cmp(&right.endpoint_id))
.then(left.key_id.cmp(&right.key_id))
.then(
left.selected_provider_model_name
.cmp(&right.selected_provider_model_name),
)
});
}
fn compare_provider_key_health_order(
left: &GatewayMinimalCandidateSelectionCandidate,
right: &GatewayMinimalCandidateSelectionCandidate,
provider_key_rpm_states: &BTreeMap<String, StoredProviderCatalogKey>,
) -> std::cmp::Ordering {
let left_bucket = candidate_provider_key_health_bucket(left, provider_key_rpm_states);
let right_bucket = candidate_provider_key_health_bucket(right, provider_key_rpm_states);
right_bucket.cmp(&left_bucket).then_with(|| {
let left_score = candidate_provider_key_health_score(left, provider_key_rpm_states);
let right_score = candidate_provider_key_health_score(right, provider_key_rpm_states);
right_score
.partial_cmp(&left_score)
.unwrap_or(std::cmp::Ordering::Equal)
})
}
fn candidate_provider_key_health_bucket(
candidate: &GatewayMinimalCandidateSelectionCandidate,
provider_key_rpm_states: &BTreeMap<String, StoredProviderCatalogKey>,
) -> Option<super::super::health::ProviderKeyHealthBucket> {
provider_key_rpm_states
.get(&candidate.key_id)
.and_then(|key| provider_key_health_bucket(key, candidate.endpoint_api_format.as_str()))
}
fn candidate_provider_key_health_score(
candidate: &GatewayMinimalCandidateSelectionCandidate,
provider_key_rpm_states: &BTreeMap<String, StoredProviderCatalogKey>,
) -> f64 {
provider_key_rpm_states
.get(&candidate.key_id)
.and_then(|key| {
effective_provider_key_health_score(key, candidate.endpoint_api_format.as_str())
})
.unwrap_or(1.0)
}
pub(super) fn should_skip_provider_quota(
quota: &StoredProviderQuotaSnapshot,
now_unix_secs: u64,
) -> bool {
let snapshot = ProviderQuotaSnapshot {
provider_id: quota.provider_id.clone(),
billing_type: ProviderBillingType::parse(&quota.billing_type),
monthly_quota_usd: quota.monthly_quota_usd,
monthly_used_usd: quota.monthly_used_usd,
quota_reset_day: quota.quota_reset_day,
quota_last_reset_at_unix_secs: quota.quota_last_reset_at_unix_secs,
quota_expires_at_unix_secs: quota.quota_expires_at_unix_secs,
is_active: quota.is_active,
};
if !snapshot.is_active || snapshot.is_expired(now_unix_secs) {
return true;
}
match snapshot.billing_type {
ProviderBillingType::MonthlyQuota | ProviderBillingType::FreeTier => snapshot
.remaining_quota_usd()
.is_some_and(|remaining| remaining <= 0.0),
ProviderBillingType::PayAsYouGo | ProviderBillingType::Unknown => false,
}
}
fn is_candidate_cooled_down(
candidate: &GatewayMinimalCandidateSelectionCandidate,
recent_candidates: &[StoredRequestCandidate],
now_unix_secs: u64,
) -> bool {
is_candidate_in_recent_failure_cooldown(
recent_candidates,
candidate.provider_id.as_str(),
candidate.endpoint_id.as_str(),
candidate.key_id.as_str(),
now_unix_secs,
)
}
pub(super) async fn is_candidate_selectable(
candidate: &GatewayMinimalCandidateSelectionCandidate,
recent_candidates: &[StoredRequestCandidate],
provider_concurrent_limits: &BTreeMap<String, usize>,
provider_key_rpm_states: &BTreeMap<String, StoredProviderCatalogKey>,
now_unix_secs: u64,
cached_affinity_target: Option<&SchedulerAffinityTarget>,
state: &AppState,
) -> Result<bool, GatewayError> {
let quota = state
.read_provider_quota_snapshot(&candidate.provider_id)
.await?;
if quota
.as_ref()
.is_some_and(|quota| should_skip_provider_quota(quota, now_unix_secs))
{
return Ok(false);
}
if is_candidate_cooled_down(candidate, recent_candidates, now_unix_secs) {
return Ok(false);
}
if provider_concurrent_limits
.get(&candidate.provider_id)
.is_some_and(|limit| {
count_recent_active_requests_for_provider(
recent_candidates,
candidate.provider_id.as_str(),
now_unix_secs,
) >= *limit
})
{
return Ok(false);
}
let is_cached_user =
cached_affinity_target.is_some_and(|target| matches_affinity_target(candidate, target));
if let Some(provider_key) = provider_key_rpm_states.get(&candidate.key_id) {
if is_provider_key_circuit_open(provider_key, candidate.endpoint_api_format.as_str()) {
return Ok(false);
}
if provider_key_health_score(provider_key, candidate.endpoint_api_format.as_str())
.is_some_and(|score| score <= 0.0)
{
return Ok(false);
}
let rpm_reset_at =
state.provider_key_rpm_reset_at(candidate.key_id.as_str(), now_unix_secs);
if !provider_key_rpm_allows_request_since(
provider_key,
recent_candidates,
now_unix_secs,
is_cached_user,
rpm_reset_at,
) {
return Ok(false);
}
}
Ok(true)
}
pub(super) async fn read_provider_concurrent_limits(
state: &AppState,
candidates: &[GatewayMinimalCandidateSelectionCandidate],
) -> Result<BTreeMap<String, usize>, GatewayError> {
let provider_ids = candidates
.iter()
.map(|candidate| candidate.provider_id.clone())
.collect::<BTreeSet<_>>()
.into_iter()
.collect::<Vec<_>>();
if provider_ids.is_empty() {
return Ok(BTreeMap::new());
}
let providers = state
.read_provider_catalog_providers_by_ids(&provider_ids)
.await?;
Ok(build_provider_concurrent_limit_map(providers))
}
fn build_provider_concurrent_limit_map(
providers: Vec<StoredProviderCatalogProvider>,
) -> BTreeMap<String, usize> {
providers
.into_iter()
.filter_map(|provider| {
provider
.concurrent_limit
.and_then(|limit| usize::try_from(limit).ok())
.filter(|limit| *limit > 0)
.map(|limit| (provider.id, limit))
})
.collect()
}
pub(super) async fn read_provider_key_rpm_states(
state: &AppState,
candidates: &[GatewayMinimalCandidateSelectionCandidate],
) -> Result<BTreeMap<String, StoredProviderCatalogKey>, GatewayError> {
let key_ids = candidates
.iter()
.map(|candidate| candidate.key_id.clone())
.collect::<BTreeSet<_>>()
.into_iter()
.collect::<Vec<_>>();
if key_ids.is_empty() {
return Ok(BTreeMap::new());
}
let keys = state.read_provider_catalog_keys_by_ids(&key_ids).await?;
Ok(keys
.into_iter()
.map(|key| (key.id.clone(), key))
.collect::<BTreeMap<_, _>>())
}

1267
apps/aether-gateway/src/scheduler/candidate/tests.rs Normal file
View File

File diff suppressed because it is too large Load Diff

316
apps/aether-gateway/src/scheduler/failover.rs Normal file
View File

@@ -0,0 +1,316 @@
use aether_contracts::ExecutionResult;
fn is_local_candidate_attempt(report_context: Option<&serde_json::Value>) -> bool {
report_context
.and_then(serde_json::Value::as_object)
.and_then(|context| context.get("candidate_index"))
.and_then(serde_json::Value::as_u64)
.is_some()
}
fn is_retryable_local_upstream_status(status_code: u16) -> bool {
status_code == 429 || status_code >= 500
}
pub(crate) fn should_retry_next_local_candidate_sync(
plan_kind: &str,
report_context: Option<&serde_json::Value>,
result: &ExecutionResult,
) -> bool {
is_local_candidate_attempt(report_context)
&& plan_kind == "openai_chat_sync"
&& is_retryable_local_upstream_status(result.status_code)
}
pub(crate) fn should_fallback_to_control_sync(
plan_kind: &str,
result: &ExecutionResult,
body_json: Option<&serde_json::Value>,
has_body_bytes: bool,
explicit_finalize: bool,
mapped_error_finalize: bool,
) -> bool {
if explicit_finalize
&& matches!(
plan_kind,
"openai_video_delete_sync" | "openai_video_cancel_sync" | "gemini_video_cancel_sync"
)
{
return false;
}
if !matches!(
plan_kind,
"openai_video_create_sync"
| "openai_video_remix_sync"
| "gemini_video_create_sync"
| "openai_chat_sync"
| "openai_cli_sync"
| "openai_compact_sync"
| "claude_chat_sync"
| "gemini_chat_sync"
| "claude_cli_sync"
| "gemini_cli_sync"
) {
return false;
}
if explicit_finalize {
return result.status_code < 400 && body_json.is_none() && !has_body_bytes;
}
if mapped_error_finalize {
return false;
}
if result.status_code >= 400 {
return true;
}
let Some(body_json) = body_json else {
return true;
};
body_json.get("error").is_some()
}
pub(crate) fn should_finalize_sync_response(report_kind: Option<&str>) -> bool {
report_kind.is_some_and(|kind| kind.ends_with("_finalize"))
}
pub(crate) fn resolve_core_sync_error_finalize_report_kind(
plan_kind: &str,
result: &ExecutionResult,
body_json: Option<&serde_json::Value>,
) -> Option<String> {
let has_embedded_error = body_json.is_some_and(|value| value.get("error").is_some());
if result.status_code < 400 && !has_embedded_error {
return None;
}
let report_kind = match plan_kind {
"openai_chat_sync" => "openai_chat_sync_finalize",
"openai_cli_sync" => "openai_cli_sync_finalize",
"openai_compact_sync" => "openai_compact_sync_finalize",
"claude_chat_sync" => "claude_chat_sync_finalize",
"gemini_chat_sync" => "gemini_chat_sync_finalize",
"claude_cli_sync" => "claude_cli_sync_finalize",
"gemini_cli_sync" => "gemini_cli_sync_finalize",
_ => return None,
};
Some(report_kind.to_string())
}
pub(crate) fn should_retry_next_local_candidate_stream(
plan_kind: &str,
report_context: Option<&serde_json::Value>,
status_code: u16,
) -> bool {
is_local_candidate_attempt(report_context)
&& plan_kind == "openai_chat_stream"
&& is_retryable_local_upstream_status(status_code)
}
pub(crate) fn should_fallback_to_control_stream(
plan_kind: &str,
status_code: u16,
mapped_error_finalize: bool,
) -> bool {
if mapped_error_finalize {
return false;
}
matches!(
plan_kind,
"openai_chat_stream"
| "claude_chat_stream"
| "gemini_chat_stream"
| "openai_cli_stream"
| "openai_compact_stream"
| "claude_cli_stream"
| "gemini_cli_stream"
) && status_code >= 400
}
pub(crate) fn resolve_core_stream_error_finalize_report_kind(
plan_kind: &str,
status_code: u16,
) -> Option<String> {
if status_code < 400 {
return None;
}
let report_kind = match plan_kind {
"openai_chat_stream" => "openai_chat_sync_finalize",
"claude_chat_stream" => "claude_chat_sync_finalize",
"gemini_chat_stream" => "gemini_chat_sync_finalize",
"openai_cli_stream" => "openai_cli_sync_finalize",
"openai_compact_stream" => "openai_compact_sync_finalize",
"claude_cli_stream" => "claude_cli_sync_finalize",
"gemini_cli_stream" => "gemini_cli_sync_finalize",
_ => return None,
};
Some(report_kind.to_string())
}
pub(crate) fn resolve_core_stream_direct_finalize_report_kind(plan_kind: &str) -> Option<String> {
let report_kind = match plan_kind {
"openai_chat_stream" => "openai_chat_sync_finalize",
"claude_chat_stream" => "claude_chat_sync_finalize",
"gemini_chat_stream" => "gemini_chat_sync_finalize",
"openai_cli_stream" => "openai_cli_sync_finalize",
"openai_compact_stream" => "openai_compact_sync_finalize",
"claude_cli_stream" => "claude_cli_sync_finalize",
"gemini_cli_stream" => "gemini_cli_sync_finalize",
_ => return None,
};
Some(report_kind.to_string())
}
#[cfg(test)]
mod tests {
use aether_contracts::ExecutionResult;
use super::{
resolve_core_stream_error_finalize_report_kind,
resolve_core_sync_error_finalize_report_kind, should_fallback_to_control_stream,
should_fallback_to_control_sync, should_retry_next_local_candidate_stream,
should_retry_next_local_candidate_sync,
};
#[test]
fn sync_failover_marks_chat_errors() {
let result = ExecutionResult {
request_id: "req-1".to_string(),
candidate_id: None,
status_code: 502,
headers: Default::default(),
body: None,
telemetry: None,
error: None,
};
assert!(should_fallback_to_control_sync(
"openai_chat_sync",
&result,
None,
false,
false,
false,
));
assert_eq!(
resolve_core_sync_error_finalize_report_kind("openai_chat_sync", &result, None),
Some("openai_chat_sync_finalize".to_string())
);
}
#[test]
fn stream_failover_marks_chat_errors() {
assert!(should_fallback_to_control_stream(
"openai_chat_stream",
502,
false,
));
assert_eq!(
resolve_core_stream_error_finalize_report_kind("openai_chat_stream", 502),
Some("openai_chat_sync_finalize".to_string())
);
}
#[test]
fn sync_retry_next_candidate_is_local_openai_chat_only() {
let result = ExecutionResult {
request_id: "req-1".to_string(),
candidate_id: None,
status_code: 502,
headers: Default::default(),
body: None,
telemetry: None,
error: None,
};
let local_report_context = serde_json::json!({
"candidate_index": 0,
"retry_index": 0,
});
assert!(should_retry_next_local_candidate_sync(
"openai_chat_sync",
Some(&local_report_context),
&result,
));
assert!(!should_retry_next_local_candidate_sync(
"openai_chat_sync",
None,
&result,
));
assert!(!should_retry_next_local_candidate_sync(
"claude_chat_sync",
None,
&result,
));
}
#[test]
fn sync_retry_next_candidate_treats_rate_limit_as_retryable() {
let result = ExecutionResult {
request_id: "req-1".to_string(),
candidate_id: None,
status_code: 429,
headers: Default::default(),
body: None,
telemetry: None,
error: None,
};
let local_report_context = serde_json::json!({
"candidate_index": 0,
"retry_index": 0,
});
assert!(should_retry_next_local_candidate_sync(
"openai_chat_sync",
Some(&local_report_context),
&result,
));
}
#[test]
fn stream_retry_next_candidate_is_local_openai_chat_only() {
let local_report_context = serde_json::json!({
"candidate_index": 0,
"retry_index": 0,
});
assert!(should_retry_next_local_candidate_stream(
"openai_chat_stream",
Some(&local_report_context),
502,
));
assert!(!should_retry_next_local_candidate_stream(
"openai_chat_stream",
None,
502,
));
assert!(!should_retry_next_local_candidate_stream(
"claude_chat_stream",
None,
502,
));
}
#[test]
fn stream_retry_next_candidate_treats_rate_limit_as_retryable() {
let local_report_context = serde_json::json!({
"candidate_index": 0,
"retry_index": 0,
});
assert!(should_retry_next_local_candidate_stream(
"openai_chat_stream",
Some(&local_report_context),
429,
));
}
}

954
apps/aether-gateway/src/scheduler/health.rs Normal file
View File

@@ -0,0 +1,954 @@
use aether_data::repository::candidates::{RequestCandidateStatus, StoredRequestCandidate};
use aether_data::repository::provider_catalog::StoredProviderCatalogKey;
const FAILURE_COOLDOWN_WINDOW_SECS: u64 = 60;
const FAILURE_COOLDOWN_THRESHOLD: usize = 2;
const ACTIVE_REQUEST_WINDOW_SECS: u64 = 300;
pub(crate) const PROVIDER_KEY_RPM_WINDOW_SECS: u64 = 60;
const PROBE_PHASE_REQUESTS: u32 = 100;
const PROBE_RESERVATION_RATIO: f64 = 0.1;
const STABLE_MIN_RESERVATION_RATIO: f64 = 0.1;
const STABLE_MAX_RESERVATION_RATIO: f64 = 0.35;
const SUCCESS_COUNT_FOR_FULL_CONFIDENCE: u32 = 50;
const COOLDOWN_HOURS_FOR_FULL_CONFIDENCE: f64 = 24.0;
const LOW_LOAD_THRESHOLD: f64 = 0.5;
const HIGH_LOAD_THRESHOLD: f64 = 0.8;
const ENFORCEMENT_CONFIDENCE_THRESHOLD: f64 = 0.6;
const HEALTH_DEGRADED_THRESHOLD: f64 = 0.8;
const HEALTH_LOW_THRESHOLD: f64 = 0.5;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub(crate) enum ProviderKeyHealthBucket {
Low,
Degraded,
Healthy,
}
impl ProviderKeyHealthBucket {
fn from_score(score: f64) -> Self {
let score = score.clamp(0.0, 1.0);
if score < HEALTH_LOW_THRESHOLD {
return Self::Low;
}
if score < HEALTH_DEGRADED_THRESHOLD {
return Self::Degraded;
}
Self::Healthy
}
}
pub(crate) fn is_candidate_in_recent_failure_cooldown(
recent_candidates: &[StoredRequestCandidate],
provider_id: &str,
endpoint_id: &str,
key_id: &str,
now_unix_secs: u64,
) -> bool {
let mut recent_failures = 0usize;
for candidate in recent_candidates {
if candidate.provider_id.as_deref() != Some(provider_id)
|| candidate.endpoint_id.as_deref() != Some(endpoint_id)
|| candidate.key_id.as_deref() != Some(key_id)
{
continue;
}
let observed_at_unix_secs = candidate
.finished_at_unix_secs
.or(candidate.started_at_unix_secs)
.unwrap_or(candidate.created_at_unix_secs);
if now_unix_secs.saturating_sub(observed_at_unix_secs) > FAILURE_COOLDOWN_WINDOW_SECS {
continue;
}
match candidate.status {
RequestCandidateStatus::Success => return false,
RequestCandidateStatus::Failed | RequestCandidateStatus::Cancelled => {
recent_failures += 1;
if recent_failures >= FAILURE_COOLDOWN_THRESHOLD {
return true;
}
}
RequestCandidateStatus::Available
| RequestCandidateStatus::Unused
| RequestCandidateStatus::Pending
| RequestCandidateStatus::Streaming
| RequestCandidateStatus::Skipped => {}
}
}
false
}
pub(crate) fn count_recent_active_requests_for_provider(
recent_candidates: &[StoredRequestCandidate],
provider_id: &str,
now_unix_secs: u64,
) -> usize {
recent_candidates
.iter()
.filter(|candidate| candidate.provider_id.as_deref() == Some(provider_id))
.filter(|candidate| is_recently_active(candidate, now_unix_secs))
.count()
}
pub(crate) fn count_recent_active_requests_for_api_key(
recent_candidates: &[StoredRequestCandidate],
api_key_id: &str,
now_unix_secs: u64,
) -> usize {
recent_candidates
.iter()
.filter(|candidate| candidate.api_key_id.as_deref() == Some(api_key_id))
.filter(|candidate| is_recently_active(candidate, now_unix_secs))
.count()
}
pub(crate) fn effective_provider_key_rpm_limit(
key: &StoredProviderCatalogKey,
now_unix_secs: u64,
) -> Option<usize> {
if let Some(limit) = key.rpm_limit.filter(|limit| *limit > 0) {
return usize::try_from(limit).ok();
}
let learned_limit = key
.learned_rpm_limit
.filter(|limit| *limit > 0)
.and_then(|limit| usize::try_from(limit).ok())?;
if provider_key_reservation_confidence(key, now_unix_secs) < ENFORCEMENT_CONFIDENCE_THRESHOLD {
return None;
}
Some(learned_limit)
}
pub(crate) fn count_recent_rpm_requests_for_provider_key(
recent_candidates: &[StoredRequestCandidate],
key_id: &str,
now_unix_secs: u64,
) -> usize {
count_recent_rpm_requests_for_provider_key_since(recent_candidates, key_id, now_unix_secs, None)
}
pub(crate) fn count_recent_rpm_requests_for_provider_key_since(
recent_candidates: &[StoredRequestCandidate],
key_id: &str,
now_unix_secs: u64,
reset_after_unix_secs: Option<u64>,
) -> usize {
let mut attempted_count = 0usize;
let mut max_observed = 0usize;
for candidate in recent_candidates {
if candidate.key_id.as_deref() != Some(key_id) {
continue;
}
if !is_recent_rpm_observation(candidate, now_unix_secs) {
continue;
}
let observed_at_unix_secs = candidate
.started_at_unix_secs
.unwrap_or(candidate.created_at_unix_secs);
if reset_after_unix_secs.is_some_and(|reset_after| observed_at_unix_secs <= reset_after) {
continue;
}
attempted_count += 1;
max_observed = max_observed.max(candidate.concurrent_requests.unwrap_or_default() as usize);
}
max_observed.max(attempted_count)
}
pub(crate) fn provider_key_rpm_allows_request(
key: &StoredProviderCatalogKey,
recent_candidates: &[StoredRequestCandidate],
now_unix_secs: u64,
is_cached_user: bool,
) -> bool {
provider_key_rpm_allows_request_since(
key,
recent_candidates,
now_unix_secs,
is_cached_user,
None,
)
}
pub(crate) fn provider_key_rpm_allows_request_since(
key: &StoredProviderCatalogKey,
recent_candidates: &[StoredRequestCandidate],
now_unix_secs: u64,
is_cached_user: bool,
reset_after_unix_secs: Option<u64>,
) -> bool {
let Some(effective_limit) = effective_provider_key_rpm_limit(key, now_unix_secs) else {
return true;
};
if effective_limit == 0 {
return false;
}
let current_usage = count_recent_rpm_requests_for_provider_key_since(
recent_candidates,
key.id.as_str(),
now_unix_secs,
reset_after_unix_secs,
);
if is_cached_user {
return current_usage < effective_limit;
}
let available_for_new = available_provider_key_rpm_slots_for_new_user(
key,
current_usage,
effective_limit,
now_unix_secs,
);
current_usage < available_for_new
}
pub(crate) fn provider_key_health_score(
key: &StoredProviderCatalogKey,
api_format: &str,
) -> Option<f64> {
let score = key
.health_by_format
.as_ref()
.and_then(serde_json::Value::as_object)
.and_then(|values| values.get(api_format))
.and_then(serde_json::Value::as_object)
.and_then(|payload| payload.get("health_score"))
.and_then(json_value_as_f64)?;
Some(score.clamp(0.0, 1.0))
}
pub(crate) fn aggregate_provider_key_health_score(key: &StoredProviderCatalogKey) -> Option<f64> {
let health_by_format = key.health_by_format.as_ref()?.as_object()?;
let mut scores = Vec::new();
for payload in health_by_format.values() {
let Some(score) = payload
.as_object()
.and_then(|payload| payload.get("health_score"))
.and_then(json_value_as_f64)
else {
continue;
};
scores.push(score.clamp(0.0, 1.0));
}
scores.into_iter().reduce(f64::min)
}
pub(crate) fn effective_provider_key_health_score(
key: &StoredProviderCatalogKey,
api_format: &str,
) -> Option<f64> {
provider_key_health_score(key, api_format).or_else(|| aggregate_provider_key_health_score(key))
}
pub(crate) fn provider_key_health_bucket(
key: &StoredProviderCatalogKey,
api_format: &str,
) -> Option<ProviderKeyHealthBucket> {
effective_provider_key_health_score(key, api_format).map(ProviderKeyHealthBucket::from_score)
}
pub(crate) fn is_provider_key_circuit_open(
key: &StoredProviderCatalogKey,
api_format: &str,
) -> bool {
key.circuit_breaker_by_format
.as_ref()
.and_then(serde_json::Value::as_object)
.and_then(|values| values.get(api_format))
.and_then(serde_json::Value::as_object)
.and_then(|payload| payload.get("open"))
.and_then(serde_json::Value::as_bool)
.unwrap_or(false)
}
fn available_provider_key_rpm_slots_for_new_user(
key: &StoredProviderCatalogKey,
current_usage: usize,
effective_limit: usize,
now_unix_secs: u64,
) -> usize {
let reservation_ratio =
provider_key_dynamic_reservation_ratio(key, current_usage, effective_limit, now_unix_secs);
usize::max(
1,
(effective_limit as f64 * (1.0 - reservation_ratio)).floor() as usize,
)
}
fn provider_key_dynamic_reservation_ratio(
key: &StoredProviderCatalogKey,
current_usage: usize,
effective_limit: usize,
now_unix_secs: u64,
) -> f64 {
let total_requests = provider_key_total_requests(key);
if total_requests < PROBE_PHASE_REQUESTS {
return PROBE_RESERVATION_RATIO;
}
let confidence = provider_key_reservation_confidence(key, now_unix_secs);
let load_ratio = provider_key_load_ratio(current_usage, effective_limit);
if load_ratio < LOW_LOAD_THRESHOLD {
return STABLE_MIN_RESERVATION_RATIO;
}
if load_ratio < HIGH_LOAD_THRESHOLD {
let load_factor =
(load_ratio - LOW_LOAD_THRESHOLD) / (HIGH_LOAD_THRESHOLD - LOW_LOAD_THRESHOLD);
return STABLE_MIN_RESERVATION_RATIO
+ confidence
* load_factor
* (STABLE_MAX_RESERVATION_RATIO - STABLE_MIN_RESERVATION_RATIO);
}
STABLE_MIN_RESERVATION_RATIO
+ confidence * (STABLE_MAX_RESERVATION_RATIO - STABLE_MIN_RESERVATION_RATIO)
}
fn is_recently_active(candidate: &StoredRequestCandidate, now_unix_secs: u64) -> bool {
if candidate.finished_at_unix_secs.is_some() {
return false;
}
if !matches!(
candidate.status,
RequestCandidateStatus::Pending | RequestCandidateStatus::Streaming
) {
return false;
}
let observed_at_unix_secs = candidate
.started_at_unix_secs
.unwrap_or(candidate.created_at_unix_secs);
now_unix_secs.saturating_sub(observed_at_unix_secs) <= ACTIVE_REQUEST_WINDOW_SECS
}
fn is_recent_rpm_observation(candidate: &StoredRequestCandidate, now_unix_secs: u64) -> bool {
if !candidate
.status
.is_attempted(candidate.started_at_unix_secs)
{
return false;
}
let observed_at_unix_secs = candidate
.started_at_unix_secs
.unwrap_or(candidate.created_at_unix_secs);
now_unix_secs.saturating_sub(observed_at_unix_secs) <= PROVIDER_KEY_RPM_WINDOW_SECS
}
fn provider_key_total_requests(key: &StoredProviderCatalogKey) -> u32 {
let request_count = key.request_count.unwrap_or_default();
if request_count > 0 {
return request_count;
}
let history_count = key
.adjustment_history
.as_ref()
.and_then(serde_json::Value::as_array)
.map(|values| values.len() as u32 * 10)
.unwrap_or_default();
key.concurrent_429_count.unwrap_or_default()
+ key.rpm_429_count.unwrap_or_default()
+ key.success_count.unwrap_or_default()
+ history_count
}
fn provider_key_load_ratio(current_usage: usize, effective_limit: usize) -> f64 {
if effective_limit == 0 {
return 0.0;
}
(current_usage as f64 / effective_limit as f64).min(1.0)
}
fn provider_key_reservation_confidence(key: &StoredProviderCatalogKey, now_unix_secs: u64) -> f64 {
let request_count = key.request_count.unwrap_or_default() as f64;
let success_count = key.success_count.unwrap_or_default() as f64;
let success_score = if request_count >= SUCCESS_COUNT_FOR_FULL_CONFIDENCE as f64 {
let success_rate = if request_count > 0.0 {
success_count / request_count
} else {
0.0
};
success_rate * 0.4
} else if request_count > 0.0 {
let success_rate = success_count / request_count;
let progress_ratio = request_count / SUCCESS_COUNT_FOR_FULL_CONFIDENCE as f64;
success_rate * progress_ratio * 0.4
} else {
0.0
};
let cooldown_score = match key.last_429_at_unix_secs {
Some(last_429_at_unix_secs) => {
let hours_since_429 =
now_unix_secs.saturating_sub(last_429_at_unix_secs) as f64 / 3600.0;
(hours_since_429 / COOLDOWN_HOURS_FOR_FULL_CONFIDENCE).min(1.0) * 0.3
}
None => 0.3,
};
let stability_score = provider_key_stability_score(key);
(success_score + cooldown_score + stability_score).min(1.0)
}
fn provider_key_stability_score(key: &StoredProviderCatalogKey) -> f64 {
let Some(history) = key
.adjustment_history
.as_ref()
.and_then(serde_json::Value::as_array)
else {
return 0.15;
};
if history.len() < 3 {
return 0.15;
}
let recent = if history.len() > 5 {
&history[history.len() - 5..]
} else {
history.as_slice()
};
let limits = recent
.iter()
.filter_map(|entry| entry.get("new_limit"))
.filter_map(json_value_as_f64)
.collect::<Vec<_>>();
if limits.len() < 2 {
return 0.15;
}
let mean = limits.iter().sum::<f64>() / limits.len() as f64;
let variance = limits
.iter()
.map(|limit| {
let delta = *limit - mean;
delta * delta
})
.sum::<f64>()
/ (limits.len() as f64 - 1.0);
let stability_ratio = (1.0 - variance / 10.0).max(0.0);
stability_ratio * 0.3
}
fn json_value_as_f64(value: &serde_json::Value) -> Option<f64> {
value
.as_f64()
.or_else(|| value.as_i64().map(|raw| raw as f64))
.or_else(|| value.as_u64().map(|raw| raw as f64))
}
#[cfg(test)]
mod tests {
use aether_data::repository::candidates::{RequestCandidateStatus, StoredRequestCandidate};
use aether_data::repository::provider_catalog::StoredProviderCatalogKey;
use super::{
aggregate_provider_key_health_score, count_recent_active_requests_for_api_key,
count_recent_active_requests_for_provider, count_recent_rpm_requests_for_provider_key,
count_recent_rpm_requests_for_provider_key_since, effective_provider_key_health_score,
effective_provider_key_rpm_limit, is_candidate_in_recent_failure_cooldown,
is_provider_key_circuit_open, provider_key_health_bucket, provider_key_health_score,
provider_key_rpm_allows_request, provider_key_rpm_allows_request_since,
ProviderKeyHealthBucket,
};
fn stored_candidate(
id: &str,
status: RequestCandidateStatus,
created_at_unix_secs: i64,
) -> StoredRequestCandidate {
StoredRequestCandidate::new(
id.to_string(),
format!("req-{id}"),
None,
None,
None,
None,
0,
0,
Some("provider-a".to_string()),
Some("endpoint-a".to_string()),
Some("key-a".to_string()),
status,
None,
false,
None,
None,
None,
None,
None,
None,
None,
created_at_unix_secs,
Some(created_at_unix_secs),
Some(created_at_unix_secs),
)
.expect("candidate should build")
}
fn provider_catalog_key(id: &str) -> StoredProviderCatalogKey {
StoredProviderCatalogKey::new(
id.to_string(),
"provider-a".to_string(),
"primary".to_string(),
"api_key".to_string(),
None,
true,
)
.expect("provider key should build")
}
#[test]
fn cooldown_triggers_after_two_recent_failures() {
let recent_candidates = vec![
stored_candidate("one", RequestCandidateStatus::Failed, 95),
stored_candidate("two", RequestCandidateStatus::Cancelled, 99),
];
assert!(is_candidate_in_recent_failure_cooldown(
&recent_candidates,
"provider-a",
"endpoint-a",
"key-a",
100,
));
}
#[test]
fn recent_success_clears_cooldown() {
let recent_candidates = vec![
stored_candidate("one", RequestCandidateStatus::Failed, 95),
stored_candidate("two", RequestCandidateStatus::Success, 99),
stored_candidate("three", RequestCandidateStatus::Cancelled, 98),
];
assert!(!is_candidate_in_recent_failure_cooldown(
&recent_candidates,
"provider-a",
"endpoint-a",
"key-a",
100,
));
}
#[test]
fn counts_only_recently_active_provider_requests() {
let recent_candidates = vec![
StoredRequestCandidate::new(
"one".to_string(),
"req-one".to_string(),
None,
Some("api-key-1".to_string()),
None,
None,
0,
0,
Some("provider-a".to_string()),
Some("endpoint-a".to_string()),
Some("key-a".to_string()),
RequestCandidateStatus::Pending,
None,
false,
None,
None,
None,
None,
None,
None,
None,
95,
Some(95),
None,
)
.expect("candidate should build"),
StoredRequestCandidate::new(
"two".to_string(),
"req-two".to_string(),
None,
Some("api-key-1".to_string()),
None,
None,
0,
0,
Some("provider-a".to_string()),
Some("endpoint-a".to_string()),
Some("key-a".to_string()),
RequestCandidateStatus::Streaming,
None,
false,
None,
None,
None,
None,
None,
None,
None,
96,
Some(96),
None,
)
.expect("candidate should build"),
StoredRequestCandidate::new(
"three".to_string(),
"req-three".to_string(),
None,
Some("api-key-1".to_string()),
None,
None,
0,
0,
Some("provider-a".to_string()),
Some("endpoint-a".to_string()),
Some("key-a".to_string()),
RequestCandidateStatus::Success,
None,
false,
None,
None,
None,
None,
None,
None,
None,
97,
Some(97),
Some(98),
)
.expect("candidate should build"),
];
assert_eq!(
count_recent_active_requests_for_provider(&recent_candidates, "provider-a", 100),
2
);
assert_eq!(
count_recent_active_requests_for_api_key(&recent_candidates, "api-key-1", 100),
2
);
}
#[test]
fn fixed_provider_key_rpm_limit_takes_precedence() {
let key = provider_catalog_key("key-a").with_rate_limit_fields(
Some(120),
Some(80),
None,
None,
None,
None,
Some(10),
Some(10),
);
assert_eq!(effective_provider_key_rpm_limit(&key, 100), Some(120));
}
#[test]
fn learned_provider_key_rpm_limit_requires_confidence() {
let low_confidence = provider_catalog_key("key-a").with_rate_limit_fields(
None,
Some(80),
Some(0),
Some(0),
Some(99),
None,
Some(5),
Some(1),
);
assert_eq!(effective_provider_key_rpm_limit(&low_confidence, 100), None);
let high_confidence = provider_catalog_key("key-a").with_rate_limit_fields(
None,
Some(80),
Some(0),
Some(0),
None,
Some(serde_json::json!([
{"new_limit": 80},
{"new_limit": 81},
{"new_limit": 80},
])),
Some(120),
Some(118),
);
assert_eq!(
effective_provider_key_rpm_limit(&high_confidence, 100),
Some(80)
);
}
#[test]
fn counts_recent_provider_key_rpm_from_snapshot_or_recent_attempts() {
let recent_candidates = vec![
StoredRequestCandidate::new(
"one".to_string(),
"req-one".to_string(),
None,
None,
None,
None,
0,
0,
Some("provider-a".to_string()),
Some("endpoint-a".to_string()),
Some("key-a".to_string()),
RequestCandidateStatus::Success,
None,
false,
Some(200),
None,
None,
Some(10),
Some(7),
None,
None,
95,
Some(95),
Some(96),
)
.expect("candidate should build"),
StoredRequestCandidate::new(
"two".to_string(),
"req-two".to_string(),
None,
None,
None,
None,
0,
0,
Some("provider-a".to_string()),
Some("endpoint-a".to_string()),
Some("key-a".to_string()),
RequestCandidateStatus::Failed,
None,
false,
Some(502),
None,
None,
Some(10),
None,
None,
None,
98,
Some(98),
Some(99),
)
.expect("candidate should build"),
];
assert_eq!(
count_recent_rpm_requests_for_provider_key(&recent_candidates, "key-a", 100),
7
);
}
#[test]
fn ignores_rpm_observations_before_reset_watermark() {
let recent_candidates = vec![
StoredRequestCandidate::new(
"one".to_string(),
"req-one".to_string(),
None,
None,
None,
None,
0,
0,
Some("provider-a".to_string()),
Some("endpoint-a".to_string()),
Some("key-a".to_string()),
RequestCandidateStatus::Success,
None,
false,
Some(200),
None,
None,
Some(10),
Some(7),
None,
None,
95,
Some(95),
Some(96),
)
.expect("candidate should build"),
StoredRequestCandidate::new(
"two".to_string(),
"req-two".to_string(),
None,
None,
None,
None,
0,
0,
Some("provider-a".to_string()),
Some("endpoint-a".to_string()),
Some("key-a".to_string()),
RequestCandidateStatus::Success,
None,
false,
Some(200),
None,
None,
Some(10),
Some(2),
None,
None,
99,
Some(99),
Some(100),
)
.expect("candidate should build"),
];
assert_eq!(
count_recent_rpm_requests_for_provider_key_since(
&recent_candidates,
"key-a",
100,
Some(98),
),
2
);
}
#[test]
fn provider_key_rpm_reserves_capacity_for_new_users() {
let key = provider_catalog_key("key-a").with_rate_limit_fields(
Some(10),
None,
None,
None,
None,
None,
Some(5),
Some(5),
);
let recent_candidates = vec![StoredRequestCandidate::new(
"one".to_string(),
"req-one".to_string(),
None,
None,
None,
None,
0,
0,
Some("provider-a".to_string()),
Some("endpoint-a".to_string()),
Some("key-a".to_string()),
RequestCandidateStatus::Success,
None,
false,
Some(200),
None,
None,
Some(10),
Some(9),
None,
None,
95,
Some(95),
Some(96),
)
.expect("candidate should build")];
assert!(!provider_key_rpm_allows_request(
&key,
&recent_candidates,
100,
false,
));
assert!(provider_key_rpm_allows_request(
&key,
&recent_candidates,
100,
true,
));
assert!(provider_key_rpm_allows_request_since(
&key,
&recent_candidates,
100,
false,
Some(97),
));
}
#[test]
fn reads_provider_key_health_and_circuit_status_for_api_format() {
let key = provider_catalog_key("key-a").with_health_fields(
Some(serde_json::json!({
"openai:chat": {"health_score": 0.25},
"openai:responses": {"health_score": 0.75}
})),
Some(serde_json::json!({
"openai:chat": {"open": true},
"openai:responses": {"open": false}
})),
);
assert_eq!(provider_key_health_score(&key, "openai:chat"), Some(0.25));
assert_eq!(
provider_key_health_score(&key, "openai:responses"),
Some(0.75)
);
assert!(is_provider_key_circuit_open(&key, "openai:chat"));
assert!(!is_provider_key_circuit_open(&key, "openai:responses"));
}
#[test]
fn aggregates_provider_key_health_score_with_lower_bound_strategy() {
let key = provider_catalog_key("key-a").with_health_fields(
Some(serde_json::json!({
"openai:chat": {"health_score": 0.85},
"openai:responses": {"health_score": 0.45},
"claude:chat": {"health_score": 0.70}
})),
None,
);
assert_eq!(aggregate_provider_key_health_score(&key), Some(0.45));
assert_eq!(
effective_provider_key_health_score(&key, "gemini:chat"),
Some(0.45)
);
}
#[test]
fn classifies_provider_key_health_bucket_from_effective_score() {
let low = provider_catalog_key("key-low").with_health_fields(
Some(serde_json::json!({"openai:chat": {"health_score": 0.30}})),
None,
);
let degraded = provider_catalog_key("key-degraded").with_health_fields(
Some(serde_json::json!({"openai:chat": {"health_score": 0.65}})),
None,
);
let healthy = provider_catalog_key("key-healthy").with_health_fields(
Some(serde_json::json!({"openai:chat": {"health_score": 0.92}})),
None,
);
assert_eq!(
provider_key_health_bucket(&low, "openai:chat"),
Some(ProviderKeyHealthBucket::Low)
);
assert_eq!(
provider_key_health_bucket(&degraded, "openai:chat"),
Some(ProviderKeyHealthBucket::Degraded)
);
assert_eq!(
provider_key_health_bucket(&healthy, "openai:chat"),
Some(ProviderKeyHealthBucket::Healthy)
);
}
}

28
apps/aether-gateway/src/scheduler/mod.rs Normal file
View File

@@ -0,0 +1,28 @@
mod candidate;
mod failover;
mod health;
mod route;
pub(crate) use candidate::{
list_selectable_candidates,
list_selectable_candidates_for_required_capability_without_requested_model,
read_cached_scheduler_affinity_target, read_minimal_candidate_selection,
GatewayMinimalCandidateSelectionCandidate,
};
pub(crate) use failover::{
resolve_core_stream_direct_finalize_report_kind,
resolve_core_stream_error_finalize_report_kind, resolve_core_sync_error_finalize_report_kind,
should_fallback_to_control_stream, should_fallback_to_control_sync,
should_finalize_sync_response, should_retry_next_local_candidate_stream,
should_retry_next_local_candidate_sync,
};
pub(crate) use health::{
count_recent_rpm_requests_for_provider_key, count_recent_rpm_requests_for_provider_key_since,
is_provider_key_circuit_open, provider_key_health_score, provider_key_rpm_allows_request_since,
PROVIDER_KEY_RPM_WINDOW_SECS,
};
pub(crate) use route::{
is_matching_stream_request, resolve_execution_runtime_stream_plan_kind,
resolve_execution_runtime_sync_plan_kind, supports_stream_scheduler_decision_kind,
supports_sync_scheduler_decision_kind,
};

355
apps/aether-gateway/src/scheduler/route.rs Normal file
View File

@@ -0,0 +1,355 @@
use crate::gateway::GatewayControlDecision;
pub(crate) fn resolve_execution_runtime_stream_plan_kind(
parts: &http::request::Parts,
decision: &GatewayControlDecision,
) -> Option<&'static str> {
if decision.route_class.as_deref() != Some("ai_public") {
return None;
}
if decision.route_family.as_deref() == Some("gemini")
&& decision.route_kind.as_deref() == Some("files")
&& parts.method == http::Method::GET
&& parts.uri.path().ends_with(":download")
{
return Some("gemini_files_download");
}
if decision.route_family.as_deref() == Some("openai")
&& decision.route_kind.as_deref() == Some("chat")
&& parts.method == http::Method::POST
&& parts.uri.path() == "/v1/chat/completions"
{
return Some("openai_chat_stream");
}
if decision.route_family.as_deref() == Some("claude")
&& decision.route_kind.as_deref() == Some("chat")
&& parts.method == http::Method::POST
&& parts.uri.path() == "/v1/messages"
{
return Some("claude_chat_stream");
}
if decision.route_family.as_deref() == Some("claude")
&& decision.route_kind.as_deref() == Some("cli")
&& parts.method == http::Method::POST
&& parts.uri.path() == "/v1/messages"
{
return Some("claude_cli_stream");
}
if decision.route_family.as_deref() == Some("gemini")
&& decision.route_kind.as_deref() == Some("chat")
&& parts.method == http::Method::POST
&& parts.uri.path().ends_with(":streamGenerateContent")
{
return Some("gemini_chat_stream");
}
if decision.route_family.as_deref() == Some("gemini")
&& decision.route_kind.as_deref() == Some("cli")
&& parts.method == http::Method::POST
&& parts.uri.path().ends_with(":streamGenerateContent")
{
return Some("gemini_cli_stream");
}
if decision.route_family.as_deref() == Some("openai")
&& decision.route_kind.as_deref() == Some("cli")
&& parts.method == http::Method::POST
&& parts.uri.path() == "/v1/responses"
{
return Some("openai_cli_stream");
}
if decision.route_family.as_deref() == Some("openai")
&& decision.route_kind.as_deref() == Some("compact")
&& parts.method == http::Method::POST
&& parts.uri.path() == "/v1/responses/compact"
{
return Some("openai_compact_stream");
}
if decision.route_family.as_deref() == Some("openai")
&& decision.route_kind.as_deref() == Some("video")
&& parts.method == http::Method::GET
&& parts.uri.path().ends_with("/content")
{
return Some("openai_video_content");
}
None
}
pub(crate) fn resolve_execution_runtime_sync_plan_kind(
parts: &http::request::Parts,
decision: &GatewayControlDecision,
) -> Option<&'static str> {
if decision.route_class.as_deref() != Some("ai_public") {
return None;
}
if decision.route_family.as_deref() == Some("openai")
&& decision.route_kind.as_deref() == Some("video")
&& parts.method == http::Method::POST
&& parts.uri.path().starts_with("/v1/videos/")
&& parts.uri.path().ends_with("/cancel")
{
return Some("openai_video_cancel_sync");
}
if decision.route_family.as_deref() == Some("openai")
&& decision.route_kind.as_deref() == Some("video")
&& parts.method == http::Method::POST
&& parts.uri.path().starts_with("/v1/videos/")
&& parts.uri.path().ends_with("/remix")
{
return Some("openai_video_remix_sync");
}
if decision.route_family.as_deref() == Some("openai")
&& decision.route_kind.as_deref() == Some("video")
&& parts.method == http::Method::POST
&& parts.uri.path() == "/v1/videos"
{
return Some("openai_video_create_sync");
}
if decision.route_family.as_deref() == Some("openai")
&& decision.route_kind.as_deref() == Some("video")
&& parts.method == http::Method::DELETE
&& parts.uri.path().starts_with("/v1/videos/")
{
return Some("openai_video_delete_sync");
}
if decision.route_family.as_deref() == Some("gemini")
&& decision.route_kind.as_deref() == Some("video")
&& parts.method == http::Method::POST
&& parts.uri.path().ends_with(":cancel")
{
return Some("gemini_video_cancel_sync");
}
if decision.route_family.as_deref() == Some("gemini")
&& decision.route_kind.as_deref() == Some("video")
&& parts.method == http::Method::POST
&& parts.uri.path().ends_with(":predictLongRunning")
{
return Some("gemini_video_create_sync");
}
if decision.route_family.as_deref() == Some("openai")
&& decision.route_kind.as_deref() == Some("chat")
&& parts.method == http::Method::POST
&& parts.uri.path() == "/v1/chat/completions"
{
return Some("openai_chat_sync");
}
if decision.route_family.as_deref() == Some("openai")
&& decision.route_kind.as_deref() == Some("cli")
&& parts.method == http::Method::POST
&& parts.uri.path() == "/v1/responses"
{
return Some("openai_cli_sync");
}
if decision.route_family.as_deref() == Some("openai")
&& decision.route_kind.as_deref() == Some("compact")
&& parts.method == http::Method::POST
&& parts.uri.path() == "/v1/responses/compact"
{
return Some("openai_compact_sync");
}
if decision.route_family.as_deref() == Some("claude")
&& decision.route_kind.as_deref() == Some("chat")
&& parts.method == http::Method::POST
&& parts.uri.path() == "/v1/messages"
{
return Some("claude_chat_sync");
}
if decision.route_family.as_deref() == Some("claude")
&& decision.route_kind.as_deref() == Some("cli")
&& parts.method == http::Method::POST
&& parts.uri.path() == "/v1/messages"
{
return Some("claude_cli_sync");
}
if decision.route_family.as_deref() == Some("gemini")
&& decision.route_kind.as_deref() == Some("chat")
&& parts.method == http::Method::POST
&& parts.uri.path().ends_with(":generateContent")
{
return Some("gemini_chat_sync");
}
if decision.route_family.as_deref() == Some("gemini")
&& decision.route_kind.as_deref() == Some("cli")
&& parts.method == http::Method::POST
&& parts.uri.path().ends_with(":generateContent")
{
return Some("gemini_cli_sync");
}
if decision.route_family.as_deref() == Some("gemini")
&& decision.route_kind.as_deref() == Some("files")
{
if parts.method == http::Method::POST && parts.uri.path() == "/upload/v1beta/files" {
return Some("gemini_files_upload");
}
if parts.method == http::Method::GET && parts.uri.path() == "/v1beta/files" {
return Some("gemini_files_list");
}
if parts.method == http::Method::GET
&& parts.uri.path().starts_with("/v1beta/files/")
&& !parts.uri.path().ends_with(":download")
{
return Some("gemini_files_get");
}
if parts.method == http::Method::DELETE
&& parts.uri.path().starts_with("/v1beta/files/")
&& !parts.uri.path().ends_with(":download")
{
return Some("gemini_files_delete");
}
}
None
}
pub(crate) fn is_matching_stream_request(
plan_kind: &str,
parts: &http::request::Parts,
body_json: &serde_json::Value,
) -> bool {
match plan_kind {
"openai_chat_stream"
| "claude_chat_stream"
| "openai_cli_stream"
| "openai_compact_stream"
| "claude_cli_stream" => body_json
.get("stream")
.and_then(|value| value.as_bool())
.unwrap_or(false),
"gemini_chat_stream" | "gemini_cli_stream" => {
parts.uri.path().ends_with(":streamGenerateContent")
}
_ => true,
}
}
pub(crate) fn supports_sync_scheduler_decision_kind(plan_kind: &str) -> bool {
matches!(
plan_kind,
"openai_chat_sync"
| "openai_cli_sync"
| "openai_compact_sync"
| "claude_chat_sync"
| "claude_cli_sync"
| "gemini_chat_sync"
| "gemini_cli_sync"
| "gemini_files_upload"
| "openai_video_create_sync"
| "openai_video_remix_sync"
| "openai_video_cancel_sync"
| "openai_video_delete_sync"
| "gemini_video_create_sync"
| "gemini_video_cancel_sync"
| "gemini_files_get"
| "gemini_files_list"
| "gemini_files_delete"
)
}
pub(crate) fn supports_stream_scheduler_decision_kind(plan_kind: &str) -> bool {
matches!(
plan_kind,
"openai_chat_stream"
| "claude_chat_stream"
| "gemini_chat_stream"
| "openai_cli_stream"
| "openai_compact_stream"
| "claude_cli_stream"
| "gemini_cli_stream"
| "gemini_files_download"
| "openai_video_content"
)
}
#[cfg(test)]
mod tests {
use axum::http::{Method, Request};
use super::{
is_matching_stream_request, resolve_execution_runtime_stream_plan_kind,
resolve_execution_runtime_sync_plan_kind, supports_stream_scheduler_decision_kind,
supports_sync_scheduler_decision_kind,
};
use crate::gateway::GatewayControlDecision;
fn sample_decision(route_family: &str, route_kind: &str) -> GatewayControlDecision {
GatewayControlDecision {
public_path: "/".to_string(),
public_query_string: None,
route_class: Some("ai_public".to_string()),
route_family: Some(route_family.to_string()),
route_kind: Some(route_kind.to_string()),
auth_context: None,
admin_principal: None,
auth_endpoint_signature: None,
execution_runtime_candidate: true,
local_auth_rejection: None,
}
}
#[test]
fn resolves_openai_chat_plan_kinds() {
let request = Request::builder()
.method(Method::POST)
.uri("/v1/chat/completions")
.body(())
.expect("request should build");
let (parts, _) = request.into_parts();
let decision = sample_decision("openai", "chat");
assert_eq!(
resolve_execution_runtime_sync_plan_kind(&parts, &decision),
Some("openai_chat_sync")
);
assert_eq!(
resolve_execution_runtime_stream_plan_kind(&parts, &decision),
Some("openai_chat_stream")
);
}
#[test]
fn stream_matching_requires_openai_stream_flag() {
let request = Request::builder()
.method(Method::POST)
.uri("/v1/chat/completions")
.body(())
.expect("request should build");
let (parts, _) = request.into_parts();
assert!(!is_matching_stream_request(
"openai_chat_stream",
&parts,
&serde_json::json!({"stream": false}),
));
assert!(is_matching_stream_request(
"openai_chat_stream",
&parts,
&serde_json::json!({"stream": true}),
));
assert!(supports_sync_scheduler_decision_kind("openai_chat_sync"));
assert!(supports_stream_scheduler_decision_kind(
"openai_chat_stream"
));
}
}