i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-03 01:40:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: extract runtime state backends

Browse Source
This commit is contained in:
fawney19
2026-05-08 00:18:12 +08:00
parent 6f620d92be
commit 6247ac3edc
111 changed files with 4358 additions and 3203 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
apps/aether-gateway/src/handlers/admin/observability/monitoring/cache_affinity.rs
View File

@@ -426,22 +426,13 @@ pub(super) async fn delete_admin_monitoring_cache_affinity_raw_keys(
return Ok(0);
}
if let Some(runner) = state.redis_kv_runner() {
let mut connection = runner
.client()
.get_multiplexed_async_connection()
.await
.map_err(|err| {
GatewayError::Internal(format!("admin monitoring redis connect failed: {err}"))
})?;
let deleted = redis::cmd("DEL")
.arg(raw_keys)
.query_async::<i64>(&mut connection)
.await
.map_err(|err| {
GatewayError::Internal(format!("admin monitoring redis delete failed: {err}"))
})?;
return Ok(usize::try_from(deleted).unwrap_or(0));
let deleted = state
.runtime_state()
.kv_delete_many(raw_keys)
.await
.map_err(|err| GatewayError::Internal(format!("runtime cache delete failed: {err}")))?;
if deleted > 0 {
return Ok(deleted);
}
Ok(delete_admin_monitoring_cache_affinity_entries_for_tests(

28
apps/aether-gateway/src/handlers/admin/observability/monitoring/cache_model_mapping.rs
View File

@@ -1,7 +1,5 @@
use super::cache_config::ADMIN_MONITORING_REDIS_CACHE_CATEGORIES;
use super::cache_store::{
admin_monitoring_has_test_redis_keys, list_admin_monitoring_namespaced_keys,
};
use super::cache_store::list_admin_monitoring_namespaced_keys;
use crate::handlers::admin::request::AdminAppState;
use crate::GatewayError;
use axum::{
@@ -14,17 +12,6 @@ use serde_json::json;
pub(super) async fn build_admin_monitoring_model_mapping_stats_response(
state: &AdminAppState<'_>,
) -> Result<Response<Body>, GatewayError> {
if state.redis_kv_runner().is_none() && !admin_monitoring_has_test_redis_keys(state) {
return Ok(Json(json!({
"status": "ok",
"data": {
"available": false,
"message": "Redis 未启用，模型映射缓存不可用",
}
}))
.into_response());
};
let model_id_keys = list_admin_monitoring_namespaced_keys(state, "model:id:*").await?;
let global_model_id_keys =
list_admin_monitoring_namespaced_keys(state, "global_model:id:*").await?;
@@ -49,6 +36,7 @@ pub(super) async fn build_admin_monitoring_model_mapping_stats_response(
"status": "ok",
"data": {
"available": true,
"backend": state.runtime_state().backend_kind().as_str(),
"ttl_seconds": 300,
"total_keys": total_keys,
"breakdown": {
@@ -69,17 +57,6 @@ pub(super) async fn build_admin_monitoring_model_mapping_stats_response(
pub(super) async fn build_admin_monitoring_redis_cache_categories_response(
state: &AdminAppState<'_>,
) -> Result<Response<Body>, GatewayError> {
if state.redis_kv_runner().is_none() && !admin_monitoring_has_test_redis_keys(state) {
return Ok(Json(json!({
"status": "ok",
"data": {
"available": false,
"message": "Redis 未启用",
}
}))
.into_response());
};
let mut categories = Vec::with_capacity(ADMIN_MONITORING_REDIS_CACHE_CATEGORIES.len());
let mut total_keys = 0usize;
@@ -101,6 +78,7 @@ pub(super) async fn build_admin_monitoring_redis_cache_categories_response(
"status": "ok",
"data": {
"available": true,
"backend": state.runtime_state().backend_kind().as_str(),
"categories": categories,
"total_keys": total_keys,
}

18
apps/aether-gateway/src/handlers/admin/observability/monitoring/cache_mutations/affinity.rs
View File

@@ -3,15 +3,8 @@ use super::super::cache_affinity::{
delete_admin_monitoring_cache_affinity_raw_keys,
};
use super::super::cache_identity::admin_monitoring_list_export_api_key_records_by_ids;
use super::super::cache_route_helpers::{
admin_monitoring_cache_affinity_delete_params_from_path,
admin_monitoring_cache_affinity_unavailable_response,
};
use super::super::cache_store::{
admin_monitoring_has_runtime_scheduler_affinity_entries,
list_admin_monitoring_cache_affinity_records_by_affinity_keys,
load_admin_monitoring_cache_affinity_entries_for_tests,
};
use super::super::cache_route_helpers::admin_monitoring_cache_affinity_delete_params_from_path;
use super::super::cache_store::list_admin_monitoring_cache_affinity_records_by_affinity_keys;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::GatewayError;
use aether_admin::observability::monitoring::{
@@ -76,13 +69,6 @@ pub(in super::super) async fn build_admin_monitoring_cache_affinity_delete_respo
));
};
if state.redis_kv_runner().is_none()
&& load_admin_monitoring_cache_affinity_entries_for_tests(state).is_empty()
&& !admin_monitoring_has_runtime_scheduler_affinity_entries(state)
{
return Ok(admin_monitoring_cache_affinity_unavailable_response());
}
let target_affinity_keys =
std::iter::once(affinity_key.clone()).collect::<std::collections::BTreeSet<_>>();
let delete_filter = admin_monitoring_cache_affinity_delete_filter_from_query(

5
apps/aether-gateway/src/handlers/admin/observability/monitoring/cache_mutations/flush.rs
View File

@@ -2,7 +2,6 @@ use super::super::cache_affinity::{
clear_admin_monitoring_scheduler_affinity_entries,
delete_admin_monitoring_cache_affinity_raw_keys,
};
use super::super::cache_route_helpers::admin_monitoring_cache_affinity_unavailable_response;
use super::super::cache_store::list_admin_monitoring_cache_affinity_records;
use crate::handlers::admin::request::AdminAppState;
use crate::GatewayError;
@@ -13,10 +12,6 @@ pub(in super::super) async fn build_admin_monitoring_cache_flush_response(
state: &AdminAppState<'_>,
) -> Result<Response<Body>, GatewayError> {
let raw_affinities = list_admin_monitoring_cache_affinity_records(state).await?;
if state.redis_kv_runner().is_none() && raw_affinities.is_empty() {
return Ok(admin_monitoring_cache_affinity_unavailable_response());
}
let raw_keys = raw_affinities
.iter()
.map(|item| item.raw_key.clone())

17
apps/aether-gateway/src/handlers/admin/observability/monitoring/cache_mutations/model_mapping.rs
View File

@@ -1,10 +1,9 @@
use super::super::cache_route_helpers::{
admin_monitoring_cache_model_mapping_provider_params_from_path,
admin_monitoring_cache_model_name_from_path, admin_monitoring_redis_unavailable_response,
admin_monitoring_cache_model_name_from_path,
};
use super::super::cache_store::{
admin_monitoring_has_test_redis_keys, delete_admin_monitoring_namespaced_keys,
list_admin_monitoring_namespaced_keys,
delete_admin_monitoring_namespaced_keys, list_admin_monitoring_namespaced_keys,
};
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::GatewayError;
@@ -19,10 +18,6 @@ use axum::{body::Body, response::Response};
pub(in super::super) async fn build_admin_monitoring_model_mapping_delete_response(
state: &AdminAppState<'_>,
) -> Result<Response<Body>, GatewayError> {
if state.redis_kv_runner().is_none() && !admin_monitoring_has_test_redis_keys(state) {
return Ok(admin_monitoring_redis_unavailable_response());
}
let mut raw_keys = list_admin_monitoring_namespaced_keys(state, "model:*").await?;
raw_keys.extend(list_admin_monitoring_namespaced_keys(state, "global_model:*").await?);
raw_keys.sort();
@@ -41,10 +36,6 @@ pub(in super::super) async fn build_admin_monitoring_model_mapping_delete_model_
else {
return Ok(admin_monitoring_bad_request_response("缺少 model_name"));
};
if state.redis_kv_runner().is_none() && !admin_monitoring_has_test_redis_keys(state) {
return Ok(admin_monitoring_redis_unavailable_response());
}
let candidate_keys = [
format!("global_model:resolve:{model_name}"),
format!("global_model:name:{model_name}"),
@@ -85,10 +76,6 @@ pub(in super::super) async fn build_admin_monitoring_model_mapping_delete_provid
"缺少 provider_id 或 global_model_id",
));
};
if state.redis_kv_runner().is_none() && !admin_monitoring_has_test_redis_keys(state) {
return Ok(admin_monitoring_redis_unavailable_response());
}
let candidate_keys = [
format!("model:provider_global:{provider_id}:{global_model_id}"),
format!("model:provider_global:hits:{provider_id}:{global_model_id}"),

9
apps/aether-gateway/src/handlers/admin/observability/monitoring/cache_mutations/provider.rs
View File

@@ -2,10 +2,7 @@ use super::super::cache_affinity::{
clear_admin_monitoring_scheduler_affinity_entries,
delete_admin_monitoring_cache_affinity_raw_keys,
};
use super::super::cache_route_helpers::{
admin_monitoring_cache_affinity_unavailable_response,
admin_monitoring_cache_provider_id_from_path,
};
use super::super::cache_route_helpers::admin_monitoring_cache_provider_id_from_path;
use super::super::cache_store::list_admin_monitoring_cache_affinity_records;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::GatewayError;
@@ -26,10 +23,6 @@ pub(in super::super) async fn build_admin_monitoring_cache_provider_delete_respo
};
let raw_affinities = list_admin_monitoring_cache_affinity_records(state).await?;
if state.redis_kv_runner().is_none() && raw_affinities.is_empty() {
return Ok(admin_monitoring_cache_affinity_unavailable_response());
}
let target_affinities = raw_affinities
.into_iter()
.filter(|item| item.provider_id.as_deref() == Some(provider_id.as_str()))

11
apps/aether-gateway/src/handlers/admin/observability/monitoring/cache_mutations/redis_keys.rs
View File

@@ -1,10 +1,7 @@
use super::super::cache_config::ADMIN_MONITORING_REDIS_CACHE_CATEGORIES;
use super::super::cache_route_helpers::{
admin_monitoring_cache_redis_category_from_path, admin_monitoring_redis_unavailable_response,
};
use super::super::cache_route_helpers::admin_monitoring_cache_redis_category_from_path;
use super::super::cache_store::{
admin_monitoring_has_test_redis_keys, delete_admin_monitoring_namespaced_keys,
list_admin_monitoring_namespaced_keys,
delete_admin_monitoring_namespaced_keys, list_admin_monitoring_namespaced_keys,
};
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::GatewayError;
@@ -31,10 +28,6 @@ pub(in super::super) async fn build_admin_monitoring_redis_keys_delete_response(
return Ok(admin_monitoring_unknown_cache_category_response(&category));
};
if state.redis_kv_runner().is_none() && !admin_monitoring_has_test_redis_keys(state) {
return Ok(admin_monitoring_redis_unavailable_response());
}
let raw_keys = list_admin_monitoring_namespaced_keys(state, pattern).await?;
let deleted_count = delete_admin_monitoring_namespaced_keys(state, &raw_keys).await?;

14
apps/aether-gateway/src/handlers/admin/observability/monitoring/cache_mutations/users.rs
View File

@@ -6,15 +6,10 @@ use super::super::cache_identity::{
admin_monitoring_find_user_summary_by_id, admin_monitoring_list_export_api_key_records_by_ids,
};
use super::super::cache_route_helpers::{
admin_monitoring_cache_affinity_unavailable_response,
admin_monitoring_cache_users_not_found_response,
admin_monitoring_cache_users_user_identifier_from_path,
};
use super::super::cache_store::{
admin_monitoring_has_runtime_scheduler_affinity_entries,
list_admin_monitoring_cache_affinity_records_by_affinity_keys,
load_admin_monitoring_cache_affinity_entries_for_tests,
};
use super::super::cache_store::list_admin_monitoring_cache_affinity_records_by_affinity_keys;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::GatewayError;
use aether_admin::observability::monitoring::{
@@ -36,13 +31,6 @@ pub(in super::super) async fn build_admin_monitoring_cache_users_delete_response
));
};
if state.redis_kv_runner().is_none()
&& load_admin_monitoring_cache_affinity_entries_for_tests(state).is_empty()
&& !admin_monitoring_has_runtime_scheduler_affinity_entries(state)
{
return Ok(admin_monitoring_cache_affinity_unavailable_response());
}
let direct_api_key_by_id = admin_monitoring_list_export_api_key_records_by_ids(
state,
std::slice::from_ref(&user_identifier),

169
apps/aether-gateway/src/handlers/admin/observability/monitoring/cache_store.rs
View File

@@ -22,41 +22,6 @@ async fn count_admin_monitoring_cache_affinity_entries(state: &AdminAppState<'_>
})
}
async fn scan_admin_monitoring_namespaced_keys(
runner: &aether_data::driver::redis::RedisKvRunner,
pattern: &str,
) -> Result<Vec<String>, GatewayError> {
let mut connection = runner
.client()
.get_multiplexed_async_connection()
.await
.map_err(|err| {
GatewayError::Internal(format!("admin monitoring redis connect failed: {err}"))
})?;
let namespaced_pattern = runner.keyspace().key(pattern);
let mut cursor = 0u64;
let mut keys = Vec::new();
loop {
let (next_cursor, batch) = redis::cmd("SCAN")
.arg(cursor)
.arg("MATCH")
.arg(&namespaced_pattern)
.arg("COUNT")
.arg(200)
.query_async::<(u64, Vec<String>)>(&mut connection)
.await
.map_err(|err| {
GatewayError::Internal(format!("admin monitoring redis scan failed: {err}"))
})?;
keys.extend(batch);
if next_cursor == 0 {
break;
}
cursor = next_cursor;
}
Ok(keys)
}
#[cfg(test)]
pub(super) fn load_admin_monitoring_cache_affinity_entries_for_tests(
state: &AdminAppState<'_>,
@@ -116,8 +81,13 @@ pub(super) async fn list_admin_monitoring_namespaced_keys(
state: &AdminAppState<'_>,
pattern: &str,
) -> Result<Vec<String>, GatewayError> {
if let Some(runner) = state.redis_kv_runner() {
return scan_admin_monitoring_namespaced_keys(&runner, pattern).await;
let keys = state
.runtime_state()
.scan_keys(pattern, 200)
.await
.map_err(|err| GatewayError::Internal(format!("runtime cache scan failed: {err}")))?;
if !keys.is_empty() {
return Ok(keys);
}
let mut keys = load_admin_monitoring_redis_keys_for_tests(state)
@@ -136,22 +106,13 @@ pub(super) async fn delete_admin_monitoring_namespaced_keys(
return Ok(0);
}
if let Some(runner) = state.redis_kv_runner() {
let mut connection = runner
.client()
.get_multiplexed_async_connection()
.await
.map_err(|err| {
GatewayError::Internal(format!("admin monitoring redis connect failed: {err}"))
})?;
let deleted = redis::cmd("DEL")
.arg(raw_keys)
.query_async::<i64>(&mut connection)
.await
.map_err(|err| {
GatewayError::Internal(format!("admin monitoring redis delete failed: {err}"))
})?;
return Ok(usize::try_from(deleted).unwrap_or(0));
let deleted = state
.runtime_state()
.kv_delete_many(raw_keys)
.await
.map_err(|err| GatewayError::Internal(format!("runtime cache delete failed: {err}")))?;
if deleted > 0 {
return Ok(deleted);
}
Ok(delete_admin_monitoring_redis_keys_for_tests(
@@ -197,85 +158,63 @@ async fn list_admin_monitoring_cache_affinity_records_matching(
}
};
if let Some(runner) = state.redis_kv_runner() {
let mut connection = runner
.client()
.get_multiplexed_async_connection()
.await
.map_err(|err| {
GatewayError::Internal(format!("admin monitoring redis connect failed: {err}"))
})?;
{
let patterns = affinity_keys
.map(|keys| {
keys.iter()
.flat_map(|affinity_key| {
[
runner
.keyspace()
.key(&format!("cache_affinity:{affinity_key}:*")),
runner
.keyspace()
.key(&format!("scheduler_affinity:{affinity_key}:*")),
runner
.keyspace()
.key(&format!("scheduler_affinity:v2:{affinity_key}:*")),
format!("cache_affinity:{affinity_key}:*"),
format!("scheduler_affinity:{affinity_key}:*"),
format!("scheduler_affinity:v2:{affinity_key}:*"),
]
})
.collect::<Vec<_>>()
})
.unwrap_or_else(|| {
vec![
runner.keyspace().key("cache_affinity:*"),
runner.keyspace().key("scheduler_affinity:*"),
"cache_affinity:*".to_string(),
"scheduler_affinity:*".to_string(),
]
});
for pattern in patterns {
let mut cursor = 0u64;
loop {
let (next_cursor, keys) = redis::cmd("SCAN")
.arg(cursor)
.arg("MATCH")
.arg(&pattern)
.arg("COUNT")
.arg(200)
.query_async::<(u64, Vec<String>)>(&mut connection)
let keys = state
.runtime_state()
.scan_keys(&pattern, 200)
.await
.map_err(|err| {
GatewayError::Internal(format!("runtime cache scan failed: {err}"))
})?;
if !keys.is_empty() {
let raw_keys = keys
.iter()
.map(|key| state.runtime_state().strip_namespace(key).to_string())
.collect::<Vec<_>>();
let values = state
.runtime_state()
.kv_get_many(&raw_keys)
.await
.map_err(|err| {
GatewayError::Internal(format!("admin monitoring redis scan failed: {err}"))
GatewayError::Internal(format!("runtime cache mget failed: {err}"))
})?;
if !keys.is_empty() {
let values = redis::cmd("MGET")
.arg(&keys)
.query_async::<Vec<Option<String>>>(&mut connection)
.await
.map_err(|err| {
GatewayError::Internal(format!(
"admin monitoring redis mget failed: {err}"
))
})?;
for (key, raw_value) in keys.into_iter().zip(values) {
let Some(raw_value) = raw_value else {
continue;
};
let record = if key.contains("scheduler_affinity:") {
admin_monitoring_scheduler_affinity_record_from_raw(&key, &raw_value)
} else {
admin_monitoring_cache_affinity_record(&key, &raw_value)
};
let Some(record) = record else {
continue;
};
if affinity_keys.is_some_and(|keys| !keys.contains(&record.affinity_key)) {
continue;
}
push_record(record);
for (key, raw_value) in keys.into_iter().zip(values) {
let Some(raw_value) = raw_value else {
continue;
};
let record = if key.contains("scheduler_affinity:") {
admin_monitoring_scheduler_affinity_record_from_raw(&key, &raw_value)
} else {
admin_monitoring_cache_affinity_record(&key, &raw_value)
};
let Some(record) = record else {
continue;
};
if affinity_keys.is_some_and(|keys| !keys.contains(&record.affinity_key)) {
continue;
}
push_record(record);
}
if next_cursor == 0 {
break;
}
cursor = next_cursor;
}
}
}
@@ -350,11 +289,7 @@ pub(super) async fn build_admin_monitoring_cache_snapshot(
round_to(cache_hits as f64 / usage_summary.total_requests as f64, 4)
};
let total_affinities = count_admin_monitoring_cache_affinity_entries(state).await;
let storage_type = if state.redis_kv_runner().is_some() {
"redis"
} else {
"memory"
};
let storage_type = state.runtime_state().backend_kind().as_str();
let scheduler_name = if scheduling_mode == "cache_affinity" {
"cache_aware".to_string()
} else {

11
apps/aether-gateway/src/handlers/admin/observability/monitoring/tests/basics.rs
View File

@@ -1,4 +1,3 @@
use super::super::cache_config::ADMIN_MONITORING_REDIS_REQUIRED_DETAIL;
use super::super::test_support::{request_context, sample_key, sample_provider, sample_usage};
use super::local_monitoring_response;
use crate::AppState;
@@ -69,7 +68,7 @@ fn admin_monitoring_matches_cache_delete_shapes_and_trailing_slashes() {
}
#[tokio::test]
async fn admin_monitoring_model_mapping_delete_requires_redis_without_runtime_or_test_entries() {
async fn admin_monitoring_model_mapping_delete_returns_empty_runtime_payload_without_entries() {
let state = AppState::new().expect("state should build");
let context = request_context(
http::Method::DELETE,
@@ -80,15 +79,13 @@ async fn admin_monitoring_model_mapping_delete_requires_redis_without_runtime_or
.expect("handler should not error")
.expect("monitoring route should be handled locally");
assert_eq!(response.status(), http::StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(response.status(), http::StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX)
.await
.expect("body should read");
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
assert_eq!(
payload,
json!({ "detail": ADMIN_MONITORING_REDIS_REQUIRED_DETAIL })
);
assert_eq!(payload["status"], json!("ok"));
assert_eq!(payload["deleted_count"], json!(0));
}
#[tokio::test]

21
apps/aether-gateway/src/handlers/admin/observability/monitoring/tests/mod.rs
View File

@@ -1086,11 +1086,9 @@ async fn admin_monitoring_model_mapping_stats_returns_local_payload_without_redi
.expect("body should read");
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
assert_eq!(payload["status"], json!("ok"));
assert_eq!(payload["data"]["available"], json!(false));
assert_eq!(
payload["data"]["message"],
json!("Redis 未启用，模型映射缓存不可用")
);
assert_eq!(payload["data"]["available"], json!(true));
assert_eq!(payload["data"]["backend"], json!("memory"));
assert_eq!(payload["data"]["total_keys"], json!(0));
}
#[tokio::test]
@@ -1193,12 +1191,13 @@ async fn admin_monitoring_redis_keys_returns_local_payload_without_redis() {
.expect("body should read");
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
assert_eq!(payload["status"], json!("ok"));
assert_eq!(payload["data"]["available"], json!(false));
assert_eq!(payload["data"]["message"], json!("Redis 未启用"));
assert_eq!(payload["data"]["available"], json!(true));
assert_eq!(payload["data"]["backend"], json!("memory"));
assert_eq!(payload["data"]["total_keys"], json!(0));
}
#[tokio::test]
async fn admin_monitoring_redis_keys_delete_returns_unavailable_without_redis() {
async fn admin_monitoring_redis_keys_delete_returns_empty_runtime_payload_without_redis() {
let state = AppState::new().expect("state should build");
let context = request_context(
http::Method::DELETE,
@@ -1210,12 +1209,14 @@ async fn admin_monitoring_redis_keys_delete_returns_unavailable_without_redis()
.expect("handler should not error")
.expect("route should be handled locally");
assert_eq!(response.status(), http::StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(response.status(), http::StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX)
.await
.expect("body should read");
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
assert_eq!(payload["detail"], json!("Redis 未启用"));
assert_eq!(payload["status"], json!("ok"));
assert_eq!(payload["category"], json!("upstream_models"));
assert_eq!(payload["deleted_count"], json!(0));
}
#[tokio::test]