i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-10-05 00:47:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: 大规模模块拆分与代码精简，新增 ai-pipeline/data-contracts 独立 crate

Browse Source 
- 新增 aether-ai-pipeline 和 aether-data-contracts crate，将 pipeline 逻辑与数据契约从 gateway 中解耦
- 重构 admin handlers：拆分单体模块为 auth/billing/endpoint/features/model/observability/provider/system 等独立子模块
- 合并 chat/cli 重复代码路径：精简 conversion、finalize、planner 中的 sync/chat/cli 分支
- 重构 scheduler/executor/data 层，引入 facade 模式降低模块间耦合
- 移除冗余的 intent 模块，将 plan_fallback/policy/stream_path/sync_path 迁移至 executor
- 前端适配：调整 admin API 调用和 provider 模型测试对话框
This commit is contained in:
fawney19
2026-04-07 02:50:19 +08:00
parent 763ff03a7b
commit 5d96d6673b
732 changed files with 28589 additions and 20662 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/aether-gateway/src/handlers/admin/endpoint/extractors.rs
+29
View File
@@ -0,0 +1,29 @@
pub(super) fn admin_health_key_id(request_path: &str) -> Option<String> {
let raw = request_path.strip_prefix("/api/admin/endpoints/health/key/")?;
let normalized = raw.trim().trim_matches('/');
if normalized.is_empty() {
None
} else {
Some(normalized.to_string())
}
}
pub(super) fn admin_recover_key_id(request_path: &str) -> Option<String> {
let raw = request_path.strip_prefix("/api/admin/endpoints/health/keys/")?;
let normalized = raw.trim().trim_matches('/');
if normalized.is_empty() {
None
} else {
Some(normalized.to_string())
}
}
pub(super) fn admin_rpm_key_id(request_path: &str) -> Option<String> {
let raw = request_path.strip_prefix("/api/admin/endpoints/rpm/key/")?;
let normalized = raw.trim().trim_matches('/');
if normalized.is_empty() {
None
} else {
Some(normalized.to_string())
}
}
apps/aether-gateway/src/handlers/admin/endpoint/health.rs
+193
View File
@@ -0,0 +1,193 @@
use super::extractors::{admin_health_key_id, admin_recover_key_id};
use super::health_builders::{
build_admin_endpoint_health_status_payload, build_admin_health_summary_payload,
build_admin_key_health_payload, recover_admin_key_health, recover_all_admin_key_health,
};
use crate::control::GatewayPublicRequestContext;
use crate::handlers::admin::shared::query_param_value;
use crate::handlers::public::{
build_api_format_health_monitor_payload, ApiFormatHealthMonitorOptions,
};
use crate::{AppState, GatewayError};
use axum::{
body::Body,
http,
response::{IntoResponse, Response},
Json,
};
use serde_json::json;
const ADMIN_ENDPOINT_HEALTH_DATA_UNAVAILABLE_DETAIL: &str =
"Admin endpoint health data unavailable";
fn build_admin_endpoint_health_data_unavailable_response() -> Response<Body> {
(
http::StatusCode::SERVICE_UNAVAILABLE,
Json(json!({ "detail": ADMIN_ENDPOINT_HEALTH_DATA_UNAVAILABLE_DETAIL })),
)
.into_response()
}
pub(super) async fn maybe_build_local_admin_endpoints_health_response(
state: &AppState,
request_context: &GatewayPublicRequestContext,
) -> Result<Option<Response<Body>>, GatewayError> {
let Some(decision) = request_context.control_decision.as_ref() else {
return Ok(None);
};
if decision.route_family.as_deref() == Some("endpoints_health")
&& decision.route_kind.as_deref() == Some("health_summary")
&& request_context.request_path == "/api/admin/endpoints/health/summary"
{
if !state.has_provider_catalog_data_reader() {
return Ok(Some(build_admin_endpoint_health_data_unavailable_response()));
}
let Some(payload) = build_admin_health_summary_payload(state).await else {
return Ok(Some(build_admin_endpoint_health_data_unavailable_response()));
};
return Ok(Some(Json(payload).into_response()));
}
if decision.route_family.as_deref() == Some("endpoints_health")
&& decision.route_kind.as_deref() == Some("key_health")
&& request_context
.request_path
.starts_with("/api/admin/endpoints/health/key/")
{
if !state.has_provider_catalog_data_reader() {
return Ok(Some(build_admin_endpoint_health_data_unavailable_response()));
}
let Some(key_id) = admin_health_key_id(&request_context.request_path) else {
return Ok(Some(
(
http::StatusCode::NOT_FOUND,
Json(json!({ "detail": "Key 不存在" })),
)
.into_response(),
));
};
let api_format = query_param_value(
request_context.request_query_string.as_deref(),
"api_format",
);
return Ok(Some(
match build_admin_key_health_payload(state, &key_id, api_format.as_deref()).await {
Some(payload) => Json(payload).into_response(),
None => (
http::StatusCode::NOT_FOUND,
Json(json!({ "detail": format!("Key {key_id} 不存在") })),
)
.into_response(),
},
));
}
if decision.route_family.as_deref() == Some("endpoints_health")
&& decision.route_kind.as_deref() == Some("recover_key_health")
&& request_context
.request_path
.starts_with("/api/admin/endpoints/health/keys/")
{
if !state.has_provider_catalog_data_reader() || !state.has_provider_catalog_data_writer() {
return Ok(Some(build_admin_endpoint_health_data_unavailable_response()));
}
let Some(key_id) = admin_recover_key_id(&request_context.request_path) else {
return Ok(Some(
(
http::StatusCode::NOT_FOUND,
Json(json!({ "detail": "Key 不存在" })),
)
.into_response(),
));
};
let api_format = query_param_value(
request_context.request_query_string.as_deref(),
"api_format",
);
return Ok(Some(
match recover_admin_key_health(state, &key_id, api_format.as_deref()).await {
Some(payload) => Json(payload).into_response(),
None => (
http::StatusCode::NOT_FOUND,
Json(json!({ "detail": format!("Key {key_id} 不存在") })),
)
.into_response(),
},
));
}
if decision.route_family.as_deref() == Some("endpoints_health")
&& decision.route_kind.as_deref() == Some("recover_all_keys_health")
&& request_context.request_path == "/api/admin/endpoints/health/keys"
{
if !state.has_provider_catalog_data_reader() || !state.has_provider_catalog_data_writer() {
return Ok(Some(build_admin_endpoint_health_data_unavailable_response()));
}
let Some(payload) = recover_all_admin_key_health(state).await else {
return Ok(Some(build_admin_endpoint_health_data_unavailable_response()));
};
return Ok(Some(Json(payload).into_response()));
}
if decision.route_family.as_deref() == Some("endpoints_health")
&& decision.route_kind.as_deref() == Some("health_api_formats")
&& request_context.request_path == "/api/admin/endpoints/health/api-formats"
{
if !state.has_provider_catalog_data_reader() || !state.has_request_candidate_data_reader() {
return Ok(Some(build_admin_endpoint_health_data_unavailable_response()));
}
let lookback_hours = query_param_value(
request_context.request_query_string.as_deref(),
"lookback_hours",
)
.and_then(|value| value.parse::<u64>().ok())
.filter(|value| (1..=72).contains(value))
.unwrap_or(6);
let per_format_limit = query_param_value(
request_context.request_query_string.as_deref(),
"per_format_limit",
)
.and_then(|value| value.parse::<usize>().ok())
.filter(|value| (10..=200).contains(value))
.unwrap_or(60);
let Some(payload) = build_api_format_health_monitor_payload(
state,
lookback_hours,
per_format_limit,
ApiFormatHealthMonitorOptions {
include_api_path: false,
include_provider_count: true,
include_key_count: true,
},
)
.await
else {
return Ok(Some(build_admin_endpoint_health_data_unavailable_response()));
};
return Ok(Some(Json(payload).into_response()));
}
if decision.route_family.as_deref() == Some("endpoints_health")
&& decision.route_kind.as_deref() == Some("health_status")
&& request_context.request_path == "/api/admin/endpoints/health/status"
{
if !state.has_provider_catalog_data_reader() || !state.has_request_candidate_data_reader() {
return Ok(Some(build_admin_endpoint_health_data_unavailable_response()));
}
let lookback_hours = query_param_value(
request_context.request_query_string.as_deref(),
"lookback_hours",
)
.and_then(|value| value.parse::<u64>().ok())
.filter(|value| (1..=72).contains(value))
.unwrap_or(6);
let Some(payload) = build_admin_endpoint_health_status_payload(state, lookback_hours).await
else {
return Ok(Some(build_admin_endpoint_health_data_unavailable_response()));
};
return Ok(Some(Json(payload).into_response()));
}
Ok(None)
}
apps/aether-gateway/src/handlers/admin/endpoint/health_builders/keys.rs
+377
View File
@@ -0,0 +1,377 @@
use crate::handlers::public::provider_key_api_formats;
use crate::AppState;
use aether_scheduler_core::count_recent_rpm_requests_for_provider_key_since;
use serde_json::json;
use std::time::{SystemTime, UNIX_EPOCH};
pub(crate) async fn build_admin_key_health_payload(
state: &AppState,
key_id: &str,
api_format: Option<&str>,
) -> Option<serde_json::Value> {
if !state.has_provider_catalog_data_reader() {
return None;
}
let key = state
.read_provider_catalog_keys_by_ids(&[key_id.to_string()])
.await
.ok()
.and_then(|mut keys| keys.drain(..).next())?;
let request_count = key.request_count.unwrap_or(0);
let success_count = key.success_count.unwrap_or(0);
let error_count = key
.error_count
.unwrap_or(request_count.saturating_sub(success_count));
let avg_response_time_ms = match (key.total_response_time_ms, success_count) {
(Some(total), successes) if successes > 0 => total as f64 / successes as f64,
_ => 0.0,
};
let mut payload = json!({
"key_id": key.id,
"key_is_active": key.is_active,
"key_statistics": {
"request_count": request_count,
"success_count": success_count,
"error_count": error_count,
"success_rate": if request_count > 0 {
success_count as f64 / request_count as f64
} else {
0.0
},
"avg_response_time_ms": avg_response_time_ms,
},
});
let health_by_format: Option<&serde_json::Map<String, serde_json::Value>> = key
.health_by_format
.as_ref()
.and_then(serde_json::Value::as_object);
let circuit_by_format: Option<&serde_json::Map<String, serde_json::Value>> = key
.circuit_breaker_by_format
.as_ref()
.and_then(serde_json::Value::as_object);
if let Some(api_format) = api_format.map(str::trim).filter(|value| !value.is_empty()) {
let health_data = health_by_format.and_then(|formats| formats.get(api_format));
let circuit_data = circuit_by_format.and_then(|formats| formats.get(api_format));
payload["api_format"] = json!(api_format);
payload["key_health_score"] = json!(health_data
.and_then(|value| value.get("health_score"))
.and_then(serde_json::Value::as_f64)
.unwrap_or(1.0));
payload["key_consecutive_failures"] = json!(health_data
.and_then(|value| value.get("consecutive_failures"))
.and_then(serde_json::Value::as_i64)
.unwrap_or(0));
payload["key_last_failure_at"] = health_data
.and_then(|value| value.get("last_failure_at"))
.cloned()
.unwrap_or(serde_json::Value::Null);
payload["circuit_breaker_open"] = json!(circuit_data
.and_then(|value| value.get("open"))
.and_then(serde_json::Value::as_bool)
.unwrap_or(false));
payload["circuit_breaker_open_at"] = circuit_data
.and_then(|value| value.get("open_at"))
.cloned()
.unwrap_or(serde_json::Value::Null);
payload["next_probe_at"] = circuit_data
.and_then(|value| value.get("next_probe_at"))
.cloned()
.unwrap_or(serde_json::Value::Null);
payload["half_open_until"] = circuit_data
.and_then(|value| value.get("half_open_until"))
.cloned()
.unwrap_or(serde_json::Value::Null);
payload["half_open_successes"] = json!(circuit_data
.and_then(|value| value.get("half_open_successes"))
.and_then(serde_json::Value::as_i64)
.unwrap_or(0));
payload["half_open_failures"] = json!(circuit_data
.and_then(|value| value.get("half_open_failures"))
.and_then(serde_json::Value::as_i64)
.unwrap_or(0));
} else {
let mut formats_payload = serde_json::Map::new();
for format_name in provider_key_api_formats(&key) {
let health_data = health_by_format.and_then(|formats| formats.get(&format_name));
let circuit_data = circuit_by_format.and_then(|formats| formats.get(&format_name));
let is_open = circuit_data
.and_then(|value| value.get("open"))
.and_then(serde_json::Value::as_bool)
.unwrap_or(false);
formats_payload.insert(
format_name.clone(),
json!({
"health_score": health_data
.and_then(|value| value.get("health_score"))
.and_then(serde_json::Value::as_f64)
.unwrap_or(1.0),
"error_rate": 0.0,
"window_size": 0,
"consecutive_failures": health_data
.and_then(|value| value.get("consecutive_failures"))
.and_then(serde_json::Value::as_i64)
.unwrap_or(0),
"last_failure_at": health_data
.and_then(|value| value.get("last_failure_at"))
.cloned()
.unwrap_or(serde_json::Value::Null),
"circuit_breaker": {
"state": if is_open { "open" } else { "closed" },
"open": is_open,
"open_at": circuit_data
.and_then(|value| value.get("open_at"))
.cloned()
.unwrap_or(serde_json::Value::Null),
"next_probe_at": circuit_data
.and_then(|value| value.get("next_probe_at"))
.cloned()
.unwrap_or(serde_json::Value::Null),
"half_open_until": circuit_data
.and_then(|value| value.get("half_open_until"))
.cloned()
.unwrap_or(serde_json::Value::Null),
"half_open_successes": circuit_data
.and_then(|value| value.get("half_open_successes"))
.and_then(serde_json::Value::as_i64)
.unwrap_or(0),
"half_open_failures": circuit_data
.and_then(|value| value.get("half_open_failures"))
.and_then(serde_json::Value::as_i64)
.unwrap_or(0),
}
}),
);
}
let key_health_score = formats_payload
.values()
.filter_map(|value| value.get("health_score"))
.filter_map(serde_json::Value::as_f64)
.reduce(f64::min)
.unwrap_or(1.0);
let any_circuit_open = formats_payload.values().any(|value| {
value
.get("circuit_breaker")
.and_then(|circuit| circuit.get("open"))
.and_then(serde_json::Value::as_bool)
.unwrap_or(false)
});
payload["key_health_score"] = json!(key_health_score);
payload["any_circuit_open"] = json!(any_circuit_open);
payload["health_by_format"] = serde_json::Value::Object(formats_payload);
}
Some(payload)
}
pub(crate) async fn build_admin_key_rpm_payload(
state: &AppState,
key_id: &str,
) -> Option<serde_json::Value> {
if !state.has_provider_catalog_data_reader() {
return None;
}
let key = state
.read_provider_catalog_keys_by_ids(&[key_id.to_string()])
.await
.ok()
.and_then(|mut keys| keys.drain(..).next())?;
let now_unix_secs = SystemTime::now()
.duration_since(UNIX_EPOCH)
.ok()
.map(|duration| duration.as_secs())
.unwrap_or(0);
let recent_candidates = state.read_recent_request_candidates(256).await.ok()?;
let reset_after_unix_secs = state.provider_key_rpm_reset_at(key.id.as_str(), now_unix_secs);
let current_rpm = count_recent_rpm_requests_for_provider_key_since(
&recent_candidates,
key.id.as_str(),
now_unix_secs,
reset_after_unix_secs,
);
Some(json!({
"key_id": key.id,
"current_rpm": current_rpm,
"rpm_limit": key.rpm_limit,
}))
}
fn default_key_health_payload() -> serde_json::Value {
json!({
"health_score": 1.0,
"consecutive_failures": 0,
"last_failure_at": serde_json::Value::Null,
})
}
fn default_key_circuit_payload() -> serde_json::Value {
json!({
"open": false,
"open_at": serde_json::Value::Null,
"next_probe_at": serde_json::Value::Null,
"half_open_until": serde_json::Value::Null,
"half_open_successes": 0,
"half_open_failures": 0,
})
}
pub(crate) async fn recover_admin_key_health(
state: &AppState,
key_id: &str,
api_format: Option<&str>,
) -> Option<serde_json::Value> {
let key = state
.read_provider_catalog_keys_by_ids(&[key_id.to_string()])
.await
.ok()
.and_then(|mut keys| keys.drain(..).next())?;
let api_format = api_format.map(str::trim).filter(|value| !value.is_empty());
let (health_by_format, circuit_breaker_by_format, message, details) =
if let Some(api_format) = api_format {
let mut health_by_format = key
.health_by_format
.as_ref()
.and_then(serde_json::Value::as_object)
.cloned()
.unwrap_or_default();
let mut circuit_breaker_by_format = key
.circuit_breaker_by_format
.as_ref()
.and_then(serde_json::Value::as_object)
.cloned()
.unwrap_or_default();
health_by_format.insert(api_format.to_string(), default_key_health_payload());
circuit_breaker_by_format.insert(api_format.to_string(), default_key_circuit_payload());
(
serde_json::Value::Object(health_by_format),
serde_json::Value::Object(circuit_breaker_by_format),
format!("Key 的 {api_format} 格式已恢复"),
json!({
"api_format": api_format,
"health_score": 1.0,
"circuit_breaker_open": false,
"is_active": true,
}),
)
} else {
(
json!({}),
json!({}),
"Key 所有格式已恢复".to_string(),
json!({
"health_score": 1.0,
"circuit_breaker_open": false,
"is_active": true,
}),
)
};
let updated = state
.update_provider_catalog_key_health_state(
key_id,
true,
Some(&health_by_format),
Some(&circuit_breaker_by_format),
)
.await
.ok()?;
if !updated {
return None;
}
Some(json!({
"message": message,
"details": details,
}))
}
pub(crate) async fn recover_all_admin_key_health(state: &AppState) -> Option<serde_json::Value> {
if !state.has_provider_catalog_data_reader() {
return None;
}
let providers = state
.list_provider_catalog_providers(false)
.await
.ok()
.unwrap_or_default();
let provider_ids = providers
.iter()
.map(|provider| provider.id.clone())
.collect::<Vec<_>>();
let keys = if provider_ids.is_empty() {
Vec::new()
} else {
state
.list_provider_catalog_keys_by_provider_ids(&provider_ids)
.await
.ok()
.unwrap_or_default()
};
let recovered_keys = keys
.into_iter()
.filter(|key| {
key.circuit_breaker_by_format
.as_ref()
.and_then(serde_json::Value::as_object)
.map(|formats| {
formats.values().any(|circuit| {
circuit
.get("open")
.and_then(serde_json::Value::as_bool)
.unwrap_or(false)
})
})
.unwrap_or(false)
})
.collect::<Vec<_>>();
if recovered_keys.is_empty() {
return Some(json!({
"message": "没有需要恢复的 Key",
"recovered_count": 0,
"recovered_keys": [],
}));
}
let empty_health = json!({});
let empty_circuit = json!({});
let mut payload_items = Vec::new();
for key in recovered_keys {
let updated = state
.update_provider_catalog_key_health_state(
&key.id,
true,
Some(&empty_health),
Some(&empty_circuit),
)
.await
.ok()?;
if !updated {
continue;
}
payload_items.push(json!({
"key_id": key.id,
"key_name": key.name,
"provider_id": key.provider_id,
"api_formats": key.api_formats.unwrap_or_else(|| json!([])),
}));
}
Some(json!({
"message": format!("已恢复 {} 个 Key", payload_items.len()),
"recovered_count": payload_items.len(),
"recovered_keys": payload_items,
}))
}
apps/aether-gateway/src/handlers/admin/endpoint/health_builders/mod.rs
+9
View File
@@ -0,0 +1,9 @@
mod keys;
mod status;
pub(super) use self::keys::{
build_admin_key_health_payload, build_admin_key_rpm_payload, recover_admin_key_health,
recover_all_admin_key_health,
};
pub(crate) use self::status::build_admin_endpoint_health_status_payload;
pub(super) use self::status::build_admin_health_summary_payload;
apps/aether-gateway/src/handlers/admin/endpoint/health_builders/status.rs
+305
View File
@@ -0,0 +1,305 @@
use crate::handlers::admin::shared::unix_secs_to_rfc3339;
use crate::handlers::public::{
api_format_display_name, build_public_health_timeline, provider_key_api_formats,
};
use crate::AppState;
use aether_data_contracts::repository::candidates::PublicHealthTimelineBucket;
use aether_scheduler_core::{is_provider_key_circuit_open, provider_key_health_score};
use serde_json::json;
use std::collections::{BTreeMap, BTreeSet};
use std::time::{SystemTime, UNIX_EPOCH};
pub(crate) async fn build_admin_endpoint_health_status_payload(
state: &AppState,
lookback_hours: u64,
) -> Option<serde_json::Value> {
if !state.has_provider_catalog_data_reader() || !state.has_request_candidate_data_reader() {
return None;
}
let now_unix_secs = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_secs())
.unwrap_or_default();
let since_unix_secs = now_unix_secs.saturating_sub(lookback_hours * 3600);
let providers = state
.list_provider_catalog_providers(true)
.await
.ok()
.unwrap_or_default();
let provider_ids = providers
.iter()
.map(|provider| provider.id.clone())
.collect::<Vec<_>>();
let active_endpoints = if provider_ids.is_empty() {
Vec::new()
} else {
state
.list_provider_catalog_endpoints_by_provider_ids(&provider_ids)
.await
.ok()
.unwrap_or_default()
.into_iter()
.filter(|endpoint| endpoint.is_active)
.collect::<Vec<_>>()
};
let mut endpoint_ids_by_format = BTreeMap::<String, Vec<String>>::new();
let mut endpoint_to_format = BTreeMap::<String, String>::new();
let mut provider_ids_by_format = BTreeMap::<String, BTreeSet<String>>::new();
let mut active_provider_formats = BTreeSet::<(String, String)>::new();
for endpoint in active_endpoints {
endpoint_to_format.insert(endpoint.id.clone(), endpoint.api_format.clone());
endpoint_ids_by_format
.entry(endpoint.api_format.clone())
.or_default()
.push(endpoint.id);
provider_ids_by_format
.entry(endpoint.api_format.clone())
.or_default()
.insert(endpoint.provider_id.clone());
active_provider_formats.insert((endpoint.provider_id, endpoint.api_format));
}
let all_endpoint_ids = endpoint_to_format.keys().cloned().collect::<Vec<_>>();
let mut total_keys_by_format = BTreeMap::<String, BTreeSet<String>>::new();
let mut active_keys_by_format = BTreeMap::<String, BTreeSet<String>>::new();
let mut health_scores_by_format = BTreeMap::<String, Vec<f64>>::new();
if !provider_ids.is_empty() {
let keys = state
.list_provider_catalog_keys_by_provider_ids(&provider_ids)
.await
.ok()
.unwrap_or_default();
for key in keys {
for api_format in provider_key_api_formats(&key) {
if !active_provider_formats.contains(&(key.provider_id.clone(), api_format.clone()))
{
continue;
}
total_keys_by_format
.entry(api_format.clone())
.or_default()
.insert(key.id.clone());
if key.is_active && !is_provider_key_circuit_open(&key, &api_format) {
let key_health_score =
provider_key_health_score(&key, &api_format).unwrap_or(1.0);
active_keys_by_format
.entry(api_format.clone())
.or_default()
.insert(key.id.clone());
health_scores_by_format
.entry(api_format)
.or_default()
.push(key_health_score);
}
}
}
}
let timeline_rows = state
.aggregate_finalized_request_candidate_timeline_by_endpoint_ids_since(
&all_endpoint_ids,
since_unix_secs,
now_unix_secs,
100,
)
.await
.ok()
.unwrap_or_default();
let mut timeline_by_format =
BTreeMap::<String, BTreeMap<u32, PublicHealthTimelineBucket>>::new();
for row in timeline_rows {
let Some(api_format) = endpoint_to_format.get(&row.endpoint_id) else {
continue;
};
let bucket = timeline_by_format
.entry(api_format.clone())
.or_default()
.entry(row.segment_idx)
.or_insert_with(|| PublicHealthTimelineBucket {
endpoint_id: api_format.clone(),
segment_idx: row.segment_idx,
total_count: 0,
success_count: 0,
failed_count: 0,
min_created_at_unix_secs: None,
max_created_at_unix_secs: None,
});
bucket.total_count += row.total_count;
bucket.success_count += row.success_count;
bucket.failed_count += row.failed_count;
bucket.min_created_at_unix_secs = match (
bucket.min_created_at_unix_secs,
row.min_created_at_unix_secs,
) {
(Some(left), Some(right)) => Some(left.min(right)),
(None, Some(right)) => Some(right),
(left, None) => left,
};
bucket.max_created_at_unix_secs = match (
bucket.max_created_at_unix_secs,
row.max_created_at_unix_secs,
) {
(Some(left), Some(right)) => Some(left.max(right)),
(None, Some(right)) => Some(right),
(left, None) => left,
};
}
let mut payload = endpoint_ids_by_format
.into_iter()
.map(|(api_format, endpoint_ids)| {
let empty_timeline = BTreeMap::new();
let timeline_source = timeline_by_format.get(&api_format).unwrap_or(&empty_timeline);
let (timeline, time_range_start, time_range_end) =
build_public_health_timeline(timeline_source, 100);
let healthy_count = timeline.iter().filter(|status| **status == "healthy").count();
let warning_count = timeline.iter().filter(|status| **status == "warning").count();
let unhealthy_count = timeline.iter().filter(|status| **status == "unhealthy").count();
let known_count = healthy_count + warning_count + unhealthy_count;
let total_keys = total_keys_by_format
.get(&api_format)
.map(BTreeSet::len)
.unwrap_or(0);
let health_score = if known_count > 0 {
(healthy_count as f64 + warning_count as f64 * 0.8) / known_count as f64
} else if let Some(scores) = health_scores_by_format.get(&api_format) {
if scores.is_empty() {
if total_keys == 0 { 0.0 } else { 0.1 }
} else {
scores.iter().copied().sum::<f64>() / scores.len() as f64
}
} else if total_keys == 0 {
0.0
} else {
0.1
};
json!({
"api_format": api_format.clone(),
"display_name": api_format_display_name(&api_format),
"health_score": health_score,
"timeline": timeline,
"time_range_start": time_range_start.and_then(unix_secs_to_rfc3339),
"time_range_end": time_range_end.or(Some(now_unix_secs)).and_then(unix_secs_to_rfc3339),
"total_endpoints": endpoint_ids.len(),
"total_keys": total_keys,
"active_keys": active_keys_by_format.get(&api_format).map(BTreeSet::len).unwrap_or(0),
"provider_count": provider_ids_by_format.get(&api_format).map(BTreeSet::len).unwrap_or(0),
})
})
.collect::<Vec<_>>();
payload.sort_by(|left, right| {
let left_score = left
.get("health_score")
.and_then(serde_json::Value::as_f64)
.unwrap_or(0.0);
let right_score = right
.get("health_score")
.and_then(serde_json::Value::as_f64)
.unwrap_or(0.0);
right_score
.partial_cmp(&left_score)
.unwrap_or(std::cmp::Ordering::Equal)
});
Some(serde_json::Value::Array(payload))
}
pub(crate) async fn build_admin_health_summary_payload(
state: &AppState,
) -> Option<serde_json::Value> {
if !state.has_provider_catalog_data_reader() {
return None;
}
let providers = state
.list_provider_catalog_providers(false)
.await
.ok()
.unwrap_or_default();
let provider_ids = providers
.iter()
.map(|provider| provider.id.clone())
.collect::<Vec<_>>();
let endpoints = if provider_ids.is_empty() {
Vec::new()
} else {
state
.list_provider_catalog_endpoints_by_provider_ids(&provider_ids)
.await
.ok()
.unwrap_or_default()
};
let keys = if provider_ids.is_empty() {
Vec::new()
} else {
state
.list_provider_catalog_keys_by_provider_ids(&provider_ids)
.await
.ok()
.unwrap_or_default()
};
let active_endpoints = endpoints
.iter()
.filter(|endpoint| endpoint.is_active)
.count();
let unhealthy_endpoints = endpoints
.iter()
.filter(|endpoint| endpoint.health_score < 0.5)
.count();
let active_keys = keys.iter().filter(|key| key.is_active).count();
let unhealthy_keys = keys
.iter()
.filter(|key| {
key.health_by_format
.as_ref()
.and_then(serde_json::Value::as_object)
.map(|formats| {
formats.values().any(|health| {
health
.get("health_score")
.and_then(serde_json::Value::as_f64)
.unwrap_or(1.0)
< 0.5
})
})
.unwrap_or(false)
})
.count();
let circuit_open_keys = keys
.iter()
.filter(|key| {
key.circuit_breaker_by_format
.as_ref()
.and_then(serde_json::Value::as_object)
.map(|formats| {
formats.values().any(|circuit| {
circuit
.get("open")
.and_then(serde_json::Value::as_bool)
.unwrap_or(false)
})
})
.unwrap_or(false)
})
.count();
Some(json!({
"endpoints": {
"total": endpoints.len(),
"active": active_endpoints,
"unhealthy": unhealthy_endpoints,
},
"keys": {
"total": keys.len(),
"active": active_keys,
"unhealthy": unhealthy_keys,
"circuit_open": circuit_open_keys,
},
}))
}
apps/aether-gateway/src/handlers/admin/endpoint/keys.rs
+1
View File
@@ -0,0 +1 @@
pub(super) use crate::handlers::admin::provider::endpoint_keys::maybe_build_local_admin_endpoints_keys_response;
apps/aether-gateway/src/handlers/admin/endpoint/mod.rs
+50
View File
@@ -0,0 +1,50 @@
use crate::control::GatewayPublicRequestContext;
use crate::{AppState, GatewayError};
use axum::body::{Body, Bytes};
use axum::http::Response;
mod extractors;
mod health;
mod health_builders;
mod keys;
mod routes;
mod rpm;
pub(crate) use self::health_builders::build_admin_endpoint_health_status_payload;
pub(crate) async fn maybe_build_local_admin_endpoints_response(
state: &AppState,
request_context: &GatewayPublicRequestContext,
request_body: Option<&Bytes>,
) -> Result<Option<Response<Body>>, GatewayError> {
if let Some(response) =
health::maybe_build_local_admin_endpoints_health_response(state, request_context).await?
{
return Ok(Some(response));
}
if let Some(response) =
rpm::maybe_build_local_admin_endpoints_rpm_response(state, request_context).await?
{
return Ok(Some(response));
}
if let Some(response) =
keys::maybe_build_local_admin_endpoints_keys_response(state, request_context, request_body)
.await?
{
return Ok(Some(response));
}
if let Some(response) = routes::maybe_build_local_admin_endpoints_routes_response(
state,
request_context,
request_body,
)
.await?
{
return Ok(Some(response));
}
Ok(None)
}
apps/aether-gateway/src/handlers/admin/endpoint/routes.rs
+1
View File
@@ -0,0 +1 @@
pub(super) use crate::handlers::admin::provider::endpoints_admin::maybe_build_local_admin_endpoints_routes_response;
apps/aether-gateway/src/handlers/admin/endpoint/rpm.rs
+94
View File
@@ -0,0 +1,94 @@
use super::extractors::admin_rpm_key_id;
use super::health_builders::build_admin_key_rpm_payload;
use crate::control::GatewayPublicRequestContext;
use crate::{AppState, GatewayError};
use axum::{
body::Body,
http,
response::{IntoResponse, Response},
Json,
};
use serde_json::json;
use std::time::{SystemTime, UNIX_EPOCH};
pub(super) async fn maybe_build_local_admin_endpoints_rpm_response(
state: &AppState,
request_context: &GatewayPublicRequestContext,
) -> Result<Option<Response<Body>>, GatewayError> {
let Some(decision) = request_context.control_decision.as_ref() else {
return Ok(None);
};
if decision.route_family.as_deref() == Some("endpoints_rpm")
&& decision.route_kind.as_deref() == Some("key_rpm")
&& request_context
.request_path
.starts_with("/api/admin/endpoints/rpm/key/")
{
let Some(key_id) = admin_rpm_key_id(&request_context.request_path) else {
return Ok(Some(
(
http::StatusCode::NOT_FOUND,
Json(json!({ "detail": "Key 不存在" })),
)
.into_response(),
));
};
return Ok(Some(
match build_admin_key_rpm_payload(state, &key_id).await {
Some(payload) => Json(payload).into_response(),
None => (
http::StatusCode::NOT_FOUND,
Json(json!({ "detail": format!("Key {key_id} 不存在") })),
)
.into_response(),
},
));
}
if decision.route_family.as_deref() == Some("endpoints_rpm")
&& decision.route_kind.as_deref() == Some("reset_key_rpm")
&& request_context.request_method == http::Method::DELETE
&& request_context
.request_path
.starts_with("/api/admin/endpoints/rpm/key/")
{
let Some(key_id) = admin_rpm_key_id(&request_context.request_path) else {
return Ok(Some(
(
http::StatusCode::NOT_FOUND,
Json(json!({ "detail": "Key 不存在" })),
)
.into_response(),
));
};
let Some(_key) = state
.read_provider_catalog_keys_by_ids(std::slice::from_ref(&key_id))
.await?
.into_iter()
.next()
else {
return Ok(Some(
(
http::StatusCode::NOT_FOUND,
Json(json!({ "detail": format!("Key {key_id} 不存在") })),
)
.into_response(),
));
};
let now_unix_secs = SystemTime::now()
.duration_since(UNIX_EPOCH)
.ok()
.map(|duration| duration.as_secs())
.unwrap_or(0);
state.mark_provider_key_rpm_reset(&key_id, now_unix_secs);
return Ok(Some(
Json(json!({
"message": "RPM 计数已重置"
}))
.into_response(),
));
}
Ok(None)
}