i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-10-04 16:37:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(models): add external catalog proxy selection

Browse Source
This commit is contained in:
elky
2026-07-31 09:32:25 +08:00
parent 12057db476
commit beb003b7ad
34 changed files with 2286 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/aether-gateway/src/handlers/admin/mod.rs
+8 -1
View File
@@ -18,6 +18,10 @@ mod shared;
pub(crate) use self::auth::maybe_build_local_admin_security_response;
pub(crate) use self::endpoint::build_admin_endpoint_health_status_payload;
pub(crate) use self::features::maybe_build_local_admin_video_tasks_response;
#[cfg(test)]
pub(crate) use self::model::{
set_admin_external_models_source_url_for_tests, ADMIN_EXTERNAL_MODELS_CONFIG_MUTATION_LOCK_KEY,
};
pub(crate) use self::observability::{
admin_stats_bad_request_response, maybe_build_local_admin_usage_response, parse_bounded_u32,
round_to, AdminStatsTimeRange, AdminStatsUsageFilter,
@@ -53,4 +57,7 @@ pub(crate) use self::request::{
};
pub(crate) use self::routes::maybe_build_local_admin_response;
#[cfg(test)]
pub(crate) use self::system::override_proxy_connectivity_probe_url_for_tests;
pub(crate) use self::system::{
clear_proxy_node_references_with_cache_failure_for_tests,
override_proxy_connectivity_probe_url_for_tests,
};
apps/aether-gateway/src/handlers/admin/model/catalog_routes.rs
+49 -3
View File
@@ -1,8 +1,9 @@
use crate::handlers::admin::model::build_admin_model_catalog_payload;
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::handlers::admin::shared::attach_admin_audit_response;
use crate::GatewayError;
use axum::{
body::Body,
body::{Body, Bytes},
http,
response::{IntoResponse, Response},
Json,
@@ -22,6 +23,7 @@ fn build_admin_model_catalog_data_unavailable_response() -> Response<Body> {
pub(crate) async fn maybe_build_local_admin_model_catalog_response(
state: &AdminAppState<'_>,
request_context: &AdminRequestContext<'_>,
request_body: Option<&Bytes>,
) -> Result<Option<Response<Body>>, GatewayError> {
let Some(decision) = request_context.decision() else {
return Ok(None);
@@ -47,12 +49,15 @@ pub(crate) async fn maybe_build_local_admin_model_catalog_response(
&& request_context.path() == "/api/admin/models/external"
{
return Ok(Some(
match state.read_admin_external_models_cache().await? {
match state
.read_admin_external_models_cache(request_context.trace_id())
.await?
{
Some(payload) => Json(payload).into_response(),
None => (
http::StatusCode::SERVICE_UNAVAILABLE,
Json(json!({
"detail": "External models catalog requires Rust admin backend"
"detail": "External models catalog unavailable"
})),
)
.into_response(),
@@ -60,6 +65,47 @@ pub(crate) async fn maybe_build_local_admin_model_catalog_response(
));
}
if decision.route_family.as_deref() == Some("model_external_manage")
&& decision.route_kind.as_deref() == Some("external_config_get")
&& request_context.method() == http::Method::GET
&& request_context.path() == "/api/admin/models/external/config"
{
return Ok(Some(
Json(state.build_admin_external_models_config_payload().await?).into_response(),
));
}
if decision.route_family.as_deref() == Some("model_external_manage")
&& decision.route_kind.as_deref() == Some("external_config_set")
&& request_context.method() == http::Method::PUT
&& request_context.path() == "/api/admin/models/external/config"
{
let Some(request_body) = request_body else {
return Ok(Some(
(
http::StatusCode::BAD_REQUEST,
Json(json!({ "detail": "请求数据验证失败" })),
)
.into_response(),
));
};
return Ok(Some(
match state
.apply_admin_external_models_config_update(request_body)
.await?
{
Ok(payload) => attach_admin_audit_response(
Json(payload).into_response(),
"admin_external_models_config_updated",
"update_external_models_config",
"external_models_catalog",
"global",
),
Err((status, payload)) => (status, Json(payload)).into_response(),
},
));
}
if decision.route_family.as_deref() == Some("model_external_manage")
&& decision.route_kind.as_deref() == Some("clear_external_cache")
&& request_context.method() == http::Method::DELETE
apps/aether-gateway/src/handlers/admin/model/external_cache.rs
+469 -63
View File
@@ -1,13 +1,72 @@
use crate::handlers::admin::request::AdminAppState;
use crate::handlers::shared::mark_external_models_official_providers;
use crate::GatewayError;
use serde_json::json;
use aether_contracts::{
ExecutionPlan, ExecutionTimeouts, RequestBody, EXECUTION_REQUEST_FOLLOW_REDIRECTS_HEADER,
};
use aether_runtime_state::RuntimeLockLease;
use axum::body::Bytes;
use axum::http;
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use std::collections::BTreeMap;
use std::time::Duration;
use tracing::warn;
const ADMIN_EXTERNAL_MODELS_CACHE_KEY: &str = "aether:external:models_dev";
const ADMIN_EXTERNAL_MODELS_LEGACY_CACHE_KEY: &str = "aether:external:models_dev";
const ADMIN_EXTERNAL_MODELS_CACHE_KEY: &str = "aether:external:models_dev:v2";
const ADMIN_EXTERNAL_MODELS_CACHE_VERSION: u8 = 2;
const ADMIN_EXTERNAL_MODELS_CACHE_TTL_SECS: u64 = 15 * 60;
const ADMIN_EXTERNAL_MODELS_SOURCE_URL_ENV: &str = "AETHER_GATEWAY_EXTERNAL_MODELS_URL";
const ADMIN_EXTERNAL_MODELS_SOURCE_URL_DEFAULT: &str = "https://models.dev/api.json";
pub(in crate::handlers::admin) const ADMIN_EXTERNAL_MODELS_PROXY_NODE_CONFIG_KEY: &str =
"external_models_proxy_node_id";
const ADMIN_EXTERNAL_MODELS_CONNECT_TIMEOUT_MS: u64 = 10_000;
const ADMIN_EXTERNAL_MODELS_TOTAL_TIMEOUT_MS: u64 = 300_000;
pub(crate) const ADMIN_EXTERNAL_MODELS_CONFIG_MUTATION_LOCK_KEY: &str =
"admin:external_models_proxy_node_config:mutation";
const ADMIN_EXTERNAL_MODELS_CONFIG_MUTATION_LOCK_TTL: Duration = Duration::from_secs(10 * 60);
#[derive(Debug, Clone, Serialize, Deserialize)]
struct AdminExternalModelsCacheEnvelope {
schema_version: u8,
proxy_node_id: Option<String>,
payload: Value,
}
#[cfg(test)]
pub(crate) struct AdminExternalModelsSourceUrlEnvGuard {
previous: Option<String>,
_lock: std::sync::MutexGuard<'static, ()>,
}
#[cfg(test)]
impl Drop for AdminExternalModelsSourceUrlEnvGuard {
fn drop(&mut self) {
if let Some(previous) = self.previous.as_deref() {
std::env::set_var(ADMIN_EXTERNAL_MODELS_SOURCE_URL_ENV, previous);
} else {
std::env::remove_var(ADMIN_EXTERNAL_MODELS_SOURCE_URL_ENV);
}
}
}
#[cfg(test)]
pub(crate) fn set_admin_external_models_source_url_for_tests(
value: &str,
) -> AdminExternalModelsSourceUrlEnvGuard {
static LOCK: std::sync::OnceLock<std::sync::Mutex<()>> = std::sync::OnceLock::new();
let lock = LOCK
.get_or_init(|| std::sync::Mutex::new(()))
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let previous = std::env::var(ADMIN_EXTERNAL_MODELS_SOURCE_URL_ENV).ok();
std::env::set_var(ADMIN_EXTERNAL_MODELS_SOURCE_URL_ENV, value);
AdminExternalModelsSourceUrlEnvGuard {
previous,
_lock: lock,
}
}
fn admin_external_models_source_url() -> String {
std::env::var(ADMIN_EXTERNAL_MODELS_SOURCE_URL_ENV)
@@ -21,12 +80,42 @@ fn normalize_admin_external_models_payload(payload: serde_json::Value) -> serde_
mark_external_models_official_providers(&payload).unwrap_or(payload)
}
fn classify_admin_external_models_transport_error(message: &str) -> &'static str {
let message = message.to_ascii_lowercase();
if message.contains("timed out") || message.contains("timeout") {
"timeout"
} else if message.contains("relay") || message.contains("tunnel") {
"relay"
} else if message.contains("proxy") {
"proxy_config"
} else if message.contains("too large") || message.contains("exceeds") {
"response_too_large"
} else if message.contains("json") {
"invalid_json"
} else if message.contains("decode") || message.contains("content-encoding") {
"response_decode"
} else if message.contains("connect") || message.contains("dns") || message.contains("tcp") {
"connect"
} else if message.contains("header") || message.contains("method") || message.contains("build")
{
"request_build"
} else {
"unknown_transport"
}
}
async fn store_admin_external_models_cache(
state: &AdminAppState<'_>,
proxy_node_id: Option<&str>,
payload: &serde_json::Value,
) -> Result<(), GatewayError> {
let envelope = AdminExternalModelsCacheEnvelope {
schema_version: ADMIN_EXTERNAL_MODELS_CACHE_VERSION,
proxy_node_id: proxy_node_id.map(ToOwned::to_owned),
payload: payload.clone(),
};
let serialized =
serde_json::to_string(payload).map_err(|err| GatewayError::Internal(err.to_string()))?;
serde_json::to_string(&envelope).map_err(|err| GatewayError::Internal(err.to_string()))?;
state
.as_ref()
.runtime_kv_setex(
@@ -38,10 +127,323 @@ async fn store_admin_external_models_cache(
Ok(())
}
fn normalize_admin_external_models_proxy_node_id(
value: Option<&Value>,
) -> Result<Option<String>, GatewayError> {
match value {
None | Some(Value::Null) => Ok(None),
Some(Value::String(value)) => {
let value = value.trim();
if value.is_empty() {
Err(GatewayError::Internal(format!(
"system config '{ADMIN_EXTERNAL_MODELS_PROXY_NODE_CONFIG_KEY}' must not be empty"
)))
} else {
Ok(Some(value.to_string()))
}
}
Some(_) => Err(GatewayError::Internal(format!(
"system config '{ADMIN_EXTERNAL_MODELS_PROXY_NODE_CONFIG_KEY}' must be a string or null"
))),
}
}
async fn read_admin_external_models_proxy_node_id(
state: &AdminAppState<'_>,
) -> Result<Option<String>, GatewayError> {
let value = state
.read_system_config_json_value_strong(ADMIN_EXTERNAL_MODELS_PROXY_NODE_CONFIG_KEY)
.await?;
normalize_admin_external_models_proxy_node_id(value.as_ref())
}
pub(crate) async fn build_admin_external_models_config_payload(
state: &AdminAppState<'_>,
) -> Result<Value, GatewayError> {
let proxy_node_id = read_admin_external_models_proxy_node_id(state).await?;
Ok(json!({
"proxy_node_id": proxy_node_id,
}))
}
fn parse_admin_external_models_config_update(
request_body: &[u8],
) -> Result<Option<String>, (http::StatusCode, Value)> {
let payload = match serde_json::from_slice::<Value>(request_body) {
Ok(Value::Object(payload)) => payload,
Ok(_) | Err(_) => {
return Err((
http::StatusCode::BAD_REQUEST,
json!({ "detail": "请求数据验证失败" }),
));
}
};
match payload.get("proxy_node_id") {
Some(Value::Null) => Ok(None),
Some(Value::String(value)) => {
let value = value.trim();
if value.is_empty() {
return Err((
http::StatusCode::BAD_REQUEST,
json!({ "detail": "proxy_node_id 不能为空" }),
));
}
Ok(Some(value.to_string()))
}
Some(_) | None => Err((
http::StatusCode::BAD_REQUEST,
json!({ "detail": "proxy_node_id 必须是字符串或 null" }),
)),
}
}
async fn clear_admin_external_models_cache_entries(
state: &AdminAppState<'_>,
) -> Result<bool, GatewayError> {
let cleared_current = state
.as_ref()
.runtime_kv_del(ADMIN_EXTERNAL_MODELS_CACHE_KEY)
.await?;
let cleared_legacy = state
.as_ref()
.runtime_kv_del(ADMIN_EXTERNAL_MODELS_LEGACY_CACHE_KEY)
.await?;
Ok(cleared_current || cleared_legacy)
}
pub(crate) async fn acquire_admin_external_models_config_mutation_lock(
state: &AdminAppState<'_>,
) -> Result<RuntimeLockLease, (http::StatusCode, Value)> {
match state
.app()
.runtime_state()
.lock_try_acquire(
ADMIN_EXTERNAL_MODELS_CONFIG_MUTATION_LOCK_KEY,
state.app().tunnel.local_instance_id(),
ADMIN_EXTERNAL_MODELS_CONFIG_MUTATION_LOCK_TTL,
)
.await
{
Ok(Some(lock)) => Ok(lock),
Ok(None) => Err((
http::StatusCode::CONFLICT,
json!({ "detail": "外部模型目录代理配置正在更新，请稍后重试" }),
)),
Err(_) => {
warn!(
runtime_backend = state.app().runtime_state_backend(),
"failed to acquire external models proxy config mutation lock"
);
Err((
http::StatusCode::SERVICE_UNAVAILABLE,
json!({ "detail": "外部模型目录代理配置暂时无法更新" }),
))
}
}
}
pub(crate) async fn release_admin_external_models_config_mutation_lock(
state: &AdminAppState<'_>,
lock: &RuntimeLockLease,
) {
if state
.app()
.runtime_state()
.lock_release(lock)
.await
.is_err()
{
warn!(
runtime_backend = state.app().runtime_state_backend(),
"failed to release external models proxy config mutation lock"
);
}
}
async fn apply_admin_external_models_config_update_locked(
state: &AdminAppState<'_>,
proxy_node_id: Option<String>,
) -> Result<Result<Value, (http::StatusCode, Value)>, GatewayError> {
if let Some(node_id) = proxy_node_id.as_deref() {
if state.find_proxy_node(node_id).await?.is_none() {
return Ok(Err((
http::StatusCode::NOT_FOUND,
json!({ "detail": format!("代理节点 '{node_id}' 不存在") }),
)));
}
if state
.resolve_admin_proxy_node_snapshot(Some(node_id))
.await
.is_none()
{
return Ok(Err((
http::StatusCode::CONFLICT,
json!({ "detail": format!("代理节点 '{node_id}' 当前不可用") }),
)));
}
}
let config_value = proxy_node_id
.as_ref()
.map_or(Value::Null, |node_id| json!(node_id));
state
.upsert_system_config_json_value(
ADMIN_EXTERNAL_MODELS_PROXY_NODE_CONFIG_KEY,
&config_value,
Some("外部模型目录代理节点 ID"),
)
.await?;
let cache_cleared = match clear_admin_external_models_cache_entries(state).await {
Ok(cache_cleared) => cache_cleared,
Err(_) => {
// The v2 cache envelope includes the selected node ID, so a stale entry cannot be
// reused after this persisted selector changes. Cache invalidation is best-effort.
warn!(
runtime_backend = state.app().runtime_state_backend(),
"failed to clear external models cache after proxy config update"
);
false
}
};
let mut payload = build_admin_external_models_config_payload(state).await?;
payload["cache_cleared"] = json!(cache_cleared);
Ok(Ok(payload))
}
pub(crate) async fn apply_admin_external_models_config_update(
state: &AdminAppState<'_>,
request_body: &Bytes,
) -> Result<Result<Value, (http::StatusCode, Value)>, GatewayError> {
let proxy_node_id = match parse_admin_external_models_config_update(request_body) {
Ok(proxy_node_id) => proxy_node_id,
Err(error) => return Ok(Err(error)),
};
if !state.app().data.has_system_config_store() {
return Ok(Err((
http::StatusCode::SERVICE_UNAVAILABLE,
json!({ "detail": "Admin system config data unavailable" }),
)));
}
if proxy_node_id.is_some() && !state.has_proxy_node_reader() {
return Ok(Err((
http::StatusCode::SERVICE_UNAVAILABLE,
json!({ "detail": "Admin proxy node data unavailable" }),
)));
}
let lock = match acquire_admin_external_models_config_mutation_lock(state).await {
Ok(lock) => lock,
Err(error) => return Ok(Err(error)),
};
let result = apply_admin_external_models_config_update_locked(state, proxy_node_id).await;
release_admin_external_models_config_mutation_lock(state, &lock).await;
result
}
async fn fetch_admin_external_models_from_source(
state: &AdminAppState<'_>,
request_id: &str,
proxy_node_id: Option<&str>,
) -> Result<serde_json::Value, GatewayError> {
let url = admin_external_models_source_url();
if let Some(node_id) = proxy_node_id {
let Some(proxy) = state.resolve_admin_proxy_node_snapshot(Some(node_id)).await else {
warn!(
request_id = %request_id,
proxy_node_id = %node_id,
proxy_mode = "unknown",
transport_error_kind = "node_unavailable",
"external models proxy execution failed"
);
return Err(GatewayError::Internal(
"external models proxy request failed".to_string(),
));
};
let proxy_mode = if proxy.mode.as_deref() == Some("tunnel") {
"tunnel"
} else if proxy.url.is_some() {
"manual"
} else {
"unknown"
};
let headers = BTreeMap::from([
("accept".to_string(), "application/json".to_string()),
(
"user-agent".to_string(),
"aether-gateway/external-models".to_string(),
),
(
EXECUTION_REQUEST_FOLLOW_REDIRECTS_HEADER.to_string(),
"true".to_string(),
),
]);
let plan = ExecutionPlan {
request_id: format!("{request_id}:external-models"),
candidate_id: None,
provider_name: Some("external_models_catalog".to_string()),
provider_id: String::new(),
endpoint_id: String::new(),
key_id: String::new(),
method: http::Method::GET.as_str().to_string(),
url,
headers,
content_type: None,
content_encoding: None,
body: RequestBody {
json_body: None,
body_bytes_b64: None,
body_ref: None,
},
stream: false,
client_api_format: "control:external_models".to_string(),
provider_api_format: "control:external_models".to_string(),
model_name: None,
proxy: Some(proxy),
transport_profile: None,
timeouts: Some(ExecutionTimeouts {
connect_ms: Some(ADMIN_EXTERNAL_MODELS_CONNECT_TIMEOUT_MS),
read_ms: Some(ADMIN_EXTERNAL_MODELS_TOTAL_TIMEOUT_MS),
write_ms: Some(ADMIN_EXTERNAL_MODELS_TOTAL_TIMEOUT_MS),
pool_ms: Some(ADMIN_EXTERNAL_MODELS_CONNECT_TIMEOUT_MS),
total_ms: Some(ADMIN_EXTERNAL_MODELS_TOTAL_TIMEOUT_MS),
..ExecutionTimeouts::default()
}),
};
let result = match state
.execute_execution_runtime_sync_plan(Some(request_id), &plan)
.await
{
Ok(result) => result,
Err(err) => {
let error_message = err.clone().into_message();
let transport_error_kind =
classify_admin_external_models_transport_error(&error_message);
warn!(
request_id = %request_id,
proxy_node_id = %node_id,
proxy_mode,
transport_error_kind,
"external models proxy execution failed"
);
return Err(GatewayError::Internal(
"external models proxy request failed".to_string(),
));
}
};
if !(200..300).contains(&result.status_code) {
return Err(GatewayError::Internal(format!(
"external models source returned HTTP {}",
result.status_code
)));
}
let payload = result.body.and_then(|body| body.json_body).ok_or_else(|| {
GatewayError::Internal(
"external models source returned a non-JSON response".to_string(),
)
})?;
return Ok(normalize_admin_external_models_payload(payload));
}
let response = state
.http_client()
.get(&url)
@@ -60,29 +462,36 @@ async fn fetch_admin_external_models_from_source(
pub(crate) async fn read_admin_external_models_cache(
state: &AdminAppState<'_>,
request_id: &str,
) -> Result<Option<serde_json::Value>, GatewayError> {
let proxy_node_id = read_admin_external_models_proxy_node_id(state).await?;
if let Some(raw) = state
.as_ref()
.runtime_kv_get(ADMIN_EXTERNAL_MODELS_CACHE_KEY)
.await?
{
match serde_json::from_str::<serde_json::Value>(&raw) {
Ok(payload) => {
let payload = normalize_admin_external_models_payload(payload);
if let Err(err) = store_admin_external_models_cache(state, &payload).await {
warn!(error = ?err, "failed to refresh external models cache ttl");
}
return Ok(Some(payload));
match serde_json::from_str::<AdminExternalModelsCacheEnvelope>(&raw) {
Ok(envelope)
if envelope.schema_version == ADMIN_EXTERNAL_MODELS_CACHE_VERSION
&& envelope.proxy_node_id == proxy_node_id =>
{
return Ok(Some(normalize_admin_external_models_payload(
envelope.payload,
)));
}
Ok(_) => {}
Err(err) => {
warn!(error = %err, "failed to parse cached external models payload");
}
}
}
match fetch_admin_external_models_from_source(state).await {
match fetch_admin_external_models_from_source(state, request_id, proxy_node_id.as_deref()).await
{
Ok(payload) => {
if let Err(err) = store_admin_external_models_cache(state, &payload).await {
if let Err(err) =
store_admin_external_models_cache(state, proxy_node_id.as_deref(), &payload).await
{
warn!(error = ?err, "failed to store fetched external models cache");
}
Ok(Some(payload))
@@ -97,10 +506,7 @@ pub(crate) async fn read_admin_external_models_cache(
pub(crate) async fn clear_admin_external_models_cache(
state: &AdminAppState<'_>,
) -> Result<serde_json::Value, GatewayError> {
let deleted = state
.as_ref()
.runtime_kv_del(ADMIN_EXTERNAL_MODELS_CACHE_KEY)
.await?;
let deleted = clear_admin_external_models_cache_entries(state).await?;
Ok(json!({
"cleared": deleted,
"message": if deleted { "缓存已清除" } else { "缓存不存在" },
@@ -110,47 +516,15 @@ pub(crate) async fn clear_admin_external_models_cache(
#[cfg(test)]
mod tests {
use super::{
admin_external_models_source_url, normalize_admin_external_models_payload,
read_admin_external_models_cache, ADMIN_EXTERNAL_MODELS_SOURCE_URL_ENV,
admin_external_models_source_url, classify_admin_external_models_transport_error,
normalize_admin_external_models_payload, normalize_admin_external_models_proxy_node_id,
read_admin_external_models_cache, set_admin_external_models_source_url_for_tests,
};
use crate::handlers::admin::request::AdminAppState;
use crate::tests::{start_server, AppState};
use axum::routing::get;
use axum::{Json, Router};
use serde_json::json;
use std::sync::{Mutex, MutexGuard, OnceLock};
fn admin_external_models_env_lock() -> &'static Mutex<()> {
static LOCK: OnceLock<Mutex<()>> = OnceLock::new();
LOCK.get_or_init(|| Mutex::new(()))
}
struct TestEnvVarGuard {
key: &'static str,
previous: Option<String>,
_lock: Option<MutexGuard<'static, ()>>,
}
impl Drop for TestEnvVarGuard {
fn drop(&mut self) {
if let Some(previous) = self.previous.as_deref() {
std::env::set_var(self.key, previous);
} else {
std::env::remove_var(self.key);
}
}
}
fn set_test_env_var(key: &'static str, value: &str) -> TestEnvVarGuard {
let lock = admin_external_models_env_lock().lock().ok();
let previous = std::env::var(key).ok();
std::env::set_var(key, value);
TestEnvVarGuard {
key,
previous,
_lock: lock,
}
}
#[test]
fn normalizes_external_models_payload_with_official_flags() {
@@ -172,11 +546,44 @@ mod tests {
}
#[test]
fn external_models_source_url_uses_env_override_when_present() {
let _guard = set_test_env_var(
ADMIN_EXTERNAL_MODELS_SOURCE_URL_ENV,
"http://127.0.0.1:12345/api",
fn external_models_proxy_config_only_treats_missing_or_null_as_direct() {
assert_eq!(
normalize_admin_external_models_proxy_node_id(None).expect("missing config is direct"),
None
);
assert_eq!(
normalize_admin_external_models_proxy_node_id(Some(&serde_json::Value::Null))
.expect("null config is direct"),
None
);
assert!(normalize_admin_external_models_proxy_node_id(Some(&json!(" "))).is_err());
assert!(normalize_admin_external_models_proxy_node_id(Some(&json!(false))).is_err());
}
#[test]
fn classifies_external_models_transport_errors_without_exposing_details() {
for (message, expected) in [
("request timeout after 300000ms", "timeout"),
("hub relay request failed", "relay"),
("invalid proxy configuration", "proxy_config"),
("upstream response body exceeds limit", "response_too_large"),
("upstream response is not valid JSON", "invalid_json"),
("failed to decode content-encoding gzip", "response_decode"),
("tcp connect error", "connect"),
("invalid upstream header value", "request_build"),
("opaque execution failure", "unknown_transport"),
] {
assert_eq!(
classify_admin_external_models_transport_error(message),
expected,
"message={message}"
);
}
}
#[test]
fn external_models_source_url_uses_env_override_when_present() {
let _guard = set_admin_external_models_source_url_for_tests("http://127.0.0.1:12345/api");
assert_eq!(
admin_external_models_source_url(),
"http://127.0.0.1:12345/api"
@@ -201,16 +608,15 @@ mod tests {
}),
);
let (upstream_url, upstream_handle) = start_server(upstream).await;
let _guard = set_test_env_var(
ADMIN_EXTERNAL_MODELS_SOURCE_URL_ENV,
&format!("{upstream_url}/api.json"),
);
let _guard =
set_admin_external_models_source_url_for_tests(&format!("{upstream_url}/api.json"));
let state = AppState::new().expect("gateway should build");
let payload = read_admin_external_models_cache(&AdminAppState::new(&state))
.await
.expect("external models read should succeed")
.expect("payload should be fetched");
let payload =
read_admin_external_models_cache(&AdminAppState::new(&state), "external-models-test")
.await
.expect("external models read should succeed")
.expect("payload should be fetched");
assert_eq!(payload["openai"]["official"], json!(true));
assert_eq!(payload["openai"]["models"]["gpt-5"]["name"], json!("GPT-5"));
apps/aether-gateway/src/handlers/admin/model/mod.rs
+8 -1
View File
@@ -11,7 +11,14 @@ mod write;
pub(super) use self::catalog_routes::maybe_build_local_admin_model_catalog_response;
pub(super) use self::external_cache::{
clear_admin_external_models_cache, read_admin_external_models_cache,
acquire_admin_external_models_config_mutation_lock, apply_admin_external_models_config_update,
build_admin_external_models_config_payload, clear_admin_external_models_cache,
read_admin_external_models_cache, release_admin_external_models_config_mutation_lock,
ADMIN_EXTERNAL_MODELS_PROXY_NODE_CONFIG_KEY,
};
#[cfg(test)]
pub(crate) use self::external_cache::{
set_admin_external_models_source_url_for_tests, ADMIN_EXTERNAL_MODELS_CONFIG_MUTATION_LOCK_KEY,
};
pub(super) use self::global::{
build_admin_global_model_payload, build_admin_global_model_providers_payload,
apps/aether-gateway/src/handlers/admin/model/routes.rs
+1
View File
@@ -7,6 +7,7 @@ pub(crate) async fn maybe_build_local_admin_model_response(
if let Some(response) = catalog_routes::maybe_build_local_admin_model_catalog_response(
&request.state(),
&request.request_context(),
request.request_body(),
)
.await?
{
apps/aether-gateway/src/handlers/admin/request/models.rs
+18 -1
View File
@@ -11,6 +11,7 @@ use aether_admin::provider::{
use aether_data_contracts::repository::global_models::{
AdminProviderModelListQuery, StoredAdminProviderModel, UpsertAdminProviderModelRecord,
};
use axum::http;
use serde_json::json;
use std::collections::{BTreeMap, BTreeSet};
use uuid::Uuid;
@@ -488,8 +489,24 @@ impl<'a> AdminAppState<'a> {
pub(crate) async fn read_admin_external_models_cache(
&self,
request_id: &str,
) -> Result<Option<serde_json::Value>, GatewayError> {
crate::handlers::admin::model::read_admin_external_models_cache(self).await
crate::handlers::admin::model::read_admin_external_models_cache(self, request_id).await
}
pub(crate) async fn build_admin_external_models_config_payload(
&self,
) -> Result<serde_json::Value, GatewayError> {
crate::handlers::admin::model::build_admin_external_models_config_payload(self).await
}
pub(crate) async fn apply_admin_external_models_config_update(
&self,
request_body: &axum::body::Bytes,
) -> Result<Result<serde_json::Value, (http::StatusCode, serde_json::Value)>, GatewayError>
{
crate::handlers::admin::model::apply_admin_external_models_config_update(self, request_body)
.await
}
pub(crate) async fn clear_admin_external_models_cache(
apps/aether-gateway/src/handlers/admin/request/system/import.rs
+97 -11
View File
@@ -2,6 +2,7 @@ use super::{AdminAppState, ADMIN_SYSTEM_DATA_EXPORT_VERSION};
use crate::ai_serving::build_provider_key_pool_score_upsert;
use crate::api::ai::admin_endpoint_signature_parts;
use crate::handlers::admin::admin_provider_pool_config;
use crate::handlers::admin::model::ADMIN_EXTERNAL_MODELS_PROXY_NODE_CONFIG_KEY;
use crate::handlers::admin::provider::endpoints_admin::payloads::AdminProviderEndpointUpdatePatch;
use crate::handlers::admin::provider::shared::payloads::{
AdminProviderCreateRequest, AdminProviderKeyCreateRequest, AdminProviderKeyUpdatePatch,
@@ -61,6 +62,15 @@ fn invalid_request(detail: impl Into<String>) -> (http::StatusCode, Value) {
)
}
fn normalize_imported_system_config_key(key: &str) -> String {
let normalized = normalize_admin_system_config_key(key);
if normalized.eq_ignore_ascii_case(ADMIN_EXTERNAL_MODELS_PROXY_NODE_CONFIG_KEY) {
ADMIN_EXTERNAL_MODELS_PROXY_NODE_CONFIG_KEY.to_string()
} else {
normalized
}
}
fn build_admin_system_data_import_part_body(
root: &Map<String, Value>,
field_name: &str,
@@ -1299,6 +1309,82 @@ impl<'a> AdminAppState<'a> {
let mut stats = AdminSystemConfigImportStats::default();
// Proxy nodes are deployment-local resources and are intentionally not imported by the
// Rust admin backend. Apply the external catalog selector before importing any other
// object, and turn a non-empty exported node reference into direct mode. This keeps a
// clean-environment restore portable and prevents a late selector validation failure from
// leaving the rest of the document partially imported.
let (imported_external_models_configs, imported_system_configs): (Vec<_>, Vec<_>) =
imported_system_configs.into_iter().partition(|item| {
normalize_imported_system_config_key(&item.value.key)
== ADMIN_EXTERNAL_MODELS_PROXY_NODE_CONFIG_KEY
});
let mut existing_system_config_keys = self
.list_system_config_entries()
.await?
.into_iter()
.map(|entry| normalize_imported_system_config_key(&entry.key))
.collect::<BTreeSet<_>>();
for imported_config_item in imported_external_models_configs {
let (_, system_config) = imported_config_item.into_parts();
let exists =
existing_system_config_keys.contains(ADMIN_EXTERNAL_MODELS_PROXY_NODE_CONFIG_KEY);
match (exists, merge_mode) {
(true, AdminImportMergeMode::Skip) => {
stats.system_configs.skipped += 1;
continue;
}
(true, AdminImportMergeMode::Error) => {
return Ok(Err(invalid_request(format!(
"SystemConfig '{ADMIN_EXTERNAL_MODELS_PROXY_NODE_CONFIG_KEY}' 已存在"
))));
}
_ => {}
}
let imported_proxy_node_id = match system_config.value {
Value::Null => None,
Value::String(value) => {
let value = value.trim();
if value.is_empty() {
return Ok(Err(invalid_request(
"external_models_proxy_node_id 不能为空",
)));
}
Some(value.to_string())
}
_ => {
return Ok(Err(invalid_request(
"external_models_proxy_node_id 必须是字符串或 null",
)))
}
};
let request_bytes = Bytes::from(
serde_json::to_vec(&json!({ "proxy_node_id": null }))
.map_err(|err| GatewayError::Internal(err.to_string()))?,
);
match self
.apply_admin_external_models_config_update(&request_bytes)
.await?
{
Ok(_) => {
if exists {
stats.system_configs.updated += 1;
} else {
stats.system_configs.created += 1;
existing_system_config_keys
.insert(ADMIN_EXTERNAL_MODELS_PROXY_NODE_CONFIG_KEY.to_string());
}
if let Some(node_id) = imported_proxy_node_id {
stats.errors.push(format!(
"外部模型目录代理节点 '{node_id}' 是当前部署的本地引用；代理节点未导入，已切换为直连"
));
}
}
Err((status, payload)) => return Ok(Err((status, payload))),
}
}
let mut global_models_by_name = self
.list_admin_global_models(&AdminGlobalModelListQuery {
offset: 0,
@@ -2231,15 +2317,14 @@ impl<'a> AdminAppState<'a> {
}
}
let mut existing_system_config_keys = self
.list_system_config_entries()
.await?
.into_iter()
.map(|entry| normalize_admin_system_config_key(&entry.key))
.collect::<BTreeSet<_>>();
for imported_config_item in imported_system_configs {
let (_, system_config) = imported_config_item.into_parts();
let normalized_key = normalize_admin_system_config_key(&system_config.key);
let ImportedSystemConfig {
key,
value,
description,
} = system_config;
let normalized_key = normalize_imported_system_config_key(&key);
let exists = existing_system_config_keys.contains(&normalized_key);
match (exists, merge_mode) {
(true, AdminImportMergeMode::Skip) => {
@@ -2256,13 +2341,14 @@ impl<'a> AdminAppState<'a> {
let request_bytes = Bytes::from(
serde_json::to_vec(&json!({
"value": system_config.value,
"description": system_config.description,
"value": value,
"description": description,
}))
.map_err(|err| GatewayError::Internal(err.to_string()))?,
);
match apply_admin_system_config_update(self, &system_config.key, &request_bytes).await?
{
let update_result =
apply_admin_system_config_update(self, &key, &request_bytes).await?;
match update_result {
Ok(_) => {
if exists {
stats.system_configs.updated += 1;
apps/aether-gateway/src/handlers/admin/request/system/mod.rs
+7
View File
@@ -29,6 +29,13 @@ impl<'a> AdminAppState<'a> {
self.app.read_system_config_json_value(key).await
}
pub(crate) async fn read_system_config_json_value_strong(
&self,
key: &str,
) -> Result<Option<serde_json::Value>, GatewayError> {
self.app.read_system_config_json_value_strong(key).await
}
pub(crate) async fn list_system_config_entries(
&self,
) -> Result<Vec<crate::data::state::StoredSystemConfigEntry>, GatewayError> {
apps/aether-gateway/src/handlers/admin/system/core/mod.rs
+1
View File
@@ -84,6 +84,7 @@ pub(crate) async fn maybe_build_local_admin_core_response(
crate::handlers::admin::model::maybe_build_local_admin_model_catalog_response(
state,
request_context,
request_body,
)
.await?
{
apps/aether-gateway/src/handlers/admin/system/mod.rs
+4 -1
View File
@@ -7,5 +7,8 @@ mod routes;
pub(super) mod shared;
#[cfg(test)]
pub(crate) use self::proxy_nodes::override_proxy_connectivity_probe_url_for_tests;
pub(crate) use self::proxy_nodes::{
clear_proxy_node_references_with_cache_failure_for_tests,
override_proxy_connectivity_probe_url_for_tests,
};
pub(super) use self::routes::maybe_build_local_admin_system_response;
apps/aether-gateway/src/handlers/admin/system/proxy_nodes.rs
+109 -10
View File
@@ -1,6 +1,11 @@
use std::future::Future;
use std::time::{Duration, Instant};
use crate::execution_runtime::transport::format_upstream_request_error;
use crate::handlers::admin::model::{
acquire_admin_external_models_config_mutation_lock,
release_admin_external_models_config_mutation_lock,
};
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
use crate::handlers::admin::shared::query_param_value;
use crate::maintenance::{
@@ -31,6 +36,7 @@ use serde::de::DeserializeOwned;
use serde::Deserialize;
use serde_json::{json, Value};
use sha2::{Digest, Sha256};
use tracing::warn;
use uuid::Uuid;
use crate::handlers::public::build_proxy_node_install_session_response;
@@ -526,27 +532,43 @@ pub(crate) async fn maybe_build_local_admin_proxy_nodes_response(
if decision.route_kind.as_deref() == Some("delete_node")
&& request_context.method() == http::Method::DELETE
{
if !state.has_proxy_node_writer() {
if !state.has_proxy_node_reader() || !state.has_proxy_node_writer() {
return Ok(Some(build_admin_proxy_nodes_data_unavailable_response()));
}
let Some(node_id) = admin_proxy_node_node_id_from_path(request_context.path()) else {
return Ok(Some(build_admin_proxy_nodes_not_found_response()));
};
let Some(_deleted_node) = state.delete_proxy_node(&node_id).await? else {
return Ok(Some(build_admin_proxy_nodes_not_found_response()));
let lock = match acquire_admin_external_models_config_mutation_lock(state).await {
Ok(lock) => lock,
Err((status, payload)) => {
return Ok(Some((status, Json(payload)).into_response()));
}
};
let cleanup = clear_deleted_proxy_node_references(state, &node_id).await?;
return Ok(Some(
Json(json!({
let delete_result: Result<Response<Body>, GatewayError> = async {
if state.find_proxy_node(&node_id).await?.is_none() {
return Ok(build_admin_proxy_nodes_not_found_response());
}
// Persistent references are cleared before the node itself. If a durable cleanup
// fails, the node remains available and the operation can be retried safely.
let cleanup = clear_proxy_node_references_before_delete(state, &node_id).await?;
let Some(_deleted_node) = state.delete_proxy_node(&node_id).await? else {
return Ok(build_admin_proxy_nodes_not_found_response());
};
Ok(Json(json!({
"message": build_delete_proxy_node_message(&cleanup),
"node_id": node_id,
"cleared_system_proxy": cleanup.cleared_system_proxy,
"cleared_external_models_proxy": cleanup.cleared_external_models_proxy,
"cleared_providers": cleanup.cleared_providers,
"cleared_endpoints": cleanup.cleared_endpoints,
"cleared_keys": cleanup.cleared_keys,
}))
.into_response(),
));
.into_response())
}
.await;
release_admin_external_models_config_mutation_lock(state, &lock).await;
return delete_result.map(Some);
}
if decision.route_kind.as_deref() == Some("test_node")
@@ -869,6 +891,8 @@ pub(crate) async fn maybe_build_local_admin_proxy_nodes_response(
#[derive(Debug, Default)]
struct DeletedProxyNodeCleanup {
cleared_system_proxy: bool,
cleared_external_models_proxy: bool,
external_models_cache_clear_succeeded: Option<bool>,
cleared_providers: usize,
cleared_endpoints: usize,
cleared_keys: usize,
@@ -895,15 +919,32 @@ struct NormalizedManualProxyEndpoint {
node_port: i32,
}
async fn clear_deleted_proxy_node_references(
async fn clear_proxy_node_references_before_delete(
state: &AdminAppState<'_>,
node_id: &str,
) -> Result<DeletedProxyNodeCleanup, GatewayError> {
let external_models_cache_clear = state.clear_admin_external_models_cache();
clear_proxy_node_references_before_delete_with_cache(
state,
node_id,
external_models_cache_clear,
)
.await
}
async fn clear_proxy_node_references_before_delete_with_cache<F>(
state: &AdminAppState<'_>,
node_id: &str,
external_models_cache_clear: F,
) -> Result<DeletedProxyNodeCleanup, GatewayError>
where
F: Future<Output = Result<Value, GatewayError>>,
{
let mut cleanup = DeletedProxyNodeCleanup::default();
if state.app().data.has_system_config_store() {
let is_system_proxy = state
.read_system_config_json_value("system_proxy_node_id")
.read_system_config_json_value_strong("system_proxy_node_id")
.await?
.and_then(|value| value.as_str().map(str::trim).map(ToOwned::to_owned))
.is_some_and(|value| value == node_id);
@@ -917,6 +958,36 @@ async fn clear_deleted_proxy_node_references(
.await?;
cleanup.cleared_system_proxy = true;
}
let is_external_models_proxy = state
.read_system_config_json_value_strong("external_models_proxy_node_id")
.await?
.and_then(|value| value.as_str().map(str::trim).map(ToOwned::to_owned))
.is_some_and(|value| value == node_id);
if is_external_models_proxy {
state
.upsert_system_config_json_value(
"external_models_proxy_node_id",
&serde_json::Value::Null,
None,
)
.await?;
cleanup.external_models_cache_clear_succeeded =
Some(match external_models_cache_clear.await {
Ok(_) => true,
Err(_) => {
// The selector is already persisted as null, and v2 cache entries carry
// their selector. A failed DEL therefore cannot route through this node.
warn!(
runtime_backend = state.app().runtime_state_backend(),
proxy_node_id = %node_id,
"failed to clear external models cache while deleting proxy node"
);
false
}
});
cleanup.cleared_external_models_proxy = true;
}
}
if state.app().has_provider_catalog_data_reader()
@@ -978,11 +1049,39 @@ async fn clear_deleted_proxy_node_references(
Ok(cleanup)
}
#[cfg(test)]
pub(crate) async fn clear_proxy_node_references_with_cache_failure_for_tests(
app: &crate::AppState,
node_id: &str,
) -> Result<Value, GatewayError> {
let state = AdminAppState::new(app);
let cleanup = clear_proxy_node_references_before_delete_with_cache(&state, node_id, async {
Err(GatewayError::Internal(
"injected cache delete failure".to_string(),
))
})
.await?;
Ok(json!({
"cleared_system_proxy": cleanup.cleared_system_proxy,
"cleared_external_models_proxy": cleanup.cleared_external_models_proxy,
"external_models_cache_clear_succeeded": cleanup.external_models_cache_clear_succeeded,
"cleared_providers": cleanup.cleared_providers,
"cleared_endpoints": cleanup.cleared_endpoints,
"cleared_keys": cleanup.cleared_keys,
}))
}
fn build_delete_proxy_node_message(cleanup: &DeletedProxyNodeCleanup) -> String {
let mut parts = vec!["deleted".to_string()];
if cleanup.cleared_system_proxy {
parts.push("system default proxy cleared".to_string());
}
if cleanup.cleared_external_models_proxy {
parts.push("external models proxy cleared".to_string());
}
if cleanup.external_models_cache_clear_succeeded == Some(false) {
parts.push("external models cache invalidation deferred".to_string());
}
if cleanup.cleared_providers > 0 || cleanup.cleared_endpoints > 0 || cleanup.cleared_keys > 0 {
parts.push(format!(
"cleared proxy refs from {} provider(s), {} endpoint(s), {} key(s)",
apps/aether-gateway/src/handlers/admin/system/shared/configs.rs
+40 -1
View File
@@ -1,3 +1,4 @@
use crate::handlers::admin::model::ADMIN_EXTERNAL_MODELS_PROXY_NODE_CONFIG_KEY;
use crate::handlers::admin::request::AdminAppState;
use crate::handlers::shared::unix_secs_to_rfc3339;
use crate::GatewayError;
@@ -17,6 +18,30 @@ use axum::body::Bytes;
use axum::http;
use serde_json::json;
const ADMIN_EXTERNAL_MODELS_CONFIG_ROUTE: &str = "/api/admin/models/external/config";
fn is_external_models_proxy_node_config_key(key: &str) -> bool {
key.trim()
.eq_ignore_ascii_case(ADMIN_EXTERNAL_MODELS_PROXY_NODE_CONFIG_KEY)
}
fn external_models_proxy_node_config_owner_error(
key: &str,
) -> Option<(http::StatusCode, serde_json::Value)> {
is_external_models_proxy_node_config_key(key).then(|| {
(
http::StatusCode::BAD_REQUEST,
json!({
"detail": format!(
"配置项 '{}' 由模型目录管理，请使用 {}",
ADMIN_EXTERNAL_MODELS_PROXY_NODE_CONFIG_KEY,
ADMIN_EXTERNAL_MODELS_CONFIG_ROUTE,
)
}),
)
})
}
fn normalize_admin_system_config_key(requested_key: &str) -> String {
normalize_admin_system_config_key_pure(requested_key)
}
@@ -51,7 +76,12 @@ fn legacy_admin_system_config_fallback_key(normalized_key: &str) -> Option<&'sta
pub(crate) fn build_admin_system_configs_payload(
entries: &[aether_data::repository::system::StoredSystemConfigEntry],
) -> serde_json::Value {
build_admin_system_configs_payload_pure(entries)
let visible_entries = entries
.iter()
.filter(|entry| !is_external_models_proxy_node_config_key(&entry.key))
.cloned()
.collect::<Vec<_>>();
build_admin_system_configs_payload_pure(&visible_entries)
}
pub(crate) async fn build_admin_system_config_detail_payload(
@@ -59,6 +89,9 @@ pub(crate) async fn build_admin_system_config_detail_payload(
requested_key: &str,
) -> Result<Result<serde_json::Value, (http::StatusCode, serde_json::Value)>, GatewayError> {
let requested_key = requested_key.trim();
if let Some(error) = external_models_proxy_node_config_owner_error(requested_key) {
return Ok(Err(error));
}
let normalized_key = normalize_admin_system_config_key(requested_key);
let mut value = state.read_system_config_json_value(&normalized_key).await?;
if value.is_none() {
@@ -78,6 +111,9 @@ pub(crate) async fn apply_admin_system_config_update(
requested_key: &str,
request_body: &Bytes,
) -> Result<Result<serde_json::Value, (http::StatusCode, serde_json::Value)>, GatewayError> {
if let Some(error) = external_models_proxy_node_config_owner_error(requested_key) {
return Ok(Err(error));
}
let update = match parse_admin_system_config_update(requested_key, request_body) {
Ok(update) => update,
Err(err) => return Ok(Err(err)),
@@ -123,6 +159,9 @@ pub(crate) async fn delete_admin_system_config(
state: &AdminAppState<'_>,
requested_key: &str,
) -> Result<Result<serde_json::Value, (http::StatusCode, serde_json::Value)>, GatewayError> {
if let Some(error) = external_models_proxy_node_config_owner_error(requested_key) {
return Ok(Err(error));
}
let delete_keys = admin_system_config_delete_keys(requested_key);
let mut deleted = false;
for key in &delete_keys {
apps/aether-gateway/src/handlers/shared/request_utils.rs
+1
View File
@@ -295,6 +295,7 @@ pub(crate) fn admin_proxy_local_requires_buffered_body(
http::Method::POST,
Some("import_from_upstream"),
)
| (Some("model_external_manage"), http::Method::PUT, Some("external_config_set"))
| (
Some("provider_ops_manage"),
http::Method::POST,