feat(proxy): 实现代理节点批量升级回滚、隧道重定向跟随及远程配置管理

核心功能:
- 新增代理节点批量升级回滚工作流,支持分批升级、健康探针、跳过/重试/取消等操作
- proxy 隧道流处理器支持 HTTP 重定向跟随(最多 10 跳),区分 307/308 可重播与不可重播请求体
- proxy 协议新增 follow_redirects / http1_only 字段,网关侧同步支持
- 新增代理节点远端配置变更接口(名称、允许端口、调度状态、升级目标等)
- 新增代理节点注册/反注册/心跳的 Admin API,及节点过期清理维护任务
- gateway 隧道 owner-relay 支持流式代理大请求体,新增 5 MiB 默认限制
- 新增 ProxyNodeRegistrationMutation / ProxyNodeRemoteConfigMutation 数据类型
- proxy 配置新增重定向重播预算、心跳间隔等参数,TUI 安装向导同步更新
- 前端 ProxyNodes 页面新增批量升级操作面板及滚动进度展示
This commit is contained in:
fawney19
2026-04-12 16:02:38 +08:00
parent 7c5bb7f383
commit 9703840a36
83 changed files with 11832 additions and 1520 deletions
@@ -88,6 +88,10 @@ impl<'a> AdminAppState<'a> {
self.app.has_proxy_node_reader()
}
pub(crate) fn has_proxy_node_writer(&self) -> bool {
self.app.has_proxy_node_writer()
}
pub(crate) fn has_auth_api_key_writer(&self) -> bool {
self.app.data.has_auth_api_key_writer()
}
@@ -1,5 +1,9 @@
use super::*;
use crate::handlers::admin::provider::oauth::errors::build_internal_control_error_response;
use aether_contracts::{
ExecutionPlan, ExecutionResult, ExecutionTimeouts, RequestBody,
EXECUTION_REQUEST_FOLLOW_REDIRECTS_HEADER,
};
use aether_data::repository::provider_oauth::{
build_provider_oauth_batch_task_status_payload, provider_oauth_batch_task_storage_key,
provider_oauth_device_session_storage_key, provider_oauth_state_storage_key,
@@ -7,11 +11,22 @@ use aether_data::repository::provider_oauth::{
PROVIDER_OAUTH_BATCH_TASK_TTL_SECS, PROVIDER_OAUTH_STATE_TTL_SECS,
};
use axum::http;
use base64::{engine::general_purpose::STANDARD, Engine as _};
use flate2::read::{DeflateDecoder, GzDecoder};
use serde_json::json;
use std::collections::BTreeMap;
use std::io::Read;
use url::Url;
const KIRO_IDC_AMZ_USER_AGENT: &str =
"aws-sdk-js/3.738.0 ua/2.1 os/other lang/js md/browser#unknown_unknown api/sso-oidc#3.738.0 m/E KiroIDE";
const ADMIN_PROVIDER_OAUTH_TIMEOUT_MS: u64 = 30_000;
pub(crate) struct AdminProviderOAuthHttpResponse {
pub(crate) status: http::StatusCode,
pub(crate) body_text: String,
pub(crate) json_body: Option<serde_json::Value>,
}
impl<'a> AdminAppState<'a> {
pub(crate) async fn update_provider_catalog_key_oauth_credentials(
@@ -117,6 +132,7 @@ impl<'a> AdminAppState<'a> {
code: &str,
state_nonce: &str,
pkce_verifier: Option<&str>,
proxy_node_id: Option<&str>,
) -> Result<serde_json::Value, Response<Body>> {
crate::handlers::admin::provider::oauth::state::exchange_admin_provider_oauth_code(
self,
@@ -124,6 +140,7 @@ impl<'a> AdminAppState<'a> {
code,
state_nonce,
pkce_verifier,
proxy_node_id,
)
.await
}
@@ -132,11 +149,13 @@ impl<'a> AdminAppState<'a> {
&self,
template: AdminProviderOAuthTemplate,
refresh_token: &str,
proxy_node_id: Option<&str>,
) -> Result<serde_json::Value, Response<Body>> {
crate::handlers::admin::provider::oauth::state::exchange_admin_provider_oauth_refresh_token(
self,
template,
refresh_token,
proxy_node_id,
)
.await
}
@@ -296,6 +315,7 @@ impl<'a> AdminAppState<'a> {
&self,
region: &str,
start_url: &str,
proxy_node_id: Option<&str>,
) -> Result<serde_json::Value, Response<Body>> {
let payload = post_kiro_device_oidc_json(
self,
@@ -317,6 +337,7 @@ impl<'a> AdminAppState<'a> {
],
"issuerUrl": start_url,
}),
proxy_node_id,
)
.await?;
if payload
@@ -343,6 +364,7 @@ impl<'a> AdminAppState<'a> {
client_id: &str,
client_secret: &str,
start_url: &str,
proxy_node_id: Option<&str>,
) -> Result<serde_json::Value, Response<Body>> {
let payload = post_kiro_device_oidc_json(
self,
@@ -353,6 +375,7 @@ impl<'a> AdminAppState<'a> {
"clientSecret": client_secret,
"startUrl": start_url,
}),
proxy_node_id,
)
.await?;
if payload
@@ -379,6 +402,7 @@ impl<'a> AdminAppState<'a> {
client_id: &str,
client_secret: &str,
device_code: &str,
proxy_node_id: Option<&str>,
) -> Result<serde_json::Value, Response<Body>> {
post_kiro_device_oidc_json(
self,
@@ -390,6 +414,7 @@ impl<'a> AdminAppState<'a> {
"grantType": "urn:ietf:params:oauth:grant-type:device_code",
"deviceCode": device_code,
}),
proxy_node_id,
)
.await
}
@@ -479,22 +504,43 @@ async fn post_kiro_device_oidc_json(
endpoint_key: &str,
default_url: String,
body: serde_json::Value,
proxy_node_id: Option<&str>,
) -> Result<serde_json::Value, Response<Body>> {
let url = state.provider_oauth_token_url(endpoint_key, &default_url);
let host = Url::parse(&url)
.ok()
.and_then(|value| value.host_str().map(ToOwned::to_owned))
.unwrap_or_default();
let headers = reqwest::header::HeaderMap::from_iter([
(
reqwest::header::CONTENT_TYPE,
reqwest::header::HeaderValue::from_static("application/json"),
),
(
reqwest::header::ACCEPT,
reqwest::header::HeaderValue::from_static("*/*"),
),
(
reqwest::header::USER_AGENT,
reqwest::header::HeaderValue::from_static("node"),
),
(
reqwest::header::HeaderName::from_static("x-amz-user-agent"),
reqwest::header::HeaderValue::from_static(KIRO_IDC_AMZ_USER_AGENT),
),
]);
let headers = maybe_insert_host_header(headers, host.as_str());
let response = state
.http_client()
.post(url)
.header("Content-Type", "application/json")
.header("Accept", "*/*")
.header("User-Agent", "node")
.header("x-amz-user-agent", KIRO_IDC_AMZ_USER_AGENT)
.header("Host", host)
.json(&body)
.send()
.execute_admin_provider_oauth_http_request(
endpoint_key,
reqwest::Method::POST,
&url,
&headers,
Some("application/json"),
Some(body),
None,
proxy_node_id,
)
.await
.map_err(|_| {
build_internal_control_error_response(
@@ -502,13 +548,8 @@ async fn post_kiro_device_oidc_json(
"发起设备授权失败: unknown",
)
})?;
let status = response.status();
let body_text = response.text().await.map_err(|_| {
build_internal_control_error_response(
http::StatusCode::BAD_REQUEST,
"发起设备授权失败: unknown",
)
})?;
let status = response.status;
let body_text = response.body_text;
Ok(
serde_json::from_str::<serde_json::Value>(&body_text).unwrap_or_else(|_| {
json!({
@@ -518,3 +559,179 @@ async fn post_kiro_device_oidc_json(
}),
)
}
impl<'a> AdminAppState<'a> {
pub(crate) async fn execute_admin_provider_oauth_http_request(
&self,
request_id: &str,
method: reqwest::Method,
url: &str,
headers: &reqwest::header::HeaderMap,
content_type: Option<&str>,
json_body: Option<serde_json::Value>,
body_bytes: Option<Vec<u8>>,
proxy_node_id: Option<&str>,
) -> Result<AdminProviderOAuthHttpResponse, String> {
let body = if let Some(json_body) = json_body {
RequestBody::from_json(json_body)
} else {
RequestBody {
json_body: None,
body_bytes_b64: body_bytes.map(|bytes| STANDARD.encode(bytes)),
body_ref: None,
}
};
let plan = ExecutionPlan {
request_id: request_id.to_string(),
candidate_id: None,
provider_name: Some("provider_oauth".to_string()),
provider_id: String::new(),
endpoint_id: String::new(),
key_id: String::new(),
method: method.as_str().to_string(),
url: url.to_string(),
headers: admin_provider_oauth_execution_headers(headers),
content_type: content_type
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned),
content_encoding: None,
body,
stream: false,
client_api_format: "provider_oauth:exchange".to_string(),
provider_api_format: "provider_oauth:exchange".to_string(),
model_name: Some("oauth-exchange".to_string()),
proxy: self.resolve_admin_proxy_node_snapshot(proxy_node_id).await,
tls_profile: None,
timeouts: Some(ExecutionTimeouts {
connect_ms: Some(ADMIN_PROVIDER_OAUTH_TIMEOUT_MS),
read_ms: Some(ADMIN_PROVIDER_OAUTH_TIMEOUT_MS),
write_ms: Some(ADMIN_PROVIDER_OAUTH_TIMEOUT_MS),
pool_ms: Some(ADMIN_PROVIDER_OAUTH_TIMEOUT_MS),
total_ms: Some(ADMIN_PROVIDER_OAUTH_TIMEOUT_MS),
..ExecutionTimeouts::default()
}),
};
let result = self
.execute_execution_runtime_sync_plan(None, &plan)
.await
.map_err(admin_provider_oauth_gateway_error_message)?;
Ok(AdminProviderOAuthHttpResponse {
status: http::StatusCode::from_u16(result.status_code)
.unwrap_or(http::StatusCode::BAD_GATEWAY),
body_text: admin_provider_oauth_execution_body_text(&result),
json_body: admin_provider_oauth_execution_json_body(&result),
})
}
}
fn maybe_insert_host_header(
mut headers: reqwest::header::HeaderMap,
host: &str,
) -> reqwest::header::HeaderMap {
let host = host.trim();
if host.is_empty() {
return headers;
}
if let Ok(value) = reqwest::header::HeaderValue::from_str(host) {
headers.insert(reqwest::header::HOST, value);
}
headers
}
fn admin_provider_oauth_execution_headers(
headers: &reqwest::header::HeaderMap,
) -> BTreeMap<String, String> {
let mut headers: BTreeMap<String, String> = headers
.iter()
.filter_map(|(name, value)| {
value
.to_str()
.ok()
.map(|text| (name.as_str().to_string(), text.to_string()))
})
.collect();
headers.insert(
EXECUTION_REQUEST_FOLLOW_REDIRECTS_HEADER.to_string(),
"true".to_string(),
);
headers
}
fn admin_provider_oauth_execution_json_body(result: &ExecutionResult) -> Option<serde_json::Value> {
result
.body
.as_ref()
.and_then(|body| body.json_body.clone())
.or_else(|| {
result
.body
.as_ref()
.and_then(|body| admin_provider_oauth_execution_body_bytes(&result.headers, body))
.and_then(|bytes| serde_json::from_slice::<serde_json::Value>(&bytes).ok())
})
}
fn admin_provider_oauth_execution_body_text(result: &ExecutionResult) -> String {
result
.body
.as_ref()
.and_then(|body| admin_provider_oauth_execution_body_bytes(&result.headers, body))
.map(|bytes| String::from_utf8_lossy(&bytes).to_string())
.or_else(|| {
result
.body
.as_ref()
.and_then(|body| body.json_body.as_ref())
.and_then(|value| serde_json::to_string(value).ok())
})
.unwrap_or_default()
}
fn admin_provider_oauth_execution_body_bytes(
headers: &BTreeMap<String, String>,
body: &aether_contracts::ResponseBody,
) -> Option<Vec<u8>> {
let bytes = body
.body_bytes_b64
.as_deref()
.and_then(|value| STANDARD.decode(value).ok())?;
admin_provider_oauth_decode_response_bytes(
&bytes,
headers.get("content-encoding").map(String::as_str),
)
.or(Some(bytes))
}
fn admin_provider_oauth_decode_response_bytes(
bytes: &[u8],
content_encoding: Option<&str>,
) -> Option<Vec<u8>> {
let encoding = content_encoding
.map(str::trim)
.filter(|value| !value.is_empty())
.map(|value| value.to_ascii_lowercase());
match encoding.as_deref() {
Some("gzip") => {
let mut decoder = GzDecoder::new(bytes);
let mut out = Vec::new();
decoder.read_to_end(&mut out).ok()?;
Some(out)
}
Some("deflate") => {
let mut decoder = DeflateDecoder::new(bytes);
let mut out = Vec::new();
decoder.read_to_end(&mut out).ok()?;
Some(out)
}
_ => None,
}
}
fn admin_provider_oauth_gateway_error_message(error: GatewayError) -> String {
match error {
GatewayError::UpstreamUnavailable { message, .. }
| GatewayError::ControlUnavailable { message, .. }
| GatewayError::Internal(message) => message,
}
}
@@ -1,4 +1,13 @@
use super::*;
use aether_contracts::ProxySnapshot;
use aether_data::repository::proxy_nodes::{proxy_node_accepts_new_tunnels, StoredProxyNode};
use aether_provider_transport::TransportTunnelAffinityLookup;
use serde_json::{json, Map, Value};
use url::Url;
const TUNNEL_BASE_URL_EXTRA_KEY: &str = "tunnel_base_url";
const TUNNEL_OWNER_INSTANCE_ID_EXTRA_KEY: &str = "tunnel_owner_instance_id";
const TUNNEL_OWNER_OBSERVED_AT_EXTRA_KEY: &str = "tunnel_owner_observed_at_unix_secs";
impl<'a> AdminAppState<'a> {
pub(crate) async fn read_provider_transport_snapshot(
@@ -100,6 +109,100 @@ impl<'a> AdminAppState<'a> {
)
}
pub(crate) async fn resolve_admin_connector_proxy_snapshot(
&self,
connector_config: Option<&Map<String, Value>>,
) -> Option<ProxySnapshot> {
let explicit_node_id = connector_config
.and_then(|config| admin_provider_transport_string_field(config, "proxy_node_id"));
if let Some(snapshot) = self
.resolve_admin_proxy_node_snapshot(explicit_node_id.as_deref())
.await
{
return Some(snapshot);
}
if explicit_node_id.is_none() {
let system_node_id = self
.read_system_config_json_value("system_proxy_node_id")
.await
.ok()
.flatten()
.and_then(|value| value.as_str().map(str::trim).map(ToOwned::to_owned))
.filter(|value| !value.is_empty());
if let Some(snapshot) = self
.resolve_admin_proxy_node_snapshot(system_node_id.as_deref())
.await
{
return Some(snapshot);
}
}
connector_config
.and_then(|config| config.get("proxy"))
.and_then(admin_provider_transport_legacy_proxy_snapshot)
}
pub(crate) async fn resolve_admin_proxy_node_snapshot(
&self,
node_id: Option<&str>,
) -> Option<ProxySnapshot> {
let node_id = node_id.map(str::trim).filter(|value| !value.is_empty())?;
let node = self.find_proxy_node(node_id).await.ok().flatten()?;
if node.status.trim() != "online" {
return None;
}
if !proxy_node_accepts_new_tunnels(&node) {
return None;
}
if node.tunnel_mode && node.tunnel_connected {
let mut extra = Map::new();
if let Ok(Some(owner)) = self.app().lookup_tunnel_attachment_owner(node_id).await {
extra.insert(
TUNNEL_BASE_URL_EXTRA_KEY.to_string(),
Value::String(owner.relay_base_url),
);
extra.insert(
TUNNEL_OWNER_INSTANCE_ID_EXTRA_KEY.to_string(),
Value::String(owner.gateway_instance_id),
);
extra.insert(
TUNNEL_OWNER_OBSERVED_AT_EXTRA_KEY.to_string(),
json!(owner.observed_at_unix_secs),
);
}
return Some(ProxySnapshot {
enabled: Some(true),
mode: Some("tunnel".to_string()),
node_id: Some(node_id.to_string()),
label: Some(node.name),
url: None,
extra: if extra.is_empty() {
None
} else {
Some(Value::Object(extra))
},
});
}
if !node.is_manual {
return None;
}
let proxy_url = node
.proxy_url
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())?;
Some(ProxySnapshot {
enabled: Some(true),
mode: admin_provider_transport_proxy_mode(Some(proxy_url)),
node_id: Some(node.id.clone()),
label: Some(node.name.clone()),
url: admin_provider_transport_proxy_url_with_node_auth(&node)
.or_else(|| Some(proxy_url.to_string())),
extra: None,
})
}
pub(crate) fn supports_local_gemini_transport_with_network(
&self,
transport: &AdminGatewayProviderTransportSnapshot,
@@ -222,3 +325,121 @@ impl<'a> AdminAppState<'a> {
crate::provider_transport::url::build_openai_chat_url(upstream_base_url, query)
}
}
fn admin_provider_transport_legacy_proxy_snapshot(value: &Value) -> Option<ProxySnapshot> {
match value {
Value::String(proxy_url) => {
let proxy_url = proxy_url.trim();
if proxy_url.is_empty() {
return None;
}
Some(ProxySnapshot {
enabled: Some(true),
mode: admin_provider_transport_proxy_mode(Some(proxy_url)),
node_id: None,
label: None,
url: Some(proxy_url.to_string()),
extra: None,
})
}
Value::Object(object) => {
if object.get("enabled").and_then(Value::as_bool) == Some(false) {
return None;
}
let proxy_url = object
.get("url")
.or_else(|| object.get("proxy_url"))
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())?;
let username = object
.get("username")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty());
let password = object
.get("password")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty());
Some(ProxySnapshot {
enabled: Some(true),
mode: object
.get("mode")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
.or_else(|| admin_provider_transport_proxy_mode(Some(proxy_url))),
node_id: None,
label: object
.get("label")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned),
url: admin_provider_transport_inject_proxy_auth(proxy_url, username, password)
.or_else(|| Some(proxy_url.to_string())),
extra: None,
})
}
_ => None,
}
}
fn admin_provider_transport_proxy_url_with_node_auth(node: &StoredProxyNode) -> Option<String> {
let proxy_url = node
.proxy_url
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())?;
let username = node
.proxy_username
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty());
let password = node
.proxy_password
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty());
admin_provider_transport_inject_proxy_auth(proxy_url, username, password)
}
fn admin_provider_transport_inject_proxy_auth(
proxy_url: &str,
username: Option<&str>,
password: Option<&str>,
) -> Option<String> {
let username = username.filter(|value| !value.is_empty())?;
let mut parsed = Url::parse(proxy_url).ok()?;
parsed.set_username(username).ok()?;
parsed.set_password(password).ok()?;
Some(parsed.to_string())
}
fn admin_provider_transport_proxy_mode(proxy_url: Option<&str>) -> Option<String> {
proxy_url
.and_then(|value| {
Url::parse(value)
.ok()
.map(|parsed| parsed.scheme().to_string())
})
.or_else(|| {
proxy_url.and_then(|value| {
value
.split_once("://")
.map(|(scheme, _)| scheme.trim().to_ascii_lowercase())
.filter(|scheme| !scheme.is_empty())
})
})
}
fn admin_provider_transport_string_field(config: &Map<String, Value>, key: &str) -> Option<String> {
config
.get(key)
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
}
@@ -1,4 +1,6 @@
use crate::handlers::admin::request::AdminAppState;
use crate::handlers::shared::unix_secs_to_rfc3339;
use crate::maintenance::{inspect_proxy_upgrade_rollout, ProxyUpgradeRolloutStatus};
use crate::GatewayError;
use aether_admin::system::{
build_admin_proxy_node_event_payload, build_admin_proxy_node_events_payload_response,
@@ -9,6 +11,20 @@ use aether_admin::system::{
use axum::{body::Body, response::Response};
impl<'a> AdminAppState<'a> {
pub(crate) async fn register_proxy_node(
&self,
mutation: &aether_data::repository::proxy_nodes::ProxyNodeRegistrationMutation,
) -> Result<Option<aether_data::repository::proxy_nodes::StoredProxyNode>, GatewayError> {
self.app.register_proxy_node(mutation).await
}
pub(crate) async fn apply_proxy_node_heartbeat(
&self,
mutation: &aether_data::repository::proxy_nodes::ProxyNodeHeartbeatMutation,
) -> Result<Option<aether_data::repository::proxy_nodes::StoredProxyNode>, GatewayError> {
self.app.apply_proxy_node_heartbeat(mutation).await
}
pub(crate) async fn build_admin_proxy_nodes_list_response(
&self,
skip: usize,
@@ -42,8 +58,12 @@ impl<'a> AdminAppState<'a> {
.take(limit)
.map(|node| build_admin_proxy_node_payload(&node))
.collect::<Vec<_>>();
let rollout = inspect_proxy_upgrade_rollout(self.app().data.as_ref())
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?
.map(build_admin_proxy_upgrade_rollout_payload);
Ok(build_admin_proxy_nodes_list_payload_response(
items, total, skip, limit,
items, total, skip, limit, rollout,
))
}
@@ -66,4 +86,55 @@ impl<'a> AdminAppState<'a> {
.collect::<Vec<_>>();
Ok(build_admin_proxy_node_events_payload_response(items))
}
pub(crate) async fn unregister_proxy_node(
&self,
node_id: &str,
) -> Result<Option<aether_data::repository::proxy_nodes::StoredProxyNode>, GatewayError> {
self.app.unregister_proxy_node(node_id).await
}
pub(crate) async fn update_proxy_node_remote_config(
&self,
mutation: &aether_data::repository::proxy_nodes::ProxyNodeRemoteConfigMutation,
) -> Result<Option<aether_data::repository::proxy_nodes::StoredProxyNode>, GatewayError> {
self.app.update_proxy_node_remote_config(mutation).await
}
}
fn build_admin_proxy_upgrade_rollout_payload(
rollout: ProxyUpgradeRolloutStatus,
) -> serde_json::Value {
serde_json::json!({
"version": rollout.version,
"batch_size": rollout.batch_size,
"cooldown_secs": rollout.cooldown_secs,
"started_at": unix_secs_to_rfc3339(rollout.started_at_unix_secs),
"last_dispatched_at": rollout
.last_dispatched_at_unix_secs
.and_then(unix_secs_to_rfc3339),
"updated_at": unix_secs_to_rfc3339(rollout.updated_at_unix_secs),
"probe": rollout.probe.map(|probe| serde_json::json!({
"url": probe.url,
"timeout_secs": probe.timeout_secs,
})),
"blocked": rollout.blocked,
"online_eligible_total": rollout.online_eligible_total,
"completed_node_ids": rollout.completed_node_ids,
"pending_node_ids": rollout.pending_node_ids,
"conflict_node_ids": rollout.conflict_node_ids,
"skipped_node_ids": rollout.skipped_node_ids,
"tracked_nodes": rollout.tracked_nodes.into_iter().map(|tracked| serde_json::json!({
"node_id": tracked.node_id,
"state": tracked.state,
"dispatched_at": unix_secs_to_rfc3339(tracked.dispatched_at_unix_secs),
"version_confirmed_at": tracked
.version_confirmed_at_unix_secs
.and_then(unix_secs_to_rfc3339),
"traffic_confirmed_at": tracked
.traffic_confirmed_at_unix_secs
.and_then(unix_secs_to_rfc3339),
"cooldown_remaining_secs": tracked.cooldown_remaining_secs,
})).collect::<Vec<_>>(),
})
}