i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 09:20:22 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(gateway): 重构 usage 数据层、迁移系统与系统导入

Browse Source 
数据库迁移:
- 引入 baseline v2 bootstrap，空库首次启动自动初始化
- 服务启动不再自动执行迁移，需显式 `--migrate` 运行
- 新增 pending migration 检测，schema 落后时拒绝启动

Usage 数据层:
- usage body 存储外部化为独立 blob 表
- 新增 HTTP audit 表拆分存储请求/响应头与 body ref
- 后台清理任务支持 legacy body ref 元数据迁移
- usage runtime 写入迁移到专用 tokio runtime（独立线程池, 8MB 栈）

系统导入/导出:
- 支持用户、API Keys、钱包数据的完整导入
- 兼容 legacy 与 v1.3+ 两种导出格式

其他改进:
- executor outcome 增加 runtime miss 诊断上下文
- 主 tokio runtime 栈大小调整为 8MB
- 前端 provider 管理支持 base URL 配置
- dev.sh 支持 --migrate 参数
This commit is contained in:
fawney19
2026-04-13 14:01:22 +08:00
parent 3698e5a833
commit 5bb08e6aa4
106 changed files with 21736 additions and 1529 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
apps/aether-gateway/src/executor/mod.rs
View File

@@ -16,8 +16,9 @@ pub(crate) use candidate_loop::{
};
pub(crate) use orchestration::*;
pub(crate) use outcome::{
build_local_execution_exhaustion, record_failed_usage_for_exhausted_request,
LocalExecutionExhaustion, LocalExecutionRequestOutcome,
build_local_execution_exhaustion, build_local_execution_runtime_miss_context,
record_failed_usage_for_exhausted_request, record_failed_usage_for_runtime_miss_request,
LocalExecutionExhaustion, LocalExecutionRequestOutcome, LocalExecutionRuntimeMissContext,
};
pub(crate) use plan_fallback::{
maybe_execute_stream_via_plan_fallback, maybe_execute_sync_via_plan_fallback,

651
apps/aether-gateway/src/executor/outcome.rs
View File

@@ -1,9 +1,13 @@
use std::collections::{BTreeMap, BTreeSet};
use std::time::Instant;
use aether_contracts::ExecutionPlan;
use aether_data_contracts::repository::candidates::{
RequestCandidateStatus, StoredRequestCandidate,
};
use aether_data_contracts::repository::provider_catalog::{
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
};
use aether_usage_runtime::{
build_usage_event_data_seed, UsageEvent, UsageEventData, UsageEventType,
};
@@ -12,7 +16,10 @@ use axum::http::{self, Response};
use serde_json::{json, Map, Value};
use tracing::warn;
use crate::constants::LOCAL_EXECUTION_RUNTIME_MISS_REASON_HEADER;
use crate::constants::{
EXECUTION_PATH_LOCAL_EXECUTION_RUNTIME_MISS, LOCAL_EXECUTION_RUNTIME_MISS_REASON_HEADER,
};
use crate::control::GatewayControlDecision;
use crate::state::LocalExecutionRuntimeMissDiagnostic;
use crate::AppState;
@@ -34,12 +41,59 @@ pub(crate) struct LocalExecutionExhaustion {
upstream_error_message: Option<String>,
}
#[derive(Debug, Clone, Default)]
pub(crate) struct LocalExecutionRuntimeMissContext {
pub(crate) auth_user_id: Option<String>,
pub(crate) auth_api_key_id: Option<String>,
pub(crate) auth_username: Option<String>,
pub(crate) auth_api_key_name: Option<String>,
candidate_contexts: Vec<RuntimeMissCandidateContext>,
}
#[derive(Debug, Clone)]
struct RuntimeMissCandidateContext {
candidate: StoredRequestCandidate,
provider_name: Option<String>,
key_name: Option<String>,
client_api_format: Option<String>,
provider_api_format: Option<String>,
global_model_name: Option<String>,
selected_provider_model_name: Option<String>,
endpoint_url: Option<String>,
}
impl LocalExecutionRequestOutcome {
pub(crate) fn responded(response: Response<Body>) -> Self {
Self::Responded(response)
}
}
impl LocalExecutionRuntimeMissContext {
pub(crate) fn persisted_candidate_count(&self) -> usize {
self.candidate_contexts.len()
}
pub(crate) fn candidate_summary(&self) -> Option<String> {
const MAX_ITEMS: usize = 5;
if self.candidate_contexts.is_empty() {
return None;
}
let mut summaries = self
.candidate_contexts
.iter()
.take(MAX_ITEMS)
.map(format_runtime_miss_candidate_summary)
.collect::<Vec<_>>();
let remaining = self.candidate_contexts.len().saturating_sub(MAX_ITEMS);
if remaining > 0 {
summaries.push(format!("+{remaining} more"));
}
Some(summaries.join(" | "))
}
}
pub(crate) async fn build_local_execution_exhaustion(
state: &AppState,
plan: &ExecutionPlan,
@@ -100,6 +154,22 @@ pub(crate) async fn build_local_execution_exhaustion(
}
}
pub(crate) async fn build_local_execution_runtime_miss_context(
state: &AppState,
request_id: &str,
decision: Option<&GatewayControlDecision>,
) -> LocalExecutionRuntimeMissContext {
let auth_context = decision.and_then(|value| value.auth_context.as_ref());
LocalExecutionRuntimeMissContext {
auth_user_id: auth_context.map(|value| value.user_id.clone()),
auth_api_key_id: auth_context.map(|value| value.api_key_id.clone()),
auth_username: auth_context.and_then(|value| value.username.clone()),
auth_api_key_name: auth_context.and_then(|value| value.api_key_name.clone()),
candidate_contexts: load_runtime_miss_candidate_contexts(state, request_id, decision).await,
}
}
pub(crate) async fn record_failed_usage_for_exhausted_request(
state: &AppState,
exhaustion: LocalExecutionExhaustion,
@@ -171,24 +241,150 @@ pub(crate) async fn record_failed_usage_for_exhausted_request(
None => Map::new(),
};
request_metadata.insert("trace_id".to_string(), Value::String(request_id.clone()));
if let Some(candidate_id) = candidate_id {
request_metadata.insert("candidate_id".to_string(), Value::String(candidate_id));
}
if let Some(candidate_index) = candidate_index {
request_metadata.insert(
"candidate_index".to_string(),
Value::Number(candidate_index.into()),
);
}
apply_runtime_miss_usage_routing(
&mut data,
&mut request_metadata,
candidate_id.as_deref(),
candidate_index,
None,
diagnostic,
None,
None,
);
data.request_metadata = Some(Value::Object(request_metadata));
state
.usage_runtime
.record_terminal_event(
state.data.as_ref(),
UsageEvent::new(UsageEventType::Failed, request_id, data),
)
.await;
state.usage_runtime.submit_terminal_event(
state.data.as_ref(),
UsageEvent::new(UsageEventType::Failed, request_id, data),
);
}
pub(crate) async fn record_failed_usage_for_runtime_miss_request(
state: &AppState,
request_id: &str,
started_at: &Instant,
local_execution_runtime_miss_detail: &str,
decision: Option<&GatewayControlDecision>,
diagnostic: Option<&LocalExecutionRuntimeMissDiagnostic>,
context: &LocalExecutionRuntimeMissContext,
) {
if !state.usage_runtime.is_enabled() {
return;
}
let selected_candidate = select_last_runtime_miss_candidate(&context.candidate_contexts);
let api_format = selected_candidate
.and_then(|value| value.client_api_format.clone())
.or_else(|| {
trimmed_non_empty(decision.and_then(|value| value.auth_endpoint_signature.as_deref()))
});
let provider_api_format = selected_candidate
.and_then(|value| value.provider_api_format.clone())
.or_else(|| api_format.clone());
let provider_name = selected_candidate
.and_then(|value| value.provider_name.clone())
.or_else(|| selected_candidate.and_then(|value| value.candidate.provider_id.clone()))
.or_else(|| trimmed_non_empty(decision.and_then(|value| value.route_family.as_deref())))
.unwrap_or_else(|| "unknown".to_string());
let model = trimmed_non_empty(diagnostic.and_then(|value| value.requested_model.as_deref()))
.or_else(|| selected_candidate.and_then(|value| value.global_model_name.clone()))
.or_else(|| selected_candidate.and_then(|value| value.selected_provider_model_name.clone()))
.unwrap_or_else(|| "unknown".to_string());
let target_model = selected_candidate
.and_then(|value| value.selected_provider_model_name.clone())
.filter(|value| !value.eq_ignore_ascii_case(model.as_str()));
let status_code = http::StatusCode::SERVICE_UNAVAILABLE.as_u16();
let client_body = json!({
"error": {
"type": "http_error",
"message": local_execution_runtime_miss_detail,
}
});
let mut client_headers = Map::from_iter([(
"content-type".to_string(),
Value::String("application/json".to_string()),
)]);
if let Some(reason) = diagnostic
.map(|value| value.reason.trim())
.filter(|value| !value.is_empty())
{
client_headers.insert(
LOCAL_EXECUTION_RUNTIME_MISS_REASON_HEADER.to_string(),
Value::String(reason.to_string()),
);
}
let mut request_metadata = Map::new();
request_metadata.insert(
"trace_id".to_string(),
Value::String(request_id.to_string()),
);
let mut data = UsageEventData {
user_id: context.auth_user_id.clone(),
api_key_id: context.auth_api_key_id.clone(),
username: context.auth_username.clone(),
api_key_name: context.auth_api_key_name.clone(),
provider_name,
model,
target_model,
provider_id: selected_candidate.and_then(|value| value.candidate.provider_id.clone()),
provider_endpoint_id: selected_candidate
.and_then(|value| value.candidate.endpoint_id.clone()),
provider_api_key_id: selected_candidate.and_then(|value| value.candidate.key_id.clone()),
request_type: Some(infer_request_type(api_format.as_deref())),
api_format: api_format.clone(),
api_family: api_format
.as_deref()
.and_then(infer_api_family)
.map(ToOwned::to_owned),
endpoint_kind: api_format
.as_deref()
.and_then(infer_endpoint_kind)
.map(ToOwned::to_owned),
endpoint_api_format: provider_api_format.clone(),
provider_api_family: provider_api_format
.as_deref()
.and_then(infer_api_family)
.map(ToOwned::to_owned),
provider_endpoint_kind: provider_api_format
.as_deref()
.and_then(infer_endpoint_kind)
.map(ToOwned::to_owned),
has_format_conversion: selected_candidate.and_then(|value| {
value
.client_api_format
.as_deref()
.zip(value.provider_api_format.as_deref())
.map(|(left, right)| !left.eq_ignore_ascii_case(right))
}),
status_code: Some(status_code),
error_message: Some(local_execution_runtime_miss_detail.to_string()),
error_category: error_category_for_failed_status(status_code),
response_time_ms: Some(started_at.elapsed().as_millis() as u64),
response_headers: Some(json_header_map()),
response_body: Some(client_body.clone()),
client_response_headers: Some(Value::Object(client_headers)),
client_response_body: Some(client_body),
..UsageEventData::default()
};
apply_runtime_miss_usage_routing(
&mut data,
&mut request_metadata,
selected_candidate.map(|value| value.candidate.id.as_str()),
selected_candidate.map(|value| value.candidate.candidate_index),
selected_candidate.and_then(|value| value.key_name.as_deref()),
diagnostic,
decision.and_then(|value| value.route_family.as_deref()),
decision.and_then(|value| value.route_kind.as_deref()),
);
data.request_metadata =
(!request_metadata.is_empty()).then_some(Value::Object(request_metadata));
state.usage_runtime.submit_terminal_event(
state.data.as_ref(),
UsageEvent::new(UsageEventType::Failed, request_id, data),
);
}
fn select_last_failed_request_candidate(
@@ -214,6 +410,22 @@ fn select_last_failed_request_candidate(
})
}
fn select_last_runtime_miss_candidate(
candidates: &[RuntimeMissCandidateContext],
) -> Option<&RuntimeMissCandidateContext> {
candidates.iter().max_by_key(|candidate| {
(
candidate.candidate.retry_index,
candidate.candidate.candidate_index,
candidate
.candidate
.finished_at_unix_ms
.or(candidate.candidate.started_at_unix_ms)
.unwrap_or(candidate.candidate.created_at_unix_ms),
)
})
}
fn error_category_for_failed_status(status_code: u16) -> Option<String> {
if status_code >= 500 {
Some("server_error".to_string())
@@ -230,3 +442,408 @@ fn json_header_map() -> Value {
Value::String("application/json".to_string()),
)]))
}
async fn load_runtime_miss_candidate_contexts(
state: &AppState,
request_id: &str,
decision: Option<&GatewayControlDecision>,
) -> Vec<RuntimeMissCandidateContext> {
let mut candidates = match state
.read_request_candidates_by_request_id(request_id)
.await
{
Ok(candidates) => candidates,
Err(err) => {
warn!(
request_id = %request_id,
error = ?err,
"gateway failed to load request candidates for local execution runtime miss"
);
return Vec::new();
}
};
if candidates.is_empty() {
return Vec::new();
}
candidates.sort_by_key(|candidate| {
(
candidate.candidate_index,
candidate.retry_index,
candidate.created_at_unix_ms,
)
});
let (providers_by_id, endpoints_by_id, keys_by_id) = if state.has_provider_catalog_data_reader()
{
let provider_ids = collect_present_ids(
candidates
.iter()
.filter_map(|value| value.provider_id.as_deref()),
);
let endpoint_ids = collect_present_ids(
candidates
.iter()
.filter_map(|value| value.endpoint_id.as_deref()),
);
let key_ids = collect_present_ids(
candidates
.iter()
.filter_map(|value| value.key_id.as_deref()),
);
let (providers_result, endpoints_result, keys_result) = tokio::join!(
state.read_provider_catalog_providers_by_ids(&provider_ids),
state.read_provider_catalog_endpoints_by_ids(&endpoint_ids),
state.read_provider_catalog_keys_by_ids(&key_ids),
);
(
match providers_result {
Ok(values) => values
.into_iter()
.map(|value| (value.id.clone(), value))
.collect::<BTreeMap<_, _>>(),
Err(err) => {
warn!(
request_id = %request_id,
error = ?err,
"gateway failed to load provider catalog providers for local execution runtime miss"
);
BTreeMap::new()
}
},
match endpoints_result {
Ok(values) => values
.into_iter()
.map(|value| (value.id.clone(), value))
.collect::<BTreeMap<_, _>>(),
Err(err) => {
warn!(
request_id = %request_id,
error = ?err,
"gateway failed to load provider catalog endpoints for local execution runtime miss"
);
BTreeMap::new()
}
},
match keys_result {
Ok(values) => values
.into_iter()
.map(|value| (value.id.clone(), value))
.collect::<BTreeMap<_, _>>(),
Err(err) => {
warn!(
request_id = %request_id,
error = ?err,
"gateway failed to load provider catalog keys for local execution runtime miss"
);
BTreeMap::new()
}
},
)
} else {
(BTreeMap::new(), BTreeMap::new(), BTreeMap::new())
};
candidates
.into_iter()
.map(|candidate| {
let provider = candidate
.provider_id
.as_deref()
.and_then(|value| providers_by_id.get(value));
let endpoint = candidate
.endpoint_id
.as_deref()
.and_then(|value| endpoints_by_id.get(value));
let key = candidate
.key_id
.as_deref()
.and_then(|value| keys_by_id.get(value));
RuntimeMissCandidateContext {
provider_name: candidate_extra_data_string(&candidate, "provider_name")
.or_else(|| provider.map(|value| value.name.clone())),
key_name: candidate_extra_data_string(&candidate, "key_name")
.or_else(|| key.map(|value| value.name.clone())),
client_api_format: candidate_extra_data_string(&candidate, "client_api_format")
.or_else(|| candidate_extra_data_string(&candidate, "client_contract")),
provider_api_format: candidate_extra_data_string(&candidate, "provider_api_format")
.or_else(|| candidate_extra_data_string(&candidate, "provider_contract"))
.or_else(|| endpoint.map(|value| value.api_format.clone())),
global_model_name: candidate_extra_data_string(&candidate, "global_model_name"),
selected_provider_model_name: candidate_extra_data_string(
&candidate,
"selected_provider_model_name",
),
endpoint_url: endpoint
.and_then(|value| build_runtime_miss_candidate_endpoint_url(value, decision)),
candidate,
}
})
.collect()
}
fn collect_present_ids<'a>(ids: impl Iterator<Item = &'a str>) -> Vec<String> {
ids.filter_map(|value| {
let trimmed = value.trim();
(!trimmed.is_empty()).then_some(trimmed.to_string())
})
.collect::<BTreeSet<_>>()
.into_iter()
.collect()
}
fn candidate_extra_data_string(candidate: &StoredRequestCandidate, key: &str) -> Option<String> {
candidate
.extra_data
.as_ref()
.and_then(Value::as_object)
.and_then(|value| value.get(key))
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
}
fn build_runtime_miss_candidate_endpoint_url(
endpoint: &StoredProviderCatalogEndpoint,
decision: Option<&GatewayControlDecision>,
) -> Option<String> {
let path = endpoint
.custom_path
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
.or_else(|| {
decision
.map(|value| value.public_path.as_str())
.map(str::trim)
.filter(|value| !value.is_empty())
});
let query = decision
.and_then(|value| value.public_query_string.as_deref())
.map(str::trim)
.filter(|value| !value.is_empty());
path.and_then(|value| {
crate::provider_transport::url::build_passthrough_path_url(
&endpoint.base_url,
value,
query,
&[],
)
})
.or_else(|| trimmed_non_empty(Some(endpoint.base_url.as_str())))
}
fn format_runtime_miss_candidate_summary(candidate: &RuntimeMissCandidateContext) -> String {
let mut parts = Vec::new();
parts.push(format!("idx={}", candidate.candidate.candidate_index));
parts.push(format!("retry={}", candidate.candidate.retry_index));
parts.push(format!(
"status={}",
request_candidate_status_label(candidate.candidate.status)
));
if let Some(provider_label) = format_name_with_id(
candidate.provider_name.as_deref(),
candidate.candidate.provider_id.as_deref(),
) {
parts.push(format!("provider={provider_label}"));
}
if let Some(endpoint_id) = candidate
.candidate
.endpoint_id
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
{
parts.push(format!("endpoint={endpoint_id}"));
}
if let Some(endpoint_url) = candidate
.endpoint_url
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
{
parts.push(format!("url={endpoint_url}"));
}
if let Some(key_label) = format_name_with_id(
candidate.key_name.as_deref(),
candidate.candidate.key_id.as_deref(),
) {
parts.push(format!("key={key_label}"));
}
if let Some(skip_reason) = candidate
.candidate
.skip_reason
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
{
parts.push(format!("skip={skip_reason}"));
}
if let Some(status_code) = candidate.candidate.status_code {
parts.push(format!("code={status_code}"));
}
if let Some(error_type) = candidate
.candidate
.error_type
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
{
parts.push(format!("error_type={error_type}"));
}
parts.join(" ")
}
fn format_name_with_id(name: Option<&str>, id: Option<&str>) -> Option<String> {
let name = name.map(str::trim).filter(|value| !value.is_empty());
let id = id.map(str::trim).filter(|value| !value.is_empty());
match (name, id) {
(Some(name), Some(id)) => Some(format!("{name}({id})")),
(Some(name), None) => Some(name.to_string()),
(None, Some(id)) => Some(id.to_string()),
(None, None) => None,
}
}
fn request_candidate_status_label(status: RequestCandidateStatus) -> &'static str {
match status {
RequestCandidateStatus::Available => "available",
RequestCandidateStatus::Unused => "unused",
RequestCandidateStatus::Pending => "pending",
RequestCandidateStatus::Streaming => "streaming",
RequestCandidateStatus::Success => "success",
RequestCandidateStatus::Failed => "failed",
RequestCandidateStatus::Cancelled => "cancelled",
RequestCandidateStatus::Skipped => "skipped",
}
}
fn infer_request_type(api_format: Option<&str>) -> String {
match infer_endpoint_kind(api_format.unwrap_or_default()) {
Some("video") => "video".to_string(),
Some("image") => "image".to_string(),
_ => "chat".to_string(),
}
}
fn infer_api_family(api_format: &str) -> Option<&str> {
api_format.split_once(':').map(|(family, _)| family)
}
fn infer_endpoint_kind(api_format: &str) -> Option<&str> {
api_format.split_once(':').map(|(_, kind)| kind)
}
fn apply_runtime_miss_usage_routing(
data: &mut UsageEventData,
request_metadata: &mut Map<String, Value>,
candidate_id: Option<&str>,
candidate_index: Option<u32>,
key_name: Option<&str>,
diagnostic: Option<&LocalExecutionRuntimeMissDiagnostic>,
route_family_fallback: Option<&str>,
route_kind_fallback: Option<&str>,
) {
data.candidate_id = data
.candidate_id
.clone()
.or_else(|| trimmed_non_empty(candidate_id));
data.candidate_index = data
.candidate_index
.or_else(|| candidate_index.map(u64::from));
data.key_name = data
.key_name
.clone()
.or_else(|| trimmed_non_empty(key_name));
data.execution_path = data
.execution_path
.clone()
.or_else(|| Some(EXECUTION_PATH_LOCAL_EXECUTION_RUNTIME_MISS.to_string()));
data.local_execution_runtime_miss_reason = data
.local_execution_runtime_miss_reason
.clone()
.or_else(|| trimmed_non_empty(diagnostic.map(|value| value.reason.as_str())));
data.route_family = data.route_family.clone().or_else(|| {
trimmed_non_empty(
diagnostic
.and_then(|value| value.route_family.as_deref())
.or(route_family_fallback),
)
});
data.route_kind = data.route_kind.clone().or_else(|| {
trimmed_non_empty(
diagnostic
.and_then(|value| value.route_kind.as_deref())
.or(route_kind_fallback),
)
});
data.planner_kind = data
.planner_kind
.clone()
.or_else(|| trimmed_non_empty(diagnostic.and_then(|value| value.plan_kind.as_deref())));
let _ = request_metadata;
}
fn trimmed_non_empty(value: Option<&str>) -> Option<String> {
value
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
}
#[cfg(test)]
mod tests {
use super::apply_runtime_miss_usage_routing;
use crate::constants::EXECUTION_PATH_LOCAL_EXECUTION_RUNTIME_MISS;
use crate::state::LocalExecutionRuntimeMissDiagnostic;
use aether_usage_runtime::UsageEventData;
use serde_json::{json, Map, Value};
#[test]
fn runtime_miss_routing_moves_to_typed_usage_fields_and_keeps_metadata_lightweight() {
let mut data = UsageEventData::default();
let mut request_metadata =
Map::from_iter([("trace_id".to_string(), Value::String("trace-1".to_string()))]);
apply_runtime_miss_usage_routing(
&mut data,
&mut request_metadata,
Some("cand-1"),
Some(2),
Some("primary"),
Some(&LocalExecutionRuntimeMissDiagnostic {
reason: "all_candidates_skipped".to_string(),
route_family: Some("claude".to_string()),
route_kind: Some("cli".to_string()),
plan_kind: Some("claude_cli_sync".to_string()),
..LocalExecutionRuntimeMissDiagnostic::default()
}),
None,
None,
);
assert_eq!(data.candidate_id.as_deref(), Some("cand-1"));
assert_eq!(data.candidate_index, Some(2));
assert_eq!(data.key_name.as_deref(), Some("primary"));
assert_eq!(data.planner_kind.as_deref(), Some("claude_cli_sync"));
assert_eq!(data.route_family.as_deref(), Some("claude"));
assert_eq!(data.route_kind.as_deref(), Some("cli"));
assert_eq!(
data.execution_path.as_deref(),
Some(EXECUTION_PATH_LOCAL_EXECUTION_RUNTIME_MISS)
);
assert_eq!(
data.local_execution_runtime_miss_reason.as_deref(),
Some("all_candidates_skipped")
);
assert_eq!(
Value::Object(request_metadata),
json!({
"trace_id": "trace-1"
})
);
}
}