feat(gateway): 重构 usage 数据层、迁移系统与系统导入

数据库迁移:
- 引入 baseline v2 bootstrap,空库首次启动自动初始化
- 服务启动不再自动执行迁移,需显式 `--migrate` 运行
- 新增 pending migration 检测,schema 落后时拒绝启动

Usage 数据层:
- usage body 存储外部化为独立 blob 表
- 新增 HTTP audit 表拆分存储请求/响应头与 body ref
- 后台清理任务支持 legacy body ref 元数据迁移
- usage runtime 写入迁移到专用 tokio runtime(独立线程池, 8MB 栈)

系统导入/导出:
- 支持用户、API Keys、钱包数据的完整导入
- 兼容 legacy 与 v1.3+ 两种导出格式

其他改进:
- executor outcome 增加 runtime miss 诊断上下文
- 主 tokio runtime 栈大小调整为 8MB
- 前端 provider 管理支持 base URL 配置
- dev.sh 支持 --migrate 参数
This commit is contained in:
fawney19
2026-04-13 14:01:22 +08:00
parent 3698e5a833
commit 5bb08e6aa4
106 changed files with 21730 additions and 1523 deletions
@@ -1060,23 +1060,21 @@ pub(super) async fn handle_dashboard_recent_requests_get(
.into_iter()
.collect::<Vec<_>>();
let users_by_id: BTreeMap<String, aether_data::repository::users::StoredUserSummary> =
if state.has_user_data_reader() && !user_ids.is_empty() {
match state.list_users_by_ids(&user_ids).await {
Ok(value) => value
.into_iter()
.map(|user| (user.id.clone(), user))
.collect(),
Err(err) => {
return build_auth_error_response(
http::StatusCode::INTERNAL_SERVER_ERROR,
format!("dashboard user lookup failed: {err:?}"),
false,
)
}
match state.resolve_auth_user_summaries_by_ids(&user_ids).await {
Ok(value) => value,
Err(err) => {
return build_auth_error_response(
http::StatusCode::INTERNAL_SERVER_ERROR,
format!("dashboard user lookup failed: {err:?}"),
false,
)
}
} else {
BTreeMap::new()
};
let mut usernames_by_id: BTreeMap<String, String> = users_by_id
.iter()
.filter(|(_, user)| !user.username.trim().is_empty())
.map(|(user_id, user)| (user_id.clone(), user.username.clone()))
.collect();
let requests = usage
.into_iter()
@@ -1084,9 +1082,13 @@ pub(super) async fn handle_dashboard_recent_requests_get(
let username = item
.user_id
.as_ref()
.and_then(|user_id| users_by_id.get(user_id))
.map(|user| user.username.clone())
.or(item.username.clone())
.and_then(|user_id| usernames_by_id.get(user_id))
.cloned()
.or_else(|| {
(!state.has_auth_user_data_reader())
.then(|| item.username.clone())
.flatten()
})
.unwrap_or_else(|| "Unknown".to_string());
json!({
"id": item.id,
@@ -530,8 +530,17 @@ pub(super) async fn handle_users_me_api_key_create(
key_hash: hash_users_me_api_key(&plaintext_key),
key_encrypted: Some(key_encrypted),
name: Some(name.clone()),
allowed_providers: None,
allowed_api_formats: None,
allowed_models: None,
rate_limit,
concurrent_limit: 5,
force_capabilities: None,
is_active: true,
expires_at_unix_secs: None,
auto_delete_on_expiry: false,
total_requests: 0,
total_cost_usd: 0.0,
};
let Some(created) = (match state.create_user_api_key(record).await {
Ok(value) => value,
@@ -145,32 +145,56 @@ fn users_me_usage_cache_hit_rate(total_input_context: u64, cache_read_tokens: u6
}
}
fn users_me_usage_matches_search(item: &StoredRequestUsageAudit, search: Option<&str>) -> bool {
fn users_me_usage_api_key_name(
item: &StoredRequestUsageAudit,
api_key_names: &BTreeMap<String, String>,
auth_api_key_reader_available: bool,
) -> Option<String> {
item.api_key_id
.as_ref()
.and_then(|api_key_id| api_key_names.get(api_key_id))
.cloned()
.or_else(|| {
(!auth_api_key_reader_available)
.then(|| item.api_key_name.clone())
.flatten()
})
}
fn users_me_usage_matches_search(
item: &StoredRequestUsageAudit,
search: Option<&str>,
api_key_names: &BTreeMap<String, String>,
auth_api_key_reader_available: bool,
) -> bool {
let Some(search) = search.map(str::trim).filter(|value| !value.is_empty()) else {
return true;
};
let model = item.model.to_ascii_lowercase();
let api_key_name = item
.api_key_name
.as_deref()
.unwrap_or_default()
.to_ascii_lowercase();
let api_key_name =
users_me_usage_api_key_name(item, api_key_names, auth_api_key_reader_available)
.as_deref()
.unwrap_or_default()
.to_ascii_lowercase();
search.split_whitespace().all(|keyword| {
let keyword = keyword.to_ascii_lowercase();
model.contains(&keyword) || api_key_name.contains(&keyword)
})
}
fn build_users_me_usage_api_key_payload(item: &StoredRequestUsageAudit) -> serde_json::Value {
fn build_users_me_usage_api_key_payload(
item: &StoredRequestUsageAudit,
api_key_names: &BTreeMap<String, String>,
auth_api_key_reader_available: bool,
) -> serde_json::Value {
let api_key_name =
users_me_usage_api_key_name(item, api_key_names, auth_api_key_reader_available);
match item.api_key_id.as_deref() {
Some(api_key_id) => json!({
"id": api_key_id,
"name": item.api_key_name.clone(),
"display": item
.api_key_name
.clone()
.unwrap_or_else(|| api_key_id.to_string()),
"name": api_key_name.clone(),
"display": api_key_name.unwrap_or_else(|| api_key_id.to_string()),
}),
None => serde_json::Value::Null,
}
@@ -179,7 +203,14 @@ fn build_users_me_usage_api_key_payload(item: &StoredRequestUsageAudit) -> serde
fn build_users_me_usage_record_payload(
item: &StoredRequestUsageAudit,
include_actual_cost: bool,
api_key_names: &BTreeMap<String, String>,
auth_api_key_reader_available: bool,
) -> serde_json::Value {
let input_price_per_1m = item.settlement_input_price_per_1m();
let output_price_per_1m = item.settlement_output_price_per_1m();
let cache_creation_price_per_1m = item.settlement_cache_creation_price_per_1m();
let cache_read_price_per_1m = item.settlement_cache_read_price_per_1m();
let rate_multiplier = item.settlement_rate_multiplier();
let mut payload = json!({
"id": item.id,
"model": item.model,
@@ -203,11 +234,15 @@ fn build_users_me_usage_record_payload(
"cache_read_input_tokens": item.cache_read_input_tokens,
"status_code": item.status_code,
"error_message": item.error_message,
"input_price_per_1m": serde_json::Value::Null,
"output_price_per_1m": item.output_price_per_1m,
"cache_creation_price_per_1m": serde_json::Value::Null,
"cache_read_price_per_1m": serde_json::Value::Null,
"api_key": build_users_me_usage_api_key_payload(item),
"input_price_per_1m": input_price_per_1m,
"output_price_per_1m": output_price_per_1m,
"cache_creation_price_per_1m": cache_creation_price_per_1m,
"cache_read_price_per_1m": cache_read_price_per_1m,
"api_key": build_users_me_usage_api_key_payload(
item,
api_key_names,
auth_api_key_reader_available,
),
});
if item.target_model.is_some() {
@@ -215,7 +250,7 @@ fn build_users_me_usage_record_payload(
}
if include_actual_cost {
payload["actual_cost"] = json!(round_to(item.actual_total_cost_usd, 6));
payload["rate_multiplier"] = serde_json::Value::Null;
payload["rate_multiplier"] = json!(rate_multiplier);
}
payload
}
@@ -233,7 +268,7 @@ fn build_users_me_usage_active_payload(item: &StoredRequestUsageAudit) -> serde_
"cache_read_input_tokens": item.cache_read_input_tokens,
"cost": round_to(item.total_cost_usd, 6),
"actual_cost": round_to(item.actual_total_cost_usd, 6),
"rate_multiplier": serde_json::Value::Null,
"rate_multiplier": item.settlement_rate_multiplier(),
"response_time_ms": item.response_time_ms,
"first_byte_time_ms": item.first_byte_time_ms,
"api_format": item.api_format,
@@ -675,6 +710,30 @@ pub(super) async fn handle_users_me_usage_get(
)
}
};
let api_key_names = if state.has_auth_api_key_data_reader() {
let api_key_ids = usage
.iter()
.filter_map(|item| item.api_key_id.clone())
.collect::<BTreeSet<_>>()
.into_iter()
.collect::<Vec<_>>();
if api_key_ids.is_empty() {
BTreeMap::new()
} else {
match state.resolve_auth_api_key_names_by_ids(&api_key_ids).await {
Ok(value) => value,
Err(err) => {
return build_auth_error_response(
http::StatusCode::INTERNAL_SERVER_ERROR,
format!("user api key name lookup failed: {err:?}"),
false,
)
}
}
}
} else {
BTreeMap::new()
};
let summary_items = usage
.iter()
@@ -738,7 +797,14 @@ pub(super) async fn handle_users_me_usage_get(
let mut records = usage
.into_iter()
.filter(|item| users_me_usage_matches_search(item, search.as_deref()))
.filter(|item| {
users_me_usage_matches_search(
item,
search.as_deref(),
&api_key_names,
state.has_auth_api_key_data_reader(),
)
})
.collect::<Vec<_>>();
records.sort_by(|left, right| {
right
@@ -751,7 +817,14 @@ pub(super) async fn handle_users_me_usage_get(
.into_iter()
.skip(offset)
.take(limit)
.map(|item| build_users_me_usage_record_payload(&item, include_actual_cost))
.map(|item| {
build_users_me_usage_record_payload(
&item,
include_actual_cost,
&api_key_names,
state.has_auth_api_key_data_reader(),
)
})
.collect::<Vec<_>>();
let wallet = state