mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 09:20:22 +08:00
refactor: 拆分 gateway 单体为独立 crate,新增 systemd 部署方案
将 gateway 内部的 model-fetch、provider-transport、scheduler-core、 usage-runtime、video-tasks-core 模块提取为独立 crate;重构 gateway 内部模块结构(state/router/cache/data/query 等);移除大量遗留模块 文件;新增 systemd 二进制部署骨架及相关文档;更新前端 usage 相关 API 和组件。
This commit is contained in:
@@ -1,4 +1,6 @@
|
||||
mod runtime;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
pub(crate) use runtime::{
|
||||
perform_provider_checkin_once, spawn_audit_cleanup_worker, spawn_db_maintenance_worker,
|
||||
|
||||
@@ -15,9 +15,9 @@ use sqlx::Row;
|
||||
use tracing::{debug, info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::gateway::gateway_data::GatewayDataState;
|
||||
use crate::gateway::handlers::admin::admin_provider_ops_local_action_response;
|
||||
use crate::gateway::{AppState, GatewayError};
|
||||
use crate::data::GatewayDataState;
|
||||
use crate::handlers::admin::provider_ops::admin_provider_ops_local_action_response;
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
#[path = "runtime/audit_cleanup.rs"]
|
||||
mod audit_cleanup;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use chrono::{DateTime, Utc};
|
||||
|
||||
use crate::gateway::gateway_data::GatewayDataState;
|
||||
use crate::data::GatewayDataState;
|
||||
|
||||
use super::{
|
||||
system_config_bool, system_config_u64, system_config_usize, DELETE_AUDIT_LOGS_BEFORE_SQL,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::gateway::gateway_data::GatewayDataState;
|
||||
use crate::data::GatewayDataState;
|
||||
|
||||
use super::{UsageCleanupSettings, UsageCleanupWindow};
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use tracing::warn;
|
||||
|
||||
use crate::gateway::gateway_data::GatewayDataState;
|
||||
use crate::data::GatewayDataState;
|
||||
|
||||
use super::{system_config_bool, DB_MAINTENANCE_TABLES};
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ use std::collections::HashSet;
|
||||
use chrono::Utc;
|
||||
use sqlx::Row;
|
||||
|
||||
use crate::gateway::gateway_data::GatewayDataState;
|
||||
use crate::data::GatewayDataState;
|
||||
|
||||
use super::{
|
||||
pending_cleanup_batch_size, pending_cleanup_timeout_minutes,
|
||||
|
||||
@@ -6,8 +6,8 @@ use aether_data::repository::provider_catalog::{
|
||||
use futures_util::stream::{self, StreamExt};
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use crate::gateway::handlers::admin::admin_provider_ops_local_action_response;
|
||||
use crate::gateway::{AppState, GatewayError};
|
||||
use crate::handlers::admin::provider_ops::admin_provider_ops_local_action_response;
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
use super::{system_config_bool, PROVIDER_CHECKIN_CONCURRENCY};
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use tracing::info;
|
||||
|
||||
use crate::gateway::gateway_data::GatewayDataState;
|
||||
use crate::data::GatewayDataState;
|
||||
|
||||
use super::{now_unix_secs, system_config_bool, system_config_u64, system_config_usize};
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use tracing::info;
|
||||
|
||||
use crate::gateway::gateway_data::GatewayDataState;
|
||||
use crate::gateway::{AppState, GatewayError};
|
||||
use crate::data::GatewayDataState;
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
use super::{
|
||||
cleanup_audit_logs_once, cleanup_expired_gemini_file_mappings_once,
|
||||
@@ -16,7 +16,13 @@ pub(super) async fn run_audit_cleanup_once(
|
||||
) -> Result<(), aether_data::DataLayerError> {
|
||||
let deleted = cleanup_audit_logs_once(data).await?;
|
||||
if deleted > 0 {
|
||||
info!(deleted, "gateway deleted expired audit logs");
|
||||
info!(
|
||||
event_name = "audit_cleanup_completed",
|
||||
log_type = "ops",
|
||||
worker = "audit_cleanup",
|
||||
deleted,
|
||||
"gateway deleted expired audit logs"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -26,7 +32,13 @@ pub(super) async fn run_gemini_file_mapping_cleanup_once(
|
||||
) -> Result<(), aether_data::DataLayerError> {
|
||||
let deleted = cleanup_expired_gemini_file_mappings_once(data).await?;
|
||||
if deleted > 0 {
|
||||
info!(deleted, "gateway deleted expired gemini file mappings");
|
||||
info!(
|
||||
event_name = "gemini_file_mapping_cleanup_completed",
|
||||
log_type = "ops",
|
||||
worker = "gemini_file_mapping_cleanup",
|
||||
deleted,
|
||||
"gateway deleted expired gemini file mappings"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -37,6 +49,9 @@ pub(super) async fn run_db_maintenance_once(
|
||||
let summary = perform_db_maintenance_once(data).await?;
|
||||
if summary.attempted > 0 {
|
||||
info!(
|
||||
event_name = "db_maintenance_completed",
|
||||
log_type = "ops",
|
||||
worker = "db_maintenance",
|
||||
attempted = summary.attempted,
|
||||
succeeded = summary.succeeded,
|
||||
failed = summary.attempted.saturating_sub(summary.succeeded),
|
||||
@@ -51,6 +66,9 @@ pub(super) async fn run_wallet_daily_usage_aggregation_once(
|
||||
) -> Result<(), aether_data::DataLayerError> {
|
||||
let summary = perform_wallet_daily_usage_aggregation_once(data).await?;
|
||||
info!(
|
||||
event_name = "wallet_daily_usage_aggregation_completed",
|
||||
log_type = "ops",
|
||||
worker = "wallet_daily_usage_aggregation",
|
||||
billing_date = %summary.billing_date,
|
||||
billing_timezone = %summary.billing_timezone,
|
||||
wallets = summary.aggregated_wallets,
|
||||
@@ -68,6 +86,9 @@ pub(super) async fn run_stats_aggregation_once(
|
||||
};
|
||||
|
||||
info!(
|
||||
event_name = "stats_daily_aggregation_completed",
|
||||
log_type = "ops",
|
||||
worker = "stats_daily_aggregation",
|
||||
day_start_utc = %summary.day_start_utc,
|
||||
total_requests = summary.total_requests,
|
||||
model_rows = summary.model_rows,
|
||||
@@ -91,6 +112,9 @@ pub(super) async fn run_usage_cleanup_once(
|
||||
|| summary.records_deleted > 0
|
||||
{
|
||||
info!(
|
||||
event_name = "usage_cleanup_completed",
|
||||
log_type = "ops",
|
||||
worker = "usage_cleanup",
|
||||
body_compressed = summary.body_compressed,
|
||||
body_cleaned = summary.body_cleaned,
|
||||
header_cleaned = summary.header_cleaned,
|
||||
@@ -108,6 +132,9 @@ pub(super) fn run_pool_monitor_once(data: &GatewayDataState) {
|
||||
};
|
||||
|
||||
info!(
|
||||
event_name = "postgres_pool_sampled",
|
||||
log_type = "ops",
|
||||
worker = "pool_monitor",
|
||||
checked_out = summary.checked_out,
|
||||
pool_size = summary.pool_size,
|
||||
idle = summary.idle,
|
||||
@@ -123,6 +150,9 @@ pub(super) async fn run_pending_cleanup_once(
|
||||
let summary = cleanup_stale_pending_requests_once(data).await?;
|
||||
if summary.failed > 0 || summary.recovered > 0 {
|
||||
info!(
|
||||
event_name = "pending_cleanup_completed",
|
||||
log_type = "ops",
|
||||
worker = "pending_cleanup",
|
||||
failed = summary.failed,
|
||||
recovered = summary.recovered,
|
||||
"gateway cleaned stale pending and streaming requests"
|
||||
@@ -139,6 +169,9 @@ pub(super) async fn run_stats_hourly_aggregation_once(
|
||||
};
|
||||
|
||||
info!(
|
||||
event_name = "stats_hourly_aggregation_completed",
|
||||
log_type = "ops",
|
||||
worker = "stats_hourly_aggregation",
|
||||
hour_utc = %summary.hour_utc,
|
||||
total_requests = summary.total_requests,
|
||||
user_rows = summary.user_rows,
|
||||
@@ -153,6 +186,9 @@ pub(super) async fn run_provider_checkin_once(state: &AppState) -> Result<(), Ga
|
||||
let summary = perform_provider_checkin_once(state).await?;
|
||||
if summary.attempted > 0 {
|
||||
info!(
|
||||
event_name = "provider_checkin_completed",
|
||||
log_type = "ops",
|
||||
worker = "provider_checkin",
|
||||
attempted = summary.attempted,
|
||||
succeeded = summary.succeeded,
|
||||
failed = summary.failed,
|
||||
|
||||
@@ -4,7 +4,7 @@ use chrono::{DateTime, Datelike, TimeZone, Timelike, Utc, Weekday};
|
||||
use chrono_tz::Tz;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::gateway::gateway_data::GatewayDataState;
|
||||
use crate::data::GatewayDataState;
|
||||
|
||||
use super::{
|
||||
system_config_string, WalletDailyUsageAggregationTarget, DB_MAINTENANCE_HOUR,
|
||||
|
||||
@@ -4,7 +4,7 @@ use chrono::{DateTime, Utc};
|
||||
use sqlx::Row;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::gateway::gateway_data::GatewayDataState;
|
||||
use crate::data::GatewayDataState;
|
||||
|
||||
use super::{
|
||||
stats_aggregation_target_day, system_config_bool, PercentileSummary, StatsAggregationSummary,
|
||||
|
||||
@@ -2,7 +2,7 @@ use chrono::{DateTime, Utc};
|
||||
use sqlx::Row;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::gateway::gateway_data::GatewayDataState;
|
||||
use crate::data::GatewayDataState;
|
||||
|
||||
use super::{
|
||||
stats_hourly_aggregation_target_hour, system_config_bool, SELECT_STATS_HOURLY_AGGREGATE_SQL,
|
||||
|
||||
@@ -276,7 +276,7 @@ fn usage_cleanup_window_uses_non_overlapping_ranges() {
|
||||
#[tokio::test]
|
||||
async fn summarize_postgres_pool_uses_busy_connections_for_usage_rate() {
|
||||
let data = GatewayDataState::from_config(
|
||||
crate::gateway::gateway_data::GatewayDataConfig::from_postgres_config(
|
||||
crate::data::GatewayDataConfig::from_postgres_config(
|
||||
aether_data::postgres::PostgresPoolConfig {
|
||||
database_url: "postgres://localhost/aether".to_string(),
|
||||
min_connections: 1,
|
||||
|
||||
@@ -6,7 +6,7 @@ use serde_json::Value;
|
||||
use sqlx::Row;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::gateway::gateway_data::GatewayDataState;
|
||||
use crate::data::GatewayDataState;
|
||||
|
||||
use super::{
|
||||
system_config_bool, usage_cleanup_settings, usage_cleanup_window, ExpiredApiKeyRow,
|
||||
|
||||
@@ -2,7 +2,7 @@ use chrono::{DateTime, Utc};
|
||||
use sqlx::Row;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::gateway::gateway_data::GatewayDataState;
|
||||
use crate::data::GatewayDataState;
|
||||
|
||||
use super::{
|
||||
maintenance_timezone, wallet_daily_usage_aggregation_target,
|
||||
|
||||
@@ -3,8 +3,8 @@ use std::sync::Arc;
|
||||
use chrono::Utc;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::gateway::gateway_data::GatewayDataState;
|
||||
use crate::gateway::AppState;
|
||||
use crate::data::GatewayDataState;
|
||||
use crate::AppState;
|
||||
|
||||
use super::{
|
||||
duration_until_next_daily_run, duration_until_next_db_maintenance_run,
|
||||
@@ -20,6 +20,21 @@ use super::{
|
||||
WALLET_DAILY_USAGE_AGGREGATION_MINUTE,
|
||||
};
|
||||
|
||||
fn log_maintenance_worker_failure(
|
||||
worker: &'static str,
|
||||
phase: &'static str,
|
||||
error: &impl std::fmt::Debug,
|
||||
) {
|
||||
warn!(
|
||||
event_name = "maintenance_worker_failed",
|
||||
log_type = "ops",
|
||||
worker,
|
||||
phase,
|
||||
error = ?error,
|
||||
"gateway maintenance worker failed"
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn spawn_audit_cleanup_worker(
|
||||
data: Arc<GatewayDataState>,
|
||||
) -> Option<tokio::task::JoinHandle<()>> {
|
||||
@@ -29,7 +44,7 @@ pub(crate) fn spawn_audit_cleanup_worker(
|
||||
|
||||
Some(tokio::spawn(async move {
|
||||
if let Err(err) = run_audit_cleanup_once(&data).await {
|
||||
warn!(error = %err, "gateway audit cleanup startup failed");
|
||||
log_maintenance_worker_failure("audit_cleanup", "startup", &err);
|
||||
}
|
||||
let mut interval = tokio::time::interval(AUDIT_LOG_CLEANUP_INTERVAL);
|
||||
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
||||
@@ -37,7 +52,7 @@ pub(crate) fn spawn_audit_cleanup_worker(
|
||||
loop {
|
||||
interval.tick().await;
|
||||
if let Err(err) = run_audit_cleanup_once(&data).await {
|
||||
warn!(error = %err, "gateway audit cleanup tick failed");
|
||||
log_maintenance_worker_failure("audit_cleanup", "tick", &err);
|
||||
}
|
||||
}
|
||||
}))
|
||||
@@ -55,7 +70,7 @@ pub(crate) fn spawn_db_maintenance_worker(
|
||||
loop {
|
||||
tokio::time::sleep(duration_until_next_db_maintenance_run(Utc::now(), timezone)).await;
|
||||
if let Err(err) = run_db_maintenance_once(&data).await {
|
||||
warn!(error = %err, "gateway db maintenance tick failed");
|
||||
log_maintenance_worker_failure("db_maintenance", "tick", &err);
|
||||
}
|
||||
}
|
||||
}))
|
||||
@@ -79,7 +94,7 @@ pub(crate) fn spawn_wallet_daily_usage_aggregation_worker(
|
||||
))
|
||||
.await;
|
||||
if let Err(err) = run_wallet_daily_usage_aggregation_once(&data).await {
|
||||
warn!(error = %err, "gateway wallet daily usage aggregation tick failed");
|
||||
log_maintenance_worker_failure("wallet_daily_usage_aggregation", "tick", &err);
|
||||
}
|
||||
}
|
||||
}))
|
||||
@@ -96,7 +111,7 @@ pub(crate) fn spawn_stats_aggregation_worker(
|
||||
loop {
|
||||
tokio::time::sleep(duration_until_next_stats_aggregation_run(Utc::now())).await;
|
||||
if let Err(err) = run_stats_aggregation_once(&data).await {
|
||||
warn!(error = %err, "gateway stats aggregation tick failed");
|
||||
log_maintenance_worker_failure("stats_daily_aggregation", "tick", &err);
|
||||
}
|
||||
}
|
||||
}))
|
||||
@@ -120,7 +135,7 @@ pub(crate) fn spawn_usage_cleanup_worker(
|
||||
))
|
||||
.await;
|
||||
if let Err(err) = run_usage_cleanup_once(&data).await {
|
||||
warn!(error = %err, "gateway usage cleanup tick failed");
|
||||
log_maintenance_worker_failure("usage_cleanup", "tick", &err);
|
||||
}
|
||||
}
|
||||
}))
|
||||
@@ -140,6 +155,10 @@ pub(crate) fn spawn_provider_checkin_worker(
|
||||
Ok(schedule) => schedule,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event_name = "maintenance_schedule_lookup_failed",
|
||||
log_type = "ops",
|
||||
worker = "provider_checkin",
|
||||
phase = "schedule_lookup",
|
||||
error = %err,
|
||||
fallback = PROVIDER_CHECKIN_DEFAULT_TIME,
|
||||
"gateway provider checkin schedule lookup failed; falling back"
|
||||
@@ -156,7 +175,7 @@ pub(crate) fn spawn_provider_checkin_worker(
|
||||
))
|
||||
.await;
|
||||
if let Err(err) = run_provider_checkin_once(&state).await {
|
||||
warn!(error = ?err, "gateway provider checkin tick failed");
|
||||
log_maintenance_worker_failure("provider_checkin", "tick", &err);
|
||||
}
|
||||
}
|
||||
}))
|
||||
@@ -171,7 +190,7 @@ pub(crate) fn spawn_gemini_file_mapping_cleanup_worker(
|
||||
|
||||
Some(tokio::spawn(async move {
|
||||
if let Err(err) = run_gemini_file_mapping_cleanup_once(&data).await {
|
||||
warn!(error = %err, "gateway gemini file mapping cleanup startup failed");
|
||||
log_maintenance_worker_failure("gemini_file_mapping_cleanup", "startup", &err);
|
||||
}
|
||||
let mut interval = tokio::time::interval(GEMINI_FILE_MAPPING_CLEANUP_INTERVAL);
|
||||
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
||||
@@ -179,7 +198,7 @@ pub(crate) fn spawn_gemini_file_mapping_cleanup_worker(
|
||||
loop {
|
||||
interval.tick().await;
|
||||
if let Err(err) = run_gemini_file_mapping_cleanup_once(&data).await {
|
||||
warn!(error = %err, "gateway gemini file mapping cleanup tick failed");
|
||||
log_maintenance_worker_failure("gemini_file_mapping_cleanup", "tick", &err);
|
||||
}
|
||||
}
|
||||
}))
|
||||
@@ -194,7 +213,7 @@ pub(crate) fn spawn_pending_cleanup_worker(
|
||||
|
||||
Some(tokio::spawn(async move {
|
||||
if let Err(err) = run_pending_cleanup_once(&data).await {
|
||||
warn!(error = %err, "gateway pending cleanup startup failed");
|
||||
log_maintenance_worker_failure("pending_cleanup", "startup", &err);
|
||||
}
|
||||
let mut interval = tokio::time::interval(PENDING_CLEANUP_INTERVAL);
|
||||
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
||||
@@ -202,7 +221,7 @@ pub(crate) fn spawn_pending_cleanup_worker(
|
||||
loop {
|
||||
interval.tick().await;
|
||||
if let Err(err) = run_pending_cleanup_once(&data).await {
|
||||
warn!(error = %err, "gateway pending cleanup tick failed");
|
||||
log_maintenance_worker_failure("pending_cleanup", "tick", &err);
|
||||
}
|
||||
}
|
||||
}))
|
||||
@@ -237,7 +256,7 @@ pub(crate) fn spawn_stats_hourly_aggregation_worker(
|
||||
loop {
|
||||
tokio::time::sleep(duration_until_next_stats_hourly_aggregation_run(Utc::now())).await;
|
||||
if let Err(err) = run_stats_hourly_aggregation_once(&data).await {
|
||||
warn!(error = %err, "gateway stats hourly aggregation tick failed");
|
||||
log_maintenance_worker_failure("stats_hourly_aggregation", "tick", &err);
|
||||
}
|
||||
}
|
||||
}))
|
||||
@@ -252,7 +271,7 @@ pub(crate) fn spawn_request_candidate_cleanup_worker(
|
||||
|
||||
Some(tokio::spawn(async move {
|
||||
if let Err(err) = run_request_candidate_cleanup_once(&data).await {
|
||||
warn!(error = %err, "gateway request candidate cleanup startup failed");
|
||||
log_maintenance_worker_failure("request_candidate_cleanup", "startup", &err);
|
||||
}
|
||||
let mut interval = tokio::time::interval(REQUEST_CANDIDATE_CLEANUP_INTERVAL);
|
||||
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
||||
@@ -260,7 +279,7 @@ pub(crate) fn spawn_request_candidate_cleanup_worker(
|
||||
loop {
|
||||
interval.tick().await;
|
||||
if let Err(err) = run_request_candidate_cleanup_once(&data).await {
|
||||
warn!(error = %err, "gateway request candidate cleanup tick failed");
|
||||
log_maintenance_worker_failure("request_candidate_cleanup", "tick", &err);
|
||||
}
|
||||
}
|
||||
}))
|
||||
|
||||
291
apps/aether-gateway/src/maintenance/tests.rs
Normal file
291
apps/aether-gateway/src/maintenance/tests.rs
Normal file
@@ -0,0 +1,291 @@
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
|
||||
use aether_data::repository::candidates::{
|
||||
InMemoryRequestCandidateRepository, RequestCandidateReadRepository, RequestCandidateStatus,
|
||||
RequestCandidateWriteRepository, UpsertRequestCandidateRecord,
|
||||
};
|
||||
use aether_data::repository::provider_catalog::{
|
||||
InMemoryProviderCatalogReadRepository, StoredProviderCatalogProvider,
|
||||
};
|
||||
use axum::http::StatusCode;
|
||||
use axum::routing::{get, post};
|
||||
use axum::{Json, Router};
|
||||
use serde_json::json;
|
||||
|
||||
use super::ProviderCheckinRunSummary;
|
||||
use crate::AppState;
|
||||
|
||||
async fn start_server(app: Router) -> (String, tokio::task::JoinHandle<()>) {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("listener should bind");
|
||||
let addr = listener.local_addr().expect("local addr should resolve");
|
||||
let handle = tokio::spawn(async move {
|
||||
axum::serve(
|
||||
listener,
|
||||
app.into_make_service_with_connect_info::<std::net::SocketAddr>(),
|
||||
)
|
||||
.await
|
||||
.expect("server should run");
|
||||
});
|
||||
(format!("http://{addr}"), handle)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn gateway_background_request_candidate_cleanup_deletes_expired_entries_in_batches() {
|
||||
fn now_unix_secs() -> u64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
}
|
||||
|
||||
async fn seed_candidate(
|
||||
repository: &InMemoryRequestCandidateRepository,
|
||||
id: &str,
|
||||
created_at_unix_secs: u64,
|
||||
) {
|
||||
repository
|
||||
.upsert(UpsertRequestCandidateRecord {
|
||||
id: id.to_string(),
|
||||
request_id: format!("req-{id}"),
|
||||
user_id: Some("user-1".to_string()),
|
||||
api_key_id: Some("api-key-1".to_string()),
|
||||
username: Some("alice".to_string()),
|
||||
api_key_name: Some("default".to_string()),
|
||||
candidate_index: 0,
|
||||
retry_index: 0,
|
||||
provider_id: Some("provider-1".to_string()),
|
||||
endpoint_id: Some("endpoint-1".to_string()),
|
||||
key_id: Some("key-1".to_string()),
|
||||
status: RequestCandidateStatus::Success,
|
||||
skip_reason: None,
|
||||
is_cached: Some(false),
|
||||
status_code: Some(200),
|
||||
error_type: None,
|
||||
error_message: None,
|
||||
latency_ms: Some(10),
|
||||
concurrent_requests: Some(1),
|
||||
extra_data: None,
|
||||
required_capabilities: None,
|
||||
created_at_unix_secs: Some(created_at_unix_secs),
|
||||
started_at_unix_secs: Some(created_at_unix_secs),
|
||||
finished_at_unix_secs: Some(created_at_unix_secs.saturating_add(1)),
|
||||
})
|
||||
.await
|
||||
.expect("candidate should seed");
|
||||
}
|
||||
|
||||
let repository = Arc::new(InMemoryRequestCandidateRepository::default());
|
||||
seed_candidate(&repository, "cand-expired-1", 1).await;
|
||||
seed_candidate(&repository, "cand-expired-2", 2).await;
|
||||
seed_candidate(&repository, "cand-active", now_unix_secs()).await;
|
||||
|
||||
let data_state =
|
||||
crate::data::GatewayDataState::with_request_candidate_repository_for_tests(
|
||||
Arc::clone(&repository),
|
||||
)
|
||||
.with_system_config_values_for_tests([
|
||||
("enable_auto_cleanup".to_string(), json!(true)),
|
||||
("cleanup_batch_size".to_string(), json!(1)),
|
||||
(
|
||||
"request_candidates_cleanup_batch_size".to_string(),
|
||||
json!(1),
|
||||
),
|
||||
("request_candidates_retention_days".to_string(), json!(30)),
|
||||
]);
|
||||
|
||||
let gateway_state = AppState::new()
|
||||
.expect("gateway state should build")
|
||||
.with_data_state_for_tests(data_state);
|
||||
let background_tasks = gateway_state.spawn_background_tasks();
|
||||
assert!(!background_tasks.is_empty(), "cleanup worker should spawn");
|
||||
|
||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_millis(500);
|
||||
loop {
|
||||
let rows = repository
|
||||
.list_recent(10)
|
||||
.await
|
||||
.expect("list recent should succeed");
|
||||
if rows.len() == 1 {
|
||||
assert_eq!(rows[0].id, "cand-active");
|
||||
break;
|
||||
}
|
||||
assert!(
|
||||
tokio::time::Instant::now() < deadline,
|
||||
"cleanup worker did not delete expired request candidates within 500ms"
|
||||
);
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
|
||||
for handle in background_tasks {
|
||||
handle.abort();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn gateway_provider_checkin_runs_local_query_balance_for_configured_provider() {
|
||||
fn sample_provider(provider_id: &str, ops_url: &str) -> StoredProviderCatalogProvider {
|
||||
StoredProviderCatalogProvider::new(
|
||||
provider_id.to_string(),
|
||||
"openai".to_string(),
|
||||
Some("https://example.com".to_string()),
|
||||
"custom".to_string(),
|
||||
)
|
||||
.expect("provider should build")
|
||||
.with_routing_fields(10)
|
||||
.with_transport_fields(
|
||||
true,
|
||||
false,
|
||||
true,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
Some(json!({
|
||||
"provider_ops": {
|
||||
"architecture_id": "generic_api",
|
||||
"base_url": ops_url,
|
||||
"connector": {
|
||||
"auth_type": "api_key",
|
||||
"config": {
|
||||
"auth_method": "bearer"
|
||||
},
|
||||
"credentials": {
|
||||
"api_key": encrypt_python_fernet_plaintext(
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
"live-secret-api-key",
|
||||
).expect("api key should encrypt"),
|
||||
}
|
||||
}
|
||||
}
|
||||
})),
|
||||
)
|
||||
}
|
||||
|
||||
let checkin_hits = Arc::new(Mutex::new(0usize));
|
||||
let checkin_hits_clone = Arc::clone(&checkin_hits);
|
||||
let balance_hits = Arc::new(Mutex::new(0usize));
|
||||
let balance_hits_clone = Arc::clone(&balance_hits);
|
||||
let ops = Router::new()
|
||||
.route(
|
||||
"/api/user/checkin",
|
||||
post(move |headers: axum::http::HeaderMap| {
|
||||
let checkin_hits_inner = Arc::clone(&checkin_hits_clone);
|
||||
async move {
|
||||
*checkin_hits_inner.lock().expect("mutex should lock") += 1;
|
||||
assert_eq!(
|
||||
headers
|
||||
.get(axum::http::header::AUTHORIZATION)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("Bearer live-secret-api-key")
|
||||
);
|
||||
(
|
||||
StatusCode::OK,
|
||||
Json(json!({
|
||||
"success": true,
|
||||
"message": "签到成功",
|
||||
})),
|
||||
)
|
||||
}
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/api/user/balance",
|
||||
get(move |headers: axum::http::HeaderMap| {
|
||||
let balance_hits_inner = Arc::clone(&balance_hits_clone);
|
||||
async move {
|
||||
*balance_hits_inner.lock().expect("mutex should lock") += 1;
|
||||
assert_eq!(
|
||||
headers
|
||||
.get(axum::http::header::AUTHORIZATION)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("Bearer live-secret-api-key")
|
||||
);
|
||||
(
|
||||
StatusCode::OK,
|
||||
Json(json!({
|
||||
"success": true,
|
||||
"data": {
|
||||
"quota": 5000000,
|
||||
"used_quota": 1000000
|
||||
}
|
||||
})),
|
||||
)
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
let (ops_url, ops_handle) = start_server(ops).await;
|
||||
let repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||
vec![sample_provider("provider-openai", &ops_url)],
|
||||
vec![],
|
||||
vec![],
|
||||
));
|
||||
let gateway_state = AppState::new()
|
||||
.expect("gateway state should build")
|
||||
.with_data_state_for_tests(
|
||||
crate::data::GatewayDataState::with_provider_catalog_repository_for_tests(
|
||||
repository,
|
||||
)
|
||||
.with_system_config_values_for_tests([
|
||||
("enable_provider_checkin".to_string(), json!(true)),
|
||||
("provider_checkin_time".to_string(), json!("01:05")),
|
||||
])
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY),
|
||||
);
|
||||
|
||||
let summary = crate::maintenance::perform_provider_checkin_once(&gateway_state)
|
||||
.await
|
||||
.expect("provider checkin should succeed");
|
||||
|
||||
assert_eq!(
|
||||
summary,
|
||||
ProviderCheckinRunSummary {
|
||||
attempted: 1,
|
||||
succeeded: 1,
|
||||
failed: 0,
|
||||
skipped: 0,
|
||||
}
|
||||
);
|
||||
assert_eq!(*checkin_hits.lock().expect("mutex should lock"), 1);
|
||||
assert_eq!(*balance_hits.lock().expect("mutex should lock"), 1);
|
||||
|
||||
ops_handle.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn gateway_provider_checkin_skips_when_disabled_via_system_config() {
|
||||
let repository = Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||
vec![],
|
||||
vec![],
|
||||
vec![],
|
||||
));
|
||||
let gateway_state = AppState::new()
|
||||
.expect("gateway state should build")
|
||||
.with_data_state_for_tests(
|
||||
crate::data::GatewayDataState::with_provider_catalog_repository_for_tests(
|
||||
repository,
|
||||
)
|
||||
.with_system_config_values_for_tests([(
|
||||
"enable_provider_checkin".to_string(),
|
||||
json!(false),
|
||||
)]),
|
||||
);
|
||||
|
||||
let summary = crate::maintenance::perform_provider_checkin_once(&gateway_state)
|
||||
.await
|
||||
.expect("provider checkin should short-circuit");
|
||||
|
||||
assert_eq!(
|
||||
summary,
|
||||
ProviderCheckinRunSummary {
|
||||
attempted: 0,
|
||||
succeeded: 0,
|
||||
failed: 0,
|
||||
skipped: 0,
|
||||
}
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user