i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-02 17:30:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: 抽离 AI pipeline 与调度共享能力逻辑

Browse Source
This commit is contained in:
fawney19
2026-04-10 01:46:14 +08:00
parent b901a6ffc7
commit 5014e2f5fd
255 changed files with 15057 additions and 3115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
apps/aether-gateway/src/handlers/admin/billing/collectors/support.rs
View File

@@ -53,7 +53,7 @@ pub(super) fn build_admin_billing_collector_payload_from_record(
"default_value": record.default_value,
"priority": record.priority,
"is_enabled": record.is_enabled,
"created_at": unix_secs_to_rfc3339(record.created_at_unix_secs),
"created_at": unix_secs_to_rfc3339(record.created_at_unix_ms),
"updated_at": unix_secs_to_rfc3339(record.updated_at_unix_secs),
})
}

6
apps/aether-gateway/src/handlers/admin/billing/payments/shared.rs
View File

@@ -213,7 +213,7 @@ pub(super) fn build_admin_payment_order_payload(
"gateway_order_id": record.gateway_order_id,
"gateway_response": record.gateway_response,
"status": admin_payment_effective_status(&record.status, record.expires_at_unix_secs),
"created_at": unix_secs_to_rfc3339(record.created_at_unix_secs),
"created_at": unix_secs_to_rfc3339(record.created_at_unix_ms),
"paid_at": record.paid_at_unix_secs.and_then(unix_secs_to_rfc3339),
"credited_at": record.credited_at_unix_secs.and_then(unix_secs_to_rfc3339),
"expires_at": record.expires_at_unix_secs.and_then(unix_secs_to_rfc3339),
@@ -236,7 +236,7 @@ pub(super) fn build_admin_payment_callback_payload(
"payload": row.try_get::<Option<serde_json::Value>, _>("payload").map_err(|err| GatewayError::Internal(err.to_string()))?,
"error_message": row.try_get::<Option<String>, _>("error_message").map_err(|err| GatewayError::Internal(err.to_string()))?,
"created_at": row
.try_get::<Option<i64>, _>("created_at_unix_secs")
.try_get::<Option<i64>, _>("created_at_unix_ms")
.map_err(|err| GatewayError::Internal(err.to_string()))?
.and_then(|value| u64::try_from(value).ok())
.and_then(unix_secs_to_rfc3339),
@@ -263,7 +263,7 @@ pub(super) fn build_admin_payment_callback_payload_from_record(
"status": record.status,
"payload": record.payload,
"error_message": record.error_message,
"created_at": unix_secs_to_rfc3339(record.created_at_unix_secs),
"created_at": unix_secs_to_rfc3339(record.created_at_unix_ms),
"processed_at": record.processed_at_unix_secs.and_then(unix_secs_to_rfc3339),
})
}

2
apps/aether-gateway/src/handlers/admin/billing/rules.rs
View File

@@ -53,7 +53,7 @@ fn build_admin_billing_rule_payload_from_record(
"variables": record.variables,
"dimension_mappings": record.dimension_mappings,
"is_enabled": record.is_enabled,
"created_at": unix_secs_to_rfc3339(record.created_at_unix_secs),
"created_at": unix_secs_to_rfc3339(record.created_at_unix_ms),
"updated_at": unix_secs_to_rfc3339(record.updated_at_unix_secs),
})
}

2
apps/aether-gateway/src/handlers/admin/billing/wallets/mutations/adjust.rs
View File

@@ -97,7 +97,7 @@ pub(in super::super) async fn build_admin_wallet_adjust_response(
transaction.link_id.as_deref(),
transaction.operator_id.as_deref(),
transaction.description.as_deref(),
unix_secs_to_rfc3339(transaction.created_at_unix_secs),
unix_secs_to_rfc3339(transaction.created_at_unix_ms),
);
let response = Json(json!({
"wallet": wallet_payload,

2
apps/aether-gateway/src/handlers/admin/billing/wallets/mutations/fail_refund.rs
View File

@@ -84,7 +84,7 @@ pub(in super::super) async fn build_admin_wallet_fail_refund_response(
transaction.link_id.as_deref(),
transaction.operator_id.as_deref(),
transaction.description.as_deref(),
unix_secs_to_rfc3339(transaction.created_at_unix_secs),
unix_secs_to_rfc3339(transaction.created_at_unix_ms),
)
})
.unwrap_or(serde_json::Value::Null),

2
apps/aether-gateway/src/handlers/admin/billing/wallets/mutations/process_refund.rs
View File

@@ -69,7 +69,7 @@ pub(in super::super) async fn build_admin_wallet_process_refund_response(
transaction.link_id.as_deref(),
transaction.operator_id.as_deref(),
transaction.description.as_deref(),
unix_secs_to_rfc3339(transaction.created_at_unix_secs),
unix_secs_to_rfc3339(transaction.created_at_unix_ms),
),
}))
.into_response();

2
apps/aether-gateway/src/handlers/admin/billing/wallets/mutations/recharge.rs
View File

@@ -87,7 +87,7 @@ pub(in super::super) async fn build_admin_wallet_recharge_response(
payment_order.amount_usd,
payment_order.payment_method,
payment_order.status,
unix_secs_to_rfc3339(payment_order.created_at_unix_secs),
unix_secs_to_rfc3339(payment_order.created_at_unix_ms),
payment_order
.credited_at_unix_secs
.and_then(unix_secs_to_rfc3339),

2
apps/aether-gateway/src/handlers/admin/billing/wallets/reads/ledger.rs
View File

@@ -69,7 +69,7 @@ pub(in super::super) async fn build_admin_wallet_ledger_response(
"operator_name": entry.operator_name,
"operator_email": entry.operator_email,
"description": entry.description,
"created_at": entry.created_at_unix_secs.and_then(unix_secs_to_rfc3339),
"created_at": entry.created_at_unix_ms.and_then(unix_secs_to_rfc3339),
})
})
.collect::<Vec<_>>();

2
apps/aether-gateway/src/handlers/admin/billing/wallets/reads/list.rs
View File

@@ -59,7 +59,7 @@ pub(in super::super) async fn build_admin_wallet_list_response(
"total_consumed": wallet.total_consumed,
"total_refunded": wallet.total_refunded,
"total_adjusted": wallet.total_adjusted,
"created_at": wallet.created_at_unix_secs.and_then(unix_secs_to_rfc3339),
"created_at": wallet.created_at_unix_ms.and_then(unix_secs_to_rfc3339),
"updated_at": wallet.updated_at_unix_secs.and_then(unix_secs_to_rfc3339),
})
})

2
apps/aether-gateway/src/handlers/admin/billing/wallets/reads/refund_requests.rs
View File

@@ -79,7 +79,7 @@ pub(in super::super) async fn build_admin_wallet_refund_requests_response(
"requested_by": refund.requested_by,
"approved_by": refund.approved_by,
"processed_by": refund.processed_by,
"created_at": refund.created_at_unix_secs.and_then(unix_secs_to_rfc3339),
"created_at": refund.created_at_unix_ms.and_then(unix_secs_to_rfc3339),
"updated_at": refund.updated_at_unix_secs.and_then(unix_secs_to_rfc3339),
"processed_at": refund.processed_at_unix_secs.and_then(unix_secs_to_rfc3339),
"completed_at": refund.completed_at_unix_secs.and_then(unix_secs_to_rfc3339),

2
apps/aether-gateway/src/handlers/admin/billing/wallets/reads/transactions.rs
View File

@@ -80,7 +80,7 @@ pub(in super::super) async fn build_admin_wallet_transactions_response(
"operator_name": operator_name,
"operator_email": operator_email,
"description": transaction.description,
"created_at": transaction.created_at_unix_secs.and_then(unix_secs_to_rfc3339),
"created_at": transaction.created_at_unix_ms.and_then(unix_secs_to_rfc3339),
}));
}

2
apps/aether-gateway/src/handlers/admin/billing/wallets/shared/payloads.rs
View File

@@ -188,7 +188,7 @@ pub(in super::super) fn build_admin_wallet_refund_payload(
"requested_by": refund.requested_by.clone(),
"approved_by": refund.approved_by.clone(),
"processed_by": refund.processed_by.clone(),
"created_at": unix_secs_to_rfc3339(refund.created_at_unix_secs),
"created_at": unix_secs_to_rfc3339(refund.created_at_unix_ms),
"updated_at": unix_secs_to_rfc3339(refund.updated_at_unix_secs),
"processed_at": refund.processed_at_unix_secs.and_then(unix_secs_to_rfc3339),
"completed_at": refund.completed_at_unix_secs.and_then(unix_secs_to_rfc3339),

37
apps/aether-gateway/src/handlers/admin/endpoint/health_builders/status.rs
View File

@@ -3,6 +3,7 @@ use crate::handlers::admin::shared::unix_secs_to_rfc3339;
use crate::handlers::public::{
api_format_display_name, build_public_health_timeline, provider_key_api_formats,
};
use crate::handlers::shared::unix_ms_to_rfc3339;
use aether_data_contracts::repository::candidates::PublicHealthTimelineBucket;
use aether_scheduler_core::{is_provider_key_circuit_open, provider_key_health_score};
use serde_json::json;
@@ -124,28 +125,24 @@ pub(crate) async fn build_admin_endpoint_health_status_payload(
total_count: 0,
success_count: 0,
failed_count: 0,
min_created_at_unix_secs: None,
max_created_at_unix_secs: None,
min_created_at_unix_ms: None,
max_created_at_unix_ms: None,
});
bucket.total_count += row.total_count;
bucket.success_count += row.success_count;
bucket.failed_count += row.failed_count;
bucket.min_created_at_unix_secs = match (
bucket.min_created_at_unix_secs,
row.min_created_at_unix_secs,
) {
(Some(left), Some(right)) => Some(left.min(right)),
(None, Some(right)) => Some(right),
(left, None) => left,
};
bucket.max_created_at_unix_secs = match (
bucket.max_created_at_unix_secs,
row.max_created_at_unix_secs,
) {
(Some(left), Some(right)) => Some(left.max(right)),
(None, Some(right)) => Some(right),
(left, None) => left,
};
bucket.min_created_at_unix_ms =
match (bucket.min_created_at_unix_ms, row.min_created_at_unix_ms) {
(Some(left), Some(right)) => Some(left.min(right)),
(None, Some(right)) => Some(right),
(left, None) => left,
};
bucket.max_created_at_unix_ms =
match (bucket.max_created_at_unix_ms, row.max_created_at_unix_ms) {
(Some(left), Some(right)) => Some(left.max(right)),
(None, Some(right)) => Some(right),
(left, None) => left,
};
}
let mut payload = endpoint_ids_by_format
@@ -182,8 +179,8 @@ pub(crate) async fn build_admin_endpoint_health_status_payload(
"display_name": api_format_display_name(&api_format),
"health_score": health_score,
"timeline": timeline,
"time_range_start": time_range_start.and_then(unix_secs_to_rfc3339),
"time_range_end": time_range_end.or(Some(now_unix_secs)).and_then(unix_secs_to_rfc3339),
"time_range_start": time_range_start.and_then(unix_ms_to_rfc3339),
"time_range_end": time_range_end.map(|ms| unix_ms_to_rfc3339(ms)).unwrap_or_else(|| unix_secs_to_rfc3339(now_unix_secs)),
"total_endpoints": endpoint_ids.len(),
"total_keys": total_keys,
"active_keys": active_keys_by_format.get(&api_format).map(BTreeSet::len).unwrap_or(0),

2
apps/aether-gateway/src/handlers/admin/features/gemini_files/read_routes.rs
View File

@@ -140,7 +140,7 @@ fn build_admin_gemini_file_mapping_payload(
"username": username,
"display_name": mapping.display_name,
"mime_type": mapping.mime_type,
"created_at": unix_secs_to_rfc3339(mapping.created_at_unix_secs),
"created_at": unix_secs_to_rfc3339(mapping.created_at_unix_ms),
"expires_at": unix_secs_to_rfc3339(mapping.expires_at_unix_secs),
"is_expired": mapping.expires_at_unix_secs <= now_unix_secs,
})

2
apps/aether-gateway/src/handlers/admin/features/video_tasks/builders.rs
View File

@@ -94,7 +94,7 @@ pub(super) fn build_admin_video_task_list_item(
"error_message": task.error_message,
"poll_count": task.poll_count,
"max_poll_count": task.max_poll_count,
"created_at": admin_video_task_timestamp(Some(task.created_at_unix_secs)),
"created_at": admin_video_task_timestamp(Some(task.created_at_unix_ms)),
"completed_at": admin_video_task_timestamp(task.completed_at_unix_secs),
"submitted_at": admin_video_task_timestamp(task.submitted_at_unix_secs),
})

2
apps/aether-gateway/src/handlers/admin/features/video_tasks/routes.rs
View File

@@ -272,7 +272,7 @@ pub(super) async fn maybe_build_local_admin_video_tasks_response(
payload.insert("max_poll_count".to_string(), json!(task.max_poll_count));
payload.insert(
"created_at".to_string(),
json!(admin_video_task_timestamp(Some(task.created_at_unix_secs))),
json!(admin_video_task_timestamp(Some(task.created_at_unix_ms))),
);
payload.insert(
"updated_at".to_string(),

2
apps/aether-gateway/src/handlers/admin/model/global/payloads.rs
View File

@@ -28,7 +28,7 @@ pub(crate) fn build_admin_global_model_response(
"config": global_model.config.clone(),
"provider_count": provider_count,
"active_provider_count": active_provider_count,
"created_at": timestamp_or_now(global_model.created_at_unix_secs, now_unix_secs),
"created_at": timestamp_or_now(global_model.created_at_unix_ms, now_unix_secs),
"updated_at": timestamp_or_now(global_model.updated_at_unix_secs, now_unix_secs),
})
}

1
apps/aether-gateway/src/handlers/admin/observability/monitoring/cache.rs
View File

@@ -49,6 +49,7 @@ pub(super) async fn build_admin_monitoring_cache_stats_response(
"affinity_stats": {
"storage_type": snapshot.storage_type,
"total_affinities": snapshot.total_affinities,
"active_affinities": snapshot.total_affinities,
"cache_hits": snapshot.cache_hits,
"cache_misses": snapshot.cache_misses,
"cache_hit_rate": snapshot.cache_hit_rate,

131
apps/aether-gateway/src/handlers/admin/observability/monitoring/cache_affinity.rs
View File

@@ -1,6 +1,8 @@
use super::cache_types::AdminMonitoringCacheAffinityRecord;
use crate::cache::SchedulerAffinityTarget;
use crate::handlers::admin::request::AdminAppState;
use crate::GatewayError;
use std::time::Duration;
fn parse_admin_monitoring_cache_affinity_key(raw_key: &str) -> Option<(String, String, String)> {
let parts = raw_key.split(':').collect::<Vec<_>>();
@@ -25,6 +27,48 @@ fn parse_admin_monitoring_cache_affinity_key(raw_key: &str) -> Option<(String, S
Some((affinity_key.to_string(), api_format, model_name))
}
fn parse_admin_monitoring_scheduler_affinity_key(
raw_key: &str,
) -> Option<(String, String, String)> {
let parts = raw_key.split(':').collect::<Vec<_>>();
let start = parts
.iter()
.position(|segment| *segment == "scheduler_affinity")?;
let affinity_key = parts.get(start + 1)?.trim();
if affinity_key.is_empty() {
return None;
}
let remaining = parts.get(start + 2..)?;
if remaining.len() < 2 {
return None;
}
let (api_format, model_name_parts) = if remaining.len() == 2 {
(remaining[0].trim().to_string(), &remaining[1..])
} else {
(
format!("{}:{}", remaining[0].trim(), remaining[1].trim()),
&remaining[2..],
)
};
if api_format.trim().is_empty() {
return None;
}
let model_name = model_name_parts
.iter()
.map(|segment| segment.trim())
.filter(|segment| !segment.is_empty())
.collect::<Vec<_>>()
.join(":");
if model_name.is_empty() {
return None;
}
Some((affinity_key.to_string(), api_format, model_name))
}
pub(super) fn admin_monitoring_scheduler_affinity_cache_key(
record: &AdminMonitoringCacheAffinityRecord,
) -> Option<String> {
@@ -92,6 +136,93 @@ pub(super) fn admin_monitoring_cache_affinity_record(
})
}
pub(super) fn admin_monitoring_scheduler_affinity_record(
cache_key: &str,
target: &SchedulerAffinityTarget,
age: Duration,
ttl: Duration,
now_unix_secs: u64,
) -> Option<AdminMonitoringCacheAffinityRecord> {
let (affinity_key, api_format, model_name) =
parse_admin_monitoring_scheduler_affinity_key(cache_key)?;
let age_secs = age.as_secs();
let created_at = now_unix_secs.saturating_sub(age_secs);
let expire_at = created_at.saturating_add(ttl.as_secs());
Some(AdminMonitoringCacheAffinityRecord {
raw_key: cache_key.to_string(),
affinity_key,
api_format,
model_name,
provider_id: Some(target.provider_id.clone()),
endpoint_id: Some(target.endpoint_id.clone()),
key_id: Some(target.key_id.clone()),
created_at: Some(serde_json::json!(created_at)),
expire_at: Some(serde_json::json!(expire_at)),
request_count: 0,
})
}
pub(super) fn admin_monitoring_scheduler_affinity_record_from_raw(
raw_key: &str,
raw_value: &str,
) -> Option<AdminMonitoringCacheAffinityRecord> {
let payload = serde_json::from_str::<serde_json::Value>(raw_value).ok()?;
let object = payload.as_object()?;
let (affinity_key, parsed_api_format, parsed_model_name) =
parse_admin_monitoring_scheduler_affinity_key(raw_key)?;
let api_format = object
.get("api_format")
.and_then(serde_json::Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.unwrap_or(parsed_api_format.as_str())
.to_string();
let model_name = object
.get("model_name")
.and_then(serde_json::Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.unwrap_or(parsed_model_name.as_str())
.to_string();
let request_count = object
.get("request_count")
.and_then(|value| {
value
.as_u64()
.or_else(|| value.as_i64().and_then(|number| u64::try_from(number).ok()))
})
.unwrap_or(0);
Some(AdminMonitoringCacheAffinityRecord {
raw_key: raw_key.to_string(),
affinity_key,
api_format,
model_name,
provider_id: object
.get("provider_id")
.and_then(serde_json::Value::as_str)
.map(ToOwned::to_owned),
endpoint_id: object
.get("endpoint_id")
.and_then(serde_json::Value::as_str)
.map(ToOwned::to_owned),
key_id: object
.get("key_id")
.and_then(serde_json::Value::as_str)
.map(ToOwned::to_owned),
created_at: object.get("created_at").cloned(),
expire_at: object.get("expire_at").cloned(),
request_count,
})
}
pub(super) fn admin_monitoring_cache_affinity_record_identity(
record: &AdminMonitoringCacheAffinityRecord,
) -> String {
admin_monitoring_scheduler_affinity_cache_key(record).unwrap_or_else(|| record.raw_key.clone())
}
pub(super) fn clear_admin_monitoring_scheduler_affinity_entries(
state: &AdminAppState<'_>,
records: &[AdminMonitoringCacheAffinityRecord],

2
apps/aether-gateway/src/handlers/admin/observability/monitoring/cache_mutations/affinity.rs
View File

@@ -8,6 +8,7 @@ use super::super::cache_route_helpers::{
admin_monitoring_cache_affinity_unavailable_response,
};
use super::super::cache_store::{
admin_monitoring_has_runtime_scheduler_affinity_entries,
list_admin_monitoring_cache_affinity_records_by_affinity_keys,
load_admin_monitoring_cache_affinity_entries_for_tests,
};
@@ -33,6 +34,7 @@ pub(in super::super) async fn build_admin_monitoring_cache_affinity_delete_respo
if state.redis_kv_runner().is_none()
&& load_admin_monitoring_cache_affinity_entries_for_tests(state).is_empty()
&& !admin_monitoring_has_runtime_scheduler_affinity_entries(state)
{
return Ok(admin_monitoring_cache_affinity_unavailable_response());
}

2
apps/aether-gateway/src/handlers/admin/observability/monitoring/cache_mutations/users.rs
View File

@@ -11,6 +11,7 @@ use super::super::cache_route_helpers::{
admin_monitoring_cache_users_user_identifier_from_path,
};
use super::super::cache_store::{
admin_monitoring_has_runtime_scheduler_affinity_entries,
list_admin_monitoring_cache_affinity_records_by_affinity_keys,
load_admin_monitoring_cache_affinity_entries_for_tests,
};
@@ -37,6 +38,7 @@ pub(in super::super) async fn build_admin_monitoring_cache_users_delete_response
if state.redis_kv_runner().is_none()
&& load_admin_monitoring_cache_affinity_entries_for_tests(state).is_empty()
&& !admin_monitoring_has_runtime_scheduler_affinity_entries(state)
{
return Ok(admin_monitoring_cache_affinity_unavailable_response());
}

118
apps/aether-gateway/src/handlers/admin/observability/monitoring/cache_store.rs
View File

@@ -1,41 +1,25 @@
use super::cache_affinity::admin_monitoring_cache_affinity_record;
use super::cache_affinity::{
admin_monitoring_cache_affinity_record, admin_monitoring_cache_affinity_record_identity,
admin_monitoring_scheduler_affinity_record,
admin_monitoring_scheduler_affinity_record_from_raw,
};
use super::cache_types::{AdminMonitoringCacheAffinityRecord, AdminMonitoringCacheSnapshot};
use crate::handlers::admin::observability::stats::round_to;
use crate::handlers::admin::request::AdminAppState;
use crate::scheduler::affinity::SCHEDULER_AFFINITY_TTL;
use crate::GatewayError;
use aether_data_contracts::repository::usage::UsageAuditListQuery;
async fn count_admin_monitoring_cache_affinity_entries(state: &AdminAppState<'_>) -> usize {
let Some(runner) = state.redis_kv_runner() else {
return 0;
};
let mut connection = match runner.client().get_multiplexed_async_connection().await {
Ok(value) => value,
Err(_) => return 0,
};
let pattern = runner.keyspace().key("cache_affinity:*");
let mut cursor = 0u64;
let mut total = 0usize;
loop {
let (next_cursor, keys) = match redis::cmd("SCAN")
.arg(cursor)
.arg("MATCH")
.arg(&pattern)
.arg("COUNT")
.arg(200)
.query_async::<(u64, Vec<String>)>(&mut connection)
.await
{
Ok(value) => value,
Err(_) => return total,
};
total += keys.len();
if next_cursor == 0 {
break;
}
cursor = next_cursor;
}
total
list_admin_monitoring_cache_affinity_records(state)
.await
.map(|items| items.len())
.unwrap_or_else(|_| {
state
.as_ref()
.list_scheduler_affinity_entries(SCHEDULER_AFFINITY_TTL)
.len()
})
}
async fn scan_admin_monitoring_namespaced_keys(
@@ -191,12 +175,27 @@ pub(super) async fn list_admin_monitoring_cache_affinity_records_by_affinity_key
list_admin_monitoring_cache_affinity_records_matching(state, Some(affinity_keys)).await
}
pub(super) fn admin_monitoring_has_runtime_scheduler_affinity_entries(
state: &AdminAppState<'_>,
) -> bool {
!state
.as_ref()
.list_scheduler_affinity_entries(SCHEDULER_AFFINITY_TTL)
.is_empty()
}
async fn list_admin_monitoring_cache_affinity_records_matching(
state: &AdminAppState<'_>,
affinity_keys: Option<&std::collections::BTreeSet<String>>,
) -> Result<Vec<AdminMonitoringCacheAffinityRecord>, GatewayError> {
let mut records = Vec::new();
let mut seen_raw_keys = std::collections::BTreeSet::new();
let mut seen_record_ids = std::collections::BTreeSet::new();
let mut push_record = |record: AdminMonitoringCacheAffinityRecord| {
if seen_record_ids.insert(admin_monitoring_cache_affinity_record_identity(&record)) {
records.push(record);
}
};
if let Some(runner) = state.redis_kv_runner() {
let mut connection = runner
@@ -209,14 +208,24 @@ async fn list_admin_monitoring_cache_affinity_records_matching(
let patterns = affinity_keys
.map(|keys| {
keys.iter()
.map(|affinity_key| {
runner
.keyspace()
.key(&format!("cache_affinity:{affinity_key}:*"))
.flat_map(|affinity_key| {
[
runner
.keyspace()
.key(&format!("cache_affinity:{affinity_key}:*")),
runner
.keyspace()
.key(&format!("scheduler_affinity:{affinity_key}:*")),
]
})
.collect::<Vec<_>>()
})
.unwrap_or_else(|| vec![runner.keyspace().key("cache_affinity:*")]);
.unwrap_or_else(|| {
vec![
runner.keyspace().key("cache_affinity:*"),
runner.keyspace().key("scheduler_affinity:*"),
]
});
for pattern in patterns {
let mut cursor = 0u64;
@@ -246,16 +255,18 @@ async fn list_admin_monitoring_cache_affinity_records_matching(
let Some(raw_value) = raw_value else {
continue;
};
let Some(record) = admin_monitoring_cache_affinity_record(&key, &raw_value)
else {
let record = if key.contains("scheduler_affinity:") {
admin_monitoring_scheduler_affinity_record_from_raw(&key, &raw_value)
} else {
admin_monitoring_cache_affinity_record(&key, &raw_value)
};
let Some(record) = record else {
continue;
};
if affinity_keys.is_some_and(|keys| !keys.contains(&record.affinity_key)) {
continue;
}
if seen_raw_keys.insert(record.raw_key.clone()) {
records.push(record);
}
push_record(record);
}
}
if next_cursor == 0 {
@@ -264,7 +275,6 @@ async fn list_admin_monitoring_cache_affinity_records_matching(
cursor = next_cursor;
}
}
return Ok(records);
}
for (key, raw_value) in load_admin_monitoring_cache_affinity_entries_for_tests(state) {
@@ -274,9 +284,27 @@ async fn list_admin_monitoring_cache_affinity_records_matching(
if affinity_keys.is_some_and(|keys| !keys.contains(&record.affinity_key)) {
continue;
}
if seen_raw_keys.insert(record.raw_key.clone()) {
records.push(record);
push_record(record);
}
let now_unix_secs = chrono::Utc::now().timestamp().max(0) as u64;
for entry in state
.as_ref()
.list_scheduler_affinity_entries(SCHEDULER_AFFINITY_TTL)
{
let Some(record) = admin_monitoring_scheduler_affinity_record(
&entry.cache_key,
&entry.target,
entry.age,
SCHEDULER_AFFINITY_TTL,
now_unix_secs,
) else {
continue;
};
if affinity_keys.is_some_and(|keys| !keys.contains(&record.affinity_key)) {
continue;
}
push_record(record);
}
Ok(records)

4
apps/aether-gateway/src/handlers/admin/observability/monitoring/resilience/snapshot.rs
View File

@@ -161,7 +161,7 @@ pub(super) async fn build_admin_monitoring_resilience_snapshot(
.filter(admin_monitoring_usage_is_error)
.collect::<Vec<_>>();
recent_usage_errors
.sort_by(|left, right| right.created_at_unix_secs.cmp(&left.created_at_unix_secs));
.sort_by(|left, right| right.created_at_unix_ms.cmp(&left.created_at_unix_ms));
let total_errors = recent_usage_errors.len();
let mut error_breakdown = BTreeMap::<String, usize>::new();
@@ -200,7 +200,7 @@ pub(super) async fn build_admin_monitoring_resilience_snapshot(
"error_id": item.id,
"error_type": error_type,
"operation": operation,
"timestamp": unix_secs_to_rfc3339(item.created_at_unix_secs),
"timestamp": unix_secs_to_rfc3339(item.created_at_unix_ms),
"context": {
"request_id": item.request_id,
"provider_id": item.provider_id,

16
apps/aether-gateway/src/handlers/admin/observability/monitoring/test_support.rs
View File

@@ -31,7 +31,7 @@ pub(super) fn sample_usage(
total_cost_usd: f64,
status: &str,
status_code: Option<i32>,
created_at_unix_secs: i64,
created_at_unix_ms: i64,
) -> StoredRequestUsageAudit {
let is_error = status_code.is_some_and(|value| value >= 400)
|| status.trim().eq_ignore_ascii_case("failed")
@@ -70,9 +70,9 @@ pub(super) fn sample_usage(
Some(30),
status.to_string(),
"billed".to_string(),
created_at_unix_secs,
created_at_unix_secs,
Some(created_at_unix_secs),
created_at_unix_ms,
created_at_unix_ms,
Some(created_at_unix_ms),
)
.expect("usage should build")
}
@@ -82,7 +82,7 @@ pub(super) fn sample_candidate(
request_id: &str,
candidate_index: i32,
status: RequestCandidateStatus,
started_at_unix_secs: Option<i64>,
started_at_unix_ms: Option<i64>,
latency_ms: Option<i32>,
status_code: Option<i32>,
) -> StoredRequestCandidate {
@@ -108,9 +108,9 @@ pub(super) fn sample_candidate(
Some(1),
None,
Some(json!({"cache_1h": true})),
100 + i64::from(candidate_index),
started_at_unix_secs,
started_at_unix_secs.map(|value| value + 1),
(100 + i64::from(candidate_index)) * 1_000,
started_at_unix_ms.map(|v| v * 1_000),
started_at_unix_ms.map(|value| (value + 1) * 1_000),
)
.expect("candidate should build")
}

50
apps/aether-gateway/src/handlers/admin/observability/monitoring/tests/basics.rs
View File

@@ -258,6 +258,56 @@ async fn admin_monitoring_resilience_status_returns_local_payload() {
assert!(payload["timestamp"].as_str().is_some());
}
#[tokio::test]
async fn admin_monitoring_cache_stats_count_runtime_scheduler_affinities() {
let state = AppState::new().expect("state should build");
let affinity_cache_key =
aether_scheduler_core::build_scheduler_affinity_cache_key_for_api_key_id(
"user-key-1",
"openai:chat",
"model-alpha",
)
.expect("scheduler affinity cache key should build");
state.scheduler_affinity_cache.insert(
affinity_cache_key,
crate::cache::SchedulerAffinityTarget {
provider_id: "provider-1".to_string(),
endpoint_id: "endpoint-1".to_string(),
key_id: "provider-key-1".to_string(),
},
crate::scheduler::affinity::SCHEDULER_AFFINITY_TTL,
128,
);
let response = local_monitoring_response(
&state,
&request_context(http::Method::GET, "/api/admin/monitoring/cache/stats"),
)
.await
.expect("handler should not error")
.expect("monitoring route should be handled locally");
assert_eq!(response.status(), http::StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX)
.await
.expect("body should read");
let payload: serde_json::Value = serde_json::from_slice(&body).expect("json body should parse");
assert_eq!(payload["status"], json!("ok"));
assert_eq!(payload["data"]["total_affinities"], json!(1));
assert_eq!(
payload["data"]["affinity_stats"]["total_affinities"],
json!(1)
);
assert_eq!(
payload["data"]["affinity_stats"]["active_affinities"],
json!(1)
);
assert_eq!(
payload["data"]["affinity_stats"]["storage_type"],
json!("memory")
);
}
#[tokio::test]
async fn admin_monitoring_cache_stats_returns_local_payload() {
let now = chrono::Utc::now().timestamp();

130
apps/aether-gateway/src/handlers/admin/observability/monitoring/tests/mod.rs
View File

@@ -209,6 +209,136 @@ async fn admin_monitoring_cache_affinities_and_affinity_return_local_payload_fro
assert_eq!(detail_payload["total_endpoints"], json!(1));
}
#[tokio::test]
async fn admin_monitoring_cache_affinities_and_delete_use_runtime_scheduler_affinity_cache() {
let user_repository = Arc::new(
InMemoryUserReadRepository::seed_auth_users(vec![sample_monitoring_auth_user("user-1")])
.with_export_users(vec![sample_monitoring_export_user("user-1")]),
);
let auth_repository = Arc::new(
InMemoryAuthApiKeySnapshotRepository::default().with_export_records(vec![
sample_monitoring_export_api_key("user-1", "user-key-1"),
]),
);
let provider_catalog = Arc::new(InMemoryProviderCatalogReadRepository::seed(
vec![sample_provider()],
vec![sample_monitoring_catalog_endpoint()],
vec![sample_monitoring_catalog_key()],
));
let state = AppState::new()
.expect("state should build")
.with_data_state_for_tests(
crate::data::GatewayDataState::with_provider_catalog_reader_for_tests(provider_catalog)
.with_user_reader(user_repository)
.with_auth_api_key_reader(auth_repository),
);
let affinity_cache_key =
aether_scheduler_core::build_scheduler_affinity_cache_key_for_api_key_id(
"user-key-1",
"openai:chat",
"model-alpha",
)
.expect("scheduler affinity cache key should build");
state.scheduler_affinity_cache.insert(
affinity_cache_key.clone(),
crate::cache::SchedulerAffinityTarget {
provider_id: "provider-1".to_string(),
endpoint_id: "endpoint-1".to_string(),
key_id: "provider-key-1".to_string(),
},
crate::scheduler::affinity::SCHEDULER_AFFINITY_TTL,
128,
);
let list_context = request_context(
http::Method::GET,
"/api/admin/monitoring/cache/affinities?keyword=alice&limit=20&offset=0",
);
let list_response = local_monitoring_response(&state, &list_context)
.await
.expect("handler should not error")
.expect("route should be handled locally");
assert_eq!(list_response.status(), http::StatusCode::OK);
let list_body = to_bytes(list_response.into_body(), usize::MAX)
.await
.expect("body should read");
let list_payload: serde_json::Value =
serde_json::from_slice(&list_body).expect("json body should parse");
assert_eq!(list_payload["status"], json!("ok"));
assert_eq!(list_payload["data"]["meta"]["total"], json!(1));
assert_eq!(list_payload["data"]["matched_user_id"], json!("user-1"));
assert_eq!(
list_payload["data"]["items"][0]["affinity_key"],
json!("user-key-1")
);
assert_eq!(
list_payload["data"]["items"][0]["api_format"],
json!("openai:chat")
);
assert_eq!(
list_payload["data"]["items"][0]["provider_name"],
json!("OpenAI")
);
assert_eq!(
list_payload["data"]["items"][0]["endpoint_url"],
json!("https://api.openai.example/v1")
);
assert_eq!(list_payload["data"]["items"][0]["request_count"], json!(0));
assert!(list_payload["data"]["items"][0]["expire_at"]
.as_u64()
.is_some_and(|value| value > 0));
let detail_context = request_context(
http::Method::GET,
"/api/admin/monitoring/cache/affinity/alice",
);
let detail_response = local_monitoring_response(&state, &detail_context)
.await
.expect("handler should not error")
.expect("route should be handled locally");
assert_eq!(detail_response.status(), http::StatusCode::OK);
let detail_body = to_bytes(detail_response.into_body(), usize::MAX)
.await
.expect("body should read");
let detail_payload: serde_json::Value =
serde_json::from_slice(&detail_body).expect("json body should parse");
assert_eq!(detail_payload["status"], json!("ok"));
assert_eq!(
detail_payload["affinities"][0]["api_format"],
json!("openai:chat")
);
assert_eq!(detail_payload["total_endpoints"], json!(1));
let delete_response = local_monitoring_response(
&state,
&request_context(
http::Method::DELETE,
"/api/admin/monitoring/cache/affinity/user-key-1/endpoint-1/model-alpha/openai:chat",
),
)
.await
.expect("handler should not error")
.expect("route should be handled locally");
assert_eq!(delete_response.status(), http::StatusCode::OK);
let delete_body = to_bytes(delete_response.into_body(), usize::MAX)
.await
.expect("body should read");
let delete_payload: serde_json::Value =
serde_json::from_slice(&delete_body).expect("json body should parse");
assert_eq!(
delete_payload["message"],
json!("已清除缓存亲和性: Alice Key")
);
assert_eq!(delete_payload["affinity_key"], json!("user-key-1"));
assert_eq!(
state.read_scheduler_affinity_target(
&affinity_cache_key,
crate::scheduler::affinity::SCHEDULER_AFFINITY_TTL,
),
None
);
}
#[tokio::test]
async fn admin_monitoring_cache_users_delete_returns_local_payload_from_test_store() {
let user_repository = Arc::new(

4
apps/aether-gateway/src/handlers/admin/observability/monitoring/trace.rs
View File

@@ -13,7 +13,7 @@ use axum::{
body::Body,
response::{IntoResponse, Response},
};
use tracing::warn;
use tracing::debug;
pub(super) async fn build_admin_monitoring_trace_request_response(
state: &AdminAppState<'_>,
@@ -38,7 +38,7 @@ pub(super) async fn build_admin_monitoring_trace_request_response(
.await
.map_err(|err| GatewayError::Internal(err.to_string()))?
else {
warn!(
debug!(
event_name = "admin_monitoring_request_trace_not_found",
log_type = "admin_monitoring",
request_id = %short_request_id(request_id.as_str()),

4
apps/aether-gateway/src/handlers/admin/observability/usage/analytics/cache_affinity.rs
View File

@@ -20,8 +20,8 @@ pub(in super::super) async fn list_recent_completed_usage_for_cache_affinity(
.await?;
items.retain(|item| item.status == "completed");
items.sort_by(|left, right| {
left.created_at_unix_secs
.cmp(&right.created_at_unix_secs)
left.created_at_unix_ms
.cmp(&right.created_at_unix_ms)
.then_with(|| left.id.cmp(&right.id))
});
Ok(items)

10
apps/aether-gateway/src/handlers/admin/observability/usage/analytics_routes/cache_affinity_interval_timeline.rs
View File

@@ -50,14 +50,14 @@ pub(super) async fn build_admin_usage_cache_affinity_interval_timeline_response(
}
}
let mut previous_created_at_unix_secs = None;
let mut previous_created_at_unix_ms = None;
for item in items {
if let Some(previous) = previous_created_at_unix_secs {
if let Some(previous) = previous_created_at_unix_ms {
let interval_minutes =
item.created_at_unix_secs.saturating_sub(previous) as f64 / 60.0;
item.created_at_unix_ms.saturating_sub(previous) as f64 / 60.0;
if interval_minutes <= 120.0 {
let mut point = json!({
"x": unix_secs_to_rfc3339(item.created_at_unix_secs),
"x": unix_secs_to_rfc3339(item.created_at_unix_ms),
"y": ((interval_minutes * 100.0).round()) / 100.0,
});
if !item.model.trim().is_empty() {
@@ -78,7 +78,7 @@ pub(super) async fn build_admin_usage_cache_affinity_interval_timeline_response(
.push(point);
}
}
previous_created_at_unix_secs = Some(item.created_at_unix_secs);
previous_created_at_unix_ms = Some(item.created_at_unix_ms);
}
}

8
apps/aether-gateway/src/handlers/admin/observability/usage/summary_routes.rs
View File

@@ -78,8 +78,8 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
.collect();
items.sort_by(|left, right| {
right
.created_at_unix_secs
.cmp(&left.created_at_unix_secs)
.created_at_unix_ms
.cmp(&left.created_at_unix_ms)
.then_with(|| left.id.cmp(&right.id))
});
if requested_ids.is_none() && items.len() > 50 {
@@ -147,8 +147,8 @@ pub(super) async fn maybe_build_local_admin_usage_summary_response(
});
usage.sort_by(|left, right| {
right
.created_at_unix_secs
.cmp(&left.created_at_unix_secs)
.created_at_unix_ms
.cmp(&left.created_at_unix_ms)
.then_with(|| left.id.cmp(&right.id))
});
let total = usage.len();

4
apps/aether-gateway/src/handlers/admin/provider/endpoints_admin/reads.rs
View File

@@ -28,9 +28,9 @@ pub(crate) async fn build_admin_provider_endpoints_payload(
.unwrap_or_default();
endpoints.sort_by(|left, right| {
right
.created_at_unix_secs
.created_at_unix_ms
.unwrap_or_default()
.cmp(&left.created_at_unix_secs.unwrap_or_default())
.cmp(&left.created_at_unix_ms.unwrap_or_default())
.then_with(|| left.id.cmp(&right.id))
});
let keys = state

2
apps/aether-gateway/src/handlers/admin/provider/oauth/dispatch/device/authorize.rs
View File

@@ -165,7 +165,7 @@ pub(super) async fn handle_admin_provider_oauth_device_authorize(
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned),
created_at_unix_secs: now_unix_secs,
created_at_unix_ms: now_unix_secs,
key_id: None,
email: None,
replaced: false,

2
apps/aether-gateway/src/handlers/admin/provider/oauth/provisioning.rs
View File

@@ -128,7 +128,7 @@ pub(crate) async fn create_provider_oauth_catalog_key(
record.total_response_time_ms = Some(0);
record.health_by_format = Some(json!({}));
record.circuit_breaker_by_format = Some(json!({}));
record.created_at_unix_secs = Some(now_unix_secs);
record.created_at_unix_ms = Some(now_unix_secs);
record.updated_at_unix_secs = Some(now_unix_secs);
state.create_provider_catalog_key(&record).await
}

2
apps/aether-gateway/src/handlers/admin/provider/pool_admin/payloads.rs
View File

@@ -803,7 +803,7 @@ pub(super) fn build_admin_pool_key_payload(
);
payload.insert(
"created_at".to_string(),
json!(key.created_at_unix_secs.and_then(unix_secs_to_rfc3339)),
json!(key.created_at_unix_ms.and_then(unix_secs_to_rfc3339)),
);
payload.insert(
"last_used_at".to_string(),

2
apps/aether-gateway/src/handlers/admin/provider/summary/aggregates.rs
View File

@@ -177,7 +177,7 @@ pub(crate) async fn build_admin_providers_summary_payload(
.is_active
.cmp(&left.is_active)
.then_with(|| left.provider_priority.cmp(&right.provider_priority))
.then_with(|| left.created_at_unix_secs.cmp(&right.created_at_unix_secs))
.then_with(|| left.created_at_unix_ms.cmp(&right.created_at_unix_ms))
});
let total = providers.len();

15
apps/aether-gateway/src/handlers/admin/provider/summary/health.rs
View File

@@ -1,6 +1,7 @@
use crate::handlers::admin::request::AdminAppState;
use crate::handlers::admin::shared::unix_secs_to_rfc3339;
use crate::handlers::public::{request_candidate_event_unix_secs, request_candidate_status_label};
use crate::handlers::public::{request_candidate_event_unix_ms, request_candidate_status_label};
use crate::handlers::shared::unix_ms_to_rfc3339;
use aether_data_contracts::repository::candidates::{
RequestCandidateStatus, StoredRequestCandidate,
};
@@ -82,14 +83,14 @@ pub(crate) async fn build_admin_provider_health_monitor_payload(
for candidates in attempts_by_endpoint.values_mut() {
candidates.sort_by(|left, right| {
right
.created_at_unix_secs
.cmp(&left.created_at_unix_secs)
.created_at_unix_ms
.cmp(&left.created_at_unix_ms)
.then_with(|| right.id.cmp(&left.id))
});
candidates.truncate(per_endpoint_limit);
candidates.sort_by(|left, right| {
request_candidate_event_unix_secs(left)
.cmp(&request_candidate_event_unix_secs(right))
request_candidate_event_unix_ms(left)
.cmp(&request_candidate_event_unix_ms(right))
.then_with(|| left.id.cmp(&right.id))
});
}
@@ -118,12 +119,12 @@ pub(crate) async fn build_admin_provider_health_monitor_payload(
};
let last_event_at = candidates
.last()
.and_then(|candidate| unix_secs_to_rfc3339(request_candidate_event_unix_secs(candidate)));
.and_then(|candidate| unix_ms_to_rfc3339(request_candidate_event_unix_ms(candidate)));
let events = candidates
.into_iter()
.filter_map(|candidate| {
Some(json!({
"timestamp": unix_secs_to_rfc3339(request_candidate_event_unix_secs(&candidate))?,
"timestamp": unix_ms_to_rfc3339(request_candidate_event_unix_ms(&candidate))?,
"status": request_candidate_status_label(candidate.status),
"status_code": candidate.status_code,
"latency_ms": candidate.latency_ms,

2
apps/aether-gateway/src/handlers/admin/provider/summary/list.rs
View File

@@ -88,7 +88,7 @@ pub(crate) async fn build_admin_providers_payload(
"api_key": has_any_key_by_provider.contains(&provider_id).then_some("***"),
"priority": provider.provider_priority,
"is_active": provider.is_active,
"created_at": provider.created_at_unix_secs.and_then(unix_secs_to_rfc3339),
"created_at": provider.created_at_unix_ms.and_then(unix_secs_to_rfc3339),
"updated_at": provider.updated_at_unix_secs.and_then(unix_secs_to_rfc3339),
})
})

4
apps/aether-gateway/src/handlers/admin/provider/summary/value.rs
View File

@@ -1,6 +1,6 @@
use crate::handlers::admin::shared::unix_secs_to_rfc3339;
use crate::handlers::public::{
provider_key_api_formats, request_candidate_event_unix_secs, request_candidate_status_label,
provider_key_api_formats, request_candidate_event_unix_ms, request_candidate_status_label,
};
use aether_data_contracts::repository::candidates::{
RequestCandidateStatus, StoredRequestCandidate,
@@ -172,7 +172,7 @@ pub(crate) fn build_admin_provider_summary_value(
"endpoint_health_details": endpoint_health_details,
"ops_configured": ops_configured,
"ops_architecture_id": ops_architecture_id,
"created_at": endpoint_timestamp_or_now(provider.created_at_unix_secs, now_unix_secs),
"created_at": endpoint_timestamp_or_now(provider.created_at_unix_ms, now_unix_secs),
"updated_at": endpoint_timestamp_or_now(provider.updated_at_unix_secs, now_unix_secs),
})
}

2
apps/aether-gateway/src/handlers/admin/provider/write/keys/create.rs
View File

@@ -176,7 +176,7 @@ pub(crate) async fn build_admin_create_provider_key_record(
normalize_string_list(payload.model_exclude_patterns).map(|value| json!(value));
key.health_by_format = Some(json!({}));
key.circuit_breaker_by_format = Some(json!({}));
key.created_at_unix_secs = Some(now_unix_secs);
key.created_at_unix_ms = Some(now_unix_secs);
key.updated_at_unix_secs = Some(now_unix_secs);
Ok(key)
}

4
apps/aether-gateway/src/handlers/admin/provider/write/keys/payload.rs
View File

@@ -24,9 +24,9 @@ pub(crate) async fn build_admin_provider_keys_payload(
left.internal_priority
.cmp(&right.internal_priority)
.then_with(|| {
left.created_at_unix_secs
left.created_at_unix_ms
.unwrap_or_default()
.cmp(&right.created_at_unix_secs.unwrap_or_default())
.cmp(&right.created_at_unix_ms.unwrap_or_default())
})
.then_with(|| left.id.cmp(&right.id))
});

4
apps/aether-gateway/src/handlers/admin/system/shared/export/providers.rs
View File

@@ -82,9 +82,9 @@ pub(crate) async fn build_admin_system_export_providers_payload(
left.internal_priority
.cmp(&right.internal_priority)
.then(
left.created_at_unix_secs
left.created_at_unix_ms
.unwrap_or(0)
.cmp(&right.created_at_unix_secs.unwrap_or(0)),
.cmp(&right.created_at_unix_ms.unwrap_or(0)),
)
.then(left.id.cmp(&right.id))
});

2
apps/aether-gateway/src/handlers/internal/gateway_helpers.rs
View File

@@ -463,7 +463,7 @@ pub(crate) fn build_management_token_payload(
"last_used_ip": token.last_used_ip,
"usage_count": token.usage_count,
"is_active": token.is_active,
"created_at": token.created_at_unix_secs.and_then(unix_secs_to_rfc3339),
"created_at": token.created_at_unix_ms.and_then(unix_secs_to_rfc3339),
"updated_at": token.updated_at_unix_secs.and_then(unix_secs_to_rfc3339),
});
if let Some(user) = user {

115
apps/aether-gateway/src/handlers/public/catalog_helpers.rs
View File

@@ -1,5 +1,7 @@
use crate::api::ai::public_api_format_local_path;
use crate::handlers::shared::{query_param_optional_bool, query_param_value, unix_secs_to_rfc3339};
use crate::handlers::shared::{
query_param_optional_bool, query_param_value, unix_ms_to_rfc3339, unix_secs_to_rfc3339,
};
use crate::AppState;
use aether_data_contracts::repository::candidates::{
PublicHealthTimelineBucket, RequestCandidateStatus, StoredRequestCandidate,
@@ -25,11 +27,11 @@ pub(crate) fn request_candidate_status_label(status: RequestCandidateStatus) ->
}
}
pub(crate) fn request_candidate_event_unix_secs(candidate: &StoredRequestCandidate) -> u64 {
pub(crate) fn request_candidate_event_unix_ms(candidate: &StoredRequestCandidate) -> u64 {
candidate
.finished_at_unix_secs
.or(candidate.started_at_unix_secs)
.unwrap_or(candidate.created_at_unix_secs)
.finished_at_unix_ms
.or(candidate.started_at_unix_ms)
.unwrap_or(candidate.created_at_unix_ms)
}
pub(crate) fn normalize_admin_base_url(base_url: &str) -> Result<String, String> {
@@ -410,28 +412,24 @@ pub(crate) async fn build_api_format_health_monitor_payload(
total_count: 0,
success_count: 0,
failed_count: 0,
min_created_at_unix_secs: None,
max_created_at_unix_secs: None,
min_created_at_unix_ms: None,
max_created_at_unix_ms: None,
});
bucket.total_count += row.total_count;
bucket.success_count += row.success_count;
bucket.failed_count += row.failed_count;
bucket.min_created_at_unix_secs = match (
bucket.min_created_at_unix_secs,
row.min_created_at_unix_secs,
) {
(Some(left), Some(right)) => Some(left.min(right)),
(None, Some(right)) => Some(right),
(left, None) => left,
};
bucket.max_created_at_unix_secs = match (
bucket.max_created_at_unix_secs,
row.max_created_at_unix_secs,
) {
(Some(left), Some(right)) => Some(left.max(right)),
(None, Some(right)) => Some(right),
(left, None) => left,
};
bucket.min_created_at_unix_ms =
match (bucket.min_created_at_unix_ms, row.min_created_at_unix_ms) {
(Some(left), Some(right)) => Some(left.min(right)),
(None, Some(right)) => Some(right),
(left, None) => left,
};
bucket.max_created_at_unix_ms =
match (bucket.max_created_at_unix_ms, row.max_created_at_unix_ms) {
(Some(left), Some(right)) => Some(left.max(right)),
(None, Some(right)) => Some(right),
(left, None) => left,
};
}
let mut formats = Vec::new();
@@ -456,19 +454,19 @@ pub(crate) async fn build_api_format_health_monitor_payload(
};
let last_event_at = attempts.first().and_then(|candidate| {
candidate
.finished_at_unix_secs
.or(candidate.started_at_unix_secs)
.or(Some(candidate.created_at_unix_secs))
.finished_at_unix_ms
.or(candidate.started_at_unix_ms)
.or(Some(candidate.created_at_unix_ms))
});
let events = attempts
.into_iter()
.filter_map(|candidate| {
let timestamp = candidate
.finished_at_unix_secs
.or(candidate.started_at_unix_secs)
.unwrap_or(candidate.created_at_unix_secs);
.finished_at_unix_ms
.or(candidate.started_at_unix_ms)
.unwrap_or(candidate.created_at_unix_ms);
Some(json!({
"timestamp": unix_secs_to_rfc3339(timestamp)?,
"timestamp": unix_ms_to_rfc3339(timestamp)?,
"status": request_candidate_status_label(candidate.status),
"status_code": candidate.status_code,
"latency_ms": candidate.latency_ms,
@@ -490,11 +488,11 @@ pub(crate) async fn build_api_format_health_monitor_payload(
"failed_count": failed_count,
"skipped_count": skipped_count,
"success_rate": success_rate,
"last_event_at": last_event_at.and_then(unix_secs_to_rfc3339),
"last_event_at": last_event_at.and_then(unix_ms_to_rfc3339),
"events": events,
"timeline": timeline,
"time_range_start": time_range_start.and_then(unix_secs_to_rfc3339),
"time_range_end": time_range_end.or(Some(now_unix_secs)).and_then(unix_secs_to_rfc3339),
"time_range_start": time_range_start.and_then(unix_ms_to_rfc3339),
"time_range_end": time_range_end.map(|ms| unix_ms_to_rfc3339(ms)).unwrap_or_else(|| unix_secs_to_rfc3339(now_unix_secs)),
});
if options.include_api_path {
format_payload["api_path"] = json!(public_api_format_local_path(&api_format));
@@ -536,12 +534,12 @@ pub(crate) fn build_public_health_timeline(
continue;
}
earliest_time = match (earliest_time, bucket.min_created_at_unix_secs) {
earliest_time = match (earliest_time, bucket.min_created_at_unix_ms) {
(Some(left), Some(right)) => Some(left.min(right)),
(None, Some(right)) => Some(right),
(left, None) => left,
};
latest_time = match (latest_time, bucket.max_created_at_unix_secs) {
latest_time = match (latest_time, bucket.max_created_at_unix_ms) {
(Some(left), Some(right)) => Some(left.max(right)),
(None, Some(right)) => Some(right),
(left, None) => left,
@@ -592,3 +590,50 @@ pub(crate) fn api_format_display_name(api_format: &str) -> String {
};
format!("{family_label} {kind_label}")
}
#[cfg(test)]
mod tests {
use super::request_candidate_event_unix_ms;
use crate::handlers::shared::unix_ms_to_rfc3339;
use aether_data_contracts::repository::candidates::{
RequestCandidateStatus, StoredRequestCandidate,
};
#[test]
fn request_candidate_event_timestamp_uses_millisecond_precision() {
let candidate = StoredRequestCandidate::new(
"cand-1".to_string(),
"req-1".to_string(),
None,
None,
None,
None,
0,
0,
Some("provider-1".to_string()),
Some("endpoint-1".to_string()),
Some("key-1".to_string()),
RequestCandidateStatus::Success,
None,
false,
Some(200),
None,
None,
Some(42),
Some(1),
None,
None,
1_700_000_000_000,
Some(1_700_000_000_111),
Some(1_700_000_000_123),
)
.expect("candidate should build");
let event_unix_ms = request_candidate_event_unix_ms(&candidate);
assert_eq!(event_unix_ms, 1_700_000_000_123);
assert_eq!(
unix_ms_to_rfc3339(event_unix_ms).as_deref(),
Some("2023-11-14T22:13:20.123Z")
);
}
}

2
apps/aether-gateway/src/handlers/public/mod.rs
View File

@@ -10,7 +10,7 @@ pub(crate) use self::catalog_helpers::{
admin_requested_force_stream, api_format_display_name, build_api_format_health_monitor_payload,
build_public_catalog_models_payload, build_public_catalog_search_models_payload,
build_public_health_timeline, build_public_providers_payload, normalize_admin_base_url,
provider_key_api_formats, request_candidate_event_unix_secs, request_candidate_status_label,
provider_key_api_formats, request_candidate_event_unix_ms, request_candidate_status_label,
ApiFormatHealthMonitorOptions,
};
pub(crate) use self::system_modules_helpers::{

2
apps/aether-gateway/src/handlers/public/support/announcements/shared.rs
View File

@@ -84,7 +84,7 @@ pub(super) fn build_public_announcement_payload(
},
"start_time": format_optional_unix_datetime(announcement.start_time_unix_secs),
"end_time": format_optional_unix_datetime(announcement.end_time_unix_secs),
"created_at": format_required_unix_datetime(announcement.created_at_unix_secs),
"created_at": format_required_unix_datetime(announcement.created_at_unix_ms),
"updated_at": format_required_unix_datetime(announcement.updated_at_unix_secs),
})
}

4
apps/aether-gateway/src/handlers/public/support/dashboard_filters.rs
View File

@@ -361,7 +361,7 @@ fn dashboard_usage_local_date(
item: &StoredRequestUsageAudit,
tz_offset_minutes: i32,
) -> Option<chrono::NaiveDate> {
let timestamp = i64::try_from(item.created_at_unix_secs).ok()?;
let timestamp = i64::try_from(item.created_at_unix_ms).ok()?;
let datetime = chrono::DateTime::<chrono::Utc>::from_timestamp(timestamp, 0)?;
Some((datetime + chrono::Duration::minutes(i64::from(tz_offset_minutes))).date_naive())
}
@@ -1008,7 +1008,7 @@ pub(super) async fn handle_dashboard_recent_requests_get(
"user": username,
"model": dashboard_non_empty_value(&item.model, "N/A"),
"tokens": item.total_tokens,
"time": dashboard_format_time_hhmm(item.created_at_unix_secs),
"time": dashboard_format_time_hhmm(item.created_at_unix_ms),
"is_stream": item.is_stream,
})
})

27
apps/aether-gateway/src/handlers/public/support/user_me_usage.rs
View File

@@ -95,7 +95,7 @@ fn users_me_usage_total_input_context(item: &StoredRequestUsageAudit) -> u64 {
fn users_me_usage_effective_unix_secs(item: &StoredRequestUsageAudit) -> u64 {
item.finalized_at_unix_secs
.unwrap_or(item.created_at_unix_secs)
.unwrap_or(item.created_at_unix_ms)
}
fn users_me_usage_cache_hit_rate(total_input_context: u64, cache_read_tokens: u64) -> f64 {
@@ -159,7 +159,7 @@ fn build_users_me_usage_record_payload(
"first_byte_time_ms": item.first_byte_time_ms,
"is_stream": item.is_stream,
"status": item.status,
"created_at": unix_secs_to_rfc3339(item.created_at_unix_secs),
"created_at": unix_secs_to_rfc3339(item.created_at_unix_ms),
"cache_creation_input_tokens": item.cache_creation_input_tokens,
"cache_read_input_tokens": item.cache_read_input_tokens,
"status_code": item.status_code,
@@ -649,8 +649,8 @@ pub(super) async fn handle_users_me_usage_get(
.collect::<Vec<_>>();
records.sort_by(|left, right| {
right
.created_at_unix_secs
.cmp(&left.created_at_unix_secs)
.created_at_unix_ms
.cmp(&left.created_at_unix_ms)
.then_with(|| left.id.cmp(&right.id))
});
let total_record_count = records.len();
@@ -735,8 +735,8 @@ pub(super) async fn handle_users_me_usage_active_get(
.collect::<Vec<_>>();
items.sort_by(|left, right| {
right
.created_at_unix_secs
.cmp(&left.created_at_unix_secs)
.created_at_unix_ms
.cmp(&left.created_at_unix_ms)
.then_with(|| left.id.cmp(&right.id))
});
if ids.is_none() && items.len() > 50 {
@@ -798,20 +798,19 @@ pub(super) async fn handle_users_me_usage_interval_timeline_get(
};
items.retain(|item| item.status == "completed");
items.sort_by(|left, right| {
left.created_at_unix_secs
.cmp(&right.created_at_unix_secs)
left.created_at_unix_ms
.cmp(&right.created_at_unix_ms)
.then_with(|| left.id.cmp(&right.id))
});
let mut points = Vec::new();
let mut previous_created_at_unix_secs = None;
let mut previous_created_at_unix_ms = None;
for item in items {
if let Some(previous) = previous_created_at_unix_secs {
let interval_minutes =
(item.created_at_unix_secs.saturating_sub(previous) as f64) / 60.0;
if let Some(previous) = previous_created_at_unix_ms {
let interval_minutes = (item.created_at_unix_ms.saturating_sub(previous) as f64) / 60.0;
if interval_minutes <= 120.0 {
points.push(json!({
"x": unix_secs_to_rfc3339(item.created_at_unix_secs),
"x": unix_secs_to_rfc3339(item.created_at_unix_ms),
"y": round_to(interval_minutes, 2),
"model": item.model,
}));
@@ -820,7 +819,7 @@ pub(super) async fn handle_users_me_usage_interval_timeline_get(
}
}
}
previous_created_at_unix_secs = Some(item.created_at_unix_secs);
previous_created_at_unix_ms = Some(item.created_at_unix_ms);
}
Json(json!({

2
apps/aether-gateway/src/handlers/public/support/wallet/reads.rs
View File

@@ -153,7 +153,7 @@ pub(super) fn wallet_transaction_payload_from_record(
"link_id": record.link_id.clone(),
"operator_id": record.operator_id.clone(),
"description": record.description.clone(),
"created_at": record.created_at_unix_secs.and_then(unix_secs_to_rfc3339),
"created_at": record.created_at_unix_ms.and_then(unix_secs_to_rfc3339),
})
}

4
apps/aether-gateway/src/handlers/public/support/wallet/recharge.rs
View File

@@ -207,7 +207,7 @@ pub(crate) fn wallet_payment_order_payload_from_row(
row: &sqlx::postgres::PgRow,
) -> Result<serde_json::Value, GatewayError> {
let created_at = row
.try_get::<Option<i64>, _>("created_at_unix_secs")
.try_get::<Option<i64>, _>("created_at_unix_ms")
.map_err(|err| GatewayError::Internal(err.to_string()))?
.and_then(|value| u64::try_from(value).ok())
.and_then(unix_secs_to_rfc3339);
@@ -280,7 +280,7 @@ fn wallet_payment_order_payload_from_record(
record.gateway_order_id.clone(),
record.gateway_response.clone(),
record.status.clone(),
Some(unix_secs_to_rfc3339(record.created_at_unix_secs)).flatten(),
Some(unix_secs_to_rfc3339(record.created_at_unix_ms)).flatten(),
record.paid_at_unix_secs.and_then(unix_secs_to_rfc3339),
record.credited_at_unix_secs.and_then(unix_secs_to_rfc3339),
record.expires_at_unix_secs.and_then(unix_secs_to_rfc3339),

6
apps/aether-gateway/src/handlers/public/support/wallet/refunds.rs
View File

@@ -98,7 +98,7 @@ fn wallet_refund_payload_from_row(
row: &sqlx::postgres::PgRow,
) -> Result<serde_json::Value, GatewayError> {
let created_at = row
.try_get::<Option<i64>, _>("created_at_unix_secs")
.try_get::<Option<i64>, _>("created_at_unix_ms")
.map_err(|err| GatewayError::Internal(err.to_string()))?
.and_then(|value| u64::try_from(value).ok())
.and_then(unix_secs_to_rfc3339);
@@ -157,7 +157,7 @@ fn wallet_refund_payload_from_record(
"payout_method": record.payout_method,
"payout_reference": record.payout_reference,
"payout_proof": record.payout_proof,
"created_at": unix_secs_to_rfc3339(record.created_at_unix_secs),
"created_at": unix_secs_to_rfc3339(record.created_at_unix_ms),
"updated_at": unix_secs_to_rfc3339(record.updated_at_unix_secs),
"processed_at": record.processed_at_unix_secs.and_then(unix_secs_to_rfc3339),
"completed_at": record.completed_at_unix_secs.and_then(unix_secs_to_rfc3339),
@@ -247,7 +247,7 @@ pub(super) async fn handle_wallet_refunds_list(
"payout_method": record.payout_method,
"payout_reference": record.payout_reference,
"payout_proof": record.payout_proof,
"created_at": unix_secs_to_rfc3339(record.created_at_unix_secs),
"created_at": unix_secs_to_rfc3339(record.created_at_unix_ms),
"updated_at": unix_secs_to_rfc3339(record.updated_at_unix_secs),
"processed_at": record.processed_at_unix_secs.and_then(unix_secs_to_rfc3339),
"completed_at": record.completed_at_unix_secs.and_then(unix_secs_to_rfc3339),

2
apps/aether-gateway/src/handlers/public/system_modules_helpers/keys_grouped.rs
View File

@@ -156,7 +156,7 @@ pub(crate) async fn build_admin_keys_grouped_by_format_payload(
.and_then(serde_json::Value::as_bool)
.unwrap_or(false),
"last_used_at": key.last_used_at_unix_secs.and_then(unix_secs_to_rfc3339),
"created_at": unix_secs_to_rfc3339(key.created_at_unix_secs.unwrap_or(now_unix_secs)),
"created_at": unix_secs_to_rfc3339(key.created_at_unix_ms.unwrap_or(now_unix_secs)),
"updated_at": unix_secs_to_rfc3339(key.updated_at_unix_secs.unwrap_or(now_unix_secs)),
}));
}

2
apps/aether-gateway/src/handlers/shared/catalog.rs
View File

@@ -598,7 +598,7 @@ pub(crate) fn build_admin_provider_key_response(
payload.insert(
"created_at".to_string(),
json!(unix_secs_to_rfc3339(
key.created_at_unix_secs.unwrap_or(now_unix_secs)
key.created_at_unix_ms.unwrap_or(now_unix_secs)
)),
);
payload.insert(

3
apps/aether-gateway/src/handlers/shared/mod.rs
View File

@@ -39,7 +39,8 @@ pub(crate) use self::request_utils::{
query_param_bool, query_param_optional_bool, query_param_value,
request_enables_control_execute, rust_auth_terminates_provider_credentials,
sanitize_upstream_path_and_query, should_strip_forwarded_provider_credential_header,
should_strip_forwarded_trusted_admin_header, strip_query_param, unix_secs_to_rfc3339,
should_strip_forwarded_trusted_admin_header, strip_query_param, unix_ms_to_rfc3339,
unix_secs_to_rfc3339,
};
pub(crate) use self::system_config_values::{
module_available_from_env, system_config_bool, system_config_string,

9
apps/aether-gateway/src/handlers/shared/request_utils.rs
View File

@@ -182,6 +182,15 @@ pub(crate) fn unix_secs_to_rfc3339(unix_secs: u64) -> Option<String> {
)
}
pub(crate) fn unix_ms_to_rfc3339(unix_ms: u64) -> Option<String> {
let secs = i64::try_from(unix_ms / 1000).ok()?;
let nanos = ((unix_ms % 1000) * 1_000_000) as u32;
Some(
chrono::DateTime::<Utc>::from_timestamp(secs, nanos)?
.to_rfc3339_opts(SecondsFormat::Millis, true),
)
}
pub(crate) fn json_string_list(value: Option<&serde_json::Value>) -> Vec<String> {
value
.and_then(serde_json::Value::as_array)