i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-10-09 02:47:45 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: revamp analytics dashboards and harden database migrations

Browse Source 
Add dashboard and overview analytics, health monitoring, provider expense tracking, and announcement updates across the gateway and frontend.

Keep schema migrations free of historical backfills while preserving automatic backfill execution. Bound migration deadlines, run schema preparation before Compose replacement, and anonymize deleted dashboard users.

Include the current documentation cleanup and regression coverage.
This commit is contained in:
elky
2026-10-01 11:48:17 +08:00
parent 60b89cc840
commit 066ea87d72
327 changed files with 31728 additions and 20645 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/aether-data/runtime/src/repository/usage/memory.rs
+150 -5
View File
@@ -44,17 +44,33 @@ use super::{
use crate::repository::auth::InMemoryAuthApiKeySnapshotRepository;
use crate::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
use crate::DataLayerError;
mod analytics;
mod dashboard_summary;
#[derive(Debug, Default)]
pub struct InMemoryUsageReadRepository {
dashboard_projection: RwLock<dashboard_summary::DashboardProjection>,
by_request_id: RwLock<BTreeMap<String, StoredRequestUsageAudit>>,
detached_bodies: RwLock<BTreeMap<String, Value>>,
provider_usage_windows: RwLock<Vec<StoredProviderUsageWindow>>,
auth_api_keys: Option<Arc<InMemoryAuthApiKeySnapshotRepository>>,
provider_catalog: Option<Arc<InMemoryProviderCatalogReadRepository>>,
analytics_users: RwLock<Vec<aether_data_contracts::repository::users::StoredUserSummary>>,
analytics_candidates:
RwLock<Vec<aether_data_contracts::repository::candidates::StoredRequestCandidate>>,
analytics_allocations: RwLock<
BTreeMap<String, aether_data_contracts::repository::usage::UsageAnalyticsAllocation>,
>,
}
impl InMemoryUsageReadRepository {
fn analytics_key_flags(&self) -> BTreeMap<String, bool> {
self.auth_api_keys
.as_ref()
.map(|repository| repository.standalone_flags())
.unwrap_or_default()
}
pub fn seed<I>(items: I) -> Self
where
I: IntoIterator<Item = StoredRequestUsageAudit>,
@@ -66,11 +82,15 @@ impl InMemoryUsageReadRepository {
by_request_id.insert(item.request_id.clone(), item);
}
Self {
dashboard_projection: Default::default(),
by_request_id: RwLock::new(by_request_id),
detached_bodies: RwLock::new(BTreeMap::new()),
provider_usage_windows: RwLock::new(Vec::new()),
auth_api_keys: None,
provider_catalog: None,
analytics_users: Default::default(),
analytics_candidates: Default::default(),
analytics_allocations: Default::default(),
}
}
@@ -119,11 +139,15 @@ impl InMemoryUsageReadRepository {
by_request_id.insert(request_id, item);
}
Self {
dashboard_projection: Default::default(),
by_request_id: RwLock::new(by_request_id),
detached_bodies: RwLock::new(detached_bodies),
provider_usage_windows: RwLock::new(Vec::new()),
auth_api_keys: None,
provider_catalog: None,
analytics_users: Default::default(),
analytics_candidates: Default::default(),
analytics_allocations: Default::default(),
}
}
@@ -132,11 +156,15 @@ impl InMemoryUsageReadRepository {
I: IntoIterator<Item = StoredProviderUsageWindow>,
{
Self {
dashboard_projection: self.dashboard_projection,
by_request_id: self.by_request_id,
detached_bodies: self.detached_bodies,
provider_usage_windows: RwLock::new(items.into_iter().collect()),
auth_api_keys: self.auth_api_keys,
provider_catalog: self.provider_catalog,
analytics_users: self.analytics_users,
analytics_candidates: self.analytics_candidates,
analytics_allocations: self.analytics_allocations,
}
}
@@ -244,7 +272,48 @@ fn usage_has_admin_unknown_model_or_provider(item: &StoredRequestUsageAudit) ->
usage_admin_unknown_label(&item.model) || usage_admin_unknown_label(&item.provider_name)
}
fn usage_matches_list_query(item: &StoredRequestUsageAudit, query: &UsageAuditListQuery) -> bool {
fn usage_matches_list_query(
item: &StoredRequestUsageAudit,
query: &UsageAuditListQuery,
keys: &BTreeMap<String, bool>,
) -> bool {
if query
.provider_id
.as_ref()
.is_some_and(|id| item.provider_id.as_ref() != Some(id))
|| query
.endpoint_kind
.as_ref()
.is_some_and(|value| item.endpoint_kind.as_ref() != Some(value))
|| query
.request_type
.as_ref()
.is_some_and(|value| item.request_type.as_ref() != Some(value))
|| query
.slow_threshold_ms
.is_some_and(|value| item.response_time_ms.is_none_or(|latency| latency < value))
|| query
.has_format_conversion
.is_some_and(|value| item.has_format_conversion != value)
|| query
.api_key_id
.as_ref()
.is_some_and(|id| item.api_key_id.as_ref() != Some(id))
|| query
.request_id
.as_ref()
.is_some_and(|id| item.request_id != *id)
|| query
.attribution_kind
.as_deref()
.is_some_and(|kind| analytics::attribution(item, keys) != kind)
|| query
.actor_user_id
.as_deref()
.is_some_and(|id| analytics::actor(item, keys) != Some(id))
{
return false;
}
// The field is historically named `created_at_unix_ms`, but usage audit rows
// across gateway handlers, SQL repositories and tests are stored as epoch seconds.
if let Some(created_from_unix_secs) = query.created_from_unix_secs {
@@ -332,7 +401,45 @@ fn usage_matches_list_query(item: &StoredRequestUsageAudit, query: &UsageAuditLi
fn usage_matches_keyword_search_query(
item: &StoredRequestUsageAudit,
query: &UsageAuditKeywordSearchQuery,
keys: &BTreeMap<String, bool>,
) -> bool {
if query
.provider_id
.as_ref()
.is_some_and(|id| item.provider_id.as_ref() != Some(id))
|| query
.endpoint_kind
.as_ref()
.is_some_and(|value| item.endpoint_kind.as_ref() != Some(value))
|| query
.request_type
.as_ref()
.is_some_and(|value| item.request_type.as_ref() != Some(value))
|| query
.slow_threshold_ms
.is_some_and(|value| item.response_time_ms.is_none_or(|latency| latency < value))
|| query
.has_format_conversion
.is_some_and(|value| item.has_format_conversion != value)
|| query
.api_key_id
.as_ref()
.is_some_and(|id| item.api_key_id.as_ref() != Some(id))
|| query
.request_id
.as_ref()
.is_some_and(|id| item.request_id != *id)
|| query
.attribution_kind
.as_deref()
.is_some_and(|kind| analytics::attribution(item, keys) != kind)
|| query
.actor_user_id
.as_deref()
.is_some_and(|id| analytics::actor(item, keys) != Some(id))
{
return false;
}
if let Some(created_from_unix_secs) = query.created_from_unix_secs {
if item.created_at_unix_ms < created_from_unix_secs {
return false;
@@ -1127,6 +1234,38 @@ fn usage_provider_aggregation_identity(
#[async_trait]
impl UsageReadRepository for InMemoryUsageReadRepository {
async fn query_dashboard_summary(
&self,
query: &aether_data_contracts::repository::usage::UsageDashboardAnalyticsQuery,
) -> Result<aether_data_contracts::repository::usage::StoredDashboardSummary, DataLayerError> {
self.dashboard_summary_query(query)
}
async fn query_dashboard_analytics(
&self,
query: &aether_data_contracts::repository::usage::UsageDashboardAnalyticsQuery,
) -> Result<
aether_data_contracts::repository::usage::StoredUsageDashboardAnalytics,
DataLayerError,
> {
self.dashboard_analytics_query(query)
}
async fn query_usage_analytics(
&self,
query: &aether_data_contracts::repository::usage::UsageAnalyticsQuery,
) -> Result<aether_data_contracts::repository::usage::StoredUsageAnalytics, DataLayerError>
{
self.analytics_query(query)
}
async fn summarize_health_observations(
&self,
query: &aether_data_contracts::repository::usage::HealthObservationQuery,
) -> Result<aether_data_contracts::repository::usage::HealthObservationSummary, DataLayerError>
{
self.health_observations(query)
}
async fn find_by_id(
&self,
id: &str,
@@ -1205,12 +1344,13 @@ impl UsageReadRepository for InMemoryUsageReadRepository {
&self,
query: &UsageAuditListQuery,
) -> Result<Vec<StoredRequestUsageAudit>, DataLayerError> {
let keys = self.analytics_key_flags();
let mut items: Vec<_> = self
.by_request_id
.read()
.expect("usage repository lock")
.values()
.filter(|item| usage_matches_list_query(item, query))
.filter(|item| usage_matches_list_query(item, query, &keys))
.cloned()
.collect();
sort_usage_items(&mut items, query.newest_first);
@@ -1231,12 +1371,13 @@ impl UsageReadRepository for InMemoryUsageReadRepository {
&self,
query: &UsageAuditKeywordSearchQuery,
) -> Result<Vec<StoredRequestUsageAudit>, DataLayerError> {
let keys = self.analytics_key_flags();
let mut items: Vec<_> = self
.by_request_id
.read()
.expect("usage repository lock")
.values()
.filter(|item| usage_matches_keyword_search_query(item, query))
.filter(|item| usage_matches_keyword_search_query(item, query, &keys))
.cloned()
.collect();
sort_usage_items(&mut items, query.newest_first);
@@ -1254,12 +1395,13 @@ impl UsageReadRepository for InMemoryUsageReadRepository {
}
async fn count_usage_audits(&self, query: &UsageAuditListQuery) -> Result<u64, DataLayerError> {
let keys = self.analytics_key_flags();
Ok(self
.by_request_id
.read()
.expect("usage repository lock")
.values()
.filter(|item| usage_matches_list_query(item, query))
.filter(|item| usage_matches_list_query(item, query, &keys))
.count() as u64)
}
@@ -1267,12 +1409,13 @@ impl UsageReadRepository for InMemoryUsageReadRepository {
&self,
query: &UsageAuditKeywordSearchQuery,
) -> Result<u64, DataLayerError> {
let keys = self.analytics_key_flags();
Ok(self
.by_request_id
.read()
.expect("usage repository lock")
.values()
.filter(|item| usage_matches_keyword_search_query(item, query))
.filter(|item| usage_matches_keyword_search_query(item, query, &keys))
.count() as u64)
}
@@ -3281,6 +3424,8 @@ impl UsageWriteRepository for InMemoryUsageReadRepository {
finalized_at_unix_secs: usage.finalized_at_unix_secs,
};
self.dashboard_projection.write().expect("dashboard projection lock")
.record(&stored, &self.analytics_key_flags());
by_request_id.insert(stored.request_id.clone(), stored.clone());
if let Some(auth_api_keys) = self.auth_api_keys.as_ref() {
let before_contribution = existing.as_ref().and_then(api_key_usage_contribution);