mirror of
https://github.com/fawney19/Aether.git
synced 2026-10-08 02:17:46 +08:00
refactor(data): remove MySQL and SQLite support
Use PostgreSQL as the only database backend across runtime, schema tooling, installation, Compose, and CI. Update regression tests and reject removed drivers explicitly.
This commit is contained in:
elky
@@ -1,9 +1,5 @@
|
||||
#[cfg(feature = "mysql")]
|
||||
mod mysql;
|
||||
#[cfg(feature = "postgres")]
|
||||
mod postgres;
|
||||
#[cfg(feature = "sqlite")]
|
||||
mod sqlite;
|
||||
|
||||
use super::{summarize_pool, DataBackends, SqlBackendRef};
|
||||
use crate::maintenance::{
|
||||
@@ -315,15 +311,7 @@ impl<'a> SqlBackendRef<'a> {
|
||||
Self::Postgres(postgres) => {
|
||||
warm_pool(postgres.pool(), postgres.config().min_connections).await
|
||||
}
|
||||
#[cfg(feature = "mysql")]
|
||||
Self::Mysql(mysql) => {
|
||||
warm_pool(mysql.pool(), mysql.config().pool.min_connections).await
|
||||
}
|
||||
#[cfg(feature = "sqlite")]
|
||||
Self::Sqlite(sqlite) => {
|
||||
warm_pool(sqlite.pool(), sqlite.config().pool.min_connections).await
|
||||
}
|
||||
#[cfg(not(any(feature = "postgres", feature = "mysql", feature = "sqlite")))]
|
||||
#[cfg(not(feature = "postgres"))]
|
||||
Self::Disabled(_) => unreachable!("a SQL backend cannot exist without a driver"),
|
||||
}
|
||||
}
|
||||
@@ -335,11 +323,7 @@ impl<'a> SqlBackendRef<'a> {
|
||||
match self {
|
||||
#[cfg(feature = "postgres")]
|
||||
Self::Postgres(postgres) => postgres.run_table_maintenance(table_names).await,
|
||||
#[cfg(feature = "mysql")]
|
||||
Self::Mysql(mysql) => mysql.run_table_maintenance(table_names).await,
|
||||
#[cfg(feature = "sqlite")]
|
||||
Self::Sqlite(sqlite) => sqlite.run_table_maintenance(table_names).await,
|
||||
#[cfg(not(any(feature = "postgres", feature = "mysql", feature = "sqlite")))]
|
||||
#[cfg(not(feature = "postgres"))]
|
||||
Self::Disabled(_) => unreachable!("a SQL backend cannot exist without a driver"),
|
||||
}
|
||||
}
|
||||
@@ -351,17 +335,7 @@ impl<'a> SqlBackendRef<'a> {
|
||||
crate::lifecycle::migrate::run_migrations(postgres.pool()).await?;
|
||||
Ok(true)
|
||||
}
|
||||
#[cfg(feature = "mysql")]
|
||||
Self::Mysql(mysql) => {
|
||||
crate::lifecycle::migrate::run_mysql_migrations(mysql.pool()).await?;
|
||||
Ok(true)
|
||||
}
|
||||
#[cfg(feature = "sqlite")]
|
||||
Self::Sqlite(sqlite) => {
|
||||
crate::lifecycle::migrate::run_sqlite_migrations(sqlite.pool()).await?;
|
||||
Ok(true)
|
||||
}
|
||||
#[cfg(not(any(feature = "postgres", feature = "mysql", feature = "sqlite")))]
|
||||
#[cfg(not(feature = "postgres"))]
|
||||
Self::Disabled(_) => unreachable!("a SQL backend cannot exist without a driver"),
|
||||
}
|
||||
}
|
||||
@@ -373,17 +347,7 @@ impl<'a> SqlBackendRef<'a> {
|
||||
crate::lifecycle::backfill::run_backfills(postgres.pool()).await?;
|
||||
Ok(true)
|
||||
}
|
||||
#[cfg(feature = "mysql")]
|
||||
Self::Mysql(mysql) => {
|
||||
crate::lifecycle::backfill::run_mysql_backfills(mysql.pool()).await?;
|
||||
Ok(true)
|
||||
}
|
||||
#[cfg(feature = "sqlite")]
|
||||
Self::Sqlite(sqlite) => {
|
||||
crate::lifecycle::backfill::run_sqlite_backfills(sqlite.pool()).await?;
|
||||
Ok(true)
|
||||
}
|
||||
#[cfg(not(any(feature = "postgres", feature = "mysql", feature = "sqlite")))]
|
||||
#[cfg(not(feature = "postgres"))]
|
||||
Self::Disabled(_) => unreachable!("a SQL backend cannot exist without a driver"),
|
||||
}
|
||||
}
|
||||
@@ -396,15 +360,7 @@ impl<'a> SqlBackendRef<'a> {
|
||||
Self::Postgres(postgres) => Ok(Some(
|
||||
crate::lifecycle::migrate::pending_migrations(postgres.pool()).await?,
|
||||
)),
|
||||
#[cfg(feature = "mysql")]
|
||||
Self::Mysql(mysql) => Ok(Some(
|
||||
crate::lifecycle::migrate::pending_mysql_migrations(mysql.pool()).await?,
|
||||
)),
|
||||
#[cfg(feature = "sqlite")]
|
||||
Self::Sqlite(sqlite) => Ok(Some(
|
||||
crate::lifecycle::migrate::pending_sqlite_migrations(sqlite.pool()).await?,
|
||||
)),
|
||||
#[cfg(not(any(feature = "postgres", feature = "mysql", feature = "sqlite")))]
|
||||
#[cfg(not(feature = "postgres"))]
|
||||
Self::Disabled(_) => unreachable!("a SQL backend cannot exist without a driver"),
|
||||
}
|
||||
}
|
||||
@@ -417,16 +373,7 @@ impl<'a> SqlBackendRef<'a> {
|
||||
Self::Postgres(postgres) => Ok(Some(
|
||||
crate::lifecycle::migrate::prepare_database_for_startup(postgres.pool()).await?,
|
||||
)),
|
||||
#[cfg(feature = "mysql")]
|
||||
Self::Mysql(mysql) => Ok(Some(
|
||||
crate::lifecycle::migrate::prepare_mysql_database_for_startup(mysql.pool()).await?,
|
||||
)),
|
||||
#[cfg(feature = "sqlite")]
|
||||
Self::Sqlite(sqlite) => Ok(Some(
|
||||
crate::lifecycle::migrate::prepare_sqlite_database_for_startup(sqlite.pool())
|
||||
.await?,
|
||||
)),
|
||||
#[cfg(not(any(feature = "postgres", feature = "mysql", feature = "sqlite")))]
|
||||
#[cfg(not(feature = "postgres"))]
|
||||
Self::Disabled(_) => unreachable!("a SQL backend cannot exist without a driver"),
|
||||
}
|
||||
}
|
||||
@@ -439,15 +386,7 @@ impl<'a> SqlBackendRef<'a> {
|
||||
Self::Postgres(postgres) => Ok(Some(
|
||||
crate::lifecycle::backfill::pending_backfills(postgres.pool()).await?,
|
||||
)),
|
||||
#[cfg(feature = "mysql")]
|
||||
Self::Mysql(mysql) => Ok(Some(
|
||||
crate::lifecycle::backfill::pending_mysql_backfills(mysql.pool()).await?,
|
||||
)),
|
||||
#[cfg(feature = "sqlite")]
|
||||
Self::Sqlite(sqlite) => Ok(Some(
|
||||
crate::lifecycle::backfill::pending_sqlite_backfills(sqlite.pool()).await?,
|
||||
)),
|
||||
#[cfg(not(any(feature = "postgres", feature = "mysql", feature = "sqlite")))]
|
||||
#[cfg(not(feature = "postgres"))]
|
||||
Self::Disabled(_) => unreachable!("a SQL backend cannot exist without a driver"),
|
||||
}
|
||||
}
|
||||
@@ -461,21 +400,7 @@ impl<'a> SqlBackendRef<'a> {
|
||||
postgres.pool().num_idle(),
|
||||
postgres.config().max_connections,
|
||||
),
|
||||
#[cfg(feature = "mysql")]
|
||||
Self::Mysql(mysql) => summarize_pool(
|
||||
crate::database::DatabaseDriver::Mysql,
|
||||
usize::try_from(mysql.pool().size()).unwrap_or(usize::MAX),
|
||||
mysql.pool().num_idle(),
|
||||
mysql.config().pool.max_connections,
|
||||
),
|
||||
#[cfg(feature = "sqlite")]
|
||||
Self::Sqlite(sqlite) => summarize_pool(
|
||||
crate::database::DatabaseDriver::Sqlite,
|
||||
usize::try_from(sqlite.pool().size()).unwrap_or(usize::MAX),
|
||||
sqlite.pool().num_idle(),
|
||||
sqlite.config().pool.max_connections,
|
||||
),
|
||||
#[cfg(not(any(feature = "postgres", feature = "mysql", feature = "sqlite")))]
|
||||
#[cfg(not(feature = "postgres"))]
|
||||
Self::Disabled(_) => unreachable!("a SQL backend cannot exist without a driver"),
|
||||
}
|
||||
}
|
||||
@@ -487,11 +412,7 @@ impl<'a> SqlBackendRef<'a> {
|
||||
match self {
|
||||
#[cfg(feature = "postgres")]
|
||||
Self::Postgres(postgres) => postgres.aggregate_wallet_daily_usage(input).await,
|
||||
#[cfg(feature = "mysql")]
|
||||
Self::Mysql(mysql) => mysql.aggregate_wallet_daily_usage(input).await,
|
||||
#[cfg(feature = "sqlite")]
|
||||
Self::Sqlite(sqlite) => sqlite.aggregate_wallet_daily_usage(input).await,
|
||||
#[cfg(not(any(feature = "postgres", feature = "mysql", feature = "sqlite")))]
|
||||
#[cfg(not(feature = "postgres"))]
|
||||
Self::Disabled(_) => unreachable!("a SQL backend cannot exist without a driver"),
|
||||
}
|
||||
}
|
||||
@@ -503,11 +424,7 @@ impl<'a> SqlBackendRef<'a> {
|
||||
match self {
|
||||
#[cfg(feature = "postgres")]
|
||||
Self::Postgres(postgres) => postgres.aggregate_stats_hourly(input).await,
|
||||
#[cfg(feature = "mysql")]
|
||||
Self::Mysql(mysql) => mysql.aggregate_stats_hourly(input).await,
|
||||
#[cfg(feature = "sqlite")]
|
||||
Self::Sqlite(sqlite) => sqlite.aggregate_stats_hourly(input).await,
|
||||
#[cfg(not(any(feature = "postgres", feature = "mysql", feature = "sqlite")))]
|
||||
#[cfg(not(feature = "postgres"))]
|
||||
Self::Disabled(_) => unreachable!("a SQL backend cannot exist without a driver"),
|
||||
}
|
||||
}
|
||||
@@ -519,11 +436,7 @@ impl<'a> SqlBackendRef<'a> {
|
||||
match self {
|
||||
#[cfg(feature = "postgres")]
|
||||
Self::Postgres(postgres) => postgres.aggregate_stats_daily(input).await,
|
||||
#[cfg(feature = "mysql")]
|
||||
Self::Mysql(mysql) => mysql.aggregate_stats_daily(input).await,
|
||||
#[cfg(feature = "sqlite")]
|
||||
Self::Sqlite(sqlite) => sqlite.aggregate_stats_daily(input).await,
|
||||
#[cfg(not(any(feature = "postgres", feature = "mysql", feature = "sqlite")))]
|
||||
#[cfg(not(feature = "postgres"))]
|
||||
Self::Disabled(_) => unreachable!("a SQL backend cannot exist without a driver"),
|
||||
}
|
||||
}
|
||||
@@ -535,11 +448,7 @@ impl<'a> SqlBackendRef<'a> {
|
||||
match self {
|
||||
#[cfg(feature = "postgres")]
|
||||
Self::Postgres(postgres) => postgres.find_system_config_value(key).await,
|
||||
#[cfg(feature = "mysql")]
|
||||
Self::Mysql(mysql) => mysql.find_system_config_value(key).await,
|
||||
#[cfg(feature = "sqlite")]
|
||||
Self::Sqlite(sqlite) => sqlite.find_system_config_value(key).await,
|
||||
#[cfg(not(any(feature = "postgres", feature = "mysql", feature = "sqlite")))]
|
||||
#[cfg(not(feature = "postgres"))]
|
||||
Self::Disabled(_) => unreachable!("a SQL backend cannot exist without a driver"),
|
||||
}
|
||||
}
|
||||
@@ -557,19 +466,7 @@ impl<'a> SqlBackendRef<'a> {
|
||||
.compare_and_set_system_config_string_value(key, expected, replacement)
|
||||
.await
|
||||
}
|
||||
#[cfg(feature = "mysql")]
|
||||
Self::Mysql(mysql) => {
|
||||
mysql
|
||||
.compare_and_set_system_config_string_value(key, expected, replacement)
|
||||
.await
|
||||
}
|
||||
#[cfg(feature = "sqlite")]
|
||||
Self::Sqlite(sqlite) => {
|
||||
sqlite
|
||||
.compare_and_set_system_config_string_value(key, expected, replacement)
|
||||
.await
|
||||
}
|
||||
#[cfg(not(any(feature = "postgres", feature = "mysql", feature = "sqlite")))]
|
||||
#[cfg(not(feature = "postgres"))]
|
||||
Self::Disabled(_) => unreachable!("a SQL backend cannot exist without a driver"),
|
||||
}
|
||||
}
|
||||
@@ -580,11 +477,7 @@ impl<'a> SqlBackendRef<'a> {
|
||||
match self {
|
||||
#[cfg(feature = "postgres")]
|
||||
Self::Postgres(postgres) => postgres.list_system_config_entries().await,
|
||||
#[cfg(feature = "mysql")]
|
||||
Self::Mysql(mysql) => mysql.list_system_config_entries().await,
|
||||
#[cfg(feature = "sqlite")]
|
||||
Self::Sqlite(sqlite) => sqlite.list_system_config_entries().await,
|
||||
#[cfg(not(any(feature = "postgres", feature = "mysql", feature = "sqlite")))]
|
||||
#[cfg(not(feature = "postgres"))]
|
||||
Self::Disabled(_) => unreachable!("a SQL backend cannot exist without a driver"),
|
||||
}
|
||||
}
|
||||
@@ -602,19 +495,7 @@ impl<'a> SqlBackendRef<'a> {
|
||||
.upsert_system_config_entry(key, value, description)
|
||||
.await
|
||||
}
|
||||
#[cfg(feature = "mysql")]
|
||||
Self::Mysql(mysql) => {
|
||||
mysql
|
||||
.upsert_system_config_entry(key, value, description)
|
||||
.await
|
||||
}
|
||||
#[cfg(feature = "sqlite")]
|
||||
Self::Sqlite(sqlite) => {
|
||||
sqlite
|
||||
.upsert_system_config_entry(key, value, description)
|
||||
.await
|
||||
}
|
||||
#[cfg(not(any(feature = "postgres", feature = "mysql", feature = "sqlite")))]
|
||||
#[cfg(not(feature = "postgres"))]
|
||||
Self::Disabled(_) => unreachable!("a SQL backend cannot exist without a driver"),
|
||||
}
|
||||
}
|
||||
@@ -623,11 +504,7 @@ impl<'a> SqlBackendRef<'a> {
|
||||
match self {
|
||||
#[cfg(feature = "postgres")]
|
||||
Self::Postgres(postgres) => postgres.delete_system_config_value(key).await,
|
||||
#[cfg(feature = "mysql")]
|
||||
Self::Mysql(mysql) => mysql.delete_system_config_value(key).await,
|
||||
#[cfg(feature = "sqlite")]
|
||||
Self::Sqlite(sqlite) => sqlite.delete_system_config_value(key).await,
|
||||
#[cfg(not(any(feature = "postgres", feature = "mysql", feature = "sqlite")))]
|
||||
#[cfg(not(feature = "postgres"))]
|
||||
Self::Disabled(_) => unreachable!("a SQL backend cannot exist without a driver"),
|
||||
}
|
||||
}
|
||||
@@ -636,11 +513,7 @@ impl<'a> SqlBackendRef<'a> {
|
||||
match self {
|
||||
#[cfg(feature = "postgres")]
|
||||
Self::Postgres(postgres) => postgres.read_admin_system_stats().await,
|
||||
#[cfg(feature = "mysql")]
|
||||
Self::Mysql(mysql) => mysql.read_admin_system_stats().await,
|
||||
#[cfg(feature = "sqlite")]
|
||||
Self::Sqlite(sqlite) => sqlite.read_admin_system_stats().await,
|
||||
#[cfg(not(any(feature = "postgres", feature = "mysql", feature = "sqlite")))]
|
||||
#[cfg(not(feature = "postgres"))]
|
||||
Self::Disabled(_) => unreachable!("a SQL backend cannot exist without a driver"),
|
||||
}
|
||||
}
|
||||
@@ -652,11 +525,7 @@ impl<'a> SqlBackendRef<'a> {
|
||||
match self {
|
||||
#[cfg(feature = "postgres")]
|
||||
Self::Postgres(postgres) => postgres.purge_admin_system_data(target).await,
|
||||
#[cfg(feature = "mysql")]
|
||||
Self::Mysql(mysql) => mysql.purge_admin_system_data(target).await,
|
||||
#[cfg(feature = "sqlite")]
|
||||
Self::Sqlite(sqlite) => sqlite.purge_admin_system_data(target).await,
|
||||
#[cfg(not(any(feature = "postgres", feature = "mysql", feature = "sqlite")))]
|
||||
#[cfg(not(feature = "postgres"))]
|
||||
Self::Disabled(_) => unreachable!("a SQL backend cannot exist without a driver"),
|
||||
}
|
||||
}
|
||||
@@ -667,11 +536,7 @@ impl<'a> SqlBackendRef<'a> {
|
||||
match self {
|
||||
#[cfg(feature = "postgres")]
|
||||
Self::Postgres(postgres) => postgres.export_admin_system_usage_aggregates().await,
|
||||
#[cfg(feature = "mysql")]
|
||||
Self::Mysql(mysql) => mysql.export_admin_system_usage_aggregates().await,
|
||||
#[cfg(feature = "sqlite")]
|
||||
Self::Sqlite(sqlite) => sqlite.export_admin_system_usage_aggregates().await,
|
||||
#[cfg(not(any(feature = "postgres", feature = "mysql", feature = "sqlite")))]
|
||||
#[cfg(not(feature = "postgres"))]
|
||||
Self::Disabled(_) => unreachable!("a SQL backend cannot exist without a driver"),
|
||||
}
|
||||
}
|
||||
@@ -695,29 +560,7 @@ impl<'a> SqlBackendRef<'a> {
|
||||
)
|
||||
.await
|
||||
}
|
||||
#[cfg(feature = "mysql")]
|
||||
Self::Mysql(mysql) => {
|
||||
mysql
|
||||
.import_admin_system_usage_aggregates(
|
||||
snapshot,
|
||||
user_id_map,
|
||||
api_key_id_map,
|
||||
mode,
|
||||
)
|
||||
.await
|
||||
}
|
||||
#[cfg(feature = "sqlite")]
|
||||
Self::Sqlite(sqlite) => {
|
||||
sqlite
|
||||
.import_admin_system_usage_aggregates(
|
||||
snapshot,
|
||||
user_id_map,
|
||||
api_key_id_map,
|
||||
mode,
|
||||
)
|
||||
.await
|
||||
}
|
||||
#[cfg(not(any(feature = "postgres", feature = "mysql", feature = "sqlite")))]
|
||||
#[cfg(not(feature = "postgres"))]
|
||||
Self::Disabled(_) => unreachable!("a SQL backend cannot exist without a driver"),
|
||||
}
|
||||
}
|
||||
@@ -729,11 +572,7 @@ impl<'a> SqlBackendRef<'a> {
|
||||
match self {
|
||||
#[cfg(feature = "postgres")]
|
||||
Self::Postgres(postgres) => postgres.purge_admin_request_bodies_batch(batch_size).await,
|
||||
#[cfg(feature = "mysql")]
|
||||
Self::Mysql(mysql) => mysql.purge_admin_request_bodies_batch(batch_size).await,
|
||||
#[cfg(feature = "sqlite")]
|
||||
Self::Sqlite(sqlite) => sqlite.purge_admin_request_bodies_batch(batch_size).await,
|
||||
#[cfg(not(any(feature = "postgres", feature = "mysql", feature = "sqlite")))]
|
||||
#[cfg(not(feature = "postgres"))]
|
||||
Self::Disabled(_) => unreachable!("a SQL backend cannot exist without a driver"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,67 +0,0 @@
|
||||
use crate::backend::MysqlBackend;
|
||||
use crate::error::SqlResultExt;
|
||||
use crate::{DataLayerError, DatabaseMaintenanceSummary};
|
||||
|
||||
use super::maintenance_identifier;
|
||||
|
||||
impl MysqlBackend {
|
||||
pub async fn run_table_maintenance(
|
||||
&self,
|
||||
table_names: &[&str],
|
||||
) -> Result<DatabaseMaintenanceSummary, DataLayerError> {
|
||||
let mut summary = DatabaseMaintenanceSummary::default();
|
||||
for table_name in table_names {
|
||||
let table_name = maintenance_identifier(table_name)?;
|
||||
summary.attempted += 1;
|
||||
let statement = format!("ANALYZE TABLE `{table_name}`");
|
||||
if sqlx::query_as::<_, (String, String, String, String)>(&statement)
|
||||
.fetch_all(self.pool())
|
||||
.await
|
||||
.map_sql_err()
|
||||
.is_ok_and(|rows| mysql_analyze_succeeded(&rows))
|
||||
{
|
||||
summary.succeeded += 1;
|
||||
}
|
||||
}
|
||||
Ok(summary)
|
||||
}
|
||||
}
|
||||
|
||||
fn mysql_analyze_succeeded(rows: &[(String, String, String, String)]) -> bool {
|
||||
!rows.is_empty()
|
||||
&& rows.iter().any(|(_, _, message_type, message)| {
|
||||
message_type.eq_ignore_ascii_case("status") && message.eq_ignore_ascii_case("ok")
|
||||
})
|
||||
&& rows
|
||||
.iter()
|
||||
.all(|(_, _, message_type, _)| !message_type.eq_ignore_ascii_case("error"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::mysql_analyze_succeeded;
|
||||
|
||||
fn row(message_type: &str, message: &str) -> (String, String, String, String) {
|
||||
(
|
||||
"aether.usage".to_string(),
|
||||
"analyze".to_string(),
|
||||
message_type.to_string(),
|
||||
message.to_string(),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn analyze_requires_an_explicit_ok_status() {
|
||||
assert!(mysql_analyze_succeeded(&[row("status", "OK")]));
|
||||
assert!(!mysql_analyze_succeeded(&[]));
|
||||
assert!(!mysql_analyze_succeeded(&[row("note", "skipped")]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn analyze_rejects_error_rows_even_when_an_ok_row_is_present() {
|
||||
assert!(!mysql_analyze_succeeded(&[
|
||||
row("Error", "Table does not exist"),
|
||||
row("status", "OK"),
|
||||
]));
|
||||
}
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
use crate::backend::SqliteBackend;
|
||||
use crate::error::SqlResultExt;
|
||||
use crate::{DataLayerError, DatabaseMaintenanceSummary};
|
||||
|
||||
use super::maintenance_identifier;
|
||||
|
||||
impl SqliteBackend {
|
||||
pub async fn run_table_maintenance(
|
||||
&self,
|
||||
table_names: &[&str],
|
||||
) -> Result<DatabaseMaintenanceSummary, DataLayerError> {
|
||||
let mut summary = DatabaseMaintenanceSummary::default();
|
||||
for table_name in table_names {
|
||||
let table_name = maintenance_identifier(table_name)?;
|
||||
summary.attempted += 1;
|
||||
let statement = format!("ANALYZE \"{table_name}\"");
|
||||
if sqlx::raw_sql(&statement)
|
||||
.execute(self.pool())
|
||||
.await
|
||||
.map_sql_err()
|
||||
.is_ok()
|
||||
{
|
||||
summary.succeeded += 1;
|
||||
}
|
||||
}
|
||||
if summary.succeeded > 0 {
|
||||
sqlx::raw_sql("PRAGMA optimize")
|
||||
.execute(self.pool())
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
}
|
||||
Ok(summary)
|
||||
}
|
||||
}
|
||||
@@ -8,17 +8,11 @@
|
||||
|
||||
mod leases;
|
||||
mod maintenance;
|
||||
#[cfg(feature = "mysql")]
|
||||
mod mysql;
|
||||
#[cfg(feature = "postgres")]
|
||||
mod postgres;
|
||||
mod read;
|
||||
mod referrals;
|
||||
#[cfg(feature = "sqlite")]
|
||||
mod sqlite;
|
||||
mod stats;
|
||||
#[cfg(any(feature = "mysql", feature = "sqlite"))]
|
||||
mod stats_common;
|
||||
mod system;
|
||||
mod transactions;
|
||||
mod wallet;
|
||||
@@ -26,8 +20,6 @@ mod write;
|
||||
|
||||
use crate::maintenance::DatabasePoolSummary;
|
||||
pub use leases::DataLeaseBackends;
|
||||
#[cfg(feature = "mysql")]
|
||||
pub use mysql::MysqlBackend;
|
||||
#[cfg(feature = "postgres")]
|
||||
pub use postgres::PostgresBackend;
|
||||
pub use read::DataReadRepositories;
|
||||
@@ -36,8 +28,6 @@ pub use referrals::{
|
||||
ReferralRelationshipListQuery, ReferralRelationshipRecord, ReferralRewardConfig,
|
||||
ReferralRewardListQuery, ReferralRewardRecord, ReferralUserDashboard,
|
||||
};
|
||||
#[cfg(feature = "sqlite")]
|
||||
pub use sqlite::SqliteBackend;
|
||||
pub use transactions::DataTransactionBackends;
|
||||
pub use write::DataWriteRepositories;
|
||||
|
||||
@@ -48,14 +38,10 @@ use crate::{DataLayerConfig, DataLayerError};
|
||||
enum SqlBackendRef<'a> {
|
||||
#[cfg(feature = "postgres")]
|
||||
Postgres(&'a PostgresBackend),
|
||||
#[cfg(feature = "mysql")]
|
||||
Mysql(&'a MysqlBackend),
|
||||
#[cfg(feature = "sqlite")]
|
||||
Sqlite(&'a SqliteBackend),
|
||||
// Keep the reference lifetime represented when this crate is built without
|
||||
// any SQL driver features. The no-driver build still exposes the
|
||||
// maintenance facade, but has no concrete backend variant to carry `'a`.
|
||||
#[cfg(not(any(feature = "postgres", feature = "mysql", feature = "sqlite")))]
|
||||
#[cfg(not(feature = "postgres"))]
|
||||
Disabled(std::marker::PhantomData<&'a ()>),
|
||||
}
|
||||
|
||||
@@ -64,10 +50,6 @@ pub struct DataBackends {
|
||||
config: DataLayerConfig,
|
||||
#[cfg(feature = "postgres")]
|
||||
postgres: Option<PostgresBackend>,
|
||||
#[cfg(feature = "mysql")]
|
||||
mysql: Option<MysqlBackend>,
|
||||
#[cfg(feature = "sqlite")]
|
||||
sqlite: Option<SqliteBackend>,
|
||||
leases: DataLeaseBackends,
|
||||
read: DataReadRepositories,
|
||||
transactions: DataTransactionBackends,
|
||||
@@ -102,18 +84,6 @@ fn ensure_driver_enabled(driver: DatabaseDriver) -> Result<(), DataLayerError> {
|
||||
DatabaseDriver::Postgres => Err(DataLayerError::InvalidInput(
|
||||
"PostgreSQL driver is not enabled for this aether-data build".to_string(),
|
||||
)),
|
||||
#[cfg(feature = "mysql")]
|
||||
DatabaseDriver::Mysql => Ok(()),
|
||||
#[cfg(not(feature = "mysql"))]
|
||||
DatabaseDriver::Mysql => Err(DataLayerError::InvalidInput(
|
||||
"MySQL driver is not enabled for this aether-data build".to_string(),
|
||||
)),
|
||||
#[cfg(feature = "sqlite")]
|
||||
DatabaseDriver::Sqlite => Ok(()),
|
||||
#[cfg(not(feature = "sqlite"))]
|
||||
DatabaseDriver::Sqlite => Err(DataLayerError::InvalidInput(
|
||||
"SQLite driver is not enabled for this aether-data build".to_string(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -123,14 +93,6 @@ impl DataBackends {
|
||||
if let Some(postgres) = self.postgres.as_ref() {
|
||||
return Some(SqlBackendRef::Postgres(postgres));
|
||||
}
|
||||
#[cfg(feature = "mysql")]
|
||||
if let Some(mysql) = self.mysql.as_ref() {
|
||||
return Some(SqlBackendRef::Mysql(mysql));
|
||||
}
|
||||
#[cfg(feature = "sqlite")]
|
||||
if let Some(sqlite) = self.sqlite.as_ref() {
|
||||
return Some(SqlBackendRef::Sqlite(sqlite));
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
@@ -148,20 +110,6 @@ impl DataBackends {
|
||||
),
|
||||
_ => None,
|
||||
};
|
||||
#[cfg(feature = "mysql")]
|
||||
let mysql = match database.clone() {
|
||||
Some(database) if database.driver == DatabaseDriver::Mysql => {
|
||||
Some(MysqlBackend::from_config(database)?)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
#[cfg(feature = "sqlite")]
|
||||
let sqlite = match database.clone() {
|
||||
Some(database) if database.driver == DatabaseDriver::Sqlite => {
|
||||
Some(SqliteBackend::from_config(database)?)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
#[cfg(feature = "postgres")]
|
||||
let leases = DataLeaseBackends::from_postgres(postgres.as_ref())?;
|
||||
#[cfg(not(feature = "postgres"))]
|
||||
@@ -169,10 +117,6 @@ impl DataBackends {
|
||||
let read = DataReadRepositories::from_backends(
|
||||
#[cfg(feature = "postgres")]
|
||||
postgres.as_ref(),
|
||||
#[cfg(feature = "mysql")]
|
||||
mysql.as_ref(),
|
||||
#[cfg(feature = "sqlite")]
|
||||
sqlite.as_ref(),
|
||||
);
|
||||
#[cfg(feature = "postgres")]
|
||||
let transactions = DataTransactionBackends::from_postgres(postgres.as_ref());
|
||||
@@ -181,20 +125,12 @@ impl DataBackends {
|
||||
let write = DataWriteRepositories::from_backends(
|
||||
#[cfg(feature = "postgres")]
|
||||
postgres.as_ref(),
|
||||
#[cfg(feature = "mysql")]
|
||||
mysql.as_ref(),
|
||||
#[cfg(feature = "sqlite")]
|
||||
sqlite.as_ref(),
|
||||
);
|
||||
|
||||
Ok(Self {
|
||||
config,
|
||||
#[cfg(feature = "postgres")]
|
||||
postgres,
|
||||
#[cfg(feature = "mysql")]
|
||||
mysql,
|
||||
#[cfg(feature = "sqlite")]
|
||||
sqlite,
|
||||
leases,
|
||||
read,
|
||||
transactions,
|
||||
@@ -217,16 +153,6 @@ impl DataBackends {
|
||||
.map(|database| database.driver)
|
||||
}
|
||||
|
||||
#[cfg(feature = "mysql")]
|
||||
pub fn mysql(&self) -> Option<&MysqlBackend> {
|
||||
self.mysql.as_ref()
|
||||
}
|
||||
|
||||
#[cfg(feature = "sqlite")]
|
||||
pub fn sqlite(&self) -> Option<&SqliteBackend> {
|
||||
self.sqlite.as_ref()
|
||||
}
|
||||
|
||||
pub fn read(&self) -> &DataReadRepositories {
|
||||
&self.read
|
||||
}
|
||||
@@ -256,31 +182,7 @@ mod tests {
|
||||
use super::DataBackends;
|
||||
#[cfg(feature = "postgres")]
|
||||
use crate::driver::postgres::PostgresPoolConfig;
|
||||
use crate::{DataLayerConfig, DatabaseDriver, SqlDatabaseConfig, SqlPoolConfig};
|
||||
|
||||
#[test]
|
||||
#[cfg(not(feature = "mysql"))]
|
||||
fn rejects_mysql_when_driver_is_not_enabled() {
|
||||
let error = DataBackends::from_config(DataLayerConfig::from_database(SqlDatabaseConfig {
|
||||
driver: DatabaseDriver::Mysql,
|
||||
url: "mysql://user:pass@localhost/aether".to_string(),
|
||||
pool: SqlPoolConfig::default(),
|
||||
}))
|
||||
.expect_err("disabled mysql should fail explicitly");
|
||||
assert!(error.to_string().contains("MySQL driver is not enabled"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(not(feature = "sqlite"))]
|
||||
fn rejects_sqlite_when_driver_is_not_enabled() {
|
||||
let error = DataBackends::from_config(DataLayerConfig::from_database(SqlDatabaseConfig {
|
||||
driver: DatabaseDriver::Sqlite,
|
||||
url: "sqlite://./data/aether.db".to_string(),
|
||||
pool: SqlPoolConfig::default(),
|
||||
}))
|
||||
.expect_err("disabled sqlite should fail explicitly");
|
||||
assert!(error.to_string().contains("SQLite driver is not enabled"));
|
||||
}
|
||||
use crate::DataLayerConfig;
|
||||
|
||||
#[test]
|
||||
fn builds_empty_backends_from_default_config() {
|
||||
@@ -290,10 +192,6 @@ mod tests {
|
||||
assert!(!backends.has_runtime_backends());
|
||||
#[cfg(feature = "postgres")]
|
||||
assert!(backends.postgres().is_none());
|
||||
#[cfg(feature = "mysql")]
|
||||
assert!(backends.mysql().is_none());
|
||||
#[cfg(feature = "sqlite")]
|
||||
assert!(backends.sqlite().is_none());
|
||||
#[cfg(feature = "postgres")]
|
||||
assert!(backends.leases().postgres().is_none());
|
||||
assert!(backends.read().auth_api_keys().is_none());
|
||||
@@ -336,10 +234,6 @@ mod tests {
|
||||
assert!(backends.has_runtime_backends());
|
||||
#[cfg(feature = "postgres")]
|
||||
assert!(backends.postgres().is_some());
|
||||
#[cfg(feature = "mysql")]
|
||||
assert!(backends.mysql().is_none());
|
||||
#[cfg(feature = "sqlite")]
|
||||
assert!(backends.sqlite().is_none());
|
||||
#[cfg(feature = "postgres")]
|
||||
assert!(backends.leases().postgres().is_some());
|
||||
assert!(backends.read().auth_api_keys().is_some());
|
||||
@@ -371,113 +265,4 @@ mod tests {
|
||||
assert!(backends.write().wallets().is_some());
|
||||
assert!(backends.config().effective_database().is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[cfg(feature = "mysql")]
|
||||
async fn builds_mysql_backend_from_database_config_with_first_core_repository() {
|
||||
let backends = DataBackends::from_config(DataLayerConfig {
|
||||
database: Some(SqlDatabaseConfig {
|
||||
driver: DatabaseDriver::Mysql,
|
||||
url: "mysql://user:pass@localhost:3306/aether".to_string(),
|
||||
pool: SqlPoolConfig::default(),
|
||||
}),
|
||||
postgres: None,
|
||||
})
|
||||
.expect("mysql backend should build");
|
||||
|
||||
assert!(backends.has_runtime_backends());
|
||||
#[cfg(feature = "postgres")]
|
||||
assert!(backends.postgres().is_none());
|
||||
#[cfg(feature = "mysql")]
|
||||
assert!(backends.mysql().is_some());
|
||||
#[cfg(feature = "sqlite")]
|
||||
assert!(backends.sqlite().is_none());
|
||||
assert!(backends.read().has_any());
|
||||
assert!(backends.read().announcements().is_some());
|
||||
assert!(backends.read().auth_api_keys().is_some());
|
||||
assert!(backends.read().auth_modules().is_some());
|
||||
assert!(backends.read().billing().is_some());
|
||||
assert!(backends.read().gemini_file_mappings().is_some());
|
||||
assert!(backends.read().global_models().is_some());
|
||||
assert!(backends.read().management_tokens().is_some());
|
||||
assert!(backends.read().minimal_candidate_selection().is_some());
|
||||
assert!(backends.read().oauth_providers().is_some());
|
||||
assert!(backends.read().provider_catalog().is_some());
|
||||
assert!(backends.read().provider_quotas().is_some());
|
||||
assert!(backends.read().proxy_nodes().is_some());
|
||||
assert!(backends.read().request_candidates().is_some());
|
||||
assert!(backends.read().users().is_some());
|
||||
assert!(backends.read().video_tasks().is_some());
|
||||
assert!(backends.has_stats_hourly_aggregation_backend());
|
||||
assert!(backends.has_stats_daily_aggregation_backend());
|
||||
assert!(backends.write().has_any());
|
||||
assert!(backends.write().announcements().is_some());
|
||||
assert!(backends.write().auth_api_keys().is_some());
|
||||
assert!(backends.write().auth_modules().is_some());
|
||||
assert!(backends.write().gemini_file_mappings().is_some());
|
||||
assert!(backends.write().global_models().is_some());
|
||||
assert!(backends.write().management_tokens().is_some());
|
||||
assert!(backends.write().oauth_providers().is_some());
|
||||
assert!(backends.write().proxy_nodes().is_some());
|
||||
assert!(backends.write().provider_catalog().is_some());
|
||||
assert!(backends.write().provider_quotas().is_some());
|
||||
assert!(backends.write().request_candidates().is_some());
|
||||
assert!(backends.write().video_tasks().is_some());
|
||||
assert!(backends.write().wallets().is_some());
|
||||
assert!(backends.config().effective_database().is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[cfg(feature = "sqlite")]
|
||||
async fn builds_sqlite_backend_from_database_config_with_first_core_repository() {
|
||||
let backends = DataBackends::from_config(DataLayerConfig {
|
||||
database: Some(SqlDatabaseConfig {
|
||||
driver: DatabaseDriver::Sqlite,
|
||||
url: "sqlite://./data/aether.db".to_string(),
|
||||
pool: SqlPoolConfig::default(),
|
||||
}),
|
||||
postgres: None,
|
||||
})
|
||||
.expect("sqlite backend should build");
|
||||
|
||||
assert!(backends.has_runtime_backends());
|
||||
#[cfg(feature = "postgres")]
|
||||
assert!(backends.postgres().is_none());
|
||||
#[cfg(feature = "mysql")]
|
||||
assert!(backends.mysql().is_none());
|
||||
#[cfg(feature = "sqlite")]
|
||||
assert!(backends.sqlite().is_some());
|
||||
assert!(backends.read().has_any());
|
||||
assert!(backends.read().announcements().is_some());
|
||||
assert!(backends.read().auth_api_keys().is_some());
|
||||
assert!(backends.read().auth_modules().is_some());
|
||||
assert!(backends.read().billing().is_some());
|
||||
assert!(backends.read().gemini_file_mappings().is_some());
|
||||
assert!(backends.read().global_models().is_some());
|
||||
assert!(backends.read().management_tokens().is_some());
|
||||
assert!(backends.read().oauth_providers().is_some());
|
||||
assert!(backends.read().provider_catalog().is_some());
|
||||
assert!(backends.read().provider_quotas().is_some());
|
||||
assert!(backends.read().proxy_nodes().is_some());
|
||||
assert!(backends.read().request_candidates().is_some());
|
||||
assert!(backends.read().users().is_some());
|
||||
assert!(backends.read().video_tasks().is_some());
|
||||
assert!(backends.has_stats_hourly_aggregation_backend());
|
||||
assert!(backends.has_stats_daily_aggregation_backend());
|
||||
assert!(backends.write().has_any());
|
||||
assert!(backends.write().announcements().is_some());
|
||||
assert!(backends.write().auth_api_keys().is_some());
|
||||
assert!(backends.write().auth_modules().is_some());
|
||||
assert!(backends.write().gemini_file_mappings().is_some());
|
||||
assert!(backends.write().global_models().is_some());
|
||||
assert!(backends.write().management_tokens().is_some());
|
||||
assert!(backends.write().oauth_providers().is_some());
|
||||
assert!(backends.write().proxy_nodes().is_some());
|
||||
assert!(backends.write().provider_catalog().is_some());
|
||||
assert!(backends.write().provider_quotas().is_some());
|
||||
assert!(backends.write().request_candidates().is_some());
|
||||
assert!(backends.write().video_tasks().is_some());
|
||||
assert!(backends.write().wallets().is_some());
|
||||
assert!(backends.config().effective_database().is_some());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,703 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::database::SqlDatabaseConfig;
|
||||
use crate::driver::mysql::{MysqlPool, MysqlPoolFactory};
|
||||
use crate::repository::announcements::{
|
||||
AnnouncementReadRepository, AnnouncementWriteRepository, MysqlAnnouncementRepository,
|
||||
};
|
||||
use crate::repository::audit::{AuditLogReadRepository, MysqlAuditLogReadRepository};
|
||||
use crate::repository::auth::{
|
||||
AuthApiKeyReadRepository, AuthApiKeyWriteRepository, MysqlAuthApiKeyReadRepository,
|
||||
};
|
||||
use crate::repository::auth_modules::{
|
||||
AuthModuleReadRepository, AuthModuleWriteRepository, MysqlAuthModuleReadRepository,
|
||||
MysqlAuthModuleRepository,
|
||||
};
|
||||
use crate::repository::background_tasks::{
|
||||
BackgroundTaskReadRepository, BackgroundTaskWriteRepository, MysqlBackgroundTaskRepository,
|
||||
};
|
||||
use crate::repository::billing::{BillingReadRepository, MysqlBillingReadRepository};
|
||||
use crate::repository::candidate_selection::{
|
||||
MinimalCandidateSelectionReadRepository, MysqlMinimalCandidateSelectionReadRepository,
|
||||
};
|
||||
use crate::repository::candidates::{
|
||||
MysqlRequestCandidateRepository, RequestCandidateReadRepository,
|
||||
RequestCandidateWriteRepository,
|
||||
};
|
||||
use crate::repository::gemini_file_mappings::{
|
||||
GeminiFileMappingReadRepository, GeminiFileMappingWriteRepository,
|
||||
MysqlGeminiFileMappingRepository,
|
||||
};
|
||||
use crate::repository::global_models::{
|
||||
GlobalModelReadRepository, GlobalModelWriteRepository, MysqlGlobalModelReadRepository,
|
||||
};
|
||||
use crate::repository::management_tokens::{
|
||||
ManagementTokenReadRepository, ManagementTokenWriteRepository, MysqlManagementTokenRepository,
|
||||
};
|
||||
use crate::repository::oauth_providers::{
|
||||
MysqlOAuthProviderRepository, OAuthProviderReadRepository, OAuthProviderWriteRepository,
|
||||
};
|
||||
use crate::repository::pool_scores::{
|
||||
MysqlPoolMemberScoreRepository, PoolMemberScoreWriteRepository, PoolScoreReadRepository,
|
||||
};
|
||||
use crate::repository::provider_catalog::{
|
||||
MysqlProviderCatalogReadRepository, ProviderCatalogReadRepository,
|
||||
ProviderCatalogWriteRepository,
|
||||
};
|
||||
use crate::repository::proxy_nodes::{
|
||||
MysqlProxyNodeReadRepository, ProxyNodeReadRepository, ProxyNodeWriteRepository,
|
||||
};
|
||||
use crate::repository::quota::{
|
||||
MysqlProviderQuotaRepository, ProviderQuotaReadRepository, ProviderQuotaWriteRepository,
|
||||
};
|
||||
use crate::repository::routing_profiles::{
|
||||
MysqlRoutingGroupRepository, RoutingGroupReadRepository, RoutingGroupWriteRepository,
|
||||
};
|
||||
use crate::repository::settlement::{MysqlSettlementRepository, SettlementWriteRepository};
|
||||
use crate::repository::usage::{
|
||||
MysqlUsageReadRepository, MysqlUsageWriteRepository, UsageReadRepository, UsageWriteRepository,
|
||||
};
|
||||
use crate::repository::users::{MysqlUserReadRepository, UserReadRepository};
|
||||
use crate::repository::video_tasks::{
|
||||
MysqlVideoTaskRepository, VideoTaskReadRepository, VideoTaskWriteRepository,
|
||||
};
|
||||
use crate::repository::wallet::{
|
||||
MysqlWalletReadRepository, WalletReadRepository, WalletWriteRepository,
|
||||
};
|
||||
use crate::DataLayerError;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MysqlBackend {
|
||||
config: SqlDatabaseConfig,
|
||||
pool: MysqlPool,
|
||||
}
|
||||
|
||||
impl MysqlBackend {
|
||||
pub fn from_config(config: SqlDatabaseConfig) -> Result<Self, DataLayerError> {
|
||||
let factory = MysqlPoolFactory::new(config.clone())?;
|
||||
let pool = factory.connect_lazy()?;
|
||||
|
||||
Ok(Self { config, pool })
|
||||
}
|
||||
|
||||
pub fn config(&self) -> &SqlDatabaseConfig {
|
||||
&self.config
|
||||
}
|
||||
|
||||
pub fn pool(&self) -> &MysqlPool {
|
||||
&self.pool
|
||||
}
|
||||
|
||||
pub fn pool_clone(&self) -> MysqlPool {
|
||||
self.pool.clone()
|
||||
}
|
||||
|
||||
pub fn auth_api_key_read_repository(&self) -> Arc<dyn AuthApiKeyReadRepository> {
|
||||
Arc::new(MysqlAuthApiKeyReadRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn announcement_read_repository(&self) -> Arc<dyn AnnouncementReadRepository> {
|
||||
Arc::new(MysqlAnnouncementRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn audit_log_read_repository(&self) -> Arc<dyn AuditLogReadRepository> {
|
||||
Arc::new(MysqlAuditLogReadRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn announcement_write_repository(&self) -> Arc<dyn AnnouncementWriteRepository> {
|
||||
Arc::new(MysqlAnnouncementRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn auth_api_key_write_repository(&self) -> Arc<dyn AuthApiKeyWriteRepository> {
|
||||
Arc::new(MysqlAuthApiKeyReadRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn management_token_read_repository(&self) -> Arc<dyn ManagementTokenReadRepository> {
|
||||
Arc::new(MysqlManagementTokenRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn management_token_write_repository(&self) -> Arc<dyn ManagementTokenWriteRepository> {
|
||||
Arc::new(MysqlManagementTokenRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn auth_module_read_repository(&self) -> Arc<dyn AuthModuleReadRepository> {
|
||||
Arc::new(MysqlAuthModuleReadRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn auth_module_write_repository(&self) -> Arc<dyn AuthModuleWriteRepository> {
|
||||
Arc::new(MysqlAuthModuleRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn billing_read_repository(&self) -> Arc<dyn BillingReadRepository> {
|
||||
Arc::new(MysqlBillingReadRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn background_task_read_repository(&self) -> Arc<dyn BackgroundTaskReadRepository> {
|
||||
Arc::new(MysqlBackgroundTaskRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn background_task_write_repository(&self) -> Arc<dyn BackgroundTaskWriteRepository> {
|
||||
Arc::new(MysqlBackgroundTaskRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn request_candidate_read_repository(&self) -> Arc<dyn RequestCandidateReadRepository> {
|
||||
Arc::new(MysqlRequestCandidateRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn request_candidate_write_repository(&self) -> Arc<dyn RequestCandidateWriteRepository> {
|
||||
Arc::new(MysqlRequestCandidateRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn minimal_candidate_selection_read_repository(
|
||||
&self,
|
||||
) -> Arc<dyn MinimalCandidateSelectionReadRepository> {
|
||||
Arc::new(MysqlMinimalCandidateSelectionReadRepository::new(
|
||||
self.pool_clone(),
|
||||
))
|
||||
}
|
||||
|
||||
pub fn gemini_file_mapping_read_repository(&self) -> Arc<dyn GeminiFileMappingReadRepository> {
|
||||
Arc::new(MysqlGeminiFileMappingRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn gemini_file_mapping_write_repository(
|
||||
&self,
|
||||
) -> Arc<dyn GeminiFileMappingWriteRepository> {
|
||||
Arc::new(MysqlGeminiFileMappingRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn global_model_read_repository(&self) -> Arc<dyn GlobalModelReadRepository> {
|
||||
Arc::new(MysqlGlobalModelReadRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn global_model_write_repository(&self) -> Arc<dyn GlobalModelWriteRepository> {
|
||||
Arc::new(MysqlGlobalModelReadRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn oauth_provider_read_repository(&self) -> Arc<dyn OAuthProviderReadRepository> {
|
||||
Arc::new(MysqlOAuthProviderRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn oauth_provider_write_repository(&self) -> Arc<dyn OAuthProviderWriteRepository> {
|
||||
Arc::new(MysqlOAuthProviderRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn provider_catalog_read_repository(&self) -> Arc<dyn ProviderCatalogReadRepository> {
|
||||
Arc::new(MysqlProviderCatalogReadRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn provider_catalog_write_repository(&self) -> Arc<dyn ProviderCatalogWriteRepository> {
|
||||
Arc::new(MysqlProviderCatalogReadRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn pool_score_read_repository(&self) -> Arc<dyn PoolScoreReadRepository> {
|
||||
Arc::new(MysqlPoolMemberScoreRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn pool_score_write_repository(&self) -> Arc<dyn PoolMemberScoreWriteRepository> {
|
||||
Arc::new(MysqlPoolMemberScoreRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn routing_group_read_repository(&self) -> Arc<dyn RoutingGroupReadRepository> {
|
||||
Arc::new(MysqlRoutingGroupRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn routing_group_write_repository(&self) -> Arc<dyn RoutingGroupWriteRepository> {
|
||||
Arc::new(MysqlRoutingGroupRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn proxy_node_read_repository(&self) -> Arc<dyn ProxyNodeReadRepository> {
|
||||
Arc::new(MysqlProxyNodeReadRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn proxy_node_write_repository(&self) -> Arc<dyn ProxyNodeWriteRepository> {
|
||||
Arc::new(MysqlProxyNodeReadRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn provider_quota_read_repository(&self) -> Arc<dyn ProviderQuotaReadRepository> {
|
||||
Arc::new(MysqlProviderQuotaRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn provider_quota_write_repository(&self) -> Arc<dyn ProviderQuotaWriteRepository> {
|
||||
Arc::new(MysqlProviderQuotaRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn settlement_write_repository(&self) -> Arc<dyn SettlementWriteRepository> {
|
||||
Arc::new(MysqlSettlementRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn usage_write_repository(&self) -> Arc<dyn UsageWriteRepository> {
|
||||
Arc::new(MysqlUsageWriteRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn usage_read_repository(&self) -> Arc<dyn UsageReadRepository> {
|
||||
Arc::new(MysqlUsageReadRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn user_read_repository(&self) -> Arc<dyn UserReadRepository> {
|
||||
Arc::new(MysqlUserReadRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn video_task_read_repository(&self) -> Arc<dyn VideoTaskReadRepository> {
|
||||
Arc::new(MysqlVideoTaskRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn video_task_write_repository(&self) -> Arc<dyn VideoTaskWriteRepository> {
|
||||
Arc::new(MysqlVideoTaskRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn wallet_read_repository(&self) -> Arc<dyn WalletReadRepository> {
|
||||
Arc::new(MysqlWalletReadRepository::new(self.pool_clone()))
|
||||
}
|
||||
|
||||
pub fn wallet_write_repository(&self) -> Arc<dyn WalletWriteRepository> {
|
||||
Arc::new(MysqlWalletReadRepository::new(self.pool_clone()))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::MysqlBackend;
|
||||
use crate::lifecycle::migrate::run_mysql_migrations;
|
||||
use crate::{
|
||||
DatabaseDriver, SqlDatabaseConfig, SqlPoolConfig, StatsDailyAggregationInput,
|
||||
StatsHourlyAggregationInput, WalletDailyUsageAggregationInput,
|
||||
};
|
||||
|
||||
#[tokio::test]
|
||||
async fn backend_retains_config_and_pool() {
|
||||
let config = SqlDatabaseConfig {
|
||||
driver: DatabaseDriver::Mysql,
|
||||
url: "mysql://user:pass@localhost:3306/aether".to_string(),
|
||||
pool: SqlPoolConfig::default(),
|
||||
};
|
||||
|
||||
let backend = MysqlBackend::from_config(config.clone()).expect("backend should build");
|
||||
|
||||
assert_eq!(backend.config(), &config);
|
||||
let _pool = backend.pool();
|
||||
let _pool_clone = backend.pool_clone();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mysql_wallet_daily_usage_aggregation_uses_settlement_wallets_when_url_is_set() {
|
||||
let Some(database_url) = std::env::var("AETHER_TEST_MYSQL_URL")
|
||||
.ok()
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
else {
|
||||
eprintln!(
|
||||
"skipping mysql wallet daily usage aggregation smoke test because AETHER_TEST_MYSQL_URL is unset"
|
||||
);
|
||||
return;
|
||||
};
|
||||
|
||||
let config = SqlDatabaseConfig {
|
||||
driver: DatabaseDriver::Mysql,
|
||||
url: database_url,
|
||||
pool: SqlPoolConfig {
|
||||
max_connections: 1,
|
||||
..SqlPoolConfig::default()
|
||||
},
|
||||
};
|
||||
let backend = MysqlBackend::from_config(config).expect("backend should build");
|
||||
run_mysql_migrations(backend.pool())
|
||||
.await
|
||||
.expect("mysql migrations should run");
|
||||
|
||||
let suffix = format!(
|
||||
"{}-{}",
|
||||
std::process::id(),
|
||||
chrono::Utc::now().timestamp_nanos_opt().unwrap_or_default()
|
||||
);
|
||||
let wallet_id = format!("wallet-daily-{suffix}");
|
||||
let stale_wallet_id = format!("wallet-daily-stale-{suffix}");
|
||||
let timezone = format!("Test/WalletDaily/{suffix}");
|
||||
let request_one = format!("request-daily-1-{suffix}");
|
||||
let request_two = format!("request-daily-2-{suffix}");
|
||||
let request_zero = format!("request-daily-zero-{suffix}");
|
||||
let request_outside = format!("request-daily-outside-{suffix}");
|
||||
let stale_ledger_id = format!("stale-ledger-{suffix}");
|
||||
let unique_offset = chrono::Utc::now()
|
||||
.timestamp_nanos_opt()
|
||||
.unwrap_or_default()
|
||||
.rem_euclid(10_000_000);
|
||||
let window_start = 4_100_000_000_i64 + unique_offset * 1_000;
|
||||
let window_end = window_start + 200;
|
||||
let first_finalized_at = window_start;
|
||||
let last_finalized_at = window_start + 100;
|
||||
let zero_finalized_at = window_start + 150;
|
||||
let outside_finalized_at = window_end;
|
||||
let seed_created_at = window_start - 100;
|
||||
let aggregated_at = window_end + 100;
|
||||
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO wallets (id, user_id, balance, gift_balance, limit_mode, created_at, updated_at)
|
||||
VALUES
|
||||
(?, ?, 10.0, 2.0, 'finite', 1, 1),
|
||||
(?, ?, 0.0, 0.0, 'finite', 1, 1)
|
||||
"#,
|
||||
)
|
||||
.bind(&wallet_id)
|
||||
.bind(format!("user-{wallet_id}"))
|
||||
.bind(&stale_wallet_id)
|
||||
.bind(format!("user-{stale_wallet_id}"))
|
||||
.execute(backend.pool())
|
||||
.await
|
||||
.expect("wallets should seed");
|
||||
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO `usage` (
|
||||
request_id, wallet_id, provider_name, model, status, billing_status,
|
||||
total_cost_usd, input_tokens, output_tokens, cache_creation_input_tokens,
|
||||
cache_read_input_tokens, finalized_at, created_at_unix_ms, updated_at_unix_secs
|
||||
) VALUES
|
||||
(?, 'wrong-wallet', 'provider', 'model', 'completed', 'pending',
|
||||
1.25, 10, 20, 3, 4, ?, ?, ?),
|
||||
(?, NULL, 'provider', 'model', 'completed', 'pending',
|
||||
2.00, 5, 7, 1, 2, ?, ?, ?),
|
||||
(?, NULL, 'provider', 'model', 'completed', 'pending',
|
||||
0.00, 100, 100, 0, 0, ?, ?, ?),
|
||||
(?, NULL, 'provider', 'model', 'completed', 'pending',
|
||||
9.00, 50, 50, 0, 0, ?, ?, ?)
|
||||
"#,
|
||||
)
|
||||
.bind(&request_one)
|
||||
.bind(seed_created_at)
|
||||
.bind(seed_created_at * 1000)
|
||||
.bind(seed_created_at)
|
||||
.bind(&request_two)
|
||||
.bind(seed_created_at + 1)
|
||||
.bind((seed_created_at + 1) * 1000)
|
||||
.bind(seed_created_at + 1)
|
||||
.bind(&request_zero)
|
||||
.bind(seed_created_at + 2)
|
||||
.bind((seed_created_at + 2) * 1000)
|
||||
.bind(seed_created_at + 2)
|
||||
.bind(&request_outside)
|
||||
.bind(seed_created_at + 3)
|
||||
.bind((seed_created_at + 3) * 1000)
|
||||
.bind(seed_created_at + 3)
|
||||
.execute(backend.pool())
|
||||
.await
|
||||
.expect("usage should seed");
|
||||
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO usage_settlement_snapshots (
|
||||
request_id, billing_status, wallet_id, finalized_at, created_at, updated_at
|
||||
) VALUES
|
||||
(?, 'settled', ?, ?, ?, ?),
|
||||
(?, 'settled', ?, ?, ?, ?),
|
||||
(?, 'settled', ?, ?, ?, ?),
|
||||
(?, 'settled', ?, ?, ?, ?)
|
||||
"#,
|
||||
)
|
||||
.bind(&request_one)
|
||||
.bind(&wallet_id)
|
||||
.bind(first_finalized_at)
|
||||
.bind(first_finalized_at)
|
||||
.bind(first_finalized_at)
|
||||
.bind(&request_two)
|
||||
.bind(&wallet_id)
|
||||
.bind(last_finalized_at)
|
||||
.bind(last_finalized_at)
|
||||
.bind(last_finalized_at)
|
||||
.bind(&request_zero)
|
||||
.bind(&wallet_id)
|
||||
.bind(zero_finalized_at)
|
||||
.bind(zero_finalized_at)
|
||||
.bind(zero_finalized_at)
|
||||
.bind(&request_outside)
|
||||
.bind(&wallet_id)
|
||||
.bind(outside_finalized_at)
|
||||
.bind(outside_finalized_at)
|
||||
.bind(outside_finalized_at)
|
||||
.execute(backend.pool())
|
||||
.await
|
||||
.expect("settlement snapshots should seed");
|
||||
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO wallet_daily_usage_ledgers (
|
||||
id, wallet_id, billing_date, billing_timezone, total_cost_usd,
|
||||
total_requests, input_tokens, output_tokens, cache_creation_tokens,
|
||||
cache_read_tokens, aggregated_at, created_at, updated_at
|
||||
) VALUES (?, ?, '2026-05-03', ?, 7.0, 3, 1, 1, 0, 0, ?, ?, ?)
|
||||
"#,
|
||||
)
|
||||
.bind(&stale_ledger_id)
|
||||
.bind(&stale_wallet_id)
|
||||
.bind(&timezone)
|
||||
.bind(seed_created_at)
|
||||
.bind(seed_created_at)
|
||||
.bind(seed_created_at)
|
||||
.execute(backend.pool())
|
||||
.await
|
||||
.expect("stale ledger should seed");
|
||||
|
||||
let summary = backend
|
||||
.aggregate_wallet_daily_usage(&WalletDailyUsageAggregationInput {
|
||||
billing_date: "2026-05-03".to_string(),
|
||||
billing_timezone: timezone.clone(),
|
||||
window_start_unix_secs: window_start as u64,
|
||||
window_end_unix_secs: window_end as u64,
|
||||
aggregated_at_unix_secs: aggregated_at as u64,
|
||||
})
|
||||
.await
|
||||
.expect("wallet daily usage aggregation should run");
|
||||
|
||||
assert_eq!(summary.aggregated_wallets, 1);
|
||||
assert_eq!(summary.deleted_stale_ledgers, 1);
|
||||
|
||||
let ledger = sqlx::query_as::<
|
||||
_,
|
||||
(
|
||||
String,
|
||||
f64,
|
||||
i64,
|
||||
i64,
|
||||
i64,
|
||||
i64,
|
||||
i64,
|
||||
Option<i64>,
|
||||
Option<i64>,
|
||||
i64,
|
||||
),
|
||||
>(
|
||||
r#"
|
||||
SELECT
|
||||
wallet_id,
|
||||
total_cost_usd,
|
||||
total_requests,
|
||||
input_tokens,
|
||||
output_tokens,
|
||||
cache_creation_tokens,
|
||||
cache_read_tokens,
|
||||
first_finalized_at,
|
||||
last_finalized_at,
|
||||
aggregated_at
|
||||
FROM wallet_daily_usage_ledgers
|
||||
WHERE wallet_id = ?
|
||||
AND billing_date = '2026-05-03'
|
||||
AND billing_timezone = ?
|
||||
"#,
|
||||
)
|
||||
.bind(&wallet_id)
|
||||
.bind(&timezone)
|
||||
.fetch_one(backend.pool())
|
||||
.await
|
||||
.expect("aggregated ledger should load");
|
||||
|
||||
assert_eq!(ledger.0, wallet_id);
|
||||
assert!((ledger.1 - 3.25).abs() < f64::EPSILON);
|
||||
assert_eq!(ledger.2, 2);
|
||||
assert_eq!(ledger.3, 15);
|
||||
assert_eq!(ledger.4, 27);
|
||||
assert_eq!(ledger.5, 4);
|
||||
assert_eq!(ledger.6, 6);
|
||||
assert_eq!(ledger.7, Some(first_finalized_at));
|
||||
assert_eq!(ledger.8, Some(last_finalized_at));
|
||||
assert_eq!(ledger.9, aggregated_at);
|
||||
|
||||
let stale_count: i64 =
|
||||
sqlx::query_scalar("SELECT COUNT(*) FROM wallet_daily_usage_ledgers WHERE id = ?")
|
||||
.bind(&stale_ledger_id)
|
||||
.fetch_one(backend.pool())
|
||||
.await
|
||||
.expect("stale ledger count should load");
|
||||
assert_eq!(stale_count, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mysql_stats_aggregation_runs_after_mysql_migrations_when_url_is_set() {
|
||||
let Some(database_url) = std::env::var("AETHER_TEST_MYSQL_URL")
|
||||
.ok()
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
else {
|
||||
eprintln!(
|
||||
"skipping mysql stats aggregation smoke test because AETHER_TEST_MYSQL_URL is unset"
|
||||
);
|
||||
return;
|
||||
};
|
||||
|
||||
let config = SqlDatabaseConfig {
|
||||
driver: DatabaseDriver::Mysql,
|
||||
url: database_url,
|
||||
pool: SqlPoolConfig {
|
||||
max_connections: 1,
|
||||
..SqlPoolConfig::default()
|
||||
},
|
||||
};
|
||||
let backend = MysqlBackend::from_config(config).expect("backend should build");
|
||||
run_mysql_migrations(backend.pool())
|
||||
.await
|
||||
.expect("mysql migrations should run");
|
||||
|
||||
for sql in [
|
||||
"DELETE FROM stats_daily WHERE `date` = 0",
|
||||
"DELETE FROM stats_hourly WHERE hour_utc = 3600",
|
||||
"DELETE FROM stats_user_summary WHERE user_id LIKE 'user-%'",
|
||||
"DELETE FROM stats_user_daily_model WHERE `date` = 0",
|
||||
"DELETE FROM stats_user_daily_provider WHERE `date` = 0",
|
||||
"DELETE FROM stats_user_daily_api_format WHERE `date` = 0",
|
||||
"DELETE FROM stats_daily_model_provider WHERE `date` = 0",
|
||||
"DELETE FROM stats_user_daily_model_provider WHERE `date` = 0",
|
||||
"DELETE FROM stats_daily_cost_savings WHERE `date` = 0",
|
||||
"DELETE FROM stats_daily_cost_savings_provider WHERE `date` = 0",
|
||||
"DELETE FROM stats_daily_cost_savings_model WHERE `date` = 0",
|
||||
"DELETE FROM stats_daily_cost_savings_model_provider WHERE `date` = 0",
|
||||
"DELETE FROM stats_user_daily_cost_savings WHERE `date` = 0",
|
||||
"DELETE FROM stats_user_daily_cost_savings_provider WHERE `date` = 0",
|
||||
"DELETE FROM stats_user_daily_cost_savings_model WHERE `date` = 0",
|
||||
"DELETE FROM stats_user_daily_cost_savings_model_provider WHERE `date` = 0",
|
||||
"DELETE FROM usage_settlement_snapshots WHERE request_id LIKE 'request-daily-%' OR request_id LIKE 'stats-%'",
|
||||
"DELETE FROM `usage` WHERE request_id LIKE 'request-%' OR request_id LIKE 'export-request-%' OR request_id LIKE 'stats-%'",
|
||||
] {
|
||||
sqlx::query(sql)
|
||||
.execute(backend.pool())
|
||||
.await
|
||||
.expect("stats smoke cleanup should run");
|
||||
}
|
||||
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO `usage` (
|
||||
request_id, user_id, api_key_id, provider_name, model, api_format, status, billing_status,
|
||||
status_code, error_category, input_tokens, output_tokens,
|
||||
cache_creation_input_tokens, cache_read_input_tokens, total_cost_usd,
|
||||
actual_total_cost_usd, cache_creation_cost_usd, cache_read_cost_usd,
|
||||
input_price_per_1m, response_time_ms, first_byte_time_ms,
|
||||
created_at_unix_ms, updated_at_unix_secs
|
||||
) VALUES
|
||||
('stats-1', 'user-1', 'key-1', 'provider-a', 'model-a', 'openai', 'completed', 'settled',
|
||||
200, NULL, 10, 20, 1, 2, 0.30, 0.25, 0.01, 0.02, 10.0, 100, 50, 3600, 3600),
|
||||
('stats-2', 'user-2', 'key-2', 'provider-b', 'model-b', 'claude', 'failed', 'void',
|
||||
500, 'upstream_error', 5, 7, 0, 1, 0.20, 0.20, 0.00, 0.01, 20.0, 300, 200, 3610, 3610),
|
||||
('stats-pending', 'user-3', 'key-3', 'provider-a', 'model-a', 'openai', 'pending', 'pending',
|
||||
NULL, NULL, 100, 100, 0, 0, 9.99, 9.99, 0.00, 0.00, 0.0, 50, 25, 3620, 3620),
|
||||
('stats-unknown-provider', 'user-4', 'key-4', 'unknown', 'model-a', 'openai', 'completed', 'settled',
|
||||
200, NULL, 100, 100, 0, 0, 9.99, 9.99, 0.00, 0.00, 0.0, 50, 25, 3630, 3630)
|
||||
"#,
|
||||
)
|
||||
.execute(backend.pool())
|
||||
.await
|
||||
.expect("usage stats rows should seed");
|
||||
|
||||
let target_hour = chrono::DateTime::<chrono::Utc>::from_timestamp(3600, 0)
|
||||
.expect("target hour should be valid");
|
||||
let aggregated_at = chrono::DateTime::<chrono::Utc>::from_timestamp(7200, 0)
|
||||
.expect("aggregation time should be valid");
|
||||
let hourly = backend
|
||||
.aggregate_stats_hourly(&StatsHourlyAggregationInput {
|
||||
target_hour_utc: target_hour,
|
||||
aggregated_at,
|
||||
})
|
||||
.await
|
||||
.expect("hourly stats aggregation should run")
|
||||
.expect("hourly bucket should aggregate");
|
||||
assert_eq!(hourly.hour_utc, target_hour);
|
||||
assert_eq!(hourly.total_requests, 2);
|
||||
assert_eq!(hourly.user_rows, 2);
|
||||
assert_eq!(hourly.user_model_rows, 2);
|
||||
assert_eq!(hourly.model_rows, 2);
|
||||
assert_eq!(hourly.provider_rows, 2);
|
||||
|
||||
let hourly_row = sqlx::query_as::<_, (i64, i64, i64, i64, f64)>(
|
||||
r#"
|
||||
SELECT total_requests, success_requests, error_requests, input_tokens, total_cost
|
||||
FROM stats_hourly
|
||||
WHERE hour_utc = 3600
|
||||
"#,
|
||||
)
|
||||
.fetch_one(backend.pool())
|
||||
.await
|
||||
.expect("hourly stats row should load");
|
||||
assert_eq!(hourly_row.0, 2);
|
||||
assert_eq!(hourly_row.1, 1);
|
||||
assert_eq!(hourly_row.2, 1);
|
||||
assert_eq!(hourly_row.3, 15);
|
||||
assert!((hourly_row.4 - 0.50).abs() < f64::EPSILON);
|
||||
|
||||
let second_hourly = backend
|
||||
.aggregate_stats_hourly(&StatsHourlyAggregationInput {
|
||||
target_hour_utc: target_hour,
|
||||
aggregated_at,
|
||||
})
|
||||
.await
|
||||
.expect("second hourly aggregation should run");
|
||||
assert!(second_hourly.is_none());
|
||||
|
||||
let target_day = chrono::DateTime::<chrono::Utc>::from_timestamp(0, 0)
|
||||
.expect("target day should be valid");
|
||||
let daily = backend
|
||||
.aggregate_stats_daily(&StatsDailyAggregationInput {
|
||||
target_day_utc: target_day,
|
||||
aggregated_at,
|
||||
})
|
||||
.await
|
||||
.expect("daily stats aggregation should run")
|
||||
.expect("daily bucket should aggregate");
|
||||
assert_eq!(daily.day_start_utc, target_day);
|
||||
assert_eq!(daily.total_requests, 2);
|
||||
assert_eq!(daily.model_rows, 2);
|
||||
assert_eq!(daily.provider_rows, 2);
|
||||
assert_eq!(daily.api_key_rows, 4);
|
||||
assert_eq!(daily.error_rows, 1);
|
||||
assert_eq!(daily.user_rows, 2);
|
||||
|
||||
let daily_row = sqlx::query_as::<_, (i64, i64, i64, i64)>(
|
||||
r#"
|
||||
SELECT total_requests, success_requests, error_requests, unique_models
|
||||
FROM stats_daily
|
||||
WHERE `date` = 0
|
||||
"#,
|
||||
)
|
||||
.fetch_one(backend.pool())
|
||||
.await
|
||||
.expect("daily stats row should load");
|
||||
assert_eq!(daily_row, (2, 1, 1, 2));
|
||||
|
||||
let enriched_daily = sqlx::query_as::<_, (i64, i64, i64, i64, i64, Option<i64>)>(
|
||||
r#"
|
||||
SELECT effective_input_tokens, total_input_context, cache_hit_total_requests,
|
||||
completed_total_requests, settled_total_requests, p50_response_time_ms
|
||||
FROM stats_daily
|
||||
WHERE `date` = 0
|
||||
"#,
|
||||
)
|
||||
.fetch_one(backend.pool())
|
||||
.await
|
||||
.expect("mysql enriched daily stats row should load");
|
||||
assert_eq!(enriched_daily, (13, 17, 4, 2, 2, None));
|
||||
|
||||
for table in [
|
||||
"stats_user_summary",
|
||||
"stats_user_daily_model",
|
||||
"stats_user_daily_provider",
|
||||
"stats_user_daily_api_format",
|
||||
"stats_daily_model_provider",
|
||||
"stats_user_daily_model_provider",
|
||||
"stats_daily_cost_savings",
|
||||
"stats_daily_cost_savings_provider",
|
||||
"stats_daily_cost_savings_model",
|
||||
"stats_daily_cost_savings_model_provider",
|
||||
"stats_user_daily_cost_savings",
|
||||
"stats_user_daily_cost_savings_provider",
|
||||
"stats_user_daily_cost_savings_model",
|
||||
"stats_user_daily_cost_savings_model_provider",
|
||||
] {
|
||||
let sql = if table == "stats_user_summary" {
|
||||
format!("SELECT COUNT(*) FROM {table} WHERE user_id IN ('user-1', 'user-2')")
|
||||
} else {
|
||||
format!("SELECT COUNT(*) FROM {table} WHERE `date` = 0")
|
||||
};
|
||||
let count: i64 = sqlx::query_scalar(&sql)
|
||||
.fetch_one(backend.pool())
|
||||
.await
|
||||
.expect("mysql advanced stats count should load");
|
||||
assert!(count > 0, "{table} should be populated");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,8 @@
|
||||
use std::fmt;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[cfg(feature = "mysql")]
|
||||
use super::MysqlBackend;
|
||||
#[cfg(feature = "postgres")]
|
||||
use super::PostgresBackend;
|
||||
#[cfg(feature = "sqlite")]
|
||||
use super::SqliteBackend;
|
||||
use crate::repository::announcements::AnnouncementReadRepository;
|
||||
use crate::repository::audit::AuditLogReadRepository;
|
||||
use crate::repository::auth::AuthApiKeyReadRepository;
|
||||
@@ -91,22 +87,12 @@ impl fmt::Debug for DataReadRepositories {
|
||||
impl DataReadRepositories {
|
||||
pub(crate) fn from_backends(
|
||||
#[cfg(feature = "postgres")] postgres: Option<&PostgresBackend>,
|
||||
#[cfg(feature = "mysql")] mysql: Option<&MysqlBackend>,
|
||||
#[cfg(feature = "sqlite")] sqlite: Option<&SqliteBackend>,
|
||||
) -> Self {
|
||||
let mut repositories = Self::default();
|
||||
#[cfg(feature = "postgres")]
|
||||
if let Some(postgres) = postgres {
|
||||
repositories.install_postgres(postgres);
|
||||
}
|
||||
#[cfg(feature = "mysql")]
|
||||
if let Some(mysql) = mysql {
|
||||
repositories.install_mysql(mysql);
|
||||
}
|
||||
#[cfg(feature = "sqlite")]
|
||||
if let Some(sqlite) = sqlite {
|
||||
repositories.install_sqlite(sqlite);
|
||||
}
|
||||
repositories
|
||||
}
|
||||
|
||||
@@ -183,157 +169,10 @@ impl DataReadRepositories {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "mysql")]
|
||||
fn install_mysql(&mut self, backend: &MysqlBackend) {
|
||||
if self.announcements.is_none() {
|
||||
self.announcements = Some(MysqlBackend::announcement_read_repository(backend));
|
||||
}
|
||||
if self.audit_logs.is_none() {
|
||||
self.audit_logs = Some(MysqlBackend::audit_log_read_repository(backend));
|
||||
}
|
||||
if self.auth_api_keys.is_none() {
|
||||
self.auth_api_keys = Some(MysqlBackend::auth_api_key_read_repository(backend));
|
||||
}
|
||||
if self.auth_modules.is_none() {
|
||||
self.auth_modules = Some(MysqlBackend::auth_module_read_repository(backend));
|
||||
}
|
||||
if self.background_tasks.is_none() {
|
||||
self.background_tasks = Some(MysqlBackend::background_task_read_repository(backend));
|
||||
}
|
||||
if self.billing.is_none() {
|
||||
self.billing = Some(MysqlBackend::billing_read_repository(backend));
|
||||
}
|
||||
if self.gemini_file_mappings.is_none() {
|
||||
self.gemini_file_mappings =
|
||||
Some(MysqlBackend::gemini_file_mapping_read_repository(backend));
|
||||
}
|
||||
if self.global_models.is_none() {
|
||||
self.global_models = Some(MysqlBackend::global_model_read_repository(backend));
|
||||
}
|
||||
if self.management_tokens.is_none() {
|
||||
self.management_tokens = Some(MysqlBackend::management_token_read_repository(backend));
|
||||
}
|
||||
if self.oauth_providers.is_none() {
|
||||
self.oauth_providers = Some(MysqlBackend::oauth_provider_read_repository(backend));
|
||||
}
|
||||
if self.pool_scores.is_none() {
|
||||
self.pool_scores = Some(MysqlBackend::pool_score_read_repository(backend));
|
||||
}
|
||||
if self.proxy_nodes.is_none() {
|
||||
self.proxy_nodes = Some(MysqlBackend::proxy_node_read_repository(backend));
|
||||
}
|
||||
if self.minimal_candidate_selection.is_none() {
|
||||
self.minimal_candidate_selection = Some(
|
||||
MysqlBackend::minimal_candidate_selection_read_repository(backend),
|
||||
);
|
||||
}
|
||||
if self.request_candidates.is_none() {
|
||||
self.request_candidates =
|
||||
Some(MysqlBackend::request_candidate_read_repository(backend));
|
||||
}
|
||||
if self.provider_catalog.is_none() {
|
||||
self.provider_catalog = Some(MysqlBackend::provider_catalog_read_repository(backend));
|
||||
}
|
||||
if self.provider_quotas.is_none() {
|
||||
self.provider_quotas = Some(MysqlBackend::provider_quota_read_repository(backend));
|
||||
}
|
||||
if self.routing_groups.is_none() {
|
||||
self.routing_groups = Some(MysqlBackend::routing_group_read_repository(backend));
|
||||
}
|
||||
if self.usage.is_none() {
|
||||
self.usage = Some(MysqlBackend::usage_read_repository(backend));
|
||||
}
|
||||
if self.users.is_none() {
|
||||
self.users = Some(MysqlBackend::user_read_repository(backend));
|
||||
}
|
||||
if self.video_tasks.is_none() {
|
||||
self.video_tasks = Some(MysqlBackend::video_task_read_repository(backend));
|
||||
}
|
||||
if self.wallets.is_none() {
|
||||
self.wallets = Some(MysqlBackend::wallet_read_repository(backend));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "sqlite")]
|
||||
fn install_sqlite(&mut self, backend: &SqliteBackend) {
|
||||
if self.announcements.is_none() {
|
||||
self.announcements = Some(SqliteBackend::announcement_read_repository(backend));
|
||||
}
|
||||
if self.audit_logs.is_none() {
|
||||
self.audit_logs = Some(SqliteBackend::audit_log_read_repository(backend));
|
||||
}
|
||||
if self.auth_api_keys.is_none() {
|
||||
self.auth_api_keys = Some(SqliteBackend::auth_api_key_read_repository(backend));
|
||||
}
|
||||
if self.auth_modules.is_none() {
|
||||
self.auth_modules = Some(SqliteBackend::auth_module_read_repository(backend));
|
||||
}
|
||||
if self.background_tasks.is_none() {
|
||||
self.background_tasks = Some(SqliteBackend::background_task_read_repository(backend));
|
||||
}
|
||||
if self.billing.is_none() {
|
||||
self.billing = Some(SqliteBackend::billing_read_repository(backend));
|
||||
}
|
||||
if self.gemini_file_mappings.is_none() {
|
||||
self.gemini_file_mappings =
|
||||
Some(SqliteBackend::gemini_file_mapping_read_repository(backend));
|
||||
}
|
||||
if self.global_models.is_none() {
|
||||
self.global_models = Some(SqliteBackend::global_model_read_repository(backend));
|
||||
}
|
||||
if self.management_tokens.is_none() {
|
||||
self.management_tokens = Some(SqliteBackend::management_token_read_repository(backend));
|
||||
}
|
||||
if self.oauth_providers.is_none() {
|
||||
self.oauth_providers = Some(SqliteBackend::oauth_provider_read_repository(backend));
|
||||
}
|
||||
if self.pool_scores.is_none() {
|
||||
self.pool_scores = Some(SqliteBackend::pool_score_read_repository(backend));
|
||||
}
|
||||
if self.proxy_nodes.is_none() {
|
||||
self.proxy_nodes = Some(SqliteBackend::proxy_node_read_repository(backend));
|
||||
}
|
||||
if self.minimal_candidate_selection.is_none() {
|
||||
self.minimal_candidate_selection = Some(
|
||||
SqliteBackend::minimal_candidate_selection_read_repository(backend),
|
||||
);
|
||||
}
|
||||
if self.request_candidates.is_none() {
|
||||
self.request_candidates =
|
||||
Some(SqliteBackend::request_candidate_read_repository(backend));
|
||||
}
|
||||
if self.provider_catalog.is_none() {
|
||||
self.provider_catalog = Some(SqliteBackend::provider_catalog_read_repository(backend));
|
||||
}
|
||||
if self.provider_quotas.is_none() {
|
||||
self.provider_quotas = Some(SqliteBackend::provider_quota_read_repository(backend));
|
||||
}
|
||||
if self.routing_groups.is_none() {
|
||||
self.routing_groups = Some(SqliteBackend::routing_group_read_repository(backend));
|
||||
}
|
||||
if self.usage.is_none() {
|
||||
self.usage = Some(SqliteBackend::usage_read_repository(backend));
|
||||
}
|
||||
if self.users.is_none() {
|
||||
self.users = Some(SqliteBackend::user_read_repository(backend));
|
||||
}
|
||||
if self.video_tasks.is_none() {
|
||||
self.video_tasks = Some(SqliteBackend::video_task_read_repository(backend));
|
||||
}
|
||||
if self.wallets.is_none() {
|
||||
self.wallets = Some(SqliteBackend::wallet_read_repository(backend));
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
#[cfg(feature = "postgres")]
|
||||
pub(crate) fn from_postgres(postgres: Option<&PostgresBackend>) -> Self {
|
||||
Self::from_backends(
|
||||
postgres,
|
||||
#[cfg(feature = "mysql")]
|
||||
None,
|
||||
#[cfg(feature = "sqlite")]
|
||||
None,
|
||||
)
|
||||
Self::from_backends(postgres)
|
||||
}
|
||||
|
||||
pub fn auth_api_keys(&self) -> Option<Arc<dyn AuthApiKeyReadRepository>> {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,8 +1,4 @@
|
||||
#[cfg(feature = "mysql")]
|
||||
pub(crate) mod mysql;
|
||||
#[cfg(feature = "postgres")]
|
||||
pub(crate) mod postgres_daily;
|
||||
#[cfg(feature = "postgres")]
|
||||
pub(crate) mod postgres_hourly;
|
||||
#[cfg(feature = "sqlite")]
|
||||
pub(crate) mod sqlite;
|
||||
|
||||
@@ -1,831 +0,0 @@
|
||||
use chrono::{DateTime, Utc};
|
||||
use sqlx::Row;
|
||||
|
||||
use crate::backend::stats_common::{stats_id, unix_secs, utc_from_unix_secs};
|
||||
use crate::backend::MysqlBackend;
|
||||
use crate::driver::mysql::MysqlPool;
|
||||
use crate::error::SqlResultExt;
|
||||
use crate::{
|
||||
DataLayerError, StatsDailyAggregationInput, StatsDailyAggregationSummary,
|
||||
StatsHourlyAggregationInput, StatsHourlyAggregationSummary,
|
||||
};
|
||||
|
||||
mod advanced;
|
||||
|
||||
impl MysqlBackend {
|
||||
pub async fn aggregate_stats_hourly(
|
||||
&self,
|
||||
input: &StatsHourlyAggregationInput,
|
||||
) -> Result<Option<StatsHourlyAggregationSummary>, DataLayerError> {
|
||||
let Some(hour_utc_unix_secs) =
|
||||
next_mysql_stats_hourly_bucket(self.pool(), input.target_hour_utc).await?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
perform_mysql_stats_hourly_aggregation(self.pool(), hour_utc_unix_secs, input.aggregated_at)
|
||||
.await
|
||||
.map(Some)
|
||||
}
|
||||
|
||||
pub async fn aggregate_stats_daily(
|
||||
&self,
|
||||
input: &StatsDailyAggregationInput,
|
||||
) -> Result<Option<StatsDailyAggregationSummary>, DataLayerError> {
|
||||
let Some(day_start_unix_secs) =
|
||||
next_mysql_stats_daily_bucket(self.pool(), input.target_day_utc).await?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
perform_mysql_stats_daily_aggregation(self.pool(), day_start_unix_secs, input.aggregated_at)
|
||||
.await
|
||||
.map(Some)
|
||||
}
|
||||
}
|
||||
|
||||
async fn next_mysql_stats_hourly_bucket(
|
||||
pool: &MysqlPool,
|
||||
target_hour_utc: DateTime<Utc>,
|
||||
) -> Result<Option<i64>, DataLayerError> {
|
||||
let latest_hour: Option<i64> =
|
||||
sqlx::query_scalar("SELECT MAX(hour_utc) FROM stats_hourly WHERE is_complete <> 0")
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
let search_from = latest_hour.map(|value| value + 3600).unwrap_or(0);
|
||||
let search_until = unix_secs(target_hour_utc) + 3600;
|
||||
if search_from >= search_until {
|
||||
return Ok(None);
|
||||
}
|
||||
let next_bucket: Option<i64> = sqlx::query_scalar(
|
||||
r#"
|
||||
SELECT CAST(MIN(FLOOR(created_at_unix_ms / 3600) * 3600) AS SIGNED)
|
||||
FROM `usage`
|
||||
WHERE created_at_unix_ms >= ?
|
||||
AND created_at_unix_ms < ?
|
||||
AND status NOT IN ('pending', 'streaming')
|
||||
AND provider_name NOT IN ('unknown', 'pending')
|
||||
"#,
|
||||
)
|
||||
.bind(search_from)
|
||||
.bind(search_until)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(next_bucket.filter(|value| *value <= unix_secs(target_hour_utc)))
|
||||
}
|
||||
|
||||
async fn next_mysql_stats_daily_bucket(
|
||||
pool: &MysqlPool,
|
||||
target_day_utc: DateTime<Utc>,
|
||||
) -> Result<Option<i64>, DataLayerError> {
|
||||
let latest_day: Option<i64> =
|
||||
sqlx::query_scalar("SELECT MAX(`date`) FROM stats_daily WHERE is_complete <> 0")
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
let search_from = latest_day.map(|value| value + 86_400).unwrap_or(0);
|
||||
let search_until = unix_secs(target_day_utc) + 86_400;
|
||||
if search_from >= search_until {
|
||||
return Ok(None);
|
||||
}
|
||||
let next_bucket: Option<i64> = sqlx::query_scalar(
|
||||
r#"
|
||||
SELECT CAST(MIN(FLOOR(created_at_unix_ms / 86400) * 86400) AS SIGNED)
|
||||
FROM `usage`
|
||||
WHERE created_at_unix_ms >= ?
|
||||
AND created_at_unix_ms < ?
|
||||
AND status NOT IN ('pending', 'streaming')
|
||||
AND provider_name NOT IN ('unknown', 'pending')
|
||||
"#,
|
||||
)
|
||||
.bind(search_from)
|
||||
.bind(search_until)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(next_bucket.filter(|value| *value <= unix_secs(target_day_utc)))
|
||||
}
|
||||
|
||||
const MYSQL_STATS_AGGREGATE_SQL: &str = r#"
|
||||
SELECT
|
||||
CAST(COUNT(*) AS SIGNED) AS total_requests,
|
||||
CAST(COALESCE(SUM(CASE
|
||||
WHEN status = 'failed'
|
||||
OR status_code >= 400
|
||||
OR error_message IS NOT NULL
|
||||
THEN 1 ELSE 0 END), 0) AS SIGNED) AS error_requests,
|
||||
CAST(COALESCE(SUM(input_tokens), 0) AS SIGNED) AS input_tokens,
|
||||
CAST(COALESCE(SUM(output_tokens), 0) AS SIGNED) AS output_tokens,
|
||||
CAST(COALESCE(SUM(cache_creation_input_tokens), 0) AS SIGNED) AS cache_creation_tokens,
|
||||
CAST(COALESCE(SUM(cache_read_input_tokens), 0) AS SIGNED) AS cache_read_tokens,
|
||||
CAST(COALESCE(SUM(total_cost_usd), 0.0) AS DOUBLE) AS total_cost,
|
||||
CAST(COALESCE(SUM(actual_total_cost_usd), 0.0) AS DOUBLE) AS actual_total_cost,
|
||||
CAST(COALESCE(AVG(response_time_ms), 0.0) AS DOUBLE) AS avg_response_time_ms
|
||||
FROM `usage`
|
||||
WHERE created_at_unix_ms >= ?
|
||||
AND created_at_unix_ms < ?
|
||||
AND status NOT IN ('pending', 'streaming')
|
||||
AND provider_name NOT IN ('unknown', 'pending')
|
||||
"#;
|
||||
|
||||
async fn perform_mysql_stats_hourly_aggregation(
|
||||
pool: &MysqlPool,
|
||||
hour_utc_unix_secs: i64,
|
||||
aggregated_at: DateTime<Utc>,
|
||||
) -> Result<StatsHourlyAggregationSummary, DataLayerError> {
|
||||
let start_unix_secs = hour_utc_unix_secs;
|
||||
let end_unix_secs = hour_utc_unix_secs + 3600;
|
||||
let aggregated_at_unix_secs = unix_secs(aggregated_at);
|
||||
let mut tx = pool.begin().await.map_sql_err()?;
|
||||
let row = sqlx::query(MYSQL_STATS_AGGREGATE_SQL)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.fetch_one(&mut *tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
let total_requests: i64 = row.try_get("total_requests").map_sql_err()?;
|
||||
let error_requests: i64 = row.try_get("error_requests").map_sql_err()?;
|
||||
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO stats_hourly (
|
||||
id, hour_utc, total_requests, success_requests, error_requests,
|
||||
input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens,
|
||||
total_cost, actual_total_cost, avg_response_time_ms, is_complete,
|
||||
aggregated_at, created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, TRUE, ?, ?, ?)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
total_requests = VALUES(total_requests),
|
||||
success_requests = VALUES(success_requests),
|
||||
error_requests = VALUES(error_requests),
|
||||
input_tokens = VALUES(input_tokens),
|
||||
output_tokens = VALUES(output_tokens),
|
||||
cache_creation_tokens = VALUES(cache_creation_tokens),
|
||||
cache_read_tokens = VALUES(cache_read_tokens),
|
||||
total_cost = VALUES(total_cost),
|
||||
actual_total_cost = VALUES(actual_total_cost),
|
||||
avg_response_time_ms = VALUES(avg_response_time_ms),
|
||||
is_complete = VALUES(is_complete),
|
||||
aggregated_at = VALUES(aggregated_at),
|
||||
updated_at = VALUES(updated_at)
|
||||
"#,
|
||||
)
|
||||
.bind(stats_id(&format!("stats-hourly:{hour_utc_unix_secs}")))
|
||||
.bind(hour_utc_unix_secs)
|
||||
.bind(total_requests)
|
||||
.bind(total_requests.saturating_sub(error_requests))
|
||||
.bind(error_requests)
|
||||
.bind(row.try_get::<i64, _>("input_tokens").map_sql_err()?)
|
||||
.bind(row.try_get::<i64, _>("output_tokens").map_sql_err()?)
|
||||
.bind(
|
||||
row.try_get::<i64, _>("cache_creation_tokens")
|
||||
.map_sql_err()?,
|
||||
)
|
||||
.bind(row.try_get::<i64, _>("cache_read_tokens").map_sql_err()?)
|
||||
.bind(row.try_get::<f64, _>("total_cost").map_sql_err()?)
|
||||
.bind(row.try_get::<f64, _>("actual_total_cost").map_sql_err()?)
|
||||
.bind(
|
||||
row.try_get::<f64, _>("avg_response_time_ms")
|
||||
.map_sql_err()?,
|
||||
)
|
||||
.bind(aggregated_at_unix_secs)
|
||||
.bind(aggregated_at_unix_secs)
|
||||
.bind(aggregated_at_unix_secs)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
|
||||
let user_rows = upsert_mysql_stats_hourly_user_rows(
|
||||
&mut tx,
|
||||
hour_utc_unix_secs,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
aggregated_at_unix_secs,
|
||||
)
|
||||
.await?;
|
||||
let user_model_rows = upsert_mysql_stats_hourly_user_model_rows(
|
||||
&mut tx,
|
||||
hour_utc_unix_secs,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
aggregated_at_unix_secs,
|
||||
)
|
||||
.await?;
|
||||
let model_rows = upsert_mysql_stats_hourly_model_rows(
|
||||
&mut tx,
|
||||
hour_utc_unix_secs,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
aggregated_at_unix_secs,
|
||||
)
|
||||
.await?;
|
||||
let provider_rows = upsert_mysql_stats_hourly_provider_rows(
|
||||
&mut tx,
|
||||
hour_utc_unix_secs,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
aggregated_at_unix_secs,
|
||||
)
|
||||
.await?;
|
||||
advanced::refresh_hourly(&mut tx, hour_utc_unix_secs, start_unix_secs, end_unix_secs).await?;
|
||||
tx.commit().await.map_sql_err()?;
|
||||
|
||||
Ok(StatsHourlyAggregationSummary {
|
||||
hour_utc: utc_from_unix_secs(hour_utc_unix_secs, "stats_hourly.hour_utc")?,
|
||||
total_requests,
|
||||
user_rows,
|
||||
user_model_rows,
|
||||
model_rows,
|
||||
provider_rows,
|
||||
})
|
||||
}
|
||||
|
||||
async fn perform_mysql_stats_daily_aggregation(
|
||||
pool: &MysqlPool,
|
||||
day_start_unix_secs: i64,
|
||||
aggregated_at: DateTime<Utc>,
|
||||
) -> Result<StatsDailyAggregationSummary, DataLayerError> {
|
||||
let start_unix_secs = day_start_unix_secs;
|
||||
let end_unix_secs = day_start_unix_secs + 86_400;
|
||||
let aggregated_at_unix_secs = unix_secs(aggregated_at);
|
||||
let mut tx = pool.begin().await.map_sql_err()?;
|
||||
let row = sqlx::query(MYSQL_STATS_AGGREGATE_SQL)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.fetch_one(&mut *tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
let total_requests: i64 = row.try_get("total_requests").map_sql_err()?;
|
||||
let error_requests: i64 = row.try_get("error_requests").map_sql_err()?;
|
||||
let unique_models =
|
||||
mysql_group_count(&mut tx, "model", start_unix_secs, end_unix_secs).await? as i64;
|
||||
let unique_providers =
|
||||
mysql_group_count(&mut tx, "provider_name", start_unix_secs, end_unix_secs).await? as i64;
|
||||
let fallback_count =
|
||||
mysql_daily_fallback_count(&mut tx, start_unix_secs, end_unix_secs).await?;
|
||||
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO stats_daily (
|
||||
id, `date`, total_requests, success_requests, error_requests,
|
||||
input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens,
|
||||
total_cost, actual_total_cost, avg_response_time_ms, fallback_count,
|
||||
unique_models, unique_providers, is_complete, aggregated_at, created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, TRUE, ?, ?, ?)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
total_requests = VALUES(total_requests),
|
||||
success_requests = VALUES(success_requests),
|
||||
error_requests = VALUES(error_requests),
|
||||
input_tokens = VALUES(input_tokens),
|
||||
output_tokens = VALUES(output_tokens),
|
||||
cache_creation_tokens = VALUES(cache_creation_tokens),
|
||||
cache_read_tokens = VALUES(cache_read_tokens),
|
||||
total_cost = VALUES(total_cost),
|
||||
actual_total_cost = VALUES(actual_total_cost),
|
||||
avg_response_time_ms = VALUES(avg_response_time_ms),
|
||||
fallback_count = VALUES(fallback_count),
|
||||
unique_models = VALUES(unique_models),
|
||||
unique_providers = VALUES(unique_providers),
|
||||
is_complete = VALUES(is_complete),
|
||||
aggregated_at = VALUES(aggregated_at),
|
||||
updated_at = VALUES(updated_at)
|
||||
"#,
|
||||
)
|
||||
.bind(stats_id(&format!("stats-daily:{day_start_unix_secs}")))
|
||||
.bind(day_start_unix_secs)
|
||||
.bind(total_requests)
|
||||
.bind(total_requests.saturating_sub(error_requests))
|
||||
.bind(error_requests)
|
||||
.bind(row.try_get::<i64, _>("input_tokens").map_sql_err()?)
|
||||
.bind(row.try_get::<i64, _>("output_tokens").map_sql_err()?)
|
||||
.bind(
|
||||
row.try_get::<i64, _>("cache_creation_tokens")
|
||||
.map_sql_err()?,
|
||||
)
|
||||
.bind(row.try_get::<i64, _>("cache_read_tokens").map_sql_err()?)
|
||||
.bind(row.try_get::<f64, _>("total_cost").map_sql_err()?)
|
||||
.bind(row.try_get::<f64, _>("actual_total_cost").map_sql_err()?)
|
||||
.bind(
|
||||
row.try_get::<f64, _>("avg_response_time_ms")
|
||||
.map_sql_err()?,
|
||||
)
|
||||
.bind(fallback_count)
|
||||
.bind(unique_models)
|
||||
.bind(unique_providers)
|
||||
.bind(aggregated_at_unix_secs)
|
||||
.bind(aggregated_at_unix_secs)
|
||||
.bind(aggregated_at_unix_secs)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
|
||||
let model_rows = upsert_mysql_stats_daily_model_rows(
|
||||
&mut tx,
|
||||
day_start_unix_secs,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
aggregated_at_unix_secs,
|
||||
)
|
||||
.await?;
|
||||
let provider_rows = upsert_mysql_stats_daily_provider_rows(
|
||||
&mut tx,
|
||||
day_start_unix_secs,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
aggregated_at_unix_secs,
|
||||
)
|
||||
.await?;
|
||||
let api_key_rows = upsert_mysql_stats_daily_api_key_rows(
|
||||
&mut tx,
|
||||
day_start_unix_secs,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
aggregated_at_unix_secs,
|
||||
)
|
||||
.await?;
|
||||
let error_rows = refresh_mysql_stats_daily_error_rows(
|
||||
&mut tx,
|
||||
day_start_unix_secs,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
aggregated_at_unix_secs,
|
||||
)
|
||||
.await?;
|
||||
let user_rows = upsert_mysql_stats_user_daily_rows(
|
||||
&mut tx,
|
||||
day_start_unix_secs,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
aggregated_at_unix_secs,
|
||||
)
|
||||
.await?;
|
||||
advanced::refresh_daily(
|
||||
&mut tx,
|
||||
day_start_unix_secs,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
aggregated_at_unix_secs,
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await.map_sql_err()?;
|
||||
|
||||
Ok(StatsDailyAggregationSummary {
|
||||
day_start_utc: utc_from_unix_secs(day_start_unix_secs, "stats_daily.date")?,
|
||||
total_requests,
|
||||
model_rows,
|
||||
provider_rows,
|
||||
api_key_rows,
|
||||
error_rows,
|
||||
user_rows,
|
||||
})
|
||||
}
|
||||
|
||||
async fn upsert_mysql_stats_hourly_user_rows(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
|
||||
hour_utc: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<usize, DataLayerError> {
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO stats_hourly_user (
|
||||
id, hour_utc, user_id, total_requests, success_requests, error_requests,
|
||||
input_tokens, output_tokens, total_cost, created_at, updated_at
|
||||
)
|
||||
SELECT
|
||||
SHA2(UUID(), 256), ?, user_id, COUNT(*),
|
||||
COUNT(*) - COALESCE(SUM(CASE
|
||||
WHEN status = 'failed' OR status_code >= 400 OR error_message IS NOT NULL
|
||||
THEN 1 ELSE 0 END), 0),
|
||||
COALESCE(SUM(CASE
|
||||
WHEN status = 'failed' OR status_code >= 400 OR error_message IS NOT NULL
|
||||
THEN 1 ELSE 0 END), 0),
|
||||
COALESCE(SUM(input_tokens), 0), COALESCE(SUM(output_tokens), 0),
|
||||
COALESCE(SUM(total_cost_usd), 0), ?, ?
|
||||
FROM `usage`
|
||||
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
|
||||
AND user_id IS NOT NULL AND user_id <> ''
|
||||
AND status NOT IN ('pending', 'streaming')
|
||||
AND provider_name NOT IN ('unknown', 'pending')
|
||||
GROUP BY user_id
|
||||
ON DUPLICATE KEY UPDATE
|
||||
total_requests = VALUES(total_requests), success_requests = VALUES(success_requests),
|
||||
error_requests = VALUES(error_requests), input_tokens = VALUES(input_tokens),
|
||||
output_tokens = VALUES(output_tokens), total_cost = VALUES(total_cost),
|
||||
updated_at = VALUES(updated_at)
|
||||
"#,
|
||||
)
|
||||
.bind(hour_utc)
|
||||
.bind(now_unix_secs)
|
||||
.bind(now_unix_secs)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
|
||||
}
|
||||
|
||||
async fn upsert_mysql_stats_hourly_user_model_rows(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
|
||||
hour_utc: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<usize, DataLayerError> {
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO stats_hourly_user_model (
|
||||
id, hour_utc, user_id, model, total_requests, input_tokens, output_tokens,
|
||||
total_cost, created_at, updated_at
|
||||
)
|
||||
SELECT SHA2(UUID(), 256), ?, user_id, model, COUNT(*),
|
||||
COALESCE(SUM(input_tokens), 0), COALESCE(SUM(output_tokens), 0),
|
||||
COALESCE(SUM(total_cost_usd), 0), ?, ?
|
||||
FROM `usage`
|
||||
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
|
||||
AND user_id IS NOT NULL AND user_id <> '' AND model IS NOT NULL AND model <> ''
|
||||
AND status NOT IN ('pending', 'streaming')
|
||||
AND provider_name NOT IN ('unknown', 'pending')
|
||||
GROUP BY user_id, model
|
||||
ON DUPLICATE KEY UPDATE
|
||||
total_requests = VALUES(total_requests), input_tokens = VALUES(input_tokens),
|
||||
output_tokens = VALUES(output_tokens), total_cost = VALUES(total_cost),
|
||||
updated_at = VALUES(updated_at)
|
||||
"#,
|
||||
)
|
||||
.bind(hour_utc)
|
||||
.bind(now_unix_secs)
|
||||
.bind(now_unix_secs)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
|
||||
}
|
||||
|
||||
async fn upsert_mysql_stats_hourly_model_rows(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
|
||||
hour_utc: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<usize, DataLayerError> {
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO stats_hourly_model (
|
||||
id, hour_utc, model, total_requests, input_tokens, output_tokens, total_cost,
|
||||
avg_response_time_ms, created_at, updated_at
|
||||
)
|
||||
SELECT SHA2(UUID(), 256), ?, model, COUNT(*),
|
||||
COALESCE(SUM(input_tokens), 0), COALESCE(SUM(output_tokens), 0),
|
||||
COALESCE(SUM(total_cost_usd), 0), COALESCE(AVG(response_time_ms), 0), ?, ?
|
||||
FROM `usage`
|
||||
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
|
||||
AND model IS NOT NULL AND model <> ''
|
||||
AND status NOT IN ('pending', 'streaming')
|
||||
AND provider_name NOT IN ('unknown', 'pending')
|
||||
GROUP BY model
|
||||
ON DUPLICATE KEY UPDATE
|
||||
total_requests = VALUES(total_requests), input_tokens = VALUES(input_tokens),
|
||||
output_tokens = VALUES(output_tokens), total_cost = VALUES(total_cost),
|
||||
avg_response_time_ms = VALUES(avg_response_time_ms), updated_at = VALUES(updated_at)
|
||||
"#,
|
||||
)
|
||||
.bind(hour_utc)
|
||||
.bind(now_unix_secs)
|
||||
.bind(now_unix_secs)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
|
||||
}
|
||||
|
||||
async fn upsert_mysql_stats_hourly_provider_rows(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
|
||||
hour_utc: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<usize, DataLayerError> {
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO stats_hourly_provider (
|
||||
id, hour_utc, provider_name, total_requests, input_tokens, output_tokens,
|
||||
total_cost, created_at, updated_at
|
||||
)
|
||||
SELECT SHA2(UUID(), 256), ?, provider_name, COUNT(*),
|
||||
COALESCE(SUM(input_tokens), 0), COALESCE(SUM(output_tokens), 0),
|
||||
COALESCE(SUM(total_cost_usd), 0), ?, ?
|
||||
FROM `usage`
|
||||
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
|
||||
AND status NOT IN ('pending', 'streaming')
|
||||
AND provider_name NOT IN ('unknown', 'pending')
|
||||
GROUP BY provider_name
|
||||
ON DUPLICATE KEY UPDATE
|
||||
total_requests = VALUES(total_requests), input_tokens = VALUES(input_tokens),
|
||||
output_tokens = VALUES(output_tokens), total_cost = VALUES(total_cost),
|
||||
updated_at = VALUES(updated_at)
|
||||
"#,
|
||||
)
|
||||
.bind(hour_utc)
|
||||
.bind(now_unix_secs)
|
||||
.bind(now_unix_secs)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
|
||||
}
|
||||
|
||||
async fn upsert_mysql_stats_daily_model_rows(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
|
||||
day_start: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<usize, DataLayerError> {
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO stats_daily_model (
|
||||
id, `date`, model, total_requests, input_tokens, output_tokens,
|
||||
cache_creation_tokens, cache_read_tokens, total_cost, avg_response_time_ms,
|
||||
created_at, updated_at
|
||||
)
|
||||
SELECT SHA2(UUID(), 256), ?, model, COUNT(*),
|
||||
COALESCE(SUM(input_tokens), 0), COALESCE(SUM(output_tokens), 0),
|
||||
COALESCE(SUM(cache_creation_input_tokens), 0),
|
||||
COALESCE(SUM(cache_read_input_tokens), 0), COALESCE(SUM(total_cost_usd), 0),
|
||||
COALESCE(AVG(response_time_ms), 0), ?, ?
|
||||
FROM `usage`
|
||||
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
|
||||
AND model IS NOT NULL AND model <> ''
|
||||
AND status NOT IN ('pending', 'streaming')
|
||||
AND provider_name NOT IN ('unknown', 'pending')
|
||||
GROUP BY model
|
||||
ON DUPLICATE KEY UPDATE
|
||||
total_requests = VALUES(total_requests), input_tokens = VALUES(input_tokens),
|
||||
output_tokens = VALUES(output_tokens), cache_creation_tokens = VALUES(cache_creation_tokens),
|
||||
cache_read_tokens = VALUES(cache_read_tokens), total_cost = VALUES(total_cost),
|
||||
avg_response_time_ms = VALUES(avg_response_time_ms), updated_at = VALUES(updated_at)
|
||||
"#,
|
||||
)
|
||||
.bind(day_start)
|
||||
.bind(now_unix_secs)
|
||||
.bind(now_unix_secs)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
|
||||
}
|
||||
|
||||
async fn upsert_mysql_stats_daily_provider_rows(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
|
||||
day_start: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<usize, DataLayerError> {
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO stats_daily_provider (
|
||||
id, `date`, provider_name, total_requests, input_tokens, output_tokens,
|
||||
cache_creation_tokens, cache_read_tokens, total_cost, created_at, updated_at
|
||||
)
|
||||
SELECT SHA2(UUID(), 256), ?, provider_name, COUNT(*),
|
||||
COALESCE(SUM(input_tokens), 0), COALESCE(SUM(output_tokens), 0),
|
||||
COALESCE(SUM(cache_creation_input_tokens), 0),
|
||||
COALESCE(SUM(cache_read_input_tokens), 0), COALESCE(SUM(total_cost_usd), 0), ?, ?
|
||||
FROM `usage`
|
||||
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
|
||||
AND status NOT IN ('pending', 'streaming')
|
||||
AND provider_name NOT IN ('unknown', 'pending')
|
||||
GROUP BY provider_name
|
||||
ON DUPLICATE KEY UPDATE
|
||||
total_requests = VALUES(total_requests), input_tokens = VALUES(input_tokens),
|
||||
output_tokens = VALUES(output_tokens), cache_creation_tokens = VALUES(cache_creation_tokens),
|
||||
cache_read_tokens = VALUES(cache_read_tokens), total_cost = VALUES(total_cost),
|
||||
updated_at = VALUES(updated_at)
|
||||
"#,
|
||||
)
|
||||
.bind(day_start)
|
||||
.bind(now_unix_secs)
|
||||
.bind(now_unix_secs)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
|
||||
}
|
||||
|
||||
async fn upsert_mysql_stats_daily_api_key_rows(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
|
||||
day_start: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<usize, DataLayerError> {
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO stats_daily_api_key (
|
||||
id, api_key_id, `date`, total_requests, success_requests, error_requests,
|
||||
input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens,
|
||||
total_cost, api_key_name, created_at, updated_at
|
||||
)
|
||||
SELECT SHA2(UUID(), 256), usage.api_key_id, ?, COUNT(*),
|
||||
COUNT(*) - COALESCE(SUM(CASE
|
||||
WHEN usage.status = 'failed' OR usage.status_code >= 400
|
||||
OR usage.error_message IS NOT NULL THEN 1 ELSE 0 END), 0),
|
||||
COALESCE(SUM(CASE
|
||||
WHEN usage.status = 'failed' OR usage.status_code >= 400
|
||||
OR usage.error_message IS NOT NULL THEN 1 ELSE 0 END), 0),
|
||||
COALESCE(SUM(usage.input_tokens), 0), COALESCE(SUM(usage.output_tokens), 0),
|
||||
COALESCE(SUM(usage.cache_creation_input_tokens), 0),
|
||||
COALESCE(SUM(usage.cache_read_input_tokens), 0),
|
||||
COALESCE(SUM(usage.total_cost_usd), 0), MAX(api_keys.name), ?, ?
|
||||
FROM `usage` AS `usage`
|
||||
LEFT JOIN api_keys ON api_keys.id = usage.api_key_id
|
||||
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
|
||||
AND usage.api_key_id IS NOT NULL AND usage.api_key_id <> ''
|
||||
GROUP BY usage.api_key_id
|
||||
ON DUPLICATE KEY UPDATE
|
||||
total_requests = VALUES(total_requests), success_requests = VALUES(success_requests),
|
||||
error_requests = VALUES(error_requests), input_tokens = VALUES(input_tokens),
|
||||
output_tokens = VALUES(output_tokens), cache_creation_tokens = VALUES(cache_creation_tokens),
|
||||
cache_read_tokens = VALUES(cache_read_tokens), total_cost = VALUES(total_cost),
|
||||
api_key_name = COALESCE(VALUES(api_key_name), stats_daily_api_key.api_key_name),
|
||||
updated_at = VALUES(updated_at)
|
||||
"#,
|
||||
)
|
||||
.bind(day_start)
|
||||
.bind(now_unix_secs)
|
||||
.bind(now_unix_secs)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
|
||||
}
|
||||
|
||||
async fn refresh_mysql_stats_daily_error_rows(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
|
||||
day_start: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<usize, DataLayerError> {
|
||||
sqlx::query("DELETE FROM stats_daily_error WHERE `date` = ?")
|
||||
.bind(day_start)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO stats_daily_error (
|
||||
id, `date`, error_category, provider_name, model, count, created_at, updated_at
|
||||
)
|
||||
SELECT SHA2(UUID(), 256), ?, error_category, provider_name, model, COUNT(*), ?, ?
|
||||
FROM `usage`
|
||||
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
|
||||
AND error_category IS NOT NULL AND error_category <> ''
|
||||
GROUP BY error_category, provider_name, model
|
||||
"#,
|
||||
)
|
||||
.bind(day_start)
|
||||
.bind(now_unix_secs)
|
||||
.bind(now_unix_secs)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
|
||||
}
|
||||
|
||||
async fn upsert_mysql_stats_user_daily_rows(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
|
||||
day_start: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<usize, DataLayerError> {
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO stats_user_daily (
|
||||
id, user_id, `date`, total_requests, success_requests, error_requests,
|
||||
input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens,
|
||||
total_cost, username, created_at, updated_at
|
||||
)
|
||||
SELECT SHA2(UUID(), 256), usage.user_id, ?, COUNT(*),
|
||||
COUNT(*) - COALESCE(SUM(CASE
|
||||
WHEN usage.status = 'failed' OR usage.status_code >= 400
|
||||
OR usage.error_message IS NOT NULL THEN 1 ELSE 0 END), 0),
|
||||
COALESCE(SUM(CASE
|
||||
WHEN usage.status = 'failed' OR usage.status_code >= 400
|
||||
OR usage.error_message IS NOT NULL THEN 1 ELSE 0 END), 0),
|
||||
COALESCE(SUM(usage.input_tokens), 0), COALESCE(SUM(usage.output_tokens), 0),
|
||||
COALESCE(SUM(usage.cache_creation_input_tokens), 0),
|
||||
COALESCE(SUM(usage.cache_read_input_tokens), 0),
|
||||
COALESCE(SUM(usage.total_cost_usd), 0), MAX(users.username), ?, ?
|
||||
FROM `usage` AS `usage`
|
||||
LEFT JOIN users ON users.id = usage.user_id
|
||||
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
|
||||
AND usage.user_id IS NOT NULL AND usage.user_id <> ''
|
||||
AND usage.status NOT IN ('pending', 'streaming')
|
||||
AND usage.provider_name NOT IN ('unknown', 'pending')
|
||||
GROUP BY usage.user_id
|
||||
ON DUPLICATE KEY UPDATE
|
||||
total_requests = VALUES(total_requests), success_requests = VALUES(success_requests),
|
||||
error_requests = VALUES(error_requests), input_tokens = VALUES(input_tokens),
|
||||
output_tokens = VALUES(output_tokens), cache_creation_tokens = VALUES(cache_creation_tokens),
|
||||
cache_read_tokens = VALUES(cache_read_tokens), total_cost = VALUES(total_cost),
|
||||
username = COALESCE(VALUES(username), stats_user_daily.username),
|
||||
updated_at = VALUES(updated_at)
|
||||
"#,
|
||||
)
|
||||
.bind(day_start)
|
||||
.bind(now_unix_secs)
|
||||
.bind(now_unix_secs)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
|
||||
}
|
||||
|
||||
async fn mysql_daily_fallback_count(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
) -> Result<i64, DataLayerError> {
|
||||
let start_unix_ms = start_unix_secs.checked_mul(1000).ok_or_else(|| {
|
||||
DataLayerError::InvalidInput("stats fallback window start overflows milliseconds".into())
|
||||
})?;
|
||||
let end_unix_ms = end_unix_secs.checked_mul(1000).ok_or_else(|| {
|
||||
DataLayerError::InvalidInput("stats fallback window end overflows milliseconds".into())
|
||||
})?;
|
||||
sqlx::query_scalar(
|
||||
r#"
|
||||
SELECT COUNT(*)
|
||||
FROM (
|
||||
SELECT request_id
|
||||
FROM request_candidates
|
||||
WHERE created_at >= ? AND created_at < ?
|
||||
AND status IN ('success', 'failed')
|
||||
GROUP BY request_id
|
||||
HAVING COUNT(id) > 1
|
||||
) AS fallback_requests
|
||||
"#,
|
||||
)
|
||||
.bind(start_unix_ms)
|
||||
.bind(end_unix_ms)
|
||||
.fetch_one(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()
|
||||
}
|
||||
|
||||
async fn mysql_group_count(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
|
||||
group_columns: &str,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
) -> Result<usize, DataLayerError> {
|
||||
let not_empty = group_columns
|
||||
.split(',')
|
||||
.map(str::trim)
|
||||
.map(|column| format!("{column} IS NOT NULL AND {column} <> ''"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(" AND ");
|
||||
let sql = format!(
|
||||
r#"
|
||||
SELECT COUNT(*)
|
||||
FROM (
|
||||
SELECT 1
|
||||
FROM `usage`
|
||||
WHERE created_at_unix_ms >= ?
|
||||
AND created_at_unix_ms < ?
|
||||
AND status NOT IN ('pending', 'streaming')
|
||||
AND provider_name NOT IN ('unknown', 'pending')
|
||||
AND {not_empty}
|
||||
GROUP BY {group_columns}
|
||||
) AS grouped
|
||||
"#
|
||||
);
|
||||
let count: i64 = sqlx::query_scalar(&sql)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.fetch_one(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(usize::try_from(count.max(0)).unwrap_or(usize::MAX))
|
||||
}
|
||||
@@ -1,973 +0,0 @@
|
||||
use sqlx::MySql;
|
||||
|
||||
use crate::backend::stats_common::stats_id;
|
||||
use crate::error::SqlResultExt;
|
||||
use crate::DataLayerError;
|
||||
|
||||
const CACHE_5M: &str = r#"GREATEST(
|
||||
COALESCE(usage.cache_creation_input_tokens_5m, 0),
|
||||
COALESCE(usage.cache_creation_ephemeral_5m_input_tokens, 0)
|
||||
)"#;
|
||||
const CACHE_1H: &str = r#"GREATEST(
|
||||
COALESCE(usage.cache_creation_input_tokens_1h, 0),
|
||||
COALESCE(usage.cache_creation_ephemeral_1h_input_tokens, 0)
|
||||
)"#;
|
||||
const CACHE_CREATION: &str = r#"CASE
|
||||
WHEN COALESCE(usage.cache_creation_input_tokens, 0) = 0
|
||||
AND ({cache_5m} + {cache_1h}) > 0
|
||||
THEN {cache_5m} + {cache_1h}
|
||||
ELSE GREATEST(COALESCE(usage.cache_creation_input_tokens, 0), 0)
|
||||
END"#;
|
||||
const EFFECTIVE_INPUT: &str = r#"CASE
|
||||
WHEN SUBSTRING_INDEX(
|
||||
LOWER(COALESCE(usage.endpoint_api_format, usage.api_format, '')), ':', 1
|
||||
) IN ('openai', 'gemini', 'google')
|
||||
AND COALESCE(usage.input_tokens, 0) > 0
|
||||
AND COALESCE(usage.cache_read_input_tokens, 0) > 0
|
||||
THEN GREATEST(COALESCE(usage.input_tokens, 0) - COALESCE(usage.cache_read_input_tokens, 0), 0)
|
||||
ELSE GREATEST(COALESCE(usage.input_tokens, 0), 0)
|
||||
END"#;
|
||||
const SUCCESS: &str = r#"CASE
|
||||
WHEN usage.status <> 'failed'
|
||||
AND (usage.status_code IS NULL OR usage.status_code < 400)
|
||||
AND usage.error_message IS NULL
|
||||
THEN 1 ELSE 0
|
||||
END"#;
|
||||
const AGGREGATABLE: &str = r#"usage.status NOT IN ('pending', 'streaming')
|
||||
AND usage.provider_name NOT IN ('unknown', 'pending')"#;
|
||||
const SETTLED: &str = r#"COALESCE(settlement.billing_status, usage.billing_status) = 'settled'
|
||||
AND COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0) > 0"#;
|
||||
|
||||
fn cache_creation_expr() -> String {
|
||||
CACHE_CREATION
|
||||
.replace("{cache_5m}", CACHE_5M)
|
||||
.replace("{cache_1h}", CACHE_1H)
|
||||
}
|
||||
|
||||
fn total_input_context_expr() -> String {
|
||||
format!(
|
||||
"({EFFECTIVE_INPUT}) + ({}) + GREATEST(COALESCE(usage.cache_read_input_tokens, 0), 0)",
|
||||
cache_creation_expr()
|
||||
)
|
||||
}
|
||||
|
||||
fn total_tokens_expr() -> String {
|
||||
format!(
|
||||
r#"COALESCE(
|
||||
NULLIF(GREATEST(COALESCE(usage.total_tokens, 0), 0), 0),
|
||||
({EFFECTIVE_INPUT})
|
||||
+ GREATEST(COALESCE(usage.output_tokens, 0), 0)
|
||||
+ ({})
|
||||
+ GREATEST(COALESCE(usage.cache_read_input_tokens, 0), 0),
|
||||
0
|
||||
)"#,
|
||||
cache_creation_expr()
|
||||
)
|
||||
}
|
||||
|
||||
fn percentile_cont(sorted: &[i64], percentile: f64) -> Option<i64> {
|
||||
if sorted.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let position = percentile * (sorted.len().saturating_sub(1) as f64);
|
||||
let lower = position.floor() as usize;
|
||||
let upper = position.ceil() as usize;
|
||||
let fraction = position - lower as f64;
|
||||
let value = sorted[lower] as f64 + (sorted[upper] - sorted[lower]) as f64 * fraction;
|
||||
Some(value.round() as i64)
|
||||
}
|
||||
|
||||
async fn load_percentiles(
|
||||
tx: &mut sqlx::Transaction<'_, MySql>,
|
||||
column: &str,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
) -> Result<(Option<i64>, Option<i64>, Option<i64>), DataLayerError> {
|
||||
let sql = format!(
|
||||
r#"
|
||||
SELECT {column}
|
||||
FROM `usage`
|
||||
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
|
||||
AND status = 'completed'
|
||||
AND provider_name NOT IN ('unknown', 'pending')
|
||||
AND {column} IS NOT NULL
|
||||
ORDER BY {column}
|
||||
"#
|
||||
);
|
||||
let values: Vec<i64> = sqlx::query_scalar(&sql)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.fetch_all(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
if values.len() < 10 {
|
||||
return Ok((None, None, None));
|
||||
}
|
||||
Ok((
|
||||
percentile_cont(&values, 0.50),
|
||||
percentile_cont(&values, 0.90),
|
||||
percentile_cont(&values, 0.99),
|
||||
))
|
||||
}
|
||||
|
||||
pub(super) async fn refresh_hourly(
|
||||
tx: &mut sqlx::Transaction<'_, MySql>,
|
||||
hour_utc: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
) -> Result<(), DataLayerError> {
|
||||
let cache_creation = cache_creation_expr();
|
||||
let total_context = total_input_context_expr();
|
||||
let sql = format!(
|
||||
r#"
|
||||
UPDATE stats_hourly AS target
|
||||
JOIN (
|
||||
SELECT
|
||||
COUNT(*) AS cache_hit_total_requests,
|
||||
COALESCE(SUM(CASE WHEN COALESCE(usage.cache_read_input_tokens, 0) > 0 THEN 1 ELSE 0 END), 0) AS cache_hit_requests,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN 1 ELSE 0 END), 0) AS completed_total_requests,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' AND COALESCE(usage.cache_read_input_tokens, 0) > 0 THEN 1 ELSE 0 END), 0) AS completed_cache_hit_requests,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN GREATEST(COALESCE(usage.input_tokens, 0), 0) ELSE 0 END), 0) AS completed_input_tokens,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN ({cache_creation}) ELSE 0 END), 0) AS completed_cache_creation_tokens,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN GREATEST(COALESCE(usage.cache_read_input_tokens, 0), 0) ELSE 0 END), 0) AS completed_cache_read_tokens,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN ({total_context}) ELSE 0 END), 0) AS completed_total_input_context,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN COALESCE(usage.cache_creation_cost_usd, 0) ELSE 0 END), 0) AS completed_cache_creation_cost,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN COALESCE(usage.cache_read_cost_usd, 0) ELSE 0 END), 0) AS completed_cache_read_cost,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0) ELSE 0 END), 0) AS settled_total_cost,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN 1 ELSE 0 END), 0) AS settled_total_requests,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN GREATEST(COALESCE(usage.input_tokens, 0), 0) ELSE 0 END), 0) AS settled_input_tokens,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN GREATEST(COALESCE(usage.output_tokens, 0), 0) ELSE 0 END), 0) AS settled_output_tokens,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN ({cache_creation}) ELSE 0 END), 0) AS settled_cache_creation_tokens,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN GREATEST(COALESCE(usage.cache_read_input_tokens, 0), 0) ELSE 0 END), 0) AS settled_cache_read_tokens,
|
||||
MIN(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_first_finalized_at_unix_secs,
|
||||
MAX(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_last_finalized_at_unix_secs,
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL AND {AGGREGATABLE} THEN GREATEST(usage.response_time_ms, 0) ELSE 0 END), 0) AS response_time_sum_ms,
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL AND {AGGREGATABLE} THEN 1 ELSE 0 END), 0) AS response_time_samples
|
||||
FROM `usage` AS `usage`
|
||||
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
|
||||
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
|
||||
) AS aggregated
|
||||
SET
|
||||
target.cache_hit_total_requests = aggregated.cache_hit_total_requests,
|
||||
target.cache_hit_requests = aggregated.cache_hit_requests,
|
||||
target.completed_total_requests = aggregated.completed_total_requests,
|
||||
target.completed_cache_hit_requests = aggregated.completed_cache_hit_requests,
|
||||
target.completed_input_tokens = aggregated.completed_input_tokens,
|
||||
target.completed_cache_creation_tokens = aggregated.completed_cache_creation_tokens,
|
||||
target.completed_cache_read_tokens = aggregated.completed_cache_read_tokens,
|
||||
target.completed_total_input_context = aggregated.completed_total_input_context,
|
||||
target.completed_cache_creation_cost = aggregated.completed_cache_creation_cost,
|
||||
target.completed_cache_read_cost = aggregated.completed_cache_read_cost,
|
||||
target.settled_total_cost = aggregated.settled_total_cost,
|
||||
target.settled_total_requests = aggregated.settled_total_requests,
|
||||
target.settled_input_tokens = aggregated.settled_input_tokens,
|
||||
target.settled_output_tokens = aggregated.settled_output_tokens,
|
||||
target.settled_cache_creation_tokens = aggregated.settled_cache_creation_tokens,
|
||||
target.settled_cache_read_tokens = aggregated.settled_cache_read_tokens,
|
||||
target.settled_first_finalized_at_unix_secs = aggregated.settled_first_finalized_at_unix_secs,
|
||||
target.settled_last_finalized_at_unix_secs = aggregated.settled_last_finalized_at_unix_secs,
|
||||
target.response_time_sum_ms = aggregated.response_time_sum_ms,
|
||||
target.response_time_samples = aggregated.response_time_samples
|
||||
WHERE target.hour_utc = ?
|
||||
"#
|
||||
);
|
||||
sqlx::query(&sql)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.bind(hour_utc)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
refresh_hourly_user(tx, hour_utc, start_unix_secs, end_unix_secs).await?;
|
||||
refresh_hourly_response_dimensions(tx, hour_utc, start_unix_secs, end_unix_secs).await
|
||||
}
|
||||
|
||||
async fn refresh_hourly_user(
|
||||
tx: &mut sqlx::Transaction<'_, MySql>,
|
||||
hour_utc: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
) -> Result<(), DataLayerError> {
|
||||
let cache_creation = cache_creation_expr();
|
||||
let sql = format!(
|
||||
r#"
|
||||
UPDATE stats_hourly_user AS target
|
||||
JOIN (
|
||||
SELECT usage.user_id,
|
||||
COALESCE(SUM({cache_creation}), 0) AS cache_creation_tokens,
|
||||
COALESCE(SUM(GREATEST(COALESCE(usage.cache_read_input_tokens, 0), 0)), 0) AS cache_read_tokens,
|
||||
COALESCE(SUM(COALESCE(settlement.billing_actual_total_cost_usd, usage.actual_total_cost_usd, 0)), 0) AS actual_total_cost,
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN GREATEST(usage.response_time_ms, 0) ELSE 0 END), 0) AS response_time_sum_ms,
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0) AS response_time_samples,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0) ELSE 0 END), 0) AS settled_total_cost,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN 1 ELSE 0 END), 0) AS settled_total_requests,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN GREATEST(COALESCE(usage.input_tokens, 0), 0) ELSE 0 END), 0) AS settled_input_tokens,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN GREATEST(COALESCE(usage.output_tokens, 0), 0) ELSE 0 END), 0) AS settled_output_tokens,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN ({cache_creation}) ELSE 0 END), 0) AS settled_cache_creation_tokens,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN GREATEST(COALESCE(usage.cache_read_input_tokens, 0), 0) ELSE 0 END), 0) AS settled_cache_read_tokens,
|
||||
MIN(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_first_finalized_at_unix_secs,
|
||||
MAX(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_last_finalized_at_unix_secs
|
||||
FROM `usage` AS `usage`
|
||||
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
|
||||
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
|
||||
AND usage.user_id IS NOT NULL AND usage.user_id <> '' AND {AGGREGATABLE}
|
||||
GROUP BY usage.user_id
|
||||
) AS aggregated ON target.user_id = aggregated.user_id
|
||||
SET target.cache_creation_tokens = aggregated.cache_creation_tokens,
|
||||
target.cache_read_tokens = aggregated.cache_read_tokens,
|
||||
target.actual_total_cost = aggregated.actual_total_cost,
|
||||
target.response_time_sum_ms = aggregated.response_time_sum_ms,
|
||||
target.response_time_samples = aggregated.response_time_samples,
|
||||
target.settled_total_cost = aggregated.settled_total_cost,
|
||||
target.settled_total_requests = aggregated.settled_total_requests,
|
||||
target.settled_input_tokens = aggregated.settled_input_tokens,
|
||||
target.settled_output_tokens = aggregated.settled_output_tokens,
|
||||
target.settled_cache_creation_tokens = aggregated.settled_cache_creation_tokens,
|
||||
target.settled_cache_read_tokens = aggregated.settled_cache_read_tokens,
|
||||
target.settled_first_finalized_at_unix_secs = aggregated.settled_first_finalized_at_unix_secs,
|
||||
target.settled_last_finalized_at_unix_secs = aggregated.settled_last_finalized_at_unix_secs
|
||||
WHERE target.hour_utc = ?
|
||||
"#
|
||||
);
|
||||
sqlx::query(&sql)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.bind(hour_utc)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn refresh_hourly_response_dimensions(
|
||||
tx: &mut sqlx::Transaction<'_, MySql>,
|
||||
hour_utc: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
) -> Result<(), DataLayerError> {
|
||||
for (table, select_dimensions, group_by, join) in [
|
||||
(
|
||||
"stats_hourly_model",
|
||||
"usage.model AS model",
|
||||
"usage.model",
|
||||
"target.model = aggregated.model",
|
||||
),
|
||||
(
|
||||
"stats_hourly_user_model",
|
||||
"usage.user_id AS user_id, usage.model AS model",
|
||||
"usage.user_id, usage.model",
|
||||
"target.user_id = aggregated.user_id AND target.model = aggregated.model",
|
||||
),
|
||||
] {
|
||||
let sql = format!(
|
||||
r#"
|
||||
UPDATE {table} AS target
|
||||
JOIN (
|
||||
SELECT {select_dimensions},
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN GREATEST(usage.response_time_ms, 0) ELSE 0 END), 0) AS response_time_sum_ms,
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0) AS response_time_samples
|
||||
FROM `usage` AS `usage`
|
||||
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ? AND {AGGREGATABLE}
|
||||
GROUP BY {group_by}
|
||||
) AS aggregated ON {join}
|
||||
SET target.response_time_sum_ms = aggregated.response_time_sum_ms,
|
||||
target.response_time_samples = aggregated.response_time_samples
|
||||
WHERE target.hour_utc = ?
|
||||
"#
|
||||
);
|
||||
sqlx::query(&sql)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.bind(hour_utc)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn refresh_daily(
|
||||
tx: &mut sqlx::Transaction<'_, MySql>,
|
||||
day_start: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<(), DataLayerError> {
|
||||
let response = load_percentiles(tx, "response_time_ms", start_unix_secs, end_unix_secs).await?;
|
||||
let first_byte =
|
||||
load_percentiles(tx, "first_byte_time_ms", start_unix_secs, end_unix_secs).await?;
|
||||
refresh_daily_root(
|
||||
tx,
|
||||
day_start,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
response,
|
||||
first_byte,
|
||||
)
|
||||
.await?;
|
||||
refresh_daily_existing_dimensions(tx, day_start, start_unix_secs, end_unix_secs).await?;
|
||||
upsert_user_dimension(
|
||||
tx,
|
||||
"stats_user_daily_model",
|
||||
"model",
|
||||
"usage.model",
|
||||
"usage.model IS NOT NULL AND usage.model <> ''",
|
||||
day_start,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
now_unix_secs,
|
||||
)
|
||||
.await?;
|
||||
upsert_user_dimension(
|
||||
tx,
|
||||
"stats_user_daily_provider",
|
||||
"provider_name",
|
||||
"usage.provider_name",
|
||||
"usage.provider_name IS NOT NULL AND usage.provider_name <> ''",
|
||||
day_start,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
now_unix_secs,
|
||||
)
|
||||
.await?;
|
||||
upsert_user_dimension(
|
||||
tx,
|
||||
"stats_user_daily_api_format",
|
||||
"api_format",
|
||||
"LOWER(COALESCE(usage.endpoint_api_format, usage.api_format, ''))",
|
||||
"COALESCE(usage.endpoint_api_format, usage.api_format, '') <> ''",
|
||||
day_start,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
now_unix_secs,
|
||||
)
|
||||
.await?;
|
||||
upsert_model_provider_rows(tx, day_start, start_unix_secs, end_unix_secs, now_unix_secs)
|
||||
.await?;
|
||||
upsert_cost_savings_rows(tx, day_start, start_unix_secs, end_unix_secs, now_unix_secs).await?;
|
||||
refresh_user_summary(tx, end_unix_secs, now_unix_secs).await
|
||||
}
|
||||
|
||||
async fn refresh_daily_root(
|
||||
tx: &mut sqlx::Transaction<'_, MySql>,
|
||||
day_start: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
response: (Option<i64>, Option<i64>, Option<i64>),
|
||||
first_byte: (Option<i64>, Option<i64>, Option<i64>),
|
||||
) -> Result<(), DataLayerError> {
|
||||
let cache_creation = cache_creation_expr();
|
||||
let total_context = total_input_context_expr();
|
||||
let sql = format!(
|
||||
r#"
|
||||
UPDATE stats_daily AS target
|
||||
JOIN (
|
||||
SELECT
|
||||
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN ({EFFECTIVE_INPUT}) ELSE 0 END), 0) AS effective_input_tokens,
|
||||
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN ({total_context}) ELSE 0 END), 0) AS total_input_context,
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL AND {AGGREGATABLE} THEN GREATEST(usage.response_time_ms, 0) ELSE 0 END), 0) AS response_time_sum_ms,
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL AND {AGGREGATABLE} THEN 1 ELSE 0 END), 0) AS response_time_samples,
|
||||
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN {CACHE_5M} ELSE 0 END), 0) AS cache_creation_ephemeral_5m_tokens,
|
||||
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN {CACHE_1H} ELSE 0 END), 0) AS cache_creation_ephemeral_1h_tokens,
|
||||
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN COALESCE(usage.input_cost_usd, 0) ELSE 0 END), 0) AS input_cost,
|
||||
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN COALESCE(usage.output_cost_usd, 0) ELSE 0 END), 0) AS output_cost,
|
||||
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN COALESCE(usage.cache_creation_cost_usd, 0) ELSE 0 END), 0) AS cache_creation_cost,
|
||||
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN COALESCE(usage.cache_read_cost_usd, 0) ELSE 0 END), 0) AS cache_read_cost,
|
||||
COUNT(*) AS cache_hit_total_requests,
|
||||
COALESCE(SUM(CASE WHEN COALESCE(usage.cache_read_input_tokens, 0) > 0 THEN 1 ELSE 0 END), 0) AS cache_hit_requests,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN 1 ELSE 0 END), 0) AS completed_total_requests,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' AND COALESCE(usage.cache_read_input_tokens, 0) > 0 THEN 1 ELSE 0 END), 0) AS completed_cache_hit_requests,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN GREATEST(COALESCE(usage.input_tokens, 0), 0) ELSE 0 END), 0) AS completed_input_tokens,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN ({cache_creation}) ELSE 0 END), 0) AS completed_cache_creation_tokens,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN GREATEST(COALESCE(usage.cache_read_input_tokens, 0), 0) ELSE 0 END), 0) AS completed_cache_read_tokens,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN ({total_context}) ELSE 0 END), 0) AS completed_total_input_context,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN COALESCE(usage.cache_creation_cost_usd, 0) ELSE 0 END), 0) AS completed_cache_creation_cost,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN COALESCE(usage.cache_read_cost_usd, 0) ELSE 0 END), 0) AS completed_cache_read_cost,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0) ELSE 0 END), 0) AS settled_total_cost,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN 1 ELSE 0 END), 0) AS settled_total_requests,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN GREATEST(COALESCE(usage.input_tokens, 0), 0) ELSE 0 END), 0) AS settled_input_tokens,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN GREATEST(COALESCE(usage.output_tokens, 0), 0) ELSE 0 END), 0) AS settled_output_tokens,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN ({cache_creation}) ELSE 0 END), 0) AS settled_cache_creation_tokens,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN GREATEST(COALESCE(usage.cache_read_input_tokens, 0), 0) ELSE 0 END), 0) AS settled_cache_read_tokens,
|
||||
MIN(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_first_finalized_at_unix_secs,
|
||||
MAX(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_last_finalized_at_unix_secs
|
||||
FROM `usage` AS `usage`
|
||||
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
|
||||
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
|
||||
) AS aggregated
|
||||
SET target.effective_input_tokens = aggregated.effective_input_tokens,
|
||||
target.total_input_context = aggregated.total_input_context,
|
||||
target.response_time_sum_ms = aggregated.response_time_sum_ms,
|
||||
target.response_time_samples = aggregated.response_time_samples,
|
||||
target.cache_creation_ephemeral_5m_tokens = aggregated.cache_creation_ephemeral_5m_tokens,
|
||||
target.cache_creation_ephemeral_1h_tokens = aggregated.cache_creation_ephemeral_1h_tokens,
|
||||
target.input_cost = aggregated.input_cost,
|
||||
target.output_cost = aggregated.output_cost,
|
||||
target.cache_creation_cost = aggregated.cache_creation_cost,
|
||||
target.cache_read_cost = aggregated.cache_read_cost,
|
||||
target.cache_hit_total_requests = aggregated.cache_hit_total_requests,
|
||||
target.cache_hit_requests = aggregated.cache_hit_requests,
|
||||
target.completed_total_requests = aggregated.completed_total_requests,
|
||||
target.completed_cache_hit_requests = aggregated.completed_cache_hit_requests,
|
||||
target.completed_input_tokens = aggregated.completed_input_tokens,
|
||||
target.completed_cache_creation_tokens = aggregated.completed_cache_creation_tokens,
|
||||
target.completed_cache_read_tokens = aggregated.completed_cache_read_tokens,
|
||||
target.completed_total_input_context = aggregated.completed_total_input_context,
|
||||
target.completed_cache_creation_cost = aggregated.completed_cache_creation_cost,
|
||||
target.completed_cache_read_cost = aggregated.completed_cache_read_cost,
|
||||
target.settled_total_cost = aggregated.settled_total_cost,
|
||||
target.settled_total_requests = aggregated.settled_total_requests,
|
||||
target.settled_input_tokens = aggregated.settled_input_tokens,
|
||||
target.settled_output_tokens = aggregated.settled_output_tokens,
|
||||
target.settled_cache_creation_tokens = aggregated.settled_cache_creation_tokens,
|
||||
target.settled_cache_read_tokens = aggregated.settled_cache_read_tokens,
|
||||
target.settled_first_finalized_at_unix_secs = aggregated.settled_first_finalized_at_unix_secs,
|
||||
target.settled_last_finalized_at_unix_secs = aggregated.settled_last_finalized_at_unix_secs,
|
||||
target.p50_response_time_ms = ?, target.p90_response_time_ms = ?, target.p99_response_time_ms = ?,
|
||||
target.p50_first_byte_time_ms = ?, target.p90_first_byte_time_ms = ?, target.p99_first_byte_time_ms = ?
|
||||
WHERE target.`date` = ?
|
||||
"#
|
||||
);
|
||||
sqlx::query(&sql)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.bind(response.0)
|
||||
.bind(response.1)
|
||||
.bind(response.2)
|
||||
.bind(first_byte.0)
|
||||
.bind(first_byte.1)
|
||||
.bind(first_byte.2)
|
||||
.bind(day_start)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn refresh_daily_existing_dimensions(
|
||||
tx: &mut sqlx::Transaction<'_, MySql>,
|
||||
day_start: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
) -> Result<(), DataLayerError> {
|
||||
let cache_creation = cache_creation_expr();
|
||||
let total_context = total_input_context_expr();
|
||||
let model_sql = format!(
|
||||
r#"
|
||||
UPDATE stats_daily_model AS target
|
||||
JOIN (
|
||||
SELECT usage.model,
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN GREATEST(usage.response_time_ms, 0) ELSE 0 END), 0) AS response_time_sum_ms,
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0) AS response_time_samples,
|
||||
COALESCE(SUM({CACHE_5M}), 0) AS cache_creation_ephemeral_5m_tokens,
|
||||
COALESCE(SUM({CACHE_1H}), 0) AS cache_creation_ephemeral_1h_tokens
|
||||
FROM `usage` AS `usage`
|
||||
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
|
||||
AND {AGGREGATABLE} AND usage.model IS NOT NULL AND usage.model <> ''
|
||||
GROUP BY usage.model
|
||||
) AS aggregated ON target.model = aggregated.model
|
||||
SET target.response_time_sum_ms = aggregated.response_time_sum_ms,
|
||||
target.response_time_samples = aggregated.response_time_samples,
|
||||
target.cache_creation_ephemeral_5m_tokens = aggregated.cache_creation_ephemeral_5m_tokens,
|
||||
target.cache_creation_ephemeral_1h_tokens = aggregated.cache_creation_ephemeral_1h_tokens
|
||||
WHERE target.`date` = ?
|
||||
"#
|
||||
);
|
||||
sqlx::query(&model_sql)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.bind(day_start)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
|
||||
let user_sql = format!(
|
||||
r#"
|
||||
UPDATE stats_user_daily AS target
|
||||
JOIN (
|
||||
SELECT usage.user_id,
|
||||
COALESCE(SUM({EFFECTIVE_INPUT}), 0) AS effective_input_tokens,
|
||||
COALESCE(SUM({total_context}), 0) AS total_input_context,
|
||||
COALESCE(SUM(COALESCE(usage.cache_creation_cost_usd, 0)), 0) AS cache_creation_cost,
|
||||
COALESCE(SUM(COALESCE(usage.cache_read_cost_usd, 0)), 0) AS cache_read_cost,
|
||||
COALESCE(SUM(COALESCE(settlement.billing_actual_total_cost_usd, usage.actual_total_cost_usd, 0)), 0) AS actual_total_cost,
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN GREATEST(usage.response_time_ms, 0) ELSE 0 END), 0) AS response_time_sum_ms,
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0) AS response_time_samples,
|
||||
COALESCE(SUM({CACHE_5M}), 0) AS cache_creation_ephemeral_5m_tokens,
|
||||
COALESCE(SUM({CACHE_1H}), 0) AS cache_creation_ephemeral_1h_tokens,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0) ELSE 0 END), 0) AS settled_total_cost,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN 1 ELSE 0 END), 0) AS settled_total_requests,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN GREATEST(COALESCE(usage.input_tokens, 0), 0) ELSE 0 END), 0) AS settled_input_tokens,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN GREATEST(COALESCE(usage.output_tokens, 0), 0) ELSE 0 END), 0) AS settled_output_tokens,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN ({cache_creation}) ELSE 0 END), 0) AS settled_cache_creation_tokens,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN GREATEST(COALESCE(usage.cache_read_input_tokens, 0), 0) ELSE 0 END), 0) AS settled_cache_read_tokens,
|
||||
MIN(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_first_finalized_at_unix_secs,
|
||||
MAX(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_last_finalized_at_unix_secs
|
||||
FROM `usage` AS `usage`
|
||||
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
|
||||
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
|
||||
AND usage.user_id IS NOT NULL AND usage.user_id <> '' AND {AGGREGATABLE}
|
||||
GROUP BY usage.user_id
|
||||
) AS aggregated ON target.user_id = aggregated.user_id
|
||||
SET target.effective_input_tokens = aggregated.effective_input_tokens,
|
||||
target.total_input_context = aggregated.total_input_context,
|
||||
target.cache_creation_cost = aggregated.cache_creation_cost,
|
||||
target.cache_read_cost = aggregated.cache_read_cost,
|
||||
target.actual_total_cost = aggregated.actual_total_cost,
|
||||
target.response_time_sum_ms = aggregated.response_time_sum_ms,
|
||||
target.response_time_samples = aggregated.response_time_samples,
|
||||
target.cache_creation_ephemeral_5m_tokens = aggregated.cache_creation_ephemeral_5m_tokens,
|
||||
target.cache_creation_ephemeral_1h_tokens = aggregated.cache_creation_ephemeral_1h_tokens,
|
||||
target.settled_total_cost = aggregated.settled_total_cost,
|
||||
target.settled_total_requests = aggregated.settled_total_requests,
|
||||
target.settled_input_tokens = aggregated.settled_input_tokens,
|
||||
target.settled_output_tokens = aggregated.settled_output_tokens,
|
||||
target.settled_cache_creation_tokens = aggregated.settled_cache_creation_tokens,
|
||||
target.settled_cache_read_tokens = aggregated.settled_cache_read_tokens,
|
||||
target.settled_first_finalized_at_unix_secs = aggregated.settled_first_finalized_at_unix_secs,
|
||||
target.settled_last_finalized_at_unix_secs = aggregated.settled_last_finalized_at_unix_secs
|
||||
WHERE target.`date` = ?
|
||||
"#
|
||||
);
|
||||
sqlx::query(&user_sql)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.bind(day_start)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn upsert_user_dimension(
|
||||
tx: &mut sqlx::Transaction<'_, MySql>,
|
||||
table: &str,
|
||||
dimension_column: &str,
|
||||
dimension_expr: &str,
|
||||
dimension_filter: &str,
|
||||
day_start: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<(), DataLayerError> {
|
||||
let cache_creation = cache_creation_expr();
|
||||
let total_context = total_input_context_expr();
|
||||
let total_tokens = total_tokens_expr();
|
||||
let sql = format!(
|
||||
r#"
|
||||
INSERT INTO {table} (
|
||||
id, user_id, username, `date`, {dimension_column}, total_requests, success_requests,
|
||||
input_tokens, effective_input_tokens, output_tokens, total_tokens, total_input_context,
|
||||
cache_creation_tokens, cache_creation_ephemeral_5m_tokens,
|
||||
cache_creation_ephemeral_1h_tokens, cache_read_tokens, total_cost, actual_total_cost,
|
||||
response_time_sum_ms, response_time_samples, successful_response_time_sum_ms,
|
||||
successful_response_time_samples, created_at, updated_at
|
||||
)
|
||||
SELECT SHA2(UUID(), 256), usage.user_id,
|
||||
MAX(COALESCE(usage.username, users.username)), ?, {dimension_expr}, COUNT(*),
|
||||
COALESCE(SUM({SUCCESS}), 0),
|
||||
COALESCE(SUM(GREATEST(COALESCE(usage.input_tokens, 0), 0)), 0),
|
||||
COALESCE(SUM({EFFECTIVE_INPUT}), 0),
|
||||
COALESCE(SUM(GREATEST(COALESCE(usage.output_tokens, 0), 0)), 0),
|
||||
COALESCE(SUM({total_tokens}), 0), COALESCE(SUM({total_context}), 0),
|
||||
COALESCE(SUM({cache_creation}), 0), COALESCE(SUM({CACHE_5M}), 0),
|
||||
COALESCE(SUM({CACHE_1H}), 0),
|
||||
COALESCE(SUM(GREATEST(COALESCE(usage.cache_read_input_tokens, 0), 0)), 0),
|
||||
COALESCE(SUM(COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0)), 0),
|
||||
COALESCE(SUM(COALESCE(settlement.billing_actual_total_cost_usd, usage.actual_total_cost_usd, 0)), 0),
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN GREATEST(usage.response_time_ms, 0) ELSE 0 END), 0),
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0),
|
||||
COALESCE(SUM(CASE WHEN ({SUCCESS}) = 1 AND usage.response_time_ms IS NOT NULL THEN GREATEST(usage.response_time_ms, 0) ELSE 0 END), 0),
|
||||
COALESCE(SUM(CASE WHEN ({SUCCESS}) = 1 AND usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0),
|
||||
?, ?
|
||||
FROM `usage` AS `usage`
|
||||
LEFT JOIN users ON users.id = usage.user_id
|
||||
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
|
||||
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
|
||||
AND usage.user_id IS NOT NULL AND usage.user_id <> ''
|
||||
AND {dimension_filter} AND {AGGREGATABLE}
|
||||
GROUP BY usage.user_id, {dimension_expr}
|
||||
ON DUPLICATE KEY UPDATE
|
||||
username = COALESCE(VALUES(username), {table}.username),
|
||||
total_requests = VALUES(total_requests), success_requests = VALUES(success_requests),
|
||||
input_tokens = VALUES(input_tokens), effective_input_tokens = VALUES(effective_input_tokens),
|
||||
output_tokens = VALUES(output_tokens), total_tokens = VALUES(total_tokens),
|
||||
total_input_context = VALUES(total_input_context),
|
||||
cache_creation_tokens = VALUES(cache_creation_tokens),
|
||||
cache_creation_ephemeral_5m_tokens = VALUES(cache_creation_ephemeral_5m_tokens),
|
||||
cache_creation_ephemeral_1h_tokens = VALUES(cache_creation_ephemeral_1h_tokens),
|
||||
cache_read_tokens = VALUES(cache_read_tokens), total_cost = VALUES(total_cost),
|
||||
actual_total_cost = VALUES(actual_total_cost),
|
||||
response_time_sum_ms = VALUES(response_time_sum_ms),
|
||||
response_time_samples = VALUES(response_time_samples),
|
||||
successful_response_time_sum_ms = VALUES(successful_response_time_sum_ms),
|
||||
successful_response_time_samples = VALUES(successful_response_time_samples),
|
||||
updated_at = VALUES(updated_at)
|
||||
"#
|
||||
);
|
||||
sqlx::query(&sql)
|
||||
.bind(day_start)
|
||||
.bind(now_unix_secs)
|
||||
.bind(now_unix_secs)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn upsert_model_provider_rows(
|
||||
tx: &mut sqlx::Transaction<'_, MySql>,
|
||||
day_start: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<(), DataLayerError> {
|
||||
let total_tokens = total_tokens_expr();
|
||||
let model_provider_sql = format!(
|
||||
r#"
|
||||
INSERT INTO stats_daily_model_provider (
|
||||
id, `date`, model, provider_name, total_requests, total_tokens, total_cost,
|
||||
response_time_sum_ms, response_time_samples, created_at, updated_at
|
||||
)
|
||||
SELECT SHA2(UUID(), 256), ?, usage.model, usage.provider_name, COUNT(*),
|
||||
COALESCE(SUM({total_tokens}), 0),
|
||||
COALESCE(SUM(COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0)), 0),
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN GREATEST(usage.response_time_ms, 0) ELSE 0 END), 0),
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0), ?, ?
|
||||
FROM `usage` AS `usage`
|
||||
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
|
||||
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
|
||||
AND usage.model IS NOT NULL AND usage.model <> '' AND {AGGREGATABLE}
|
||||
GROUP BY usage.model, usage.provider_name
|
||||
ON DUPLICATE KEY UPDATE
|
||||
total_requests = VALUES(total_requests), total_tokens = VALUES(total_tokens),
|
||||
total_cost = VALUES(total_cost), response_time_sum_ms = VALUES(response_time_sum_ms),
|
||||
response_time_samples = VALUES(response_time_samples), updated_at = VALUES(updated_at)
|
||||
"#
|
||||
);
|
||||
sqlx::query(&model_provider_sql)
|
||||
.bind(day_start)
|
||||
.bind(now_unix_secs)
|
||||
.bind(now_unix_secs)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
|
||||
let user_model_provider_sql = format!(
|
||||
r#"
|
||||
INSERT INTO stats_user_daily_model_provider (
|
||||
id, user_id, username, `date`, model, provider_name, total_requests, total_tokens,
|
||||
total_cost, response_time_sum_ms, response_time_samples, created_at, updated_at
|
||||
)
|
||||
SELECT SHA2(UUID(), 256), usage.user_id, MAX(COALESCE(usage.username, users.username)),
|
||||
?, usage.model, usage.provider_name, COUNT(*), COALESCE(SUM({total_tokens}), 0),
|
||||
COALESCE(SUM(COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0)), 0),
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN GREATEST(usage.response_time_ms, 0) ELSE 0 END), 0),
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0), ?, ?
|
||||
FROM `usage` AS `usage`
|
||||
LEFT JOIN users ON users.id = usage.user_id
|
||||
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
|
||||
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
|
||||
AND usage.user_id IS NOT NULL AND usage.user_id <> ''
|
||||
AND usage.model IS NOT NULL AND usage.model <> '' AND {AGGREGATABLE}
|
||||
GROUP BY usage.user_id, usage.model, usage.provider_name
|
||||
ON DUPLICATE KEY UPDATE
|
||||
username = COALESCE(VALUES(username), stats_user_daily_model_provider.username),
|
||||
total_requests = VALUES(total_requests), total_tokens = VALUES(total_tokens),
|
||||
total_cost = VALUES(total_cost), response_time_sum_ms = VALUES(response_time_sum_ms),
|
||||
response_time_samples = VALUES(response_time_samples), updated_at = VALUES(updated_at)
|
||||
"#
|
||||
);
|
||||
sqlx::query(&user_model_provider_sql)
|
||||
.bind(day_start)
|
||||
.bind(now_unix_secs)
|
||||
.bind(now_unix_secs)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn upsert_cost_savings_rows(
|
||||
tx: &mut sqlx::Transaction<'_, MySql>,
|
||||
day_start: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<(), DataLayerError> {
|
||||
for (table, dimensions) in [
|
||||
("stats_daily_cost_savings", Vec::new()),
|
||||
(
|
||||
"stats_daily_cost_savings_provider",
|
||||
vec![("provider_name", "COALESCE(usage.provider_name, '')")],
|
||||
),
|
||||
(
|
||||
"stats_daily_cost_savings_model",
|
||||
vec![("model", "COALESCE(usage.model, '')")],
|
||||
),
|
||||
(
|
||||
"stats_daily_cost_savings_model_provider",
|
||||
vec![
|
||||
("model", "COALESCE(usage.model, '')"),
|
||||
("provider_name", "COALESCE(usage.provider_name, '')"),
|
||||
],
|
||||
),
|
||||
] {
|
||||
upsert_cost_savings_dimension(
|
||||
tx,
|
||||
table,
|
||||
false,
|
||||
&dimensions,
|
||||
day_start,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
now_unix_secs,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
for (table, dimensions) in [
|
||||
("stats_user_daily_cost_savings", Vec::new()),
|
||||
(
|
||||
"stats_user_daily_cost_savings_provider",
|
||||
vec![("provider_name", "COALESCE(usage.provider_name, '')")],
|
||||
),
|
||||
(
|
||||
"stats_user_daily_cost_savings_model",
|
||||
vec![("model", "COALESCE(usage.model, '')")],
|
||||
),
|
||||
(
|
||||
"stats_user_daily_cost_savings_model_provider",
|
||||
vec![
|
||||
("model", "COALESCE(usage.model, '')"),
|
||||
("provider_name", "COALESCE(usage.provider_name, '')"),
|
||||
],
|
||||
),
|
||||
] {
|
||||
upsert_cost_savings_dimension(
|
||||
tx,
|
||||
table,
|
||||
true,
|
||||
&dimensions,
|
||||
day_start,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
now_unix_secs,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn upsert_cost_savings_dimension(
|
||||
tx: &mut sqlx::Transaction<'_, MySql>,
|
||||
table: &str,
|
||||
per_user: bool,
|
||||
dimensions: &[(&str, &str)],
|
||||
day_start: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<(), DataLayerError> {
|
||||
let dimension_columns = dimensions
|
||||
.iter()
|
||||
.map(|(column, _)| *column)
|
||||
.collect::<Vec<_>>();
|
||||
let dimension_exprs = dimensions
|
||||
.iter()
|
||||
.map(|(_, expression)| *expression)
|
||||
.collect::<Vec<_>>();
|
||||
let user_columns = if per_user { "user_id, username, " } else { "" };
|
||||
let user_select = if per_user {
|
||||
"usage.user_id, MAX(COALESCE(usage.username, users.username)), "
|
||||
} else {
|
||||
""
|
||||
};
|
||||
let user_join = if per_user {
|
||||
"LEFT JOIN users ON users.id = usage.user_id"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
let user_filter = if per_user {
|
||||
"AND usage.user_id IS NOT NULL AND usage.user_id <> ''"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
let mut group_by = Vec::new();
|
||||
if per_user {
|
||||
group_by.push("usage.user_id");
|
||||
}
|
||||
group_by.extend(dimension_exprs.iter().copied());
|
||||
let dimension_columns_sql = if dimension_columns.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!("{}, ", dimension_columns.join(", "))
|
||||
};
|
||||
let dimension_select_sql = if dimension_exprs.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!("{}, ", dimension_exprs.join(", "))
|
||||
};
|
||||
let group_by_sql = if group_by.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!("GROUP BY {}", group_by.join(", "))
|
||||
};
|
||||
let sql = format!(
|
||||
r#"
|
||||
INSERT INTO {table} (
|
||||
id, {user_columns}`date`, {dimension_columns_sql}cache_read_tokens,
|
||||
cache_read_cost, cache_creation_cost, estimated_full_cost, created_at, updated_at
|
||||
)
|
||||
SELECT SHA2(UUID(), 256), {user_select}?, {dimension_select_sql}
|
||||
COALESCE(SUM(GREATEST(COALESCE(usage.cache_read_input_tokens, 0), 0)), 0),
|
||||
COALESCE(SUM(COALESCE(usage.cache_read_cost_usd, 0)), 0),
|
||||
COALESCE(SUM(COALESCE(usage.cache_creation_cost_usd, 0)), 0),
|
||||
COALESCE(SUM(
|
||||
COALESCE(settlement.input_price_per_1m, usage.input_price_per_1m, 0)
|
||||
* GREATEST(COALESCE(usage.cache_read_input_tokens, 0), 0) / 1000000.0
|
||||
), 0), ?, ?
|
||||
FROM `usage` AS `usage`
|
||||
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
|
||||
{user_join}
|
||||
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ? {user_filter}
|
||||
{group_by_sql}
|
||||
ON DUPLICATE KEY UPDATE
|
||||
{}cache_read_tokens = VALUES(cache_read_tokens),
|
||||
cache_read_cost = VALUES(cache_read_cost),
|
||||
cache_creation_cost = VALUES(cache_creation_cost),
|
||||
estimated_full_cost = VALUES(estimated_full_cost), updated_at = VALUES(updated_at)
|
||||
"#,
|
||||
if per_user {
|
||||
format!("username = COALESCE(VALUES(username), {table}.username), ")
|
||||
} else {
|
||||
String::new()
|
||||
}
|
||||
);
|
||||
sqlx::query(&sql)
|
||||
.bind(day_start)
|
||||
.bind(now_unix_secs)
|
||||
.bind(now_unix_secs)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn refresh_user_summary(
|
||||
tx: &mut sqlx::Transaction<'_, MySql>,
|
||||
cutoff_date: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<(), DataLayerError> {
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO stats_user_summary (
|
||||
id, user_id, username, cutoff_date, all_time_requests, all_time_success_requests,
|
||||
all_time_error_requests, all_time_input_tokens, all_time_output_tokens,
|
||||
all_time_cache_creation_tokens, all_time_cache_read_tokens, all_time_cost,
|
||||
all_time_actual_cost, active_days, first_active_date, last_active_date,
|
||||
created_at, updated_at
|
||||
)
|
||||
SELECT SHA2(UUID(), 256), user_id, MAX(username), ?,
|
||||
COALESCE(SUM(total_requests), 0), COALESCE(SUM(success_requests), 0),
|
||||
COALESCE(SUM(error_requests), 0), COALESCE(SUM(input_tokens), 0),
|
||||
COALESCE(SUM(output_tokens), 0), COALESCE(SUM(cache_creation_tokens), 0),
|
||||
COALESCE(SUM(cache_read_tokens), 0), COALESCE(SUM(total_cost), 0),
|
||||
COALESCE(SUM(actual_total_cost), 0),
|
||||
COALESCE(SUM(CASE WHEN total_requests > 0 THEN 1 ELSE 0 END), 0),
|
||||
MIN(CASE WHEN total_requests > 0 THEN `date` END),
|
||||
MAX(CASE WHEN total_requests > 0 THEN `date` END), ?, ?
|
||||
FROM stats_user_daily
|
||||
WHERE `date` < ?
|
||||
GROUP BY user_id
|
||||
ON DUPLICATE KEY UPDATE
|
||||
username = COALESCE(VALUES(username), stats_user_summary.username),
|
||||
cutoff_date = VALUES(cutoff_date), all_time_requests = VALUES(all_time_requests),
|
||||
all_time_success_requests = VALUES(all_time_success_requests),
|
||||
all_time_error_requests = VALUES(all_time_error_requests),
|
||||
all_time_input_tokens = VALUES(all_time_input_tokens),
|
||||
all_time_output_tokens = VALUES(all_time_output_tokens),
|
||||
all_time_cache_creation_tokens = VALUES(all_time_cache_creation_tokens),
|
||||
all_time_cache_read_tokens = VALUES(all_time_cache_read_tokens),
|
||||
all_time_cost = VALUES(all_time_cost), all_time_actual_cost = VALUES(all_time_actual_cost),
|
||||
active_days = VALUES(active_days), first_active_date = VALUES(first_active_date),
|
||||
last_active_date = VALUES(last_active_date), updated_at = VALUES(updated_at)
|
||||
"#,
|
||||
)
|
||||
.bind(cutoff_date)
|
||||
.bind(now_unix_secs)
|
||||
.bind(now_unix_secs)
|
||||
.bind(cutoff_date)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
refresh_global_summary(tx, cutoff_date, now_unix_secs).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn refresh_global_summary(
|
||||
tx: &mut sqlx::Transaction<'_, MySql>,
|
||||
cutoff_date: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<(), DataLayerError> {
|
||||
let existing_id: Option<String> =
|
||||
sqlx::query_scalar("SELECT id FROM stats_summary ORDER BY created_at, id LIMIT 1")
|
||||
.fetch_optional(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
let summary_id = existing_id.unwrap_or_else(|| stats_id("stats-summary"));
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO stats_summary (
|
||||
id, cutoff_date, all_time_requests, all_time_success_requests,
|
||||
all_time_error_requests, all_time_input_tokens, all_time_output_tokens,
|
||||
all_time_cache_creation_tokens, all_time_cache_read_tokens, all_time_cost,
|
||||
all_time_actual_cost, total_users, active_users, total_api_keys,
|
||||
active_api_keys, created_at, updated_at
|
||||
)
|
||||
SELECT ?, ?, COALESCE(SUM(total_requests), 0), COALESCE(SUM(success_requests), 0),
|
||||
COALESCE(SUM(error_requests), 0), COALESCE(SUM(input_tokens), 0),
|
||||
COALESCE(SUM(output_tokens), 0), COALESCE(SUM(cache_creation_tokens), 0),
|
||||
COALESCE(SUM(cache_read_tokens), 0), COALESCE(SUM(total_cost), 0),
|
||||
COALESCE(SUM(actual_total_cost), 0),
|
||||
(SELECT COUNT(*) FROM users),
|
||||
(SELECT COUNT(*) FROM users WHERE is_active <> 0),
|
||||
(SELECT COUNT(*) FROM api_keys),
|
||||
(SELECT COUNT(*) FROM api_keys WHERE is_active <> 0), ?, ?
|
||||
FROM stats_daily
|
||||
WHERE `date` < ?
|
||||
ON DUPLICATE KEY UPDATE
|
||||
cutoff_date = VALUES(cutoff_date),
|
||||
all_time_requests = VALUES(all_time_requests),
|
||||
all_time_success_requests = VALUES(all_time_success_requests),
|
||||
all_time_error_requests = VALUES(all_time_error_requests),
|
||||
all_time_input_tokens = VALUES(all_time_input_tokens),
|
||||
all_time_output_tokens = VALUES(all_time_output_tokens),
|
||||
all_time_cache_creation_tokens = VALUES(all_time_cache_creation_tokens),
|
||||
all_time_cache_read_tokens = VALUES(all_time_cache_read_tokens),
|
||||
all_time_cost = VALUES(all_time_cost),
|
||||
all_time_actual_cost = VALUES(all_time_actual_cost),
|
||||
total_users = VALUES(total_users), active_users = VALUES(active_users),
|
||||
total_api_keys = VALUES(total_api_keys), active_api_keys = VALUES(active_api_keys),
|
||||
updated_at = VALUES(updated_at)
|
||||
"#,
|
||||
)
|
||||
.bind(summary_id)
|
||||
.bind(cutoff_date)
|
||||
.bind(now_unix_secs)
|
||||
.bind(now_unix_secs)
|
||||
.bind(cutoff_date)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,871 +0,0 @@
|
||||
use chrono::{DateTime, Utc};
|
||||
use sqlx::Row;
|
||||
|
||||
use crate::backend::stats_common::{stats_id, unix_secs, utc_from_unix_secs};
|
||||
use crate::backend::SqliteBackend;
|
||||
use crate::driver::sqlite::{sqlite_real, SqlitePool};
|
||||
use crate::error::SqlResultExt;
|
||||
use crate::{
|
||||
DataLayerError, StatsDailyAggregationInput, StatsDailyAggregationSummary,
|
||||
StatsHourlyAggregationInput, StatsHourlyAggregationSummary,
|
||||
};
|
||||
|
||||
mod advanced;
|
||||
|
||||
impl SqliteBackend {
|
||||
pub async fn aggregate_stats_hourly(
|
||||
&self,
|
||||
input: &StatsHourlyAggregationInput,
|
||||
) -> Result<Option<StatsHourlyAggregationSummary>, DataLayerError> {
|
||||
let Some(hour_utc_unix_secs) =
|
||||
next_sqlite_stats_hourly_bucket(self.pool(), input.target_hour_utc).await?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
perform_sqlite_stats_hourly_aggregation(
|
||||
self.pool(),
|
||||
hour_utc_unix_secs,
|
||||
input.aggregated_at,
|
||||
)
|
||||
.await
|
||||
.map(Some)
|
||||
}
|
||||
|
||||
pub async fn aggregate_stats_daily(
|
||||
&self,
|
||||
input: &StatsDailyAggregationInput,
|
||||
) -> Result<Option<StatsDailyAggregationSummary>, DataLayerError> {
|
||||
let Some(day_start_unix_secs) =
|
||||
next_sqlite_stats_daily_bucket(self.pool(), input.target_day_utc).await?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
perform_sqlite_stats_daily_aggregation(
|
||||
self.pool(),
|
||||
day_start_unix_secs,
|
||||
input.aggregated_at,
|
||||
)
|
||||
.await
|
||||
.map(Some)
|
||||
}
|
||||
}
|
||||
|
||||
async fn next_sqlite_stats_hourly_bucket(
|
||||
pool: &SqlitePool,
|
||||
target_hour_utc: DateTime<Utc>,
|
||||
) -> Result<Option<i64>, DataLayerError> {
|
||||
let latest_hour: Option<i64> =
|
||||
sqlx::query_scalar("SELECT MAX(hour_utc) FROM stats_hourly WHERE is_complete <> 0")
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
let search_from = latest_hour.map(|value| value + 3600).unwrap_or(0);
|
||||
let search_until = unix_secs(target_hour_utc) + 3600;
|
||||
if search_from >= search_until {
|
||||
return Ok(None);
|
||||
}
|
||||
let next_bucket: Option<i64> = sqlx::query_scalar(
|
||||
r#"
|
||||
SELECT MIN(CAST(created_at_unix_ms / 3600 AS INTEGER) * 3600)
|
||||
FROM "usage"
|
||||
WHERE created_at_unix_ms >= ?
|
||||
AND created_at_unix_ms < ?
|
||||
AND status NOT IN ('pending', 'streaming')
|
||||
AND provider_name NOT IN ('unknown', 'pending')
|
||||
"#,
|
||||
)
|
||||
.bind(search_from)
|
||||
.bind(search_until)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(next_bucket.filter(|value| *value <= unix_secs(target_hour_utc)))
|
||||
}
|
||||
|
||||
async fn next_sqlite_stats_daily_bucket(
|
||||
pool: &SqlitePool,
|
||||
target_day_utc: DateTime<Utc>,
|
||||
) -> Result<Option<i64>, DataLayerError> {
|
||||
let latest_day: Option<i64> =
|
||||
sqlx::query_scalar(r#"SELECT MAX("date") FROM stats_daily WHERE is_complete <> 0"#)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
let search_from = latest_day.map(|value| value + 86_400).unwrap_or(0);
|
||||
let search_until = unix_secs(target_day_utc) + 86_400;
|
||||
if search_from >= search_until {
|
||||
return Ok(None);
|
||||
}
|
||||
let next_bucket: Option<i64> = sqlx::query_scalar(
|
||||
r#"
|
||||
SELECT MIN(CAST(created_at_unix_ms / 86400 AS INTEGER) * 86400)
|
||||
FROM "usage"
|
||||
WHERE created_at_unix_ms >= ?
|
||||
AND created_at_unix_ms < ?
|
||||
AND status NOT IN ('pending', 'streaming')
|
||||
AND provider_name NOT IN ('unknown', 'pending')
|
||||
"#,
|
||||
)
|
||||
.bind(search_from)
|
||||
.bind(search_until)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(next_bucket.filter(|value| *value <= unix_secs(target_day_utc)))
|
||||
}
|
||||
|
||||
const SQLITE_STATS_AGGREGATE_SQL: &str = r#"
|
||||
SELECT
|
||||
COUNT(*) AS total_requests,
|
||||
COALESCE(SUM(CASE
|
||||
WHEN status = 'failed'
|
||||
OR status_code >= 400
|
||||
OR error_message IS NOT NULL
|
||||
THEN 1 ELSE 0 END), 0) AS error_requests,
|
||||
COALESCE(SUM(input_tokens), 0) AS input_tokens,
|
||||
COALESCE(SUM(output_tokens), 0) AS output_tokens,
|
||||
COALESCE(SUM(cache_creation_input_tokens), 0) AS cache_creation_tokens,
|
||||
COALESCE(SUM(cache_read_input_tokens), 0) AS cache_read_tokens,
|
||||
CAST(COALESCE(SUM(total_cost_usd), 0) AS REAL) AS total_cost,
|
||||
CAST(COALESCE(SUM(actual_total_cost_usd), 0) AS REAL) AS actual_total_cost,
|
||||
CAST(COALESCE(AVG(response_time_ms), 0) AS REAL) AS avg_response_time_ms
|
||||
FROM "usage"
|
||||
WHERE created_at_unix_ms >= ?
|
||||
AND created_at_unix_ms < ?
|
||||
AND status NOT IN ('pending', 'streaming')
|
||||
AND provider_name NOT IN ('unknown', 'pending')
|
||||
"#;
|
||||
|
||||
async fn perform_sqlite_stats_hourly_aggregation(
|
||||
pool: &SqlitePool,
|
||||
hour_utc_unix_secs: i64,
|
||||
aggregated_at: DateTime<Utc>,
|
||||
) -> Result<StatsHourlyAggregationSummary, DataLayerError> {
|
||||
let start_unix_secs = hour_utc_unix_secs;
|
||||
let end_unix_secs = hour_utc_unix_secs + 3600;
|
||||
let aggregated_at_unix_secs = unix_secs(aggregated_at);
|
||||
let mut tx = pool.begin().await.map_sql_err()?;
|
||||
let row = sqlx::query(SQLITE_STATS_AGGREGATE_SQL)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.fetch_one(&mut *tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
let total_requests: i64 = row.try_get("total_requests").map_sql_err()?;
|
||||
let error_requests: i64 = row.try_get("error_requests").map_sql_err()?;
|
||||
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO stats_hourly (
|
||||
id, hour_utc, total_requests, success_requests, error_requests,
|
||||
input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens,
|
||||
total_cost, actual_total_cost, avg_response_time_ms, is_complete,
|
||||
aggregated_at, created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?)
|
||||
ON CONFLICT (hour_utc) DO UPDATE SET
|
||||
total_requests = excluded.total_requests,
|
||||
success_requests = excluded.success_requests,
|
||||
error_requests = excluded.error_requests,
|
||||
input_tokens = excluded.input_tokens,
|
||||
output_tokens = excluded.output_tokens,
|
||||
cache_creation_tokens = excluded.cache_creation_tokens,
|
||||
cache_read_tokens = excluded.cache_read_tokens,
|
||||
total_cost = excluded.total_cost,
|
||||
actual_total_cost = excluded.actual_total_cost,
|
||||
avg_response_time_ms = excluded.avg_response_time_ms,
|
||||
is_complete = excluded.is_complete,
|
||||
aggregated_at = excluded.aggregated_at,
|
||||
updated_at = excluded.updated_at
|
||||
"#,
|
||||
)
|
||||
.bind(stats_id(&format!("stats-hourly:{hour_utc_unix_secs}")))
|
||||
.bind(hour_utc_unix_secs)
|
||||
.bind(total_requests)
|
||||
.bind(total_requests.saturating_sub(error_requests))
|
||||
.bind(error_requests)
|
||||
.bind(row.try_get::<i64, _>("input_tokens").map_sql_err()?)
|
||||
.bind(row.try_get::<i64, _>("output_tokens").map_sql_err()?)
|
||||
.bind(
|
||||
row.try_get::<i64, _>("cache_creation_tokens")
|
||||
.map_sql_err()?,
|
||||
)
|
||||
.bind(row.try_get::<i64, _>("cache_read_tokens").map_sql_err()?)
|
||||
.bind(sqlite_real(&row, "total_cost")?)
|
||||
.bind(sqlite_real(&row, "actual_total_cost")?)
|
||||
.bind(sqlite_real(&row, "avg_response_time_ms")?)
|
||||
.bind(aggregated_at_unix_secs)
|
||||
.bind(aggregated_at_unix_secs)
|
||||
.bind(aggregated_at_unix_secs)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
|
||||
let user_rows = upsert_sqlite_stats_hourly_user_rows(
|
||||
&mut tx,
|
||||
hour_utc_unix_secs,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
aggregated_at_unix_secs,
|
||||
)
|
||||
.await?;
|
||||
let user_model_rows = upsert_sqlite_stats_hourly_user_model_rows(
|
||||
&mut tx,
|
||||
hour_utc_unix_secs,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
aggregated_at_unix_secs,
|
||||
)
|
||||
.await?;
|
||||
let model_rows = upsert_sqlite_stats_hourly_model_rows(
|
||||
&mut tx,
|
||||
hour_utc_unix_secs,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
aggregated_at_unix_secs,
|
||||
)
|
||||
.await?;
|
||||
let provider_rows = upsert_sqlite_stats_hourly_provider_rows(
|
||||
&mut tx,
|
||||
hour_utc_unix_secs,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
aggregated_at_unix_secs,
|
||||
)
|
||||
.await?;
|
||||
advanced::refresh_hourly(&mut tx, hour_utc_unix_secs, start_unix_secs, end_unix_secs).await?;
|
||||
tx.commit().await.map_sql_err()?;
|
||||
|
||||
Ok(StatsHourlyAggregationSummary {
|
||||
hour_utc: utc_from_unix_secs(hour_utc_unix_secs, "stats_hourly.hour_utc")?,
|
||||
total_requests,
|
||||
user_rows,
|
||||
user_model_rows,
|
||||
model_rows,
|
||||
provider_rows,
|
||||
})
|
||||
}
|
||||
|
||||
async fn perform_sqlite_stats_daily_aggregation(
|
||||
pool: &SqlitePool,
|
||||
day_start_unix_secs: i64,
|
||||
aggregated_at: DateTime<Utc>,
|
||||
) -> Result<StatsDailyAggregationSummary, DataLayerError> {
|
||||
let start_unix_secs = day_start_unix_secs;
|
||||
let end_unix_secs = day_start_unix_secs + 86_400;
|
||||
let aggregated_at_unix_secs = unix_secs(aggregated_at);
|
||||
let mut tx = pool.begin().await.map_sql_err()?;
|
||||
let row = sqlx::query(SQLITE_STATS_AGGREGATE_SQL)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.fetch_one(&mut *tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
let total_requests: i64 = row.try_get("total_requests").map_sql_err()?;
|
||||
let error_requests: i64 = row.try_get("error_requests").map_sql_err()?;
|
||||
let unique_models =
|
||||
sqlite_group_count(&mut tx, "model", start_unix_secs, end_unix_secs).await? as i64;
|
||||
let unique_providers =
|
||||
sqlite_group_count(&mut tx, "provider_name", start_unix_secs, end_unix_secs).await? as i64;
|
||||
let fallback_count =
|
||||
sqlite_daily_fallback_count(&mut tx, start_unix_secs, end_unix_secs).await?;
|
||||
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO stats_daily (
|
||||
id, "date", total_requests, success_requests, error_requests,
|
||||
input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens,
|
||||
total_cost, actual_total_cost, avg_response_time_ms, fallback_count,
|
||||
unique_models, unique_providers, is_complete, aggregated_at, created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?)
|
||||
ON CONFLICT ("date") DO UPDATE SET
|
||||
total_requests = excluded.total_requests,
|
||||
success_requests = excluded.success_requests,
|
||||
error_requests = excluded.error_requests,
|
||||
input_tokens = excluded.input_tokens,
|
||||
output_tokens = excluded.output_tokens,
|
||||
cache_creation_tokens = excluded.cache_creation_tokens,
|
||||
cache_read_tokens = excluded.cache_read_tokens,
|
||||
total_cost = excluded.total_cost,
|
||||
actual_total_cost = excluded.actual_total_cost,
|
||||
avg_response_time_ms = excluded.avg_response_time_ms,
|
||||
fallback_count = excluded.fallback_count,
|
||||
unique_models = excluded.unique_models,
|
||||
unique_providers = excluded.unique_providers,
|
||||
is_complete = excluded.is_complete,
|
||||
aggregated_at = excluded.aggregated_at,
|
||||
updated_at = excluded.updated_at
|
||||
"#,
|
||||
)
|
||||
.bind(stats_id(&format!("stats-daily:{day_start_unix_secs}")))
|
||||
.bind(day_start_unix_secs)
|
||||
.bind(total_requests)
|
||||
.bind(total_requests.saturating_sub(error_requests))
|
||||
.bind(error_requests)
|
||||
.bind(row.try_get::<i64, _>("input_tokens").map_sql_err()?)
|
||||
.bind(row.try_get::<i64, _>("output_tokens").map_sql_err()?)
|
||||
.bind(
|
||||
row.try_get::<i64, _>("cache_creation_tokens")
|
||||
.map_sql_err()?,
|
||||
)
|
||||
.bind(row.try_get::<i64, _>("cache_read_tokens").map_sql_err()?)
|
||||
.bind(sqlite_real(&row, "total_cost")?)
|
||||
.bind(sqlite_real(&row, "actual_total_cost")?)
|
||||
.bind(sqlite_real(&row, "avg_response_time_ms")?)
|
||||
.bind(fallback_count)
|
||||
.bind(unique_models)
|
||||
.bind(unique_providers)
|
||||
.bind(aggregated_at_unix_secs)
|
||||
.bind(aggregated_at_unix_secs)
|
||||
.bind(aggregated_at_unix_secs)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
|
||||
let model_rows = upsert_sqlite_stats_daily_model_rows(
|
||||
&mut tx,
|
||||
day_start_unix_secs,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
aggregated_at_unix_secs,
|
||||
)
|
||||
.await?;
|
||||
let provider_rows = upsert_sqlite_stats_daily_provider_rows(
|
||||
&mut tx,
|
||||
day_start_unix_secs,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
aggregated_at_unix_secs,
|
||||
)
|
||||
.await?;
|
||||
let api_key_rows = upsert_sqlite_stats_daily_api_key_rows(
|
||||
&mut tx,
|
||||
day_start_unix_secs,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
aggregated_at_unix_secs,
|
||||
)
|
||||
.await?;
|
||||
let error_rows = refresh_sqlite_stats_daily_error_rows(
|
||||
&mut tx,
|
||||
day_start_unix_secs,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
aggregated_at_unix_secs,
|
||||
)
|
||||
.await?;
|
||||
let user_rows = upsert_sqlite_stats_user_daily_rows(
|
||||
&mut tx,
|
||||
day_start_unix_secs,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
aggregated_at_unix_secs,
|
||||
)
|
||||
.await?;
|
||||
advanced::refresh_daily(
|
||||
&mut tx,
|
||||
day_start_unix_secs,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
aggregated_at_unix_secs,
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await.map_sql_err()?;
|
||||
|
||||
Ok(StatsDailyAggregationSummary {
|
||||
day_start_utc: utc_from_unix_secs(day_start_unix_secs, "stats_daily.date")?,
|
||||
total_requests,
|
||||
model_rows,
|
||||
provider_rows,
|
||||
api_key_rows,
|
||||
error_rows,
|
||||
user_rows,
|
||||
})
|
||||
}
|
||||
|
||||
async fn upsert_sqlite_stats_hourly_user_rows(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
|
||||
hour_utc: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<usize, DataLayerError> {
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO stats_hourly_user (
|
||||
id, hour_utc, user_id, total_requests, success_requests, error_requests,
|
||||
input_tokens, output_tokens, total_cost, created_at, updated_at
|
||||
)
|
||||
SELECT
|
||||
lower(hex(randomblob(32))), ?, user_id, COUNT(*),
|
||||
COUNT(*) - COALESCE(SUM(CASE
|
||||
WHEN status = 'failed' OR status_code >= 400 OR error_message IS NOT NULL
|
||||
THEN 1 ELSE 0 END), 0),
|
||||
COALESCE(SUM(CASE
|
||||
WHEN status = 'failed' OR status_code >= 400 OR error_message IS NOT NULL
|
||||
THEN 1 ELSE 0 END), 0),
|
||||
COALESCE(SUM(input_tokens), 0), COALESCE(SUM(output_tokens), 0),
|
||||
CAST(COALESCE(SUM(total_cost_usd), 0) AS REAL), ?, ?
|
||||
FROM "usage"
|
||||
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
|
||||
AND user_id IS NOT NULL AND user_id <> ''
|
||||
AND status NOT IN ('pending', 'streaming')
|
||||
AND provider_name NOT IN ('unknown', 'pending')
|
||||
GROUP BY user_id
|
||||
ON CONFLICT (hour_utc, user_id) DO UPDATE SET
|
||||
total_requests = excluded.total_requests,
|
||||
success_requests = excluded.success_requests,
|
||||
error_requests = excluded.error_requests,
|
||||
input_tokens = excluded.input_tokens,
|
||||
output_tokens = excluded.output_tokens,
|
||||
total_cost = excluded.total_cost,
|
||||
updated_at = excluded.updated_at
|
||||
"#,
|
||||
)
|
||||
.bind(hour_utc)
|
||||
.bind(now_unix_secs)
|
||||
.bind(now_unix_secs)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
|
||||
}
|
||||
|
||||
async fn upsert_sqlite_stats_hourly_user_model_rows(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
|
||||
hour_utc: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<usize, DataLayerError> {
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO stats_hourly_user_model (
|
||||
id, hour_utc, user_id, model, total_requests, input_tokens, output_tokens,
|
||||
total_cost, created_at, updated_at
|
||||
)
|
||||
SELECT
|
||||
lower(hex(randomblob(32))), ?, user_id, model, COUNT(*),
|
||||
COALESCE(SUM(input_tokens), 0), COALESCE(SUM(output_tokens), 0),
|
||||
CAST(COALESCE(SUM(total_cost_usd), 0) AS REAL), ?, ?
|
||||
FROM "usage"
|
||||
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
|
||||
AND user_id IS NOT NULL AND user_id <> ''
|
||||
AND model IS NOT NULL AND model <> ''
|
||||
AND status NOT IN ('pending', 'streaming')
|
||||
AND provider_name NOT IN ('unknown', 'pending')
|
||||
GROUP BY user_id, model
|
||||
ON CONFLICT (hour_utc, user_id, model) DO UPDATE SET
|
||||
total_requests = excluded.total_requests,
|
||||
input_tokens = excluded.input_tokens,
|
||||
output_tokens = excluded.output_tokens,
|
||||
total_cost = excluded.total_cost,
|
||||
updated_at = excluded.updated_at
|
||||
"#,
|
||||
)
|
||||
.bind(hour_utc)
|
||||
.bind(now_unix_secs)
|
||||
.bind(now_unix_secs)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
|
||||
}
|
||||
|
||||
async fn upsert_sqlite_stats_hourly_model_rows(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
|
||||
hour_utc: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<usize, DataLayerError> {
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO stats_hourly_model (
|
||||
id, hour_utc, model, total_requests, input_tokens, output_tokens, total_cost,
|
||||
avg_response_time_ms, created_at, updated_at
|
||||
)
|
||||
SELECT
|
||||
lower(hex(randomblob(32))), ?, model, COUNT(*),
|
||||
COALESCE(SUM(input_tokens), 0), COALESCE(SUM(output_tokens), 0),
|
||||
CAST(COALESCE(SUM(total_cost_usd), 0) AS REAL),
|
||||
CAST(COALESCE(AVG(response_time_ms), 0) AS REAL), ?, ?
|
||||
FROM "usage"
|
||||
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
|
||||
AND model IS NOT NULL AND model <> ''
|
||||
AND status NOT IN ('pending', 'streaming')
|
||||
AND provider_name NOT IN ('unknown', 'pending')
|
||||
GROUP BY model
|
||||
ON CONFLICT (hour_utc, model) DO UPDATE SET
|
||||
total_requests = excluded.total_requests,
|
||||
input_tokens = excluded.input_tokens,
|
||||
output_tokens = excluded.output_tokens,
|
||||
total_cost = excluded.total_cost,
|
||||
avg_response_time_ms = excluded.avg_response_time_ms,
|
||||
updated_at = excluded.updated_at
|
||||
"#,
|
||||
)
|
||||
.bind(hour_utc)
|
||||
.bind(now_unix_secs)
|
||||
.bind(now_unix_secs)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
|
||||
}
|
||||
|
||||
async fn upsert_sqlite_stats_hourly_provider_rows(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
|
||||
hour_utc: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<usize, DataLayerError> {
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO stats_hourly_provider (
|
||||
id, hour_utc, provider_name, total_requests, input_tokens, output_tokens,
|
||||
total_cost, created_at, updated_at
|
||||
)
|
||||
SELECT
|
||||
lower(hex(randomblob(32))), ?, provider_name, COUNT(*),
|
||||
COALESCE(SUM(input_tokens), 0), COALESCE(SUM(output_tokens), 0),
|
||||
CAST(COALESCE(SUM(total_cost_usd), 0) AS REAL), ?, ?
|
||||
FROM "usage"
|
||||
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
|
||||
AND status NOT IN ('pending', 'streaming')
|
||||
AND provider_name NOT IN ('unknown', 'pending')
|
||||
GROUP BY provider_name
|
||||
ON CONFLICT (hour_utc, provider_name) DO UPDATE SET
|
||||
total_requests = excluded.total_requests,
|
||||
input_tokens = excluded.input_tokens,
|
||||
output_tokens = excluded.output_tokens,
|
||||
total_cost = excluded.total_cost,
|
||||
updated_at = excluded.updated_at
|
||||
"#,
|
||||
)
|
||||
.bind(hour_utc)
|
||||
.bind(now_unix_secs)
|
||||
.bind(now_unix_secs)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
|
||||
}
|
||||
|
||||
async fn upsert_sqlite_stats_daily_model_rows(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
|
||||
day_start: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<usize, DataLayerError> {
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO stats_daily_model (
|
||||
id, "date", model, total_requests, input_tokens, output_tokens,
|
||||
cache_creation_tokens, cache_read_tokens, total_cost, avg_response_time_ms,
|
||||
created_at, updated_at
|
||||
)
|
||||
SELECT
|
||||
lower(hex(randomblob(32))), ?, model, COUNT(*),
|
||||
COALESCE(SUM(input_tokens), 0), COALESCE(SUM(output_tokens), 0),
|
||||
COALESCE(SUM(cache_creation_input_tokens), 0),
|
||||
COALESCE(SUM(cache_read_input_tokens), 0),
|
||||
CAST(COALESCE(SUM(total_cost_usd), 0) AS REAL),
|
||||
CAST(COALESCE(AVG(response_time_ms), 0) AS REAL), ?, ?
|
||||
FROM "usage"
|
||||
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
|
||||
AND model IS NOT NULL AND model <> ''
|
||||
AND status NOT IN ('pending', 'streaming')
|
||||
AND provider_name NOT IN ('unknown', 'pending')
|
||||
GROUP BY model
|
||||
ON CONFLICT ("date", model) DO UPDATE SET
|
||||
total_requests = excluded.total_requests,
|
||||
input_tokens = excluded.input_tokens,
|
||||
output_tokens = excluded.output_tokens,
|
||||
cache_creation_tokens = excluded.cache_creation_tokens,
|
||||
cache_read_tokens = excluded.cache_read_tokens,
|
||||
total_cost = excluded.total_cost,
|
||||
avg_response_time_ms = excluded.avg_response_time_ms,
|
||||
updated_at = excluded.updated_at
|
||||
"#,
|
||||
)
|
||||
.bind(day_start)
|
||||
.bind(now_unix_secs)
|
||||
.bind(now_unix_secs)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
|
||||
}
|
||||
|
||||
async fn upsert_sqlite_stats_daily_provider_rows(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
|
||||
day_start: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<usize, DataLayerError> {
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO stats_daily_provider (
|
||||
id, "date", provider_name, total_requests, input_tokens, output_tokens,
|
||||
cache_creation_tokens, cache_read_tokens, total_cost, created_at, updated_at
|
||||
)
|
||||
SELECT
|
||||
lower(hex(randomblob(32))), ?, provider_name, COUNT(*),
|
||||
COALESCE(SUM(input_tokens), 0), COALESCE(SUM(output_tokens), 0),
|
||||
COALESCE(SUM(cache_creation_input_tokens), 0),
|
||||
COALESCE(SUM(cache_read_input_tokens), 0),
|
||||
CAST(COALESCE(SUM(total_cost_usd), 0) AS REAL), ?, ?
|
||||
FROM "usage"
|
||||
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
|
||||
AND status NOT IN ('pending', 'streaming')
|
||||
AND provider_name NOT IN ('unknown', 'pending')
|
||||
GROUP BY provider_name
|
||||
ON CONFLICT ("date", provider_name) DO UPDATE SET
|
||||
total_requests = excluded.total_requests,
|
||||
input_tokens = excluded.input_tokens,
|
||||
output_tokens = excluded.output_tokens,
|
||||
cache_creation_tokens = excluded.cache_creation_tokens,
|
||||
cache_read_tokens = excluded.cache_read_tokens,
|
||||
total_cost = excluded.total_cost,
|
||||
updated_at = excluded.updated_at
|
||||
"#,
|
||||
)
|
||||
.bind(day_start)
|
||||
.bind(now_unix_secs)
|
||||
.bind(now_unix_secs)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
|
||||
}
|
||||
|
||||
async fn upsert_sqlite_stats_daily_api_key_rows(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
|
||||
day_start: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<usize, DataLayerError> {
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO stats_daily_api_key (
|
||||
id, api_key_id, "date", total_requests, success_requests, error_requests,
|
||||
input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens,
|
||||
total_cost, api_key_name, created_at, updated_at
|
||||
)
|
||||
SELECT
|
||||
lower(hex(randomblob(32))), usage.api_key_id, ?, COUNT(*),
|
||||
COUNT(*) - COALESCE(SUM(CASE
|
||||
WHEN usage.status = 'failed' OR usage.status_code >= 400
|
||||
OR usage.error_message IS NOT NULL THEN 1 ELSE 0 END), 0),
|
||||
COALESCE(SUM(CASE
|
||||
WHEN usage.status = 'failed' OR usage.status_code >= 400
|
||||
OR usage.error_message IS NOT NULL THEN 1 ELSE 0 END), 0),
|
||||
COALESCE(SUM(usage.input_tokens), 0), COALESCE(SUM(usage.output_tokens), 0),
|
||||
COALESCE(SUM(usage.cache_creation_input_tokens), 0),
|
||||
COALESCE(SUM(usage.cache_read_input_tokens), 0),
|
||||
CAST(COALESCE(SUM(usage.total_cost_usd), 0) AS REAL), MAX(api_keys.name), ?, ?
|
||||
FROM "usage" AS usage
|
||||
LEFT JOIN api_keys ON api_keys.id = usage.api_key_id
|
||||
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
|
||||
AND usage.api_key_id IS NOT NULL AND usage.api_key_id <> ''
|
||||
GROUP BY usage.api_key_id
|
||||
ON CONFLICT ("date", api_key_id) DO UPDATE SET
|
||||
total_requests = excluded.total_requests,
|
||||
success_requests = excluded.success_requests,
|
||||
error_requests = excluded.error_requests,
|
||||
input_tokens = excluded.input_tokens,
|
||||
output_tokens = excluded.output_tokens,
|
||||
cache_creation_tokens = excluded.cache_creation_tokens,
|
||||
cache_read_tokens = excluded.cache_read_tokens,
|
||||
total_cost = excluded.total_cost,
|
||||
api_key_name = COALESCE(excluded.api_key_name, stats_daily_api_key.api_key_name),
|
||||
updated_at = excluded.updated_at
|
||||
"#,
|
||||
)
|
||||
.bind(day_start)
|
||||
.bind(now_unix_secs)
|
||||
.bind(now_unix_secs)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
|
||||
}
|
||||
|
||||
async fn refresh_sqlite_stats_daily_error_rows(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
|
||||
day_start: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<usize, DataLayerError> {
|
||||
sqlx::query(r#"DELETE FROM stats_daily_error WHERE "date" = ?"#)
|
||||
.bind(day_start)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO stats_daily_error (
|
||||
id, "date", error_category, provider_name, model, count, created_at, updated_at
|
||||
)
|
||||
SELECT
|
||||
lower(hex(randomblob(32))), ?, error_category, provider_name, model,
|
||||
COUNT(*), ?, ?
|
||||
FROM "usage"
|
||||
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
|
||||
AND error_category IS NOT NULL AND error_category <> ''
|
||||
GROUP BY error_category, provider_name, model
|
||||
"#,
|
||||
)
|
||||
.bind(day_start)
|
||||
.bind(now_unix_secs)
|
||||
.bind(now_unix_secs)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
|
||||
}
|
||||
|
||||
async fn upsert_sqlite_stats_user_daily_rows(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
|
||||
day_start: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<usize, DataLayerError> {
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO stats_user_daily (
|
||||
id, user_id, "date", total_requests, success_requests, error_requests,
|
||||
input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens,
|
||||
total_cost, username, created_at, updated_at
|
||||
)
|
||||
SELECT
|
||||
lower(hex(randomblob(32))), usage.user_id, ?, COUNT(*),
|
||||
COUNT(*) - COALESCE(SUM(CASE
|
||||
WHEN usage.status = 'failed' OR usage.status_code >= 400
|
||||
OR usage.error_message IS NOT NULL THEN 1 ELSE 0 END), 0),
|
||||
COALESCE(SUM(CASE
|
||||
WHEN usage.status = 'failed' OR usage.status_code >= 400
|
||||
OR usage.error_message IS NOT NULL THEN 1 ELSE 0 END), 0),
|
||||
COALESCE(SUM(usage.input_tokens), 0), COALESCE(SUM(usage.output_tokens), 0),
|
||||
COALESCE(SUM(usage.cache_creation_input_tokens), 0),
|
||||
COALESCE(SUM(usage.cache_read_input_tokens), 0),
|
||||
CAST(COALESCE(SUM(usage.total_cost_usd), 0) AS REAL), MAX(users.username), ?, ?
|
||||
FROM "usage" AS usage
|
||||
LEFT JOIN users ON users.id = usage.user_id
|
||||
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
|
||||
AND usage.user_id IS NOT NULL AND usage.user_id <> ''
|
||||
AND usage.status NOT IN ('pending', 'streaming')
|
||||
AND usage.provider_name NOT IN ('unknown', 'pending')
|
||||
GROUP BY usage.user_id
|
||||
ON CONFLICT ("date", user_id) DO UPDATE SET
|
||||
total_requests = excluded.total_requests,
|
||||
success_requests = excluded.success_requests,
|
||||
error_requests = excluded.error_requests,
|
||||
input_tokens = excluded.input_tokens,
|
||||
output_tokens = excluded.output_tokens,
|
||||
cache_creation_tokens = excluded.cache_creation_tokens,
|
||||
cache_read_tokens = excluded.cache_read_tokens,
|
||||
total_cost = excluded.total_cost,
|
||||
username = COALESCE(excluded.username, stats_user_daily.username),
|
||||
updated_at = excluded.updated_at
|
||||
"#,
|
||||
)
|
||||
.bind(day_start)
|
||||
.bind(now_unix_secs)
|
||||
.bind(now_unix_secs)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
|
||||
}
|
||||
|
||||
async fn sqlite_daily_fallback_count(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
) -> Result<i64, DataLayerError> {
|
||||
let start_unix_ms = start_unix_secs.checked_mul(1000).ok_or_else(|| {
|
||||
DataLayerError::InvalidInput("stats fallback window start overflows milliseconds".into())
|
||||
})?;
|
||||
let end_unix_ms = end_unix_secs.checked_mul(1000).ok_or_else(|| {
|
||||
DataLayerError::InvalidInput("stats fallback window end overflows milliseconds".into())
|
||||
})?;
|
||||
sqlx::query_scalar(
|
||||
r#"
|
||||
SELECT COUNT(*)
|
||||
FROM (
|
||||
SELECT request_id
|
||||
FROM request_candidates
|
||||
WHERE created_at >= ? AND created_at < ?
|
||||
AND status IN ('success', 'failed')
|
||||
GROUP BY request_id
|
||||
HAVING COUNT(id) > 1
|
||||
)
|
||||
"#,
|
||||
)
|
||||
.bind(start_unix_ms)
|
||||
.bind(end_unix_ms)
|
||||
.fetch_one(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()
|
||||
}
|
||||
|
||||
async fn sqlite_group_count(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
|
||||
group_columns: &str,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
) -> Result<usize, DataLayerError> {
|
||||
let not_empty = group_columns
|
||||
.split(',')
|
||||
.map(str::trim)
|
||||
.map(|column| format!("{column} IS NOT NULL AND {column} <> ''"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(" AND ");
|
||||
let sql = format!(
|
||||
r#"
|
||||
SELECT COUNT(*)
|
||||
FROM (
|
||||
SELECT 1
|
||||
FROM "usage"
|
||||
WHERE created_at_unix_ms >= ?
|
||||
AND created_at_unix_ms < ?
|
||||
AND status NOT IN ('pending', 'streaming')
|
||||
AND provider_name NOT IN ('unknown', 'pending')
|
||||
AND {not_empty}
|
||||
GROUP BY {group_columns}
|
||||
)
|
||||
"#
|
||||
);
|
||||
let count: i64 = sqlx::query_scalar(&sql)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.fetch_one(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(usize::try_from(count.max(0)).unwrap_or(usize::MAX))
|
||||
}
|
||||
@@ -1,986 +0,0 @@
|
||||
use sqlx::Sqlite;
|
||||
|
||||
use crate::backend::stats_common::stats_id;
|
||||
use crate::error::SqlResultExt;
|
||||
use crate::DataLayerError;
|
||||
|
||||
const CACHE_5M: &str = r#"MAX(
|
||||
COALESCE(usage.cache_creation_input_tokens_5m, 0),
|
||||
COALESCE(usage.cache_creation_ephemeral_5m_input_tokens, 0)
|
||||
)"#;
|
||||
const CACHE_1H: &str = r#"MAX(
|
||||
COALESCE(usage.cache_creation_input_tokens_1h, 0),
|
||||
COALESCE(usage.cache_creation_ephemeral_1h_input_tokens, 0)
|
||||
)"#;
|
||||
const CACHE_CREATION: &str = r#"CASE
|
||||
WHEN COALESCE(usage.cache_creation_input_tokens, 0) = 0
|
||||
AND ({cache_5m} + {cache_1h}) > 0
|
||||
THEN {cache_5m} + {cache_1h}
|
||||
ELSE MAX(COALESCE(usage.cache_creation_input_tokens, 0), 0)
|
||||
END"#;
|
||||
const EFFECTIVE_INPUT: &str = r#"CASE
|
||||
WHEN (
|
||||
LOWER(COALESCE(usage.endpoint_api_format, usage.api_format, '')) IN ('openai', 'gemini', 'google')
|
||||
OR LOWER(COALESCE(usage.endpoint_api_format, usage.api_format, '')) LIKE 'openai:%'
|
||||
OR LOWER(COALESCE(usage.endpoint_api_format, usage.api_format, '')) LIKE 'gemini:%'
|
||||
OR LOWER(COALESCE(usage.endpoint_api_format, usage.api_format, '')) LIKE 'google:%'
|
||||
)
|
||||
AND COALESCE(usage.input_tokens, 0) > 0
|
||||
AND COALESCE(usage.cache_read_input_tokens, 0) > 0
|
||||
THEN MAX(COALESCE(usage.input_tokens, 0) - COALESCE(usage.cache_read_input_tokens, 0), 0)
|
||||
ELSE MAX(COALESCE(usage.input_tokens, 0), 0)
|
||||
END"#;
|
||||
const SUCCESS: &str = r#"CASE
|
||||
WHEN usage.status <> 'failed'
|
||||
AND (usage.status_code IS NULL OR usage.status_code < 400)
|
||||
AND usage.error_message IS NULL
|
||||
THEN 1 ELSE 0
|
||||
END"#;
|
||||
const AGGREGATABLE: &str = r#"usage.status NOT IN ('pending', 'streaming')
|
||||
AND usage.provider_name NOT IN ('unknown', 'pending')"#;
|
||||
const SETTLED: &str = r#"COALESCE(settlement.billing_status, usage.billing_status) = 'settled'
|
||||
AND COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0) > 0"#;
|
||||
|
||||
fn cache_creation_expr() -> String {
|
||||
CACHE_CREATION
|
||||
.replace("{cache_5m}", CACHE_5M)
|
||||
.replace("{cache_1h}", CACHE_1H)
|
||||
}
|
||||
|
||||
fn total_input_context_expr() -> String {
|
||||
format!(
|
||||
"({EFFECTIVE_INPUT}) + ({}) + MAX(COALESCE(usage.cache_read_input_tokens, 0), 0)",
|
||||
cache_creation_expr()
|
||||
)
|
||||
}
|
||||
|
||||
fn total_tokens_expr() -> String {
|
||||
format!(
|
||||
r#"COALESCE(
|
||||
NULLIF(MAX(COALESCE(usage.total_tokens, 0), 0), 0),
|
||||
({EFFECTIVE_INPUT})
|
||||
+ MAX(COALESCE(usage.output_tokens, 0), 0)
|
||||
+ ({})
|
||||
+ MAX(COALESCE(usage.cache_read_input_tokens, 0), 0),
|
||||
0
|
||||
)"#,
|
||||
cache_creation_expr()
|
||||
)
|
||||
}
|
||||
|
||||
fn percentile_cont(sorted: &[i64], percentile: f64) -> Option<i64> {
|
||||
if sorted.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let position = percentile * (sorted.len().saturating_sub(1) as f64);
|
||||
let lower = position.floor() as usize;
|
||||
let upper = position.ceil() as usize;
|
||||
let fraction = position - lower as f64;
|
||||
let value = sorted[lower] as f64 + (sorted[upper] - sorted[lower]) as f64 * fraction;
|
||||
Some(value.round() as i64)
|
||||
}
|
||||
|
||||
async fn load_percentiles(
|
||||
tx: &mut sqlx::Transaction<'_, Sqlite>,
|
||||
column: &str,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
) -> Result<(Option<i64>, Option<i64>, Option<i64>), DataLayerError> {
|
||||
let sql = format!(
|
||||
r#"
|
||||
SELECT {column}
|
||||
FROM "usage"
|
||||
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
|
||||
AND status = 'completed'
|
||||
AND provider_name NOT IN ('unknown', 'pending')
|
||||
AND {column} IS NOT NULL
|
||||
ORDER BY {column}
|
||||
"#
|
||||
);
|
||||
let values: Vec<i64> = sqlx::query_scalar(&sql)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.fetch_all(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
if values.len() < 10 {
|
||||
return Ok((None, None, None));
|
||||
}
|
||||
Ok((
|
||||
percentile_cont(&values, 0.50),
|
||||
percentile_cont(&values, 0.90),
|
||||
percentile_cont(&values, 0.99),
|
||||
))
|
||||
}
|
||||
|
||||
pub(super) async fn refresh_hourly(
|
||||
tx: &mut sqlx::Transaction<'_, Sqlite>,
|
||||
hour_utc: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
) -> Result<(), DataLayerError> {
|
||||
let cache_creation = cache_creation_expr();
|
||||
let total_context = total_input_context_expr();
|
||||
let sql = format!(
|
||||
r#"
|
||||
UPDATE stats_hourly AS target
|
||||
SET
|
||||
cache_hit_total_requests = aggregated.cache_hit_total_requests,
|
||||
cache_hit_requests = aggregated.cache_hit_requests,
|
||||
completed_total_requests = aggregated.completed_total_requests,
|
||||
completed_cache_hit_requests = aggregated.completed_cache_hit_requests,
|
||||
completed_input_tokens = aggregated.completed_input_tokens,
|
||||
completed_cache_creation_tokens = aggregated.completed_cache_creation_tokens,
|
||||
completed_cache_read_tokens = aggregated.completed_cache_read_tokens,
|
||||
completed_total_input_context = aggregated.completed_total_input_context,
|
||||
completed_cache_creation_cost = aggregated.completed_cache_creation_cost,
|
||||
completed_cache_read_cost = aggregated.completed_cache_read_cost,
|
||||
settled_total_cost = aggregated.settled_total_cost,
|
||||
settled_total_requests = aggregated.settled_total_requests,
|
||||
settled_input_tokens = aggregated.settled_input_tokens,
|
||||
settled_output_tokens = aggregated.settled_output_tokens,
|
||||
settled_cache_creation_tokens = aggregated.settled_cache_creation_tokens,
|
||||
settled_cache_read_tokens = aggregated.settled_cache_read_tokens,
|
||||
settled_first_finalized_at_unix_secs = aggregated.settled_first_finalized_at_unix_secs,
|
||||
settled_last_finalized_at_unix_secs = aggregated.settled_last_finalized_at_unix_secs,
|
||||
response_time_sum_ms = aggregated.response_time_sum_ms,
|
||||
response_time_samples = aggregated.response_time_samples
|
||||
FROM (
|
||||
SELECT
|
||||
COUNT(*) AS cache_hit_total_requests,
|
||||
COALESCE(SUM(CASE WHEN COALESCE(usage.cache_read_input_tokens, 0) > 0 THEN 1 ELSE 0 END), 0) AS cache_hit_requests,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN 1 ELSE 0 END), 0) AS completed_total_requests,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' AND COALESCE(usage.cache_read_input_tokens, 0) > 0 THEN 1 ELSE 0 END), 0) AS completed_cache_hit_requests,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN MAX(COALESCE(usage.input_tokens, 0), 0) ELSE 0 END), 0) AS completed_input_tokens,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN ({cache_creation}) ELSE 0 END), 0) AS completed_cache_creation_tokens,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN MAX(COALESCE(usage.cache_read_input_tokens, 0), 0) ELSE 0 END), 0) AS completed_cache_read_tokens,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN ({total_context}) ELSE 0 END), 0) AS completed_total_input_context,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN COALESCE(usage.cache_creation_cost_usd, 0) ELSE 0 END), 0) AS completed_cache_creation_cost,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN COALESCE(usage.cache_read_cost_usd, 0) ELSE 0 END), 0) AS completed_cache_read_cost,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0) ELSE 0 END), 0) AS settled_total_cost,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN 1 ELSE 0 END), 0) AS settled_total_requests,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN MAX(COALESCE(usage.input_tokens, 0), 0) ELSE 0 END), 0) AS settled_input_tokens,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN MAX(COALESCE(usage.output_tokens, 0), 0) ELSE 0 END), 0) AS settled_output_tokens,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN ({cache_creation}) ELSE 0 END), 0) AS settled_cache_creation_tokens,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN MAX(COALESCE(usage.cache_read_input_tokens, 0), 0) ELSE 0 END), 0) AS settled_cache_read_tokens,
|
||||
MIN(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_first_finalized_at_unix_secs,
|
||||
MAX(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_last_finalized_at_unix_secs,
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL AND {AGGREGATABLE} THEN MAX(usage.response_time_ms, 0) ELSE 0 END), 0) AS response_time_sum_ms,
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL AND {AGGREGATABLE} THEN 1 ELSE 0 END), 0) AS response_time_samples
|
||||
FROM "usage" AS usage
|
||||
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
|
||||
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
|
||||
) AS aggregated
|
||||
WHERE target.hour_utc = ?
|
||||
"#
|
||||
);
|
||||
sqlx::query(&sql)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.bind(hour_utc)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
|
||||
refresh_hourly_user(tx, hour_utc, start_unix_secs, end_unix_secs).await?;
|
||||
refresh_hourly_response_dimensions(tx, hour_utc, start_unix_secs, end_unix_secs).await
|
||||
}
|
||||
|
||||
async fn refresh_hourly_user(
|
||||
tx: &mut sqlx::Transaction<'_, Sqlite>,
|
||||
hour_utc: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
) -> Result<(), DataLayerError> {
|
||||
let cache_creation = cache_creation_expr();
|
||||
let sql = format!(
|
||||
r#"
|
||||
UPDATE stats_hourly_user AS target
|
||||
SET
|
||||
cache_creation_tokens = aggregated.cache_creation_tokens,
|
||||
cache_read_tokens = aggregated.cache_read_tokens,
|
||||
actual_total_cost = aggregated.actual_total_cost,
|
||||
response_time_sum_ms = aggregated.response_time_sum_ms,
|
||||
response_time_samples = aggregated.response_time_samples,
|
||||
settled_total_cost = aggregated.settled_total_cost,
|
||||
settled_total_requests = aggregated.settled_total_requests,
|
||||
settled_input_tokens = aggregated.settled_input_tokens,
|
||||
settled_output_tokens = aggregated.settled_output_tokens,
|
||||
settled_cache_creation_tokens = aggregated.settled_cache_creation_tokens,
|
||||
settled_cache_read_tokens = aggregated.settled_cache_read_tokens,
|
||||
settled_first_finalized_at_unix_secs = aggregated.settled_first_finalized_at_unix_secs,
|
||||
settled_last_finalized_at_unix_secs = aggregated.settled_last_finalized_at_unix_secs
|
||||
FROM (
|
||||
SELECT
|
||||
usage.user_id,
|
||||
COALESCE(SUM({cache_creation}), 0) AS cache_creation_tokens,
|
||||
COALESCE(SUM(MAX(COALESCE(usage.cache_read_input_tokens, 0), 0)), 0) AS cache_read_tokens,
|
||||
COALESCE(SUM(COALESCE(settlement.billing_actual_total_cost_usd, usage.actual_total_cost_usd, 0)), 0) AS actual_total_cost,
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN MAX(usage.response_time_ms, 0) ELSE 0 END), 0) AS response_time_sum_ms,
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0) AS response_time_samples,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0) ELSE 0 END), 0) AS settled_total_cost,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN 1 ELSE 0 END), 0) AS settled_total_requests,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN MAX(COALESCE(usage.input_tokens, 0), 0) ELSE 0 END), 0) AS settled_input_tokens,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN MAX(COALESCE(usage.output_tokens, 0), 0) ELSE 0 END), 0) AS settled_output_tokens,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN ({cache_creation}) ELSE 0 END), 0) AS settled_cache_creation_tokens,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN MAX(COALESCE(usage.cache_read_input_tokens, 0), 0) ELSE 0 END), 0) AS settled_cache_read_tokens,
|
||||
MIN(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_first_finalized_at_unix_secs,
|
||||
MAX(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_last_finalized_at_unix_secs
|
||||
FROM "usage" AS usage
|
||||
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
|
||||
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
|
||||
AND usage.user_id IS NOT NULL AND usage.user_id <> '' AND {AGGREGATABLE}
|
||||
GROUP BY usage.user_id
|
||||
) AS aggregated
|
||||
WHERE target.hour_utc = ? AND target.user_id = aggregated.user_id
|
||||
"#
|
||||
);
|
||||
sqlx::query(&sql)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.bind(hour_utc)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn refresh_hourly_response_dimensions(
|
||||
tx: &mut sqlx::Transaction<'_, Sqlite>,
|
||||
hour_utc: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
) -> Result<(), DataLayerError> {
|
||||
for (table, dimensions, group_by, target_match) in [
|
||||
(
|
||||
"stats_hourly_model",
|
||||
"usage.model AS model",
|
||||
"usage.model",
|
||||
"target.model = aggregated.model",
|
||||
),
|
||||
(
|
||||
"stats_hourly_user_model",
|
||||
"usage.user_id AS user_id, usage.model AS model",
|
||||
"usage.user_id, usage.model",
|
||||
"target.user_id = aggregated.user_id AND target.model = aggregated.model",
|
||||
),
|
||||
] {
|
||||
let sql = format!(
|
||||
r#"
|
||||
UPDATE {table} AS target
|
||||
SET response_time_sum_ms = aggregated.response_time_sum_ms,
|
||||
response_time_samples = aggregated.response_time_samples
|
||||
FROM (
|
||||
SELECT {dimensions},
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN MAX(usage.response_time_ms, 0) ELSE 0 END), 0) AS response_time_sum_ms,
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0) AS response_time_samples
|
||||
FROM "usage" AS usage
|
||||
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
|
||||
AND {AGGREGATABLE}
|
||||
GROUP BY {group_by}
|
||||
) AS aggregated
|
||||
WHERE target.hour_utc = ? AND {target_match}
|
||||
"#
|
||||
);
|
||||
sqlx::query(&sql)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.bind(hour_utc)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn refresh_daily(
|
||||
tx: &mut sqlx::Transaction<'_, Sqlite>,
|
||||
day_start: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<(), DataLayerError> {
|
||||
let response = load_percentiles(tx, "response_time_ms", start_unix_secs, end_unix_secs).await?;
|
||||
let first_byte =
|
||||
load_percentiles(tx, "first_byte_time_ms", start_unix_secs, end_unix_secs).await?;
|
||||
refresh_daily_root(
|
||||
tx,
|
||||
day_start,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
response,
|
||||
first_byte,
|
||||
)
|
||||
.await?;
|
||||
refresh_daily_existing_dimensions(tx, day_start, start_unix_secs, end_unix_secs).await?;
|
||||
upsert_user_dimension(
|
||||
tx,
|
||||
"stats_user_daily_model",
|
||||
"model",
|
||||
"usage.model",
|
||||
"usage.model IS NOT NULL AND usage.model <> ''",
|
||||
day_start,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
now_unix_secs,
|
||||
)
|
||||
.await?;
|
||||
upsert_user_dimension(
|
||||
tx,
|
||||
"stats_user_daily_provider",
|
||||
"provider_name",
|
||||
"usage.provider_name",
|
||||
"usage.provider_name IS NOT NULL AND usage.provider_name <> ''",
|
||||
day_start,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
now_unix_secs,
|
||||
)
|
||||
.await?;
|
||||
upsert_user_dimension(
|
||||
tx,
|
||||
"stats_user_daily_api_format",
|
||||
"api_format",
|
||||
"LOWER(COALESCE(usage.endpoint_api_format, usage.api_format, ''))",
|
||||
"COALESCE(usage.endpoint_api_format, usage.api_format, '') <> ''",
|
||||
day_start,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
now_unix_secs,
|
||||
)
|
||||
.await?;
|
||||
upsert_model_provider_rows(tx, day_start, start_unix_secs, end_unix_secs, now_unix_secs)
|
||||
.await?;
|
||||
upsert_cost_savings_rows(tx, day_start, start_unix_secs, end_unix_secs, now_unix_secs).await?;
|
||||
refresh_user_summary(tx, end_unix_secs, now_unix_secs).await
|
||||
}
|
||||
|
||||
async fn refresh_daily_root(
|
||||
tx: &mut sqlx::Transaction<'_, Sqlite>,
|
||||
day_start: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
response: (Option<i64>, Option<i64>, Option<i64>),
|
||||
first_byte: (Option<i64>, Option<i64>, Option<i64>),
|
||||
) -> Result<(), DataLayerError> {
|
||||
let cache_creation = cache_creation_expr();
|
||||
let total_context = total_input_context_expr();
|
||||
let sql = format!(
|
||||
r#"
|
||||
UPDATE stats_daily AS target
|
||||
SET
|
||||
effective_input_tokens = aggregated.effective_input_tokens,
|
||||
total_input_context = aggregated.total_input_context,
|
||||
response_time_sum_ms = aggregated.response_time_sum_ms,
|
||||
response_time_samples = aggregated.response_time_samples,
|
||||
cache_creation_ephemeral_5m_tokens = aggregated.cache_creation_ephemeral_5m_tokens,
|
||||
cache_creation_ephemeral_1h_tokens = aggregated.cache_creation_ephemeral_1h_tokens,
|
||||
input_cost = aggregated.input_cost,
|
||||
output_cost = aggregated.output_cost,
|
||||
cache_creation_cost = aggregated.cache_creation_cost,
|
||||
cache_read_cost = aggregated.cache_read_cost,
|
||||
cache_hit_total_requests = aggregated.cache_hit_total_requests,
|
||||
cache_hit_requests = aggregated.cache_hit_requests,
|
||||
completed_total_requests = aggregated.completed_total_requests,
|
||||
completed_cache_hit_requests = aggregated.completed_cache_hit_requests,
|
||||
completed_input_tokens = aggregated.completed_input_tokens,
|
||||
completed_cache_creation_tokens = aggregated.completed_cache_creation_tokens,
|
||||
completed_cache_read_tokens = aggregated.completed_cache_read_tokens,
|
||||
completed_total_input_context = aggregated.completed_total_input_context,
|
||||
completed_cache_creation_cost = aggregated.completed_cache_creation_cost,
|
||||
completed_cache_read_cost = aggregated.completed_cache_read_cost,
|
||||
settled_total_cost = aggregated.settled_total_cost,
|
||||
settled_total_requests = aggregated.settled_total_requests,
|
||||
settled_input_tokens = aggregated.settled_input_tokens,
|
||||
settled_output_tokens = aggregated.settled_output_tokens,
|
||||
settled_cache_creation_tokens = aggregated.settled_cache_creation_tokens,
|
||||
settled_cache_read_tokens = aggregated.settled_cache_read_tokens,
|
||||
settled_first_finalized_at_unix_secs = aggregated.settled_first_finalized_at_unix_secs,
|
||||
settled_last_finalized_at_unix_secs = aggregated.settled_last_finalized_at_unix_secs,
|
||||
p50_response_time_ms = ?, p90_response_time_ms = ?, p99_response_time_ms = ?,
|
||||
p50_first_byte_time_ms = ?, p90_first_byte_time_ms = ?, p99_first_byte_time_ms = ?
|
||||
FROM (
|
||||
SELECT
|
||||
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN ({EFFECTIVE_INPUT}) ELSE 0 END), 0) AS effective_input_tokens,
|
||||
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN ({total_context}) ELSE 0 END), 0) AS total_input_context,
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL AND {AGGREGATABLE} THEN MAX(usage.response_time_ms, 0) ELSE 0 END), 0) AS response_time_sum_ms,
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL AND {AGGREGATABLE} THEN 1 ELSE 0 END), 0) AS response_time_samples,
|
||||
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN {CACHE_5M} ELSE 0 END), 0) AS cache_creation_ephemeral_5m_tokens,
|
||||
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN {CACHE_1H} ELSE 0 END), 0) AS cache_creation_ephemeral_1h_tokens,
|
||||
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN COALESCE(usage.input_cost_usd, 0) ELSE 0 END), 0) AS input_cost,
|
||||
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN COALESCE(usage.output_cost_usd, 0) ELSE 0 END), 0) AS output_cost,
|
||||
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN COALESCE(usage.cache_creation_cost_usd, 0) ELSE 0 END), 0) AS cache_creation_cost,
|
||||
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN COALESCE(usage.cache_read_cost_usd, 0) ELSE 0 END), 0) AS cache_read_cost,
|
||||
COUNT(*) AS cache_hit_total_requests,
|
||||
COALESCE(SUM(CASE WHEN COALESCE(usage.cache_read_input_tokens, 0) > 0 THEN 1 ELSE 0 END), 0) AS cache_hit_requests,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN 1 ELSE 0 END), 0) AS completed_total_requests,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' AND COALESCE(usage.cache_read_input_tokens, 0) > 0 THEN 1 ELSE 0 END), 0) AS completed_cache_hit_requests,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN MAX(COALESCE(usage.input_tokens, 0), 0) ELSE 0 END), 0) AS completed_input_tokens,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN ({cache_creation}) ELSE 0 END), 0) AS completed_cache_creation_tokens,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN MAX(COALESCE(usage.cache_read_input_tokens, 0), 0) ELSE 0 END), 0) AS completed_cache_read_tokens,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN ({total_context}) ELSE 0 END), 0) AS completed_total_input_context,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN COALESCE(usage.cache_creation_cost_usd, 0) ELSE 0 END), 0) AS completed_cache_creation_cost,
|
||||
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN COALESCE(usage.cache_read_cost_usd, 0) ELSE 0 END), 0) AS completed_cache_read_cost,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0) ELSE 0 END), 0) AS settled_total_cost,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN 1 ELSE 0 END), 0) AS settled_total_requests,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN MAX(COALESCE(usage.input_tokens, 0), 0) ELSE 0 END), 0) AS settled_input_tokens,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN MAX(COALESCE(usage.output_tokens, 0), 0) ELSE 0 END), 0) AS settled_output_tokens,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN ({cache_creation}) ELSE 0 END), 0) AS settled_cache_creation_tokens,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN MAX(COALESCE(usage.cache_read_input_tokens, 0), 0) ELSE 0 END), 0) AS settled_cache_read_tokens,
|
||||
MIN(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_first_finalized_at_unix_secs,
|
||||
MAX(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_last_finalized_at_unix_secs
|
||||
FROM "usage" AS usage
|
||||
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
|
||||
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
|
||||
) AS aggregated
|
||||
WHERE target."date" = ?
|
||||
"#
|
||||
);
|
||||
sqlx::query(&sql)
|
||||
.bind(response.0)
|
||||
.bind(response.1)
|
||||
.bind(response.2)
|
||||
.bind(first_byte.0)
|
||||
.bind(first_byte.1)
|
||||
.bind(first_byte.2)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.bind(day_start)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn refresh_daily_existing_dimensions(
|
||||
tx: &mut sqlx::Transaction<'_, Sqlite>,
|
||||
day_start: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
) -> Result<(), DataLayerError> {
|
||||
let cache_creation = cache_creation_expr();
|
||||
let total_context = total_input_context_expr();
|
||||
let daily_model_sql = format!(
|
||||
r#"
|
||||
UPDATE stats_daily_model AS target
|
||||
SET response_time_sum_ms = aggregated.response_time_sum_ms,
|
||||
response_time_samples = aggregated.response_time_samples,
|
||||
cache_creation_ephemeral_5m_tokens = aggregated.cache_creation_ephemeral_5m_tokens,
|
||||
cache_creation_ephemeral_1h_tokens = aggregated.cache_creation_ephemeral_1h_tokens
|
||||
FROM (
|
||||
SELECT usage.model,
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN MAX(usage.response_time_ms, 0) ELSE 0 END), 0) AS response_time_sum_ms,
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0) AS response_time_samples,
|
||||
COALESCE(SUM({CACHE_5M}), 0) AS cache_creation_ephemeral_5m_tokens,
|
||||
COALESCE(SUM({CACHE_1H}), 0) AS cache_creation_ephemeral_1h_tokens
|
||||
FROM "usage" AS usage
|
||||
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
|
||||
AND {AGGREGATABLE} AND usage.model IS NOT NULL AND usage.model <> ''
|
||||
GROUP BY usage.model
|
||||
) AS aggregated
|
||||
WHERE target."date" = ? AND target.model = aggregated.model
|
||||
"#
|
||||
);
|
||||
sqlx::query(&daily_model_sql)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.bind(day_start)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
|
||||
let user_sql = format!(
|
||||
r#"
|
||||
UPDATE stats_user_daily AS target
|
||||
SET effective_input_tokens = aggregated.effective_input_tokens,
|
||||
total_input_context = aggregated.total_input_context,
|
||||
cache_creation_cost = aggregated.cache_creation_cost,
|
||||
cache_read_cost = aggregated.cache_read_cost,
|
||||
actual_total_cost = aggregated.actual_total_cost,
|
||||
response_time_sum_ms = aggregated.response_time_sum_ms,
|
||||
response_time_samples = aggregated.response_time_samples,
|
||||
cache_creation_ephemeral_5m_tokens = aggregated.cache_creation_ephemeral_5m_tokens,
|
||||
cache_creation_ephemeral_1h_tokens = aggregated.cache_creation_ephemeral_1h_tokens,
|
||||
settled_total_cost = aggregated.settled_total_cost,
|
||||
settled_total_requests = aggregated.settled_total_requests,
|
||||
settled_input_tokens = aggregated.settled_input_tokens,
|
||||
settled_output_tokens = aggregated.settled_output_tokens,
|
||||
settled_cache_creation_tokens = aggregated.settled_cache_creation_tokens,
|
||||
settled_cache_read_tokens = aggregated.settled_cache_read_tokens,
|
||||
settled_first_finalized_at_unix_secs = aggregated.settled_first_finalized_at_unix_secs,
|
||||
settled_last_finalized_at_unix_secs = aggregated.settled_last_finalized_at_unix_secs
|
||||
FROM (
|
||||
SELECT usage.user_id,
|
||||
COALESCE(SUM({EFFECTIVE_INPUT}), 0) AS effective_input_tokens,
|
||||
COALESCE(SUM({total_context}), 0) AS total_input_context,
|
||||
COALESCE(SUM(COALESCE(usage.cache_creation_cost_usd, 0)), 0) AS cache_creation_cost,
|
||||
COALESCE(SUM(COALESCE(usage.cache_read_cost_usd, 0)), 0) AS cache_read_cost,
|
||||
COALESCE(SUM(COALESCE(settlement.billing_actual_total_cost_usd, usage.actual_total_cost_usd, 0)), 0) AS actual_total_cost,
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN MAX(usage.response_time_ms, 0) ELSE 0 END), 0) AS response_time_sum_ms,
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0) AS response_time_samples,
|
||||
COALESCE(SUM({CACHE_5M}), 0) AS cache_creation_ephemeral_5m_tokens,
|
||||
COALESCE(SUM({CACHE_1H}), 0) AS cache_creation_ephemeral_1h_tokens,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0) ELSE 0 END), 0) AS settled_total_cost,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN 1 ELSE 0 END), 0) AS settled_total_requests,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN MAX(COALESCE(usage.input_tokens, 0), 0) ELSE 0 END), 0) AS settled_input_tokens,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN MAX(COALESCE(usage.output_tokens, 0), 0) ELSE 0 END), 0) AS settled_output_tokens,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN ({cache_creation}) ELSE 0 END), 0) AS settled_cache_creation_tokens,
|
||||
COALESCE(SUM(CASE WHEN {SETTLED} THEN MAX(COALESCE(usage.cache_read_input_tokens, 0), 0) ELSE 0 END), 0) AS settled_cache_read_tokens,
|
||||
MIN(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_first_finalized_at_unix_secs,
|
||||
MAX(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_last_finalized_at_unix_secs
|
||||
FROM "usage" AS usage
|
||||
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
|
||||
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
|
||||
AND usage.user_id IS NOT NULL AND usage.user_id <> '' AND {AGGREGATABLE}
|
||||
GROUP BY usage.user_id
|
||||
) AS aggregated
|
||||
WHERE target."date" = ? AND target.user_id = aggregated.user_id
|
||||
"#
|
||||
);
|
||||
sqlx::query(&user_sql)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.bind(day_start)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn upsert_user_dimension(
|
||||
tx: &mut sqlx::Transaction<'_, Sqlite>,
|
||||
table: &str,
|
||||
dimension_column: &str,
|
||||
dimension_expr: &str,
|
||||
dimension_filter: &str,
|
||||
day_start: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<(), DataLayerError> {
|
||||
let cache_creation = cache_creation_expr();
|
||||
let total_context = total_input_context_expr();
|
||||
let total_tokens = total_tokens_expr();
|
||||
let sql = format!(
|
||||
r#"
|
||||
INSERT INTO {table} (
|
||||
id, user_id, username, "date", {dimension_column}, total_requests, success_requests,
|
||||
input_tokens, effective_input_tokens, output_tokens, total_tokens, total_input_context,
|
||||
cache_creation_tokens, cache_creation_ephemeral_5m_tokens,
|
||||
cache_creation_ephemeral_1h_tokens, cache_read_tokens, total_cost, actual_total_cost,
|
||||
response_time_sum_ms, response_time_samples, successful_response_time_sum_ms,
|
||||
successful_response_time_samples, created_at, updated_at
|
||||
)
|
||||
SELECT lower(hex(randomblob(32))), usage.user_id,
|
||||
MAX(COALESCE(usage.username, users.username)), ?, {dimension_expr}, COUNT(*),
|
||||
COALESCE(SUM({SUCCESS}), 0),
|
||||
COALESCE(SUM(MAX(COALESCE(usage.input_tokens, 0), 0)), 0),
|
||||
COALESCE(SUM({EFFECTIVE_INPUT}), 0),
|
||||
COALESCE(SUM(MAX(COALESCE(usage.output_tokens, 0), 0)), 0),
|
||||
COALESCE(SUM({total_tokens}), 0), COALESCE(SUM({total_context}), 0),
|
||||
COALESCE(SUM({cache_creation}), 0), COALESCE(SUM({CACHE_5M}), 0),
|
||||
COALESCE(SUM({CACHE_1H}), 0),
|
||||
COALESCE(SUM(MAX(COALESCE(usage.cache_read_input_tokens, 0), 0)), 0),
|
||||
COALESCE(SUM(COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0)), 0),
|
||||
COALESCE(SUM(COALESCE(settlement.billing_actual_total_cost_usd, usage.actual_total_cost_usd, 0)), 0),
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN MAX(usage.response_time_ms, 0) ELSE 0 END), 0),
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0),
|
||||
COALESCE(SUM(CASE WHEN ({SUCCESS}) = 1 AND usage.response_time_ms IS NOT NULL THEN MAX(usage.response_time_ms, 0) ELSE 0 END), 0),
|
||||
COALESCE(SUM(CASE WHEN ({SUCCESS}) = 1 AND usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0),
|
||||
?, ?
|
||||
FROM "usage" AS usage
|
||||
LEFT JOIN users ON users.id = usage.user_id
|
||||
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
|
||||
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
|
||||
AND usage.user_id IS NOT NULL AND usage.user_id <> ''
|
||||
AND {dimension_filter} AND {AGGREGATABLE}
|
||||
GROUP BY usage.user_id, {dimension_expr}
|
||||
ON CONFLICT (user_id, "date", {dimension_column}) DO UPDATE SET
|
||||
username = COALESCE(excluded.username, {table}.username),
|
||||
total_requests = excluded.total_requests, success_requests = excluded.success_requests,
|
||||
input_tokens = excluded.input_tokens, effective_input_tokens = excluded.effective_input_tokens,
|
||||
output_tokens = excluded.output_tokens, total_tokens = excluded.total_tokens,
|
||||
total_input_context = excluded.total_input_context,
|
||||
cache_creation_tokens = excluded.cache_creation_tokens,
|
||||
cache_creation_ephemeral_5m_tokens = excluded.cache_creation_ephemeral_5m_tokens,
|
||||
cache_creation_ephemeral_1h_tokens = excluded.cache_creation_ephemeral_1h_tokens,
|
||||
cache_read_tokens = excluded.cache_read_tokens, total_cost = excluded.total_cost,
|
||||
actual_total_cost = excluded.actual_total_cost,
|
||||
response_time_sum_ms = excluded.response_time_sum_ms,
|
||||
response_time_samples = excluded.response_time_samples,
|
||||
successful_response_time_sum_ms = excluded.successful_response_time_sum_ms,
|
||||
successful_response_time_samples = excluded.successful_response_time_samples,
|
||||
updated_at = excluded.updated_at
|
||||
"#
|
||||
);
|
||||
sqlx::query(&sql)
|
||||
.bind(day_start)
|
||||
.bind(now_unix_secs)
|
||||
.bind(now_unix_secs)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn upsert_model_provider_rows(
|
||||
tx: &mut sqlx::Transaction<'_, Sqlite>,
|
||||
day_start: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<(), DataLayerError> {
|
||||
let total_tokens = total_tokens_expr();
|
||||
let model_provider_sql = format!(
|
||||
r#"
|
||||
INSERT INTO stats_daily_model_provider (
|
||||
id, "date", model, provider_name, total_requests, total_tokens, total_cost,
|
||||
response_time_sum_ms, response_time_samples, created_at, updated_at
|
||||
)
|
||||
SELECT lower(hex(randomblob(32))), ?, usage.model, usage.provider_name, COUNT(*),
|
||||
COALESCE(SUM({total_tokens}), 0),
|
||||
COALESCE(SUM(COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0)), 0),
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN MAX(usage.response_time_ms, 0) ELSE 0 END), 0),
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0), ?, ?
|
||||
FROM "usage" AS usage
|
||||
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
|
||||
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
|
||||
AND usage.model IS NOT NULL AND usage.model <> '' AND {AGGREGATABLE}
|
||||
GROUP BY usage.model, usage.provider_name
|
||||
ON CONFLICT ("date", model, provider_name) DO UPDATE SET
|
||||
total_requests = excluded.total_requests, total_tokens = excluded.total_tokens,
|
||||
total_cost = excluded.total_cost, response_time_sum_ms = excluded.response_time_sum_ms,
|
||||
response_time_samples = excluded.response_time_samples, updated_at = excluded.updated_at
|
||||
"#
|
||||
);
|
||||
sqlx::query(&model_provider_sql)
|
||||
.bind(day_start)
|
||||
.bind(now_unix_secs)
|
||||
.bind(now_unix_secs)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
|
||||
let user_model_provider_sql = format!(
|
||||
r#"
|
||||
INSERT INTO stats_user_daily_model_provider (
|
||||
id, user_id, username, "date", model, provider_name, total_requests, total_tokens,
|
||||
total_cost, response_time_sum_ms, response_time_samples, created_at, updated_at
|
||||
)
|
||||
SELECT lower(hex(randomblob(32))), usage.user_id, MAX(COALESCE(usage.username, users.username)),
|
||||
?, usage.model, usage.provider_name, COUNT(*), COALESCE(SUM({total_tokens}), 0),
|
||||
COALESCE(SUM(COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0)), 0),
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN MAX(usage.response_time_ms, 0) ELSE 0 END), 0),
|
||||
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0), ?, ?
|
||||
FROM "usage" AS usage
|
||||
LEFT JOIN users ON users.id = usage.user_id
|
||||
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
|
||||
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
|
||||
AND usage.user_id IS NOT NULL AND usage.user_id <> ''
|
||||
AND usage.model IS NOT NULL AND usage.model <> '' AND {AGGREGATABLE}
|
||||
GROUP BY usage.user_id, usage.model, usage.provider_name
|
||||
ON CONFLICT (user_id, "date", model, provider_name) DO UPDATE SET
|
||||
username = COALESCE(excluded.username, stats_user_daily_model_provider.username),
|
||||
total_requests = excluded.total_requests, total_tokens = excluded.total_tokens,
|
||||
total_cost = excluded.total_cost, response_time_sum_ms = excluded.response_time_sum_ms,
|
||||
response_time_samples = excluded.response_time_samples, updated_at = excluded.updated_at
|
||||
"#
|
||||
);
|
||||
sqlx::query(&user_model_provider_sql)
|
||||
.bind(day_start)
|
||||
.bind(now_unix_secs)
|
||||
.bind(now_unix_secs)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn upsert_cost_savings_rows(
|
||||
tx: &mut sqlx::Transaction<'_, Sqlite>,
|
||||
day_start: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<(), DataLayerError> {
|
||||
for (table, dimensions) in [
|
||||
("stats_daily_cost_savings", Vec::new()),
|
||||
(
|
||||
"stats_daily_cost_savings_provider",
|
||||
vec![("provider_name", "COALESCE(usage.provider_name, '')")],
|
||||
),
|
||||
(
|
||||
"stats_daily_cost_savings_model",
|
||||
vec![("model", "COALESCE(usage.model, '')")],
|
||||
),
|
||||
(
|
||||
"stats_daily_cost_savings_model_provider",
|
||||
vec![
|
||||
("model", "COALESCE(usage.model, '')"),
|
||||
("provider_name", "COALESCE(usage.provider_name, '')"),
|
||||
],
|
||||
),
|
||||
] {
|
||||
upsert_cost_savings_dimension(
|
||||
tx,
|
||||
table,
|
||||
false,
|
||||
&dimensions,
|
||||
day_start,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
now_unix_secs,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
for (table, dimensions) in [
|
||||
("stats_user_daily_cost_savings", Vec::new()),
|
||||
(
|
||||
"stats_user_daily_cost_savings_provider",
|
||||
vec![("provider_name", "COALESCE(usage.provider_name, '')")],
|
||||
),
|
||||
(
|
||||
"stats_user_daily_cost_savings_model",
|
||||
vec![("model", "COALESCE(usage.model, '')")],
|
||||
),
|
||||
(
|
||||
"stats_user_daily_cost_savings_model_provider",
|
||||
vec![
|
||||
("model", "COALESCE(usage.model, '')"),
|
||||
("provider_name", "COALESCE(usage.provider_name, '')"),
|
||||
],
|
||||
),
|
||||
] {
|
||||
upsert_cost_savings_dimension(
|
||||
tx,
|
||||
table,
|
||||
true,
|
||||
&dimensions,
|
||||
day_start,
|
||||
start_unix_secs,
|
||||
end_unix_secs,
|
||||
now_unix_secs,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn upsert_cost_savings_dimension(
|
||||
tx: &mut sqlx::Transaction<'_, Sqlite>,
|
||||
table: &str,
|
||||
per_user: bool,
|
||||
dimensions: &[(&str, &str)],
|
||||
day_start: i64,
|
||||
start_unix_secs: i64,
|
||||
end_unix_secs: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<(), DataLayerError> {
|
||||
let dimension_columns = dimensions
|
||||
.iter()
|
||||
.map(|(column, _)| *column)
|
||||
.collect::<Vec<_>>();
|
||||
let dimension_exprs = dimensions
|
||||
.iter()
|
||||
.map(|(_, expression)| *expression)
|
||||
.collect::<Vec<_>>();
|
||||
let user_columns = if per_user { "user_id, username, " } else { "" };
|
||||
let user_select = if per_user {
|
||||
"usage.user_id, MAX(COALESCE(usage.username, users.username)), "
|
||||
} else {
|
||||
""
|
||||
};
|
||||
let user_join = if per_user {
|
||||
"LEFT JOIN users ON users.id = usage.user_id"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
let user_filter = if per_user {
|
||||
"AND usage.user_id IS NOT NULL AND usage.user_id <> ''"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
let mut conflict_columns = vec!["\"date\""];
|
||||
let mut group_by = Vec::new();
|
||||
if per_user {
|
||||
conflict_columns.insert(0, "user_id");
|
||||
group_by.push("usage.user_id");
|
||||
}
|
||||
conflict_columns.extend(dimension_columns.iter().copied());
|
||||
group_by.extend(dimension_exprs.iter().copied());
|
||||
let dimension_columns_sql = if dimension_columns.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!("{}, ", dimension_columns.join(", "))
|
||||
};
|
||||
let dimension_select_sql = if dimension_exprs.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!("{}, ", dimension_exprs.join(", "))
|
||||
};
|
||||
let group_by_sql = if group_by.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!("GROUP BY {}", group_by.join(", "))
|
||||
};
|
||||
let sql = format!(
|
||||
r#"
|
||||
INSERT INTO {table} (
|
||||
id, {user_columns}"date", {dimension_columns_sql}cache_read_tokens,
|
||||
cache_read_cost, cache_creation_cost, estimated_full_cost, created_at, updated_at
|
||||
)
|
||||
SELECT lower(hex(randomblob(32))), {user_select}?, {dimension_select_sql}
|
||||
COALESCE(SUM(MAX(COALESCE(usage.cache_read_input_tokens, 0), 0)), 0),
|
||||
COALESCE(SUM(COALESCE(usage.cache_read_cost_usd, 0)), 0),
|
||||
COALESCE(SUM(COALESCE(usage.cache_creation_cost_usd, 0)), 0),
|
||||
COALESCE(SUM(
|
||||
COALESCE(settlement.input_price_per_1m, usage.input_price_per_1m, 0)
|
||||
* MAX(COALESCE(usage.cache_read_input_tokens, 0), 0) / 1000000.0
|
||||
), 0), ?, ?
|
||||
FROM "usage" AS usage
|
||||
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
|
||||
{user_join}
|
||||
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ? {user_filter}
|
||||
{group_by_sql}
|
||||
ON CONFLICT ({}) DO UPDATE SET
|
||||
{}cache_read_tokens = excluded.cache_read_tokens,
|
||||
cache_read_cost = excluded.cache_read_cost,
|
||||
cache_creation_cost = excluded.cache_creation_cost,
|
||||
estimated_full_cost = excluded.estimated_full_cost,
|
||||
updated_at = excluded.updated_at
|
||||
"#,
|
||||
conflict_columns.join(", "),
|
||||
if per_user {
|
||||
format!("username = COALESCE(excluded.username, {table}.username), ")
|
||||
} else {
|
||||
String::new()
|
||||
}
|
||||
);
|
||||
sqlx::query(&sql)
|
||||
.bind(day_start)
|
||||
.bind(now_unix_secs)
|
||||
.bind(now_unix_secs)
|
||||
.bind(start_unix_secs)
|
||||
.bind(end_unix_secs)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn refresh_user_summary(
|
||||
tx: &mut sqlx::Transaction<'_, Sqlite>,
|
||||
cutoff_date: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<(), DataLayerError> {
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO stats_user_summary (
|
||||
id, user_id, username, cutoff_date, all_time_requests, all_time_success_requests,
|
||||
all_time_error_requests, all_time_input_tokens, all_time_output_tokens,
|
||||
all_time_cache_creation_tokens, all_time_cache_read_tokens, all_time_cost,
|
||||
all_time_actual_cost, active_days, first_active_date, last_active_date,
|
||||
created_at, updated_at
|
||||
)
|
||||
SELECT lower(hex(randomblob(32))), user_id, MAX(username), ?,
|
||||
COALESCE(SUM(total_requests), 0), COALESCE(SUM(success_requests), 0),
|
||||
COALESCE(SUM(error_requests), 0), COALESCE(SUM(input_tokens), 0),
|
||||
COALESCE(SUM(output_tokens), 0), COALESCE(SUM(cache_creation_tokens), 0),
|
||||
COALESCE(SUM(cache_read_tokens), 0), COALESCE(SUM(total_cost), 0),
|
||||
COALESCE(SUM(actual_total_cost), 0),
|
||||
COALESCE(SUM(CASE WHEN total_requests > 0 THEN 1 ELSE 0 END), 0),
|
||||
MIN(CASE WHEN total_requests > 0 THEN "date" END),
|
||||
MAX(CASE WHEN total_requests > 0 THEN "date" END), ?, ?
|
||||
FROM stats_user_daily
|
||||
WHERE "date" < ?
|
||||
GROUP BY user_id
|
||||
ON CONFLICT (user_id) DO UPDATE SET
|
||||
username = COALESCE(excluded.username, stats_user_summary.username),
|
||||
cutoff_date = excluded.cutoff_date, all_time_requests = excluded.all_time_requests,
|
||||
all_time_success_requests = excluded.all_time_success_requests,
|
||||
all_time_error_requests = excluded.all_time_error_requests,
|
||||
all_time_input_tokens = excluded.all_time_input_tokens,
|
||||
all_time_output_tokens = excluded.all_time_output_tokens,
|
||||
all_time_cache_creation_tokens = excluded.all_time_cache_creation_tokens,
|
||||
all_time_cache_read_tokens = excluded.all_time_cache_read_tokens,
|
||||
all_time_cost = excluded.all_time_cost, all_time_actual_cost = excluded.all_time_actual_cost,
|
||||
active_days = excluded.active_days, first_active_date = excluded.first_active_date,
|
||||
last_active_date = excluded.last_active_date, updated_at = excluded.updated_at
|
||||
"#,
|
||||
)
|
||||
.bind(cutoff_date)
|
||||
.bind(now_unix_secs)
|
||||
.bind(now_unix_secs)
|
||||
.bind(cutoff_date)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
refresh_global_summary(tx, cutoff_date, now_unix_secs).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn refresh_global_summary(
|
||||
tx: &mut sqlx::Transaction<'_, Sqlite>,
|
||||
cutoff_date: i64,
|
||||
now_unix_secs: i64,
|
||||
) -> Result<(), DataLayerError> {
|
||||
let existing_id: Option<String> =
|
||||
sqlx::query_scalar("SELECT id FROM stats_summary ORDER BY created_at, id LIMIT 1")
|
||||
.fetch_optional(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
let summary_id = existing_id.unwrap_or_else(|| stats_id("stats-summary"));
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO stats_summary (
|
||||
id, cutoff_date, all_time_requests, all_time_success_requests,
|
||||
all_time_error_requests, all_time_input_tokens, all_time_output_tokens,
|
||||
all_time_cache_creation_tokens, all_time_cache_read_tokens, all_time_cost,
|
||||
all_time_actual_cost, total_users, active_users, total_api_keys,
|
||||
active_api_keys, created_at, updated_at
|
||||
)
|
||||
SELECT ?, ?, COALESCE(SUM(total_requests), 0), COALESCE(SUM(success_requests), 0),
|
||||
COALESCE(SUM(error_requests), 0), COALESCE(SUM(input_tokens), 0),
|
||||
COALESCE(SUM(output_tokens), 0), COALESCE(SUM(cache_creation_tokens), 0),
|
||||
COALESCE(SUM(cache_read_tokens), 0), COALESCE(SUM(total_cost), 0),
|
||||
COALESCE(SUM(actual_total_cost), 0),
|
||||
(SELECT COUNT(*) FROM users),
|
||||
(SELECT COUNT(*) FROM users WHERE is_active <> 0),
|
||||
(SELECT COUNT(*) FROM api_keys),
|
||||
(SELECT COUNT(*) FROM api_keys WHERE is_active <> 0), ?, ?
|
||||
FROM stats_daily
|
||||
WHERE "date" < ?
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
cutoff_date = excluded.cutoff_date,
|
||||
all_time_requests = excluded.all_time_requests,
|
||||
all_time_success_requests = excluded.all_time_success_requests,
|
||||
all_time_error_requests = excluded.all_time_error_requests,
|
||||
all_time_input_tokens = excluded.all_time_input_tokens,
|
||||
all_time_output_tokens = excluded.all_time_output_tokens,
|
||||
all_time_cache_creation_tokens = excluded.all_time_cache_creation_tokens,
|
||||
all_time_cache_read_tokens = excluded.all_time_cache_read_tokens,
|
||||
all_time_cost = excluded.all_time_cost,
|
||||
all_time_actual_cost = excluded.all_time_actual_cost,
|
||||
total_users = excluded.total_users, active_users = excluded.active_users,
|
||||
total_api_keys = excluded.total_api_keys, active_api_keys = excluded.active_api_keys,
|
||||
updated_at = excluded.updated_at
|
||||
"#,
|
||||
)
|
||||
.bind(summary_id)
|
||||
.bind(cutoff_date)
|
||||
.bind(now_unix_secs)
|
||||
.bind(now_unix_secs)
|
||||
.bind(cutoff_date)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
use chrono::{DateTime, Utc};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::DataLayerError;
|
||||
|
||||
pub(crate) fn unix_secs(value: DateTime<Utc>) -> i64 {
|
||||
value.timestamp().max(0)
|
||||
}
|
||||
|
||||
pub(crate) fn utc_from_unix_secs(
|
||||
value: i64,
|
||||
field_name: &str,
|
||||
) -> Result<DateTime<Utc>, DataLayerError> {
|
||||
DateTime::<Utc>::from_timestamp(value, 0).ok_or_else(|| {
|
||||
DataLayerError::UnexpectedValue(format!("{field_name} contains invalid timestamp {value}"))
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn stats_id(value: &str) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(value.as_bytes());
|
||||
hasher
|
||||
.finalize()
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect()
|
||||
}
|
||||
@@ -1,11 +1,7 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
#[cfg(feature = "mysql")]
|
||||
use super::MysqlBackend;
|
||||
#[cfg(feature = "postgres")]
|
||||
use super::PostgresBackend;
|
||||
#[cfg(feature = "sqlite")]
|
||||
use super::SqliteBackend;
|
||||
use crate::repository::system::{
|
||||
AdminSystemPurgeSummary, AdminSystemPurgeTarget, AdminSystemUsageAggregateImportMode,
|
||||
AdminSystemUsageAggregateImportSummary, AdminSystemUsageAggregateSnapshot,
|
||||
@@ -13,12 +9,8 @@ use crate::repository::system::{
|
||||
};
|
||||
use crate::DataLayerError;
|
||||
|
||||
#[cfg(feature = "mysql")]
|
||||
mod mysql;
|
||||
#[cfg(feature = "postgres")]
|
||||
mod postgres;
|
||||
#[cfg(feature = "sqlite")]
|
||||
mod sqlite;
|
||||
|
||||
const ADMIN_CONFIG_PURGE_TABLES: &[&str] = &[
|
||||
"api_key_provider_mappings",
|
||||
@@ -124,11 +116,6 @@ fn checked_sql_identifier(value: &str) -> Result<&str, DataLayerError> {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "mysql", feature = "sqlite"))]
|
||||
fn current_unix_secs() -> u64 {
|
||||
chrono::Utc::now().timestamp().max(0) as u64
|
||||
}
|
||||
|
||||
fn i64_from_u64(value: u64, field_name: &str) -> Result<i64, DataLayerError> {
|
||||
i64::try_from(value)
|
||||
.map_err(|_| DataLayerError::InvalidInput(format!("{field_name} exceeds i64 range")))
|
||||
@@ -176,17 +163,3 @@ fn should_skip_imported_aggregate(
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "mysql", feature = "sqlite"))]
|
||||
fn serialize_json_value(value: &serde_json::Value) -> Result<String, DataLayerError> {
|
||||
serde_json::to_string(value).map_err(|err| {
|
||||
DataLayerError::UnexpectedValue(format!("invalid system config JSON value: {err}"))
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "mysql", feature = "sqlite"))]
|
||||
fn parse_json_value(value: String) -> Result<serde_json::Value, DataLayerError> {
|
||||
serde_json::from_str(&value).map_err(|err| {
|
||||
DataLayerError::UnexpectedValue(format!("invalid system config JSON value: {err}"))
|
||||
})
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,14 +1,7 @@
|
||||
//! Driver-specific wallet usage aggregation adapters.
|
||||
|
||||
#[cfg(feature = "mysql")]
|
||||
mod mysql;
|
||||
#[cfg(feature = "postgres")]
|
||||
mod postgres;
|
||||
#[cfg(feature = "sqlite")]
|
||||
mod sqlite;
|
||||
|
||||
#[cfg(any(feature = "mysql", feature = "sqlite"))]
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::DataLayerError;
|
||||
|
||||
@@ -27,43 +20,9 @@ pub(super) fn unix_secs_to_utc(
|
||||
.ok_or_else(|| DataLayerError::InvalidInput(format!("invalid {field_name}: {value}")))
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "mysql", feature = "sqlite"))]
|
||||
pub(super) fn wallet_daily_usage_id(
|
||||
wallet_id: &str,
|
||||
billing_date: &str,
|
||||
billing_timezone: &str,
|
||||
) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(b"wallet-daily-usage:");
|
||||
hasher.update(wallet_id.as_bytes());
|
||||
hasher.update(b":");
|
||||
hasher.update(billing_date.as_bytes());
|
||||
hasher.update(b":");
|
||||
hasher.update(billing_timezone.as_bytes());
|
||||
hasher
|
||||
.finalize()
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::u64_to_i64;
|
||||
#[cfg(any(feature = "mysql", feature = "sqlite"))]
|
||||
use super::wallet_daily_usage_id;
|
||||
|
||||
#[cfg(any(feature = "mysql", feature = "sqlite"))]
|
||||
#[test]
|
||||
fn wallet_daily_usage_ids_are_stable_and_partition_specific() {
|
||||
let first = wallet_daily_usage_id("wallet-1", "2026-07-13", "UTC");
|
||||
let same = wallet_daily_usage_id("wallet-1", "2026-07-13", "UTC");
|
||||
let other_day = wallet_daily_usage_id("wallet-1", "2026-07-14", "UTC");
|
||||
|
||||
assert_eq!(first, same);
|
||||
assert_ne!(first, other_day);
|
||||
assert_eq!(first.len(), 64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_timestamps_outside_i64_range() {
|
||||
|
||||
@@ -1,155 +0,0 @@
|
||||
use sqlx::Row;
|
||||
|
||||
use crate::backend::MysqlBackend;
|
||||
use crate::error::SqlResultExt;
|
||||
use crate::{DataLayerError, WalletDailyUsageAggregationInput, WalletDailyUsageAggregationResult};
|
||||
|
||||
use super::{u64_to_i64, wallet_daily_usage_id};
|
||||
|
||||
const SELECT_WALLET_DAILY_USAGE_AGGREGATES_SQL: &str = r#"
|
||||
SELECT
|
||||
usage_settlement_snapshots.wallet_id AS wallet_id,
|
||||
CAST(COUNT(*) AS SIGNED) AS total_requests,
|
||||
CAST(COALESCE(SUM(`usage`.total_cost_usd), 0) AS DOUBLE) AS total_cost_usd,
|
||||
CAST(COALESCE(SUM(`usage`.input_tokens), 0) AS SIGNED) AS input_tokens,
|
||||
CAST(COALESCE(SUM(`usage`.output_tokens), 0) AS SIGNED) AS output_tokens,
|
||||
CAST(COALESCE(SUM(`usage`.cache_creation_input_tokens), 0) AS SIGNED) AS cache_creation_tokens,
|
||||
CAST(COALESCE(SUM(`usage`.cache_read_input_tokens), 0) AS SIGNED) AS cache_read_tokens,
|
||||
MIN(COALESCE(usage_settlement_snapshots.finalized_at, `usage`.finalized_at)) AS first_finalized_at,
|
||||
MAX(COALESCE(usage_settlement_snapshots.finalized_at, `usage`.finalized_at)) AS last_finalized_at
|
||||
FROM `usage`
|
||||
JOIN usage_settlement_snapshots
|
||||
ON usage_settlement_snapshots.request_id = `usage`.request_id
|
||||
WHERE usage_settlement_snapshots.wallet_id IS NOT NULL
|
||||
AND usage_settlement_snapshots.wallet_id <> ''
|
||||
AND COALESCE(usage_settlement_snapshots.billing_status, `usage`.billing_status) = 'settled'
|
||||
AND `usage`.total_cost_usd > 0
|
||||
AND COALESCE(usage_settlement_snapshots.finalized_at, `usage`.finalized_at) >= ?
|
||||
AND COALESCE(usage_settlement_snapshots.finalized_at, `usage`.finalized_at) < ?
|
||||
GROUP BY usage_settlement_snapshots.wallet_id
|
||||
"#;
|
||||
|
||||
impl MysqlBackend {
|
||||
pub async fn aggregate_wallet_daily_usage(
|
||||
&self,
|
||||
input: &WalletDailyUsageAggregationInput,
|
||||
) -> Result<WalletDailyUsageAggregationResult, DataLayerError> {
|
||||
let window_start = u64_to_i64(input.window_start_unix_secs, "window_start")?;
|
||||
let window_end = u64_to_i64(input.window_end_unix_secs, "window_end")?;
|
||||
let aggregated_at = u64_to_i64(input.aggregated_at_unix_secs, "aggregated_at")?;
|
||||
let mut tx = self.pool().begin().await.map_sql_err()?;
|
||||
|
||||
let rows = sqlx::query(SELECT_WALLET_DAILY_USAGE_AGGREGATES_SQL)
|
||||
.bind(window_start)
|
||||
.bind(window_end)
|
||||
.fetch_all(&mut *tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
|
||||
let mut aggregated_wallets = 0usize;
|
||||
for row in rows {
|
||||
let wallet_id: String = row.try_get("wallet_id").map_sql_err()?;
|
||||
sqlx::query(
|
||||
r#"
|
||||
DELETE FROM wallet_daily_usage_ledgers
|
||||
WHERE wallet_id = ?
|
||||
AND billing_date = ?
|
||||
AND billing_timezone = ?
|
||||
"#,
|
||||
)
|
||||
.bind(&wallet_id)
|
||||
.bind(&input.billing_date)
|
||||
.bind(&input.billing_timezone)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO wallet_daily_usage_ledgers (
|
||||
id,
|
||||
wallet_id,
|
||||
billing_date,
|
||||
billing_timezone,
|
||||
total_cost_usd,
|
||||
total_requests,
|
||||
input_tokens,
|
||||
output_tokens,
|
||||
cache_creation_tokens,
|
||||
cache_read_tokens,
|
||||
first_finalized_at,
|
||||
last_finalized_at,
|
||||
aggregated_at,
|
||||
created_at,
|
||||
updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
"#,
|
||||
)
|
||||
.bind(wallet_daily_usage_id(
|
||||
&wallet_id,
|
||||
&input.billing_date,
|
||||
&input.billing_timezone,
|
||||
))
|
||||
.bind(&wallet_id)
|
||||
.bind(&input.billing_date)
|
||||
.bind(&input.billing_timezone)
|
||||
.bind(row.try_get::<f64, _>("total_cost_usd").map_sql_err()?)
|
||||
.bind(row.try_get::<i64, _>("total_requests").map_sql_err()?)
|
||||
.bind(row.try_get::<i64, _>("input_tokens").map_sql_err()?)
|
||||
.bind(row.try_get::<i64, _>("output_tokens").map_sql_err()?)
|
||||
.bind(
|
||||
row.try_get::<i64, _>("cache_creation_tokens")
|
||||
.map_sql_err()?,
|
||||
)
|
||||
.bind(row.try_get::<i64, _>("cache_read_tokens").map_sql_err()?)
|
||||
.bind(
|
||||
row.try_get::<Option<i64>, _>("first_finalized_at")
|
||||
.map_sql_err()?,
|
||||
)
|
||||
.bind(
|
||||
row.try_get::<Option<i64>, _>("last_finalized_at")
|
||||
.map_sql_err()?,
|
||||
)
|
||||
.bind(aggregated_at)
|
||||
.bind(aggregated_at)
|
||||
.bind(aggregated_at)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
aggregated_wallets += 1;
|
||||
}
|
||||
|
||||
let deleted_stale_ledgers = sqlx::query(
|
||||
r#"
|
||||
DELETE FROM wallet_daily_usage_ledgers
|
||||
WHERE billing_date = ?
|
||||
AND billing_timezone = ?
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM `usage`
|
||||
JOIN usage_settlement_snapshots
|
||||
ON usage_settlement_snapshots.request_id = `usage`.request_id
|
||||
WHERE usage_settlement_snapshots.wallet_id = wallet_daily_usage_ledgers.wallet_id
|
||||
AND COALESCE(usage_settlement_snapshots.billing_status, `usage`.billing_status) = 'settled'
|
||||
AND `usage`.total_cost_usd > 0
|
||||
AND COALESCE(usage_settlement_snapshots.finalized_at, `usage`.finalized_at) >= ?
|
||||
AND COALESCE(usage_settlement_snapshots.finalized_at, `usage`.finalized_at) < ?
|
||||
)
|
||||
"#,
|
||||
)
|
||||
.bind(&input.billing_date)
|
||||
.bind(&input.billing_timezone)
|
||||
.bind(window_start)
|
||||
.bind(window_end)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_sql_err()?
|
||||
.rows_affected();
|
||||
|
||||
tx.commit().await.map_sql_err()?;
|
||||
Ok(WalletDailyUsageAggregationResult {
|
||||
aggregated_wallets,
|
||||
deleted_stale_ledgers: usize::try_from(deleted_stale_ledgers).unwrap_or(usize::MAX),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,156 +0,0 @@
|
||||
use sqlx::Row;
|
||||
|
||||
use crate::backend::SqliteBackend;
|
||||
use crate::driver::sqlite::sqlite_real;
|
||||
use crate::error::SqlResultExt;
|
||||
use crate::{DataLayerError, WalletDailyUsageAggregationInput, WalletDailyUsageAggregationResult};
|
||||
|
||||
use super::{u64_to_i64, wallet_daily_usage_id};
|
||||
|
||||
const SELECT_WALLET_DAILY_USAGE_AGGREGATES_SQL: &str = r#"
|
||||
SELECT
|
||||
usage_settlement_snapshots.wallet_id AS wallet_id,
|
||||
COUNT(*) AS total_requests,
|
||||
CAST(COALESCE(SUM("usage".total_cost_usd), 0) AS REAL) AS total_cost_usd,
|
||||
COALESCE(SUM("usage".input_tokens), 0) AS input_tokens,
|
||||
COALESCE(SUM("usage".output_tokens), 0) AS output_tokens,
|
||||
COALESCE(SUM("usage".cache_creation_input_tokens), 0) AS cache_creation_tokens,
|
||||
COALESCE(SUM("usage".cache_read_input_tokens), 0) AS cache_read_tokens,
|
||||
MIN(COALESCE(usage_settlement_snapshots.finalized_at, "usage".finalized_at)) AS first_finalized_at,
|
||||
MAX(COALESCE(usage_settlement_snapshots.finalized_at, "usage".finalized_at)) AS last_finalized_at
|
||||
FROM "usage"
|
||||
JOIN usage_settlement_snapshots
|
||||
ON usage_settlement_snapshots.request_id = "usage".request_id
|
||||
WHERE usage_settlement_snapshots.wallet_id IS NOT NULL
|
||||
AND usage_settlement_snapshots.wallet_id <> ''
|
||||
AND COALESCE(usage_settlement_snapshots.billing_status, "usage".billing_status) = 'settled'
|
||||
AND "usage".total_cost_usd > 0
|
||||
AND COALESCE(usage_settlement_snapshots.finalized_at, "usage".finalized_at) >= ?
|
||||
AND COALESCE(usage_settlement_snapshots.finalized_at, "usage".finalized_at) < ?
|
||||
GROUP BY usage_settlement_snapshots.wallet_id
|
||||
"#;
|
||||
|
||||
impl SqliteBackend {
|
||||
pub async fn aggregate_wallet_daily_usage(
|
||||
&self,
|
||||
input: &WalletDailyUsageAggregationInput,
|
||||
) -> Result<WalletDailyUsageAggregationResult, DataLayerError> {
|
||||
let window_start = u64_to_i64(input.window_start_unix_secs, "window_start")?;
|
||||
let window_end = u64_to_i64(input.window_end_unix_secs, "window_end")?;
|
||||
let aggregated_at = u64_to_i64(input.aggregated_at_unix_secs, "aggregated_at")?;
|
||||
let mut tx = self.pool().begin().await.map_sql_err()?;
|
||||
|
||||
let rows = sqlx::query(SELECT_WALLET_DAILY_USAGE_AGGREGATES_SQL)
|
||||
.bind(window_start)
|
||||
.bind(window_end)
|
||||
.fetch_all(&mut *tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
|
||||
let mut aggregated_wallets = 0usize;
|
||||
for row in rows {
|
||||
let wallet_id: String = row.try_get("wallet_id").map_sql_err()?;
|
||||
sqlx::query(
|
||||
r#"
|
||||
DELETE FROM wallet_daily_usage_ledgers
|
||||
WHERE wallet_id = ?
|
||||
AND billing_date = ?
|
||||
AND billing_timezone = ?
|
||||
"#,
|
||||
)
|
||||
.bind(&wallet_id)
|
||||
.bind(&input.billing_date)
|
||||
.bind(&input.billing_timezone)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO wallet_daily_usage_ledgers (
|
||||
id,
|
||||
wallet_id,
|
||||
billing_date,
|
||||
billing_timezone,
|
||||
total_cost_usd,
|
||||
total_requests,
|
||||
input_tokens,
|
||||
output_tokens,
|
||||
cache_creation_tokens,
|
||||
cache_read_tokens,
|
||||
first_finalized_at,
|
||||
last_finalized_at,
|
||||
aggregated_at,
|
||||
created_at,
|
||||
updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
"#,
|
||||
)
|
||||
.bind(wallet_daily_usage_id(
|
||||
&wallet_id,
|
||||
&input.billing_date,
|
||||
&input.billing_timezone,
|
||||
))
|
||||
.bind(&wallet_id)
|
||||
.bind(&input.billing_date)
|
||||
.bind(&input.billing_timezone)
|
||||
.bind(sqlite_real(&row, "total_cost_usd")?)
|
||||
.bind(row.try_get::<i64, _>("total_requests").map_sql_err()?)
|
||||
.bind(row.try_get::<i64, _>("input_tokens").map_sql_err()?)
|
||||
.bind(row.try_get::<i64, _>("output_tokens").map_sql_err()?)
|
||||
.bind(
|
||||
row.try_get::<i64, _>("cache_creation_tokens")
|
||||
.map_sql_err()?,
|
||||
)
|
||||
.bind(row.try_get::<i64, _>("cache_read_tokens").map_sql_err()?)
|
||||
.bind(
|
||||
row.try_get::<Option<i64>, _>("first_finalized_at")
|
||||
.map_sql_err()?,
|
||||
)
|
||||
.bind(
|
||||
row.try_get::<Option<i64>, _>("last_finalized_at")
|
||||
.map_sql_err()?,
|
||||
)
|
||||
.bind(aggregated_at)
|
||||
.bind(aggregated_at)
|
||||
.bind(aggregated_at)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
aggregated_wallets += 1;
|
||||
}
|
||||
|
||||
let deleted_stale_ledgers = sqlx::query(
|
||||
r#"
|
||||
DELETE FROM wallet_daily_usage_ledgers
|
||||
WHERE billing_date = ?
|
||||
AND billing_timezone = ?
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM "usage"
|
||||
JOIN usage_settlement_snapshots
|
||||
ON usage_settlement_snapshots.request_id = "usage".request_id
|
||||
WHERE usage_settlement_snapshots.wallet_id = wallet_daily_usage_ledgers.wallet_id
|
||||
AND COALESCE(usage_settlement_snapshots.billing_status, "usage".billing_status) = 'settled'
|
||||
AND "usage".total_cost_usd > 0
|
||||
AND COALESCE(usage_settlement_snapshots.finalized_at, "usage".finalized_at) >= ?
|
||||
AND COALESCE(usage_settlement_snapshots.finalized_at, "usage".finalized_at) < ?
|
||||
)
|
||||
"#,
|
||||
)
|
||||
.bind(&input.billing_date)
|
||||
.bind(&input.billing_timezone)
|
||||
.bind(window_start)
|
||||
.bind(window_end)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_sql_err()?
|
||||
.rows_affected();
|
||||
|
||||
tx.commit().await.map_sql_err()?;
|
||||
Ok(WalletDailyUsageAggregationResult {
|
||||
aggregated_wallets,
|
||||
deleted_stale_ledgers: usize::try_from(deleted_stale_ledgers).unwrap_or(usize::MAX),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,8 @@
|
||||
use std::fmt;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[cfg(feature = "mysql")]
|
||||
use super::MysqlBackend;
|
||||
#[cfg(feature = "postgres")]
|
||||
use super::PostgresBackend;
|
||||
#[cfg(feature = "sqlite")]
|
||||
use super::SqliteBackend;
|
||||
use crate::repository::announcements::AnnouncementWriteRepository;
|
||||
use crate::repository::auth::AuthApiKeyWriteRepository;
|
||||
use crate::repository::auth_modules::AuthModuleWriteRepository;
|
||||
@@ -79,22 +75,12 @@ impl fmt::Debug for DataWriteRepositories {
|
||||
impl DataWriteRepositories {
|
||||
pub(crate) fn from_backends(
|
||||
#[cfg(feature = "postgres")] postgres: Option<&PostgresBackend>,
|
||||
#[cfg(feature = "mysql")] mysql: Option<&MysqlBackend>,
|
||||
#[cfg(feature = "sqlite")] sqlite: Option<&SqliteBackend>,
|
||||
) -> Self {
|
||||
let mut repositories = Self::default();
|
||||
#[cfg(feature = "postgres")]
|
||||
if let Some(postgres) = postgres {
|
||||
repositories.install_postgres(postgres);
|
||||
}
|
||||
#[cfg(feature = "mysql")]
|
||||
if let Some(mysql) = mysql {
|
||||
repositories.install_mysql(mysql);
|
||||
}
|
||||
#[cfg(feature = "sqlite")]
|
||||
if let Some(sqlite) = sqlite {
|
||||
repositories.install_sqlite(sqlite);
|
||||
}
|
||||
repositories
|
||||
}
|
||||
|
||||
@@ -162,136 +148,10 @@ impl DataWriteRepositories {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "mysql")]
|
||||
fn install_mysql(&mut self, backend: &MysqlBackend) {
|
||||
if self.announcements.is_none() {
|
||||
self.announcements = Some(MysqlBackend::announcement_write_repository(backend));
|
||||
}
|
||||
if self.auth_api_keys.is_none() {
|
||||
self.auth_api_keys = Some(MysqlBackend::auth_api_key_write_repository(backend));
|
||||
}
|
||||
if self.auth_modules.is_none() {
|
||||
self.auth_modules = Some(MysqlBackend::auth_module_write_repository(backend));
|
||||
}
|
||||
if self.background_tasks.is_none() {
|
||||
self.background_tasks = Some(MysqlBackend::background_task_write_repository(backend));
|
||||
}
|
||||
if self.request_candidates.is_none() {
|
||||
self.request_candidates =
|
||||
Some(MysqlBackend::request_candidate_write_repository(backend));
|
||||
}
|
||||
if self.gemini_file_mappings.is_none() {
|
||||
self.gemini_file_mappings =
|
||||
Some(MysqlBackend::gemini_file_mapping_write_repository(backend));
|
||||
}
|
||||
if self.global_models.is_none() {
|
||||
self.global_models = Some(MysqlBackend::global_model_write_repository(backend));
|
||||
}
|
||||
if self.management_tokens.is_none() {
|
||||
self.management_tokens = Some(MysqlBackend::management_token_write_repository(backend));
|
||||
}
|
||||
if self.oauth_providers.is_none() {
|
||||
self.oauth_providers = Some(MysqlBackend::oauth_provider_write_repository(backend));
|
||||
}
|
||||
if self.pool_scores.is_none() {
|
||||
self.pool_scores = Some(MysqlBackend::pool_score_write_repository(backend));
|
||||
}
|
||||
if self.proxy_nodes.is_none() {
|
||||
self.proxy_nodes = Some(MysqlBackend::proxy_node_write_repository(backend));
|
||||
}
|
||||
if self.provider_catalog.is_none() {
|
||||
self.provider_catalog = Some(MysqlBackend::provider_catalog_write_repository(backend));
|
||||
}
|
||||
if self.provider_quotas.is_none() {
|
||||
self.provider_quotas = Some(MysqlBackend::provider_quota_write_repository(backend));
|
||||
}
|
||||
if self.routing_groups.is_none() {
|
||||
self.routing_groups = Some(MysqlBackend::routing_group_write_repository(backend));
|
||||
}
|
||||
if self.settlement.is_none() {
|
||||
self.settlement = Some(MysqlBackend::settlement_write_repository(backend));
|
||||
}
|
||||
if self.usage.is_none() {
|
||||
self.usage = Some(MysqlBackend::usage_write_repository(backend));
|
||||
}
|
||||
if self.video_tasks.is_none() {
|
||||
self.video_tasks = Some(MysqlBackend::video_task_write_repository(backend));
|
||||
}
|
||||
if self.wallets.is_none() {
|
||||
self.wallets = Some(MysqlBackend::wallet_write_repository(backend));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "sqlite")]
|
||||
fn install_sqlite(&mut self, backend: &SqliteBackend) {
|
||||
if self.announcements.is_none() {
|
||||
self.announcements = Some(SqliteBackend::announcement_write_repository(backend));
|
||||
}
|
||||
if self.auth_api_keys.is_none() {
|
||||
self.auth_api_keys = Some(SqliteBackend::auth_api_key_write_repository(backend));
|
||||
}
|
||||
if self.auth_modules.is_none() {
|
||||
self.auth_modules = Some(SqliteBackend::auth_module_write_repository(backend));
|
||||
}
|
||||
if self.background_tasks.is_none() {
|
||||
self.background_tasks = Some(SqliteBackend::background_task_write_repository(backend));
|
||||
}
|
||||
if self.request_candidates.is_none() {
|
||||
self.request_candidates =
|
||||
Some(SqliteBackend::request_candidate_write_repository(backend));
|
||||
}
|
||||
if self.gemini_file_mappings.is_none() {
|
||||
self.gemini_file_mappings =
|
||||
Some(SqliteBackend::gemini_file_mapping_write_repository(backend));
|
||||
}
|
||||
if self.global_models.is_none() {
|
||||
self.global_models = Some(SqliteBackend::global_model_write_repository(backend));
|
||||
}
|
||||
if self.management_tokens.is_none() {
|
||||
self.management_tokens =
|
||||
Some(SqliteBackend::management_token_write_repository(backend));
|
||||
}
|
||||
if self.oauth_providers.is_none() {
|
||||
self.oauth_providers = Some(SqliteBackend::oauth_provider_write_repository(backend));
|
||||
}
|
||||
if self.pool_scores.is_none() {
|
||||
self.pool_scores = Some(SqliteBackend::pool_score_write_repository(backend));
|
||||
}
|
||||
if self.proxy_nodes.is_none() {
|
||||
self.proxy_nodes = Some(SqliteBackend::proxy_node_write_repository(backend));
|
||||
}
|
||||
if self.provider_catalog.is_none() {
|
||||
self.provider_catalog = Some(SqliteBackend::provider_catalog_write_repository(backend));
|
||||
}
|
||||
if self.provider_quotas.is_none() {
|
||||
self.provider_quotas = Some(SqliteBackend::provider_quota_write_repository(backend));
|
||||
}
|
||||
if self.routing_groups.is_none() {
|
||||
self.routing_groups = Some(SqliteBackend::routing_group_write_repository(backend));
|
||||
}
|
||||
if self.settlement.is_none() {
|
||||
self.settlement = Some(SqliteBackend::settlement_write_repository(backend));
|
||||
}
|
||||
if self.usage.is_none() {
|
||||
self.usage = Some(SqliteBackend::usage_write_repository(backend));
|
||||
}
|
||||
if self.video_tasks.is_none() {
|
||||
self.video_tasks = Some(SqliteBackend::video_task_write_repository(backend));
|
||||
}
|
||||
if self.wallets.is_none() {
|
||||
self.wallets = Some(SqliteBackend::wallet_write_repository(backend));
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
#[cfg(feature = "postgres")]
|
||||
pub(crate) fn from_postgres(postgres: Option<&PostgresBackend>) -> Self {
|
||||
Self::from_backends(
|
||||
postgres,
|
||||
#[cfg(feature = "mysql")]
|
||||
None,
|
||||
#[cfg(feature = "sqlite")]
|
||||
None,
|
||||
)
|
||||
Self::from_backends(postgres)
|
||||
}
|
||||
|
||||
pub fn announcements(&self) -> Option<Arc<dyn AnnouncementWriteRepository>> {
|
||||
|
||||
Reference in New Issue
Block a user