i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-09-12 14:10:19 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(data): complete portable SQL backend parity

Browse Source 
Align MySQL and SQLite schemas, migrations, usage, stats, export, and backfill behavior with the shared data contracts. Extend gateway startup and maintenance support across all SQL drivers.
This commit is contained in:
elky
2026-07-25 21:28:21 +08:00
parent 764e9fd131
commit 778cfb1a5c
85 changed files with 32096 additions and 2328 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/aether-data/runtime/backfills/mysql/20260422120000_backfill_api_key_usage_stats.sql
+36
View File
@@ -0,0 +1,36 @@
UPDATE api_keys AS target
LEFT JOIN (
SELECT
`usage`.api_key_id,
COUNT(*) AS total_requests,
COALESCE(
SUM(
GREATEST(
COALESCE(
`usage`.total_tokens,
COALESCE(`usage`.input_tokens, 0) + COALESCE(`usage`.output_tokens, 0)
),
0
)
),
0
) AS total_tokens,
COALESCE(SUM(COALESCE(`usage`.total_cost_usd, 0)), 0) AS total_cost_usd,
MAX(
COALESCE(
`usage`.created_at,
`usage`.created_at_unix_ms,
`usage`.updated_at_unix_secs
)
) AS last_used_at
FROM `usage`
WHERE `usage`.api_key_id IS NOT NULL
AND TRIM(`usage`.api_key_id) <> ''
GROUP BY `usage`.api_key_id
) AS aggregated
ON aggregated.api_key_id = target.id
SET
target.total_requests = COALESCE(aggregated.total_requests, 0),
target.total_tokens = COALESCE(aggregated.total_tokens, 0),
target.total_cost_usd = COALESCE(aggregated.total_cost_usd, 0),
target.last_used_at = aggregated.last_used_at;
crates/aether-data/runtime/backfills/mysql/20260505120000_rebuild_global_model_usage_count.sql
+15
View File
@@ -0,0 +1,15 @@
UPDATE global_models AS target
LEFT JOIN (
SELECT
`usage`.model,
COUNT(*) AS usage_count
FROM `usage`
WHERE `usage`.model IS NOT NULL
AND TRIM(`usage`.model) <> ''
AND `usage`.status NOT IN ('pending', 'streaming')
GROUP BY `usage`.model
) AS aggregated
ON aggregated.model = target.name
SET
target.usage_count = COALESCE(aggregated.usage_count, 0),
target.updated_at = UNIX_TIMESTAMP();
crates/aether-data/runtime/backfills/mysql/20260517012000_sync_legacy_enabled_active_flags.sql
+11
View File
@@ -0,0 +1,11 @@
UPDATE providers
SET enabled = is_active
WHERE enabled <> is_active;
UPDATE provider_endpoints
SET enabled = is_active
WHERE enabled <> is_active;
UPDATE models
SET enabled = is_active
WHERE enabled <> is_active;
crates/aether-data/runtime/backfills/mysql/20260716010000_rebuild_canonical_usage_token_totals.sql
+229
View File
@@ -0,0 +1,229 @@
UPDATE api_keys AS target
LEFT JOIN (
SELECT
source.api_key_id,
COALESCE(SUM(source.canonical_total_tokens), 0) AS total_tokens
FROM (
SELECT
`usage`.api_key_id,
GREATEST(
COALESCE(
CASE
WHEN settlement.billing_effective_input_tokens IS NOT NULL THEN
GREATEST(settlement.billing_effective_input_tokens, 0)
+ GREATEST(COALESCE(settlement.billing_output_tokens, `usage`.output_tokens, 0), 0)
+ GREATEST(
COALESCE(
settlement.billing_cache_creation_tokens,
CASE
WHEN settlement.billing_cache_creation_5m_tokens IS NOT NULL
OR settlement.billing_cache_creation_1h_tokens IS NOT NULL
THEN COALESCE(settlement.billing_cache_creation_5m_tokens, 0)
+ COALESCE(settlement.billing_cache_creation_1h_tokens, 0)
END,
CASE
WHEN COALESCE(`usage`.cache_creation_input_tokens, 0) = 0
AND (
GREATEST(
COALESCE(`usage`.cache_creation_input_tokens_5m, 0),
COALESCE(`usage`.cache_creation_ephemeral_5m_input_tokens, 0)
)
+ GREATEST(
COALESCE(`usage`.cache_creation_input_tokens_1h, 0),
COALESCE(`usage`.cache_creation_ephemeral_1h_input_tokens, 0)
)
) > 0
THEN GREATEST(
COALESCE(`usage`.cache_creation_input_tokens_5m, 0),
COALESCE(`usage`.cache_creation_ephemeral_5m_input_tokens, 0)
)
+ GREATEST(
COALESCE(`usage`.cache_creation_input_tokens_1h, 0),
COALESCE(`usage`.cache_creation_ephemeral_1h_input_tokens, 0)
)
ELSE COALESCE(`usage`.cache_creation_input_tokens, 0)
END,
0
),
0
)
+ GREATEST(
COALESCE(
settlement.billing_cache_read_tokens,
`usage`.cache_read_input_tokens,
0
),
0
)
WHEN settlement.billing_total_input_context IS NOT NULL THEN
GREATEST(settlement.billing_total_input_context, 0)
+ GREATEST(COALESCE(settlement.billing_output_tokens, `usage`.output_tokens, 0), 0)
END,
NULLIF(GREATEST(COALESCE(`usage`.total_tokens, 0), 0), 0),
CASE
WHEN SUBSTRING_INDEX(
LOWER(COALESCE(`usage`.endpoint_api_format, `usage`.api_format, '')),
':',
1
) IN ('openai', 'gemini', 'google')
THEN GREATEST(COALESCE(`usage`.input_tokens, 0), 0)
+ GREATEST(COALESCE(`usage`.output_tokens, 0), 0)
ELSE GREATEST(COALESCE(`usage`.input_tokens, 0), 0)
+ GREATEST(COALESCE(`usage`.output_tokens, 0), 0)
+ GREATEST(
CASE
WHEN COALESCE(`usage`.cache_creation_input_tokens, 0) = 0
AND (
GREATEST(
COALESCE(`usage`.cache_creation_input_tokens_5m, 0),
COALESCE(`usage`.cache_creation_ephemeral_5m_input_tokens, 0)
)
+ GREATEST(
COALESCE(`usage`.cache_creation_input_tokens_1h, 0),
COALESCE(`usage`.cache_creation_ephemeral_1h_input_tokens, 0)
)
) > 0
THEN GREATEST(
COALESCE(`usage`.cache_creation_input_tokens_5m, 0),
COALESCE(`usage`.cache_creation_ephemeral_5m_input_tokens, 0)
)
+ GREATEST(
COALESCE(`usage`.cache_creation_input_tokens_1h, 0),
COALESCE(`usage`.cache_creation_ephemeral_1h_input_tokens, 0)
)
ELSE COALESCE(`usage`.cache_creation_input_tokens, 0)
END,
0
)
+ GREATEST(COALESCE(`usage`.cache_read_input_tokens, 0), 0)
END,
0
),
0
) AS canonical_total_tokens
FROM `usage`
LEFT JOIN usage_settlement_snapshots AS settlement
ON settlement.request_id = `usage`.request_id
WHERE `usage`.status NOT IN ('pending', 'streaming')
) AS source
WHERE source.api_key_id IS NOT NULL
AND TRIM(source.api_key_id) <> ''
GROUP BY source.api_key_id
) AS aggregated
ON aggregated.api_key_id = target.id
SET target.total_tokens = COALESCE(aggregated.total_tokens, 0);
UPDATE provider_api_keys AS target
LEFT JOIN (
SELECT
source.provider_api_key_id,
COALESCE(SUM(source.canonical_total_tokens), 0) AS total_tokens
FROM (
SELECT
`usage`.provider_api_key_id,
GREATEST(
COALESCE(
CASE
WHEN settlement.billing_effective_input_tokens IS NOT NULL THEN
GREATEST(settlement.billing_effective_input_tokens, 0)
+ GREATEST(COALESCE(settlement.billing_output_tokens, `usage`.output_tokens, 0), 0)
+ GREATEST(
COALESCE(
settlement.billing_cache_creation_tokens,
CASE
WHEN settlement.billing_cache_creation_5m_tokens IS NOT NULL
OR settlement.billing_cache_creation_1h_tokens IS NOT NULL
THEN COALESCE(settlement.billing_cache_creation_5m_tokens, 0)
+ COALESCE(settlement.billing_cache_creation_1h_tokens, 0)
END,
CASE
WHEN COALESCE(`usage`.cache_creation_input_tokens, 0) = 0
AND (
GREATEST(
COALESCE(`usage`.cache_creation_input_tokens_5m, 0),
COALESCE(`usage`.cache_creation_ephemeral_5m_input_tokens, 0)
)
+ GREATEST(
COALESCE(`usage`.cache_creation_input_tokens_1h, 0),
COALESCE(`usage`.cache_creation_ephemeral_1h_input_tokens, 0)
)
) > 0
THEN GREATEST(
COALESCE(`usage`.cache_creation_input_tokens_5m, 0),
COALESCE(`usage`.cache_creation_ephemeral_5m_input_tokens, 0)
)
+ GREATEST(
COALESCE(`usage`.cache_creation_input_tokens_1h, 0),
COALESCE(`usage`.cache_creation_ephemeral_1h_input_tokens, 0)
)
ELSE COALESCE(`usage`.cache_creation_input_tokens, 0)
END,
0
),
0
)
+ GREATEST(
COALESCE(
settlement.billing_cache_read_tokens,
`usage`.cache_read_input_tokens,
0
),
0
)
WHEN settlement.billing_total_input_context IS NOT NULL THEN
GREATEST(settlement.billing_total_input_context, 0)
+ GREATEST(COALESCE(settlement.billing_output_tokens, `usage`.output_tokens, 0), 0)
END,
NULLIF(GREATEST(COALESCE(`usage`.total_tokens, 0), 0), 0),
CASE
WHEN SUBSTRING_INDEX(
LOWER(COALESCE(`usage`.endpoint_api_format, `usage`.api_format, '')),
':',
1
) IN ('openai', 'gemini', 'google')
THEN GREATEST(COALESCE(`usage`.input_tokens, 0), 0)
+ GREATEST(COALESCE(`usage`.output_tokens, 0), 0)
ELSE GREATEST(COALESCE(`usage`.input_tokens, 0), 0)
+ GREATEST(COALESCE(`usage`.output_tokens, 0), 0)
+ GREATEST(
CASE
WHEN COALESCE(`usage`.cache_creation_input_tokens, 0) = 0
AND (
GREATEST(
COALESCE(`usage`.cache_creation_input_tokens_5m, 0),
COALESCE(`usage`.cache_creation_ephemeral_5m_input_tokens, 0)
)
+ GREATEST(
COALESCE(`usage`.cache_creation_input_tokens_1h, 0),
COALESCE(`usage`.cache_creation_ephemeral_1h_input_tokens, 0)
)
) > 0
THEN GREATEST(
COALESCE(`usage`.cache_creation_input_tokens_5m, 0),
COALESCE(`usage`.cache_creation_ephemeral_5m_input_tokens, 0)
)
+ GREATEST(
COALESCE(`usage`.cache_creation_input_tokens_1h, 0),
COALESCE(`usage`.cache_creation_ephemeral_1h_input_tokens, 0)
)
ELSE COALESCE(`usage`.cache_creation_input_tokens, 0)
END,
0
)
+ GREATEST(COALESCE(`usage`.cache_read_input_tokens, 0), 0)
END,
0
),
0
) AS canonical_total_tokens
FROM `usage`
LEFT JOIN usage_settlement_snapshots AS settlement
ON settlement.request_id = `usage`.request_id
WHERE `usage`.status NOT IN ('pending', 'streaming')
) AS source
WHERE source.provider_api_key_id IS NOT NULL
AND TRIM(source.provider_api_key_id) <> ''
GROUP BY source.provider_api_key_id
) AS aggregated
ON aggregated.provider_api_key_id = target.id
SET target.total_tokens = COALESCE(aggregated.total_tokens, 0);
crates/aether-data/runtime/backfills/mysql/README.md
+2 -1
View File
@@ -1 +1,2 @@
MySQL-specific backfills live here when they are needed.
MySQL-specific lifecycle backfills live here. Versions intentionally match the
equivalent PostgreSQL backfills when they repair the same logical data.
crates/aether-data/runtime/backfills/sqlite/20260422120000_backfill_api_key_usage_stats.sql
+28
View File
@@ -0,0 +1,28 @@
UPDATE api_keys AS target
SET
total_requests = (
SELECT COUNT(*)
FROM "usage"
WHERE "usage".api_key_id = target.id
),
total_tokens = COALESCE((
SELECT SUM(MAX(COALESCE("usage".total_tokens, 0), 0))
FROM "usage"
WHERE "usage".api_key_id = target.id
), 0),
total_cost_usd = COALESCE((
SELECT SUM(COALESCE("usage".total_cost_usd, 0))
FROM "usage"
WHERE "usage".api_key_id = target.id
), 0),
last_used_at = (
SELECT MAX(
COALESCE(
"usage".created_at,
"usage".created_at_unix_ms,
"usage".updated_at_unix_secs
)
)
FROM "usage"
WHERE "usage".api_key_id = target.id
);
crates/aether-data/runtime/backfills/sqlite/20260505120000_rebuild_global_model_usage_count.sql
+9
View File
@@ -0,0 +1,9 @@
UPDATE global_models AS target
SET
usage_count = (
SELECT COUNT(*)
FROM "usage"
WHERE "usage".model = target.name
AND "usage".status NOT IN ('pending', 'streaming')
),
updated_at = CAST(strftime('%s', 'now') AS INTEGER);
crates/aether-data/runtime/backfills/sqlite/20260517012000_sync_legacy_enabled_active_flags.sql
+11
View File
@@ -0,0 +1,11 @@
UPDATE providers
SET enabled = is_active
WHERE enabled <> is_active;
UPDATE provider_endpoints
SET enabled = is_active
WHERE enabled <> is_active;
UPDATE models
SET enabled = is_active
WHERE enabled <> is_active;
crates/aether-data/runtime/backfills/sqlite/20260716010000_rebuild_canonical_usage_token_totals.sql
+225
View File
@@ -0,0 +1,225 @@
WITH canonical_usage AS (
SELECT
"usage".api_key_id,
MAX(
COALESCE(
CASE
WHEN settlement.billing_effective_input_tokens IS NOT NULL THEN
MAX(settlement.billing_effective_input_tokens, 0)
+ MAX(COALESCE(settlement.billing_output_tokens, "usage".output_tokens, 0), 0)
+ MAX(
COALESCE(
settlement.billing_cache_creation_tokens,
CASE
WHEN settlement.billing_cache_creation_5m_tokens IS NOT NULL
OR settlement.billing_cache_creation_1h_tokens IS NOT NULL
THEN COALESCE(settlement.billing_cache_creation_5m_tokens, 0)
+ COALESCE(settlement.billing_cache_creation_1h_tokens, 0)
END,
CASE
WHEN COALESCE("usage".cache_creation_input_tokens, 0) = 0
AND (
MAX(
COALESCE("usage".cache_creation_input_tokens_5m, 0),
COALESCE("usage".cache_creation_ephemeral_5m_input_tokens, 0)
)
+ MAX(
COALESCE("usage".cache_creation_input_tokens_1h, 0),
COALESCE("usage".cache_creation_ephemeral_1h_input_tokens, 0)
)
) > 0
THEN MAX(
COALESCE("usage".cache_creation_input_tokens_5m, 0),
COALESCE("usage".cache_creation_ephemeral_5m_input_tokens, 0)
)
+ MAX(
COALESCE("usage".cache_creation_input_tokens_1h, 0),
COALESCE("usage".cache_creation_ephemeral_1h_input_tokens, 0)
)
ELSE COALESCE("usage".cache_creation_input_tokens, 0)
END,
0
),
0
)
+ MAX(
COALESCE(
settlement.billing_cache_read_tokens,
"usage".cache_read_input_tokens,
0
),
0
)
WHEN settlement.billing_total_input_context IS NOT NULL THEN
MAX(settlement.billing_total_input_context, 0)
+ MAX(COALESCE(settlement.billing_output_tokens, "usage".output_tokens, 0), 0)
END,
NULLIF(MAX(COALESCE("usage".total_tokens, 0), 0), 0),
CASE
WHEN LOWER(COALESCE("usage".endpoint_api_format, "usage".api_format, ''))
IN ('openai', 'gemini', 'google')
OR LOWER(COALESCE("usage".endpoint_api_format, "usage".api_format, ''))
LIKE 'openai:%'
OR LOWER(COALESCE("usage".endpoint_api_format, "usage".api_format, ''))
LIKE 'gemini:%'
OR LOWER(COALESCE("usage".endpoint_api_format, "usage".api_format, ''))
LIKE 'google:%'
THEN MAX(COALESCE("usage".input_tokens, 0), 0)
+ MAX(COALESCE("usage".output_tokens, 0), 0)
ELSE MAX(COALESCE("usage".input_tokens, 0), 0)
+ MAX(COALESCE("usage".output_tokens, 0), 0)
+ MAX(
CASE
WHEN COALESCE("usage".cache_creation_input_tokens, 0) = 0
AND (
MAX(
COALESCE("usage".cache_creation_input_tokens_5m, 0),
COALESCE("usage".cache_creation_ephemeral_5m_input_tokens, 0)
)
+ MAX(
COALESCE("usage".cache_creation_input_tokens_1h, 0),
COALESCE("usage".cache_creation_ephemeral_1h_input_tokens, 0)
)
) > 0
THEN MAX(
COALESCE("usage".cache_creation_input_tokens_5m, 0),
COALESCE("usage".cache_creation_ephemeral_5m_input_tokens, 0)
)
+ MAX(
COALESCE("usage".cache_creation_input_tokens_1h, 0),
COALESCE("usage".cache_creation_ephemeral_1h_input_tokens, 0)
)
ELSE COALESCE("usage".cache_creation_input_tokens, 0)
END,
0
)
+ MAX(COALESCE("usage".cache_read_input_tokens, 0), 0)
END,
0
),
0
) AS canonical_total_tokens
FROM "usage"
LEFT JOIN usage_settlement_snapshots AS settlement
ON settlement.request_id = "usage".request_id
WHERE "usage".status NOT IN ('pending', 'streaming')
)
UPDATE api_keys AS target
SET total_tokens = COALESCE((
SELECT SUM(canonical_usage.canonical_total_tokens)
FROM canonical_usage
WHERE canonical_usage.api_key_id = target.id
), 0);
WITH canonical_usage AS (
SELECT
"usage".provider_api_key_id,
MAX(
COALESCE(
CASE
WHEN settlement.billing_effective_input_tokens IS NOT NULL THEN
MAX(settlement.billing_effective_input_tokens, 0)
+ MAX(COALESCE(settlement.billing_output_tokens, "usage".output_tokens, 0), 0)
+ MAX(
COALESCE(
settlement.billing_cache_creation_tokens,
CASE
WHEN settlement.billing_cache_creation_5m_tokens IS NOT NULL
OR settlement.billing_cache_creation_1h_tokens IS NOT NULL
THEN COALESCE(settlement.billing_cache_creation_5m_tokens, 0)
+ COALESCE(settlement.billing_cache_creation_1h_tokens, 0)
END,
CASE
WHEN COALESCE("usage".cache_creation_input_tokens, 0) = 0
AND (
MAX(
COALESCE("usage".cache_creation_input_tokens_5m, 0),
COALESCE("usage".cache_creation_ephemeral_5m_input_tokens, 0)
)
+ MAX(
COALESCE("usage".cache_creation_input_tokens_1h, 0),
COALESCE("usage".cache_creation_ephemeral_1h_input_tokens, 0)
)
) > 0
THEN MAX(
COALESCE("usage".cache_creation_input_tokens_5m, 0),
COALESCE("usage".cache_creation_ephemeral_5m_input_tokens, 0)
)
+ MAX(
COALESCE("usage".cache_creation_input_tokens_1h, 0),
COALESCE("usage".cache_creation_ephemeral_1h_input_tokens, 0)
)
ELSE COALESCE("usage".cache_creation_input_tokens, 0)
END,
0
),
0
)
+ MAX(
COALESCE(
settlement.billing_cache_read_tokens,
"usage".cache_read_input_tokens,
0
),
0
)
WHEN settlement.billing_total_input_context IS NOT NULL THEN
MAX(settlement.billing_total_input_context, 0)
+ MAX(COALESCE(settlement.billing_output_tokens, "usage".output_tokens, 0), 0)
END,
NULLIF(MAX(COALESCE("usage".total_tokens, 0), 0), 0),
CASE
WHEN LOWER(COALESCE("usage".endpoint_api_format, "usage".api_format, ''))
IN ('openai', 'gemini', 'google')
OR LOWER(COALESCE("usage".endpoint_api_format, "usage".api_format, ''))
LIKE 'openai:%'
OR LOWER(COALESCE("usage".endpoint_api_format, "usage".api_format, ''))
LIKE 'gemini:%'
OR LOWER(COALESCE("usage".endpoint_api_format, "usage".api_format, ''))
LIKE 'google:%'
THEN MAX(COALESCE("usage".input_tokens, 0), 0)
+ MAX(COALESCE("usage".output_tokens, 0), 0)
ELSE MAX(COALESCE("usage".input_tokens, 0), 0)
+ MAX(COALESCE("usage".output_tokens, 0), 0)
+ MAX(
CASE
WHEN COALESCE("usage".cache_creation_input_tokens, 0) = 0
AND (
MAX(
COALESCE("usage".cache_creation_input_tokens_5m, 0),
COALESCE("usage".cache_creation_ephemeral_5m_input_tokens, 0)
)
+ MAX(
COALESCE("usage".cache_creation_input_tokens_1h, 0),
COALESCE("usage".cache_creation_ephemeral_1h_input_tokens, 0)
)
) > 0
THEN MAX(
COALESCE("usage".cache_creation_input_tokens_5m, 0),
COALESCE("usage".cache_creation_ephemeral_5m_input_tokens, 0)
)
+ MAX(
COALESCE("usage".cache_creation_input_tokens_1h, 0),
COALESCE("usage".cache_creation_ephemeral_1h_input_tokens, 0)
)
ELSE COALESCE("usage".cache_creation_input_tokens, 0)
END,
0
)
+ MAX(COALESCE("usage".cache_read_input_tokens, 0), 0)
END,
0
),
0
) AS canonical_total_tokens
FROM "usage"
LEFT JOIN usage_settlement_snapshots AS settlement
ON settlement.request_id = "usage".request_id
WHERE "usage".status NOT IN ('pending', 'streaming')
)
UPDATE provider_api_keys AS target
SET total_tokens = COALESCE((
SELECT SUM(canonical_usage.canonical_total_tokens)
FROM canonical_usage
WHERE canonical_usage.provider_api_key_id = target.id
), 0);
crates/aether-data/runtime/backfills/sqlite/README.md
+2 -1
View File
@@ -1 +1,2 @@
SQLite-specific backfills live here when they are needed.
SQLite-specific lifecycle backfills live here. Versions intentionally match
the equivalent PostgreSQL backfills when they repair the same logical data.
crates/aether-data/runtime/schema/generated/mysql/baseline/006_usage.sql
+87 -1
View File
@@ -111,6 +111,63 @@ CREATE TABLE IF NOT EXISTS `usage` (
KEY usage_wallet_id_idx (`wallet_id`)
);
CREATE TABLE IF NOT EXISTS usage_body_blobs (
`body_ref` VARCHAR(160) NOT NULL,
`request_id` VARCHAR(128) NOT NULL,
`body_field` VARCHAR(50) NOT NULL,
`payload_gzip` LONGBLOB NOT NULL,
`created_at` BIGINT NOT NULL,
`updated_at` BIGINT NOT NULL,
PRIMARY KEY (`body_ref`),
UNIQUE KEY usage_body_blobs_request_id_field_key (`request_id`, `body_field`),
KEY ix_usage_body_blobs_request_id (`request_id`),
CONSTRAINT usage_body_blobs_request_id_fkey FOREIGN KEY (`request_id`) REFERENCES usage (`request_id`) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS usage_http_audits (
`request_id` VARCHAR(128) NOT NULL,
`request_headers` JSON,
`provider_request_headers` JSON,
`response_headers` JSON,
`client_response_headers` JSON,
`request_body_ref` VARCHAR(160),
`provider_request_body_ref` VARCHAR(160),
`response_body_ref` VARCHAR(160),
`client_response_body_ref` VARCHAR(160),
`request_body_state` VARCHAR(32),
`provider_request_body_state` VARCHAR(32),
`response_body_state` VARCHAR(32),
`client_response_body_state` VARCHAR(32),
`body_capture_mode` VARCHAR(32) NOT NULL DEFAULT 'none',
`created_at` BIGINT NOT NULL,
`updated_at` BIGINT NOT NULL,
PRIMARY KEY (`request_id`),
KEY ix_usage_http_audits_updated_at (`updated_at`),
CONSTRAINT usage_http_audits_request_id_fkey FOREIGN KEY (`request_id`) REFERENCES usage (`request_id`) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS usage_routing_snapshots (
`request_id` VARCHAR(128) NOT NULL,
`candidate_id` VARCHAR(160),
`candidate_index` BIGINT,
`key_name` VARCHAR(255),
`planner_kind` VARCHAR(120),
`route_family` VARCHAR(80),
`route_kind` VARCHAR(80),
`execution_path` VARCHAR(80),
`local_execution_runtime_miss_reason` VARCHAR(255),
`selected_provider_id` VARCHAR(100),
`selected_endpoint_id` VARCHAR(100),
`selected_provider_api_key_id` VARCHAR(100),
`has_format_conversion` TINYINT(1),
`created_at` BIGINT NOT NULL,
`updated_at` BIGINT NOT NULL,
PRIMARY KEY (`request_id`),
KEY ix_usage_routing_snapshots_route_family_kind (`route_family`, `route_kind`),
KEY ix_usage_routing_snapshots_candidate_id (`candidate_id`),
CONSTRAINT usage_routing_snapshots_request_id_fkey FOREIGN KEY (`request_id`) REFERENCES usage (`request_id`) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS usage_counter_deltas (
`id` VARCHAR(36) NOT NULL,
`request_id` VARCHAR(128) NOT NULL,
@@ -149,11 +206,40 @@ CREATE TABLE IF NOT EXISTS usage_settlement_snapshots (
`wallet_gift_balance_before` DOUBLE,
`wallet_gift_balance_after` DOUBLE,
`provider_monthly_used_usd` DOUBLE,
`billing_snapshot_schema_version` VARCHAR(20),
`billing_snapshot_status` VARCHAR(20),
`rate_multiplier` DOUBLE,
`is_free_tier` TINYINT(1),
`input_price_per_1m` DOUBLE,
`output_price_per_1m` DOUBLE,
`cache_creation_price_per_1m` DOUBLE,
`cache_read_price_per_1m` DOUBLE,
`price_per_request` DOUBLE,
`settlement_snapshot_schema_version` VARCHAR(20),
`settlement_snapshot` JSON,
`billing_dimensions` JSON,
`billing_input_tokens` BIGINT,
`billing_effective_input_tokens` BIGINT,
`billing_output_tokens` BIGINT,
`billing_cache_creation_tokens` BIGINT,
`billing_cache_creation_5m_tokens` BIGINT,
`billing_cache_creation_1h_tokens` BIGINT,
`billing_cache_read_tokens` BIGINT,
`billing_total_input_context` BIGINT,
`billing_cache_creation_cost_usd` DOUBLE,
`billing_cache_read_cost_usd` DOUBLE,
`billing_total_cost_usd` DOUBLE,
`billing_actual_total_cost_usd` DOUBLE,
`billing_pricing_source` VARCHAR(50),
`billing_rule_id` VARCHAR(100),
`billing_rule_version` VARCHAR(50),
`finalized_at` BIGINT,
`created_at` BIGINT NOT NULL,
`updated_at` BIGINT NOT NULL,
PRIMARY KEY (`request_id`),
KEY usage_settlement_snapshots_billing_status_idx (`billing_status`),
KEY usage_settlement_snapshots_wallet_id_idx (`wallet_id`)
KEY usage_settlement_snapshots_wallet_id_idx (`wallet_id`),
KEY ix_usage_settlement_snapshots_schema_version (`settlement_snapshot_schema_version`),
KEY ix_usage_settlement_snapshots_pricing_source (`billing_pricing_source`)
);
crates/aether-data/runtime/schema/generated/mysql/baseline/007_stats.sql
+363
View File
@@ -18,6 +18,26 @@ CREATE TABLE IF NOT EXISTS stats_hourly (
`aggregated_at` BIGINT,
`created_at` BIGINT NOT NULL,
`updated_at` BIGINT NOT NULL,
`response_time_sum_ms` DOUBLE NOT NULL DEFAULT 0,
`response_time_samples` BIGINT NOT NULL DEFAULT 0,
`cache_hit_total_requests` BIGINT NOT NULL DEFAULT 0,
`cache_hit_requests` BIGINT NOT NULL DEFAULT 0,
`completed_total_requests` BIGINT NOT NULL DEFAULT 0,
`completed_cache_hit_requests` BIGINT NOT NULL DEFAULT 0,
`completed_input_tokens` BIGINT NOT NULL DEFAULT 0,
`completed_cache_creation_tokens` BIGINT NOT NULL DEFAULT 0,
`completed_cache_read_tokens` BIGINT NOT NULL DEFAULT 0,
`completed_total_input_context` BIGINT NOT NULL DEFAULT 0,
`completed_cache_creation_cost` DOUBLE NOT NULL DEFAULT 0,
`completed_cache_read_cost` DOUBLE NOT NULL DEFAULT 0,
`settled_total_cost` DOUBLE NOT NULL DEFAULT 0,
`settled_total_requests` BIGINT NOT NULL DEFAULT 0,
`settled_input_tokens` BIGINT NOT NULL DEFAULT 0,
`settled_output_tokens` BIGINT NOT NULL DEFAULT 0,
`settled_cache_creation_tokens` BIGINT NOT NULL DEFAULT 0,
`settled_cache_read_tokens` BIGINT NOT NULL DEFAULT 0,
`settled_first_finalized_at_unix_secs` BIGINT,
`settled_last_finalized_at_unix_secs` BIGINT,
PRIMARY KEY (`id`),
UNIQUE KEY uq_stats_hourly_hour (`hour_utc`)
);
@@ -55,6 +75,19 @@ CREATE TABLE IF NOT EXISTS stats_hourly_user (
`total_cost` DOUBLE NOT NULL DEFAULT 0,
`created_at` BIGINT NOT NULL,
`updated_at` BIGINT NOT NULL,
`cache_creation_tokens` BIGINT NOT NULL DEFAULT 0,
`cache_read_tokens` BIGINT NOT NULL DEFAULT 0,
`actual_total_cost` DOUBLE NOT NULL DEFAULT 0,
`response_time_sum_ms` DOUBLE NOT NULL DEFAULT 0,
`response_time_samples` BIGINT NOT NULL DEFAULT 0,
`settled_total_cost` DOUBLE NOT NULL DEFAULT 0,
`settled_total_requests` BIGINT NOT NULL DEFAULT 0,
`settled_input_tokens` BIGINT NOT NULL DEFAULT 0,
`settled_output_tokens` BIGINT NOT NULL DEFAULT 0,
`settled_cache_creation_tokens` BIGINT NOT NULL DEFAULT 0,
`settled_cache_read_tokens` BIGINT NOT NULL DEFAULT 0,
`settled_first_finalized_at_unix_secs` BIGINT,
`settled_last_finalized_at_unix_secs` BIGINT,
PRIMARY KEY (`id`),
UNIQUE KEY uq_stats_hourly_user (`hour_utc`, `user_id`)
);
@@ -70,6 +103,8 @@ CREATE TABLE IF NOT EXISTS stats_hourly_user_model (
`total_cost` DOUBLE NOT NULL DEFAULT 0,
`created_at` BIGINT NOT NULL,
`updated_at` BIGINT NOT NULL,
`response_time_sum_ms` DOUBLE NOT NULL DEFAULT 0,
`response_time_samples` BIGINT NOT NULL DEFAULT 0,
PRIMARY KEY (`id`),
UNIQUE KEY uq_stats_hourly_user_model (`hour_utc`, `user_id`, `model`)
);
@@ -98,6 +133,8 @@ CREATE TABLE IF NOT EXISTS stats_hourly_model (
`avg_response_time_ms` DOUBLE NOT NULL DEFAULT 0,
`created_at` BIGINT NOT NULL,
`updated_at` BIGINT NOT NULL,
`response_time_sum_ms` DOUBLE NOT NULL DEFAULT 0,
`response_time_samples` BIGINT NOT NULL DEFAULT 0,
PRIMARY KEY (`id`),
UNIQUE KEY uq_stats_hourly_model (`hour_utc`, `model`)
);
@@ -146,6 +183,30 @@ CREATE TABLE IF NOT EXISTS stats_daily (
`p50_first_byte_time_ms` BIGINT,
`p90_first_byte_time_ms` BIGINT,
`p99_first_byte_time_ms` BIGINT,
`effective_input_tokens` BIGINT NOT NULL DEFAULT 0,
`total_input_context` BIGINT NOT NULL DEFAULT 0,
`response_time_sum_ms` DOUBLE NOT NULL DEFAULT 0,
`response_time_samples` BIGINT NOT NULL DEFAULT 0,
`cache_creation_ephemeral_5m_tokens` BIGINT NOT NULL DEFAULT 0,
`cache_creation_ephemeral_1h_tokens` BIGINT NOT NULL DEFAULT 0,
`cache_hit_total_requests` BIGINT NOT NULL DEFAULT 0,
`cache_hit_requests` BIGINT NOT NULL DEFAULT 0,
`completed_total_requests` BIGINT NOT NULL DEFAULT 0,
`completed_cache_hit_requests` BIGINT NOT NULL DEFAULT 0,
`completed_input_tokens` BIGINT NOT NULL DEFAULT 0,
`completed_cache_creation_tokens` BIGINT NOT NULL DEFAULT 0,
`completed_cache_read_tokens` BIGINT NOT NULL DEFAULT 0,
`completed_total_input_context` BIGINT NOT NULL DEFAULT 0,
`completed_cache_creation_cost` DOUBLE NOT NULL DEFAULT 0,
`completed_cache_read_cost` DOUBLE NOT NULL DEFAULT 0,
`settled_total_cost` DOUBLE NOT NULL DEFAULT 0,
`settled_total_requests` BIGINT NOT NULL DEFAULT 0,
`settled_input_tokens` BIGINT NOT NULL DEFAULT 0,
`settled_output_tokens` BIGINT NOT NULL DEFAULT 0,
`settled_cache_creation_tokens` BIGINT NOT NULL DEFAULT 0,
`settled_cache_read_tokens` BIGINT NOT NULL DEFAULT 0,
`settled_first_finalized_at_unix_secs` BIGINT,
`settled_last_finalized_at_unix_secs` BIGINT,
PRIMARY KEY (`id`),
UNIQUE KEY uq_stats_daily_date (`date`)
);
@@ -163,6 +224,10 @@ CREATE TABLE IF NOT EXISTS stats_daily_model (
`avg_response_time_ms` DOUBLE NOT NULL DEFAULT 0,
`created_at` BIGINT NOT NULL,
`updated_at` BIGINT NOT NULL,
`response_time_sum_ms` DOUBLE NOT NULL DEFAULT 0,
`response_time_samples` BIGINT NOT NULL DEFAULT 0,
`cache_creation_ephemeral_5m_tokens` BIGINT NOT NULL DEFAULT 0,
`cache_creation_ephemeral_1h_tokens` BIGINT NOT NULL DEFAULT 0,
PRIMARY KEY (`id`),
UNIQUE KEY uq_stats_daily_model (`date`, `model`)
);
@@ -230,7 +295,305 @@ CREATE TABLE IF NOT EXISTS stats_user_daily (
`username` VARCHAR(255),
`created_at` BIGINT NOT NULL,
`updated_at` BIGINT NOT NULL,
`actual_total_cost` DOUBLE NOT NULL DEFAULT 0,
`response_time_sum_ms` DOUBLE NOT NULL DEFAULT 0,
`response_time_samples` BIGINT NOT NULL DEFAULT 0,
`effective_input_tokens` BIGINT NOT NULL DEFAULT 0,
`total_input_context` BIGINT NOT NULL DEFAULT 0,
`cache_creation_cost` DOUBLE NOT NULL DEFAULT 0,
`cache_read_cost` DOUBLE NOT NULL DEFAULT 0,
`cache_creation_ephemeral_5m_tokens` BIGINT NOT NULL DEFAULT 0,
`cache_creation_ephemeral_1h_tokens` BIGINT NOT NULL DEFAULT 0,
`settled_total_cost` DOUBLE NOT NULL DEFAULT 0,
`settled_total_requests` BIGINT NOT NULL DEFAULT 0,
`settled_input_tokens` BIGINT NOT NULL DEFAULT 0,
`settled_output_tokens` BIGINT NOT NULL DEFAULT 0,
`settled_cache_creation_tokens` BIGINT NOT NULL DEFAULT 0,
`settled_cache_read_tokens` BIGINT NOT NULL DEFAULT 0,
`settled_first_finalized_at_unix_secs` BIGINT,
`settled_last_finalized_at_unix_secs` BIGINT,
PRIMARY KEY (`id`),
UNIQUE KEY uq_stats_user_daily (`date`, `user_id`)
);
CREATE TABLE IF NOT EXISTS stats_user_summary (
`id` VARCHAR(64) NOT NULL,
`user_id` VARCHAR(64) NOT NULL,
`username` VARCHAR(255),
`cutoff_date` BIGINT NOT NULL,
`all_time_requests` BIGINT NOT NULL DEFAULT 0,
`all_time_success_requests` BIGINT NOT NULL DEFAULT 0,
`all_time_error_requests` BIGINT NOT NULL DEFAULT 0,
`all_time_input_tokens` BIGINT NOT NULL DEFAULT 0,
`all_time_output_tokens` BIGINT NOT NULL DEFAULT 0,
`all_time_cache_creation_tokens` BIGINT NOT NULL DEFAULT 0,
`all_time_cache_read_tokens` BIGINT NOT NULL DEFAULT 0,
`all_time_cost` DOUBLE NOT NULL DEFAULT 0,
`all_time_actual_cost` DOUBLE NOT NULL DEFAULT 0,
`active_days` BIGINT NOT NULL DEFAULT 0,
`first_active_date` BIGINT,
`last_active_date` BIGINT,
`created_at` BIGINT NOT NULL,
`updated_at` BIGINT NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY uq_stats_user_summary_user_id (`user_id`),
KEY idx_stats_user_summary_cutoff_date (`cutoff_date`)
);
CREATE TABLE IF NOT EXISTS stats_user_daily_model (
`id` VARCHAR(64) NOT NULL,
`user_id` VARCHAR(64) NOT NULL,
`username` VARCHAR(255),
`date` BIGINT NOT NULL,
`model` VARCHAR(255) NOT NULL,
`total_requests` BIGINT NOT NULL DEFAULT 0,
`success_requests` BIGINT NOT NULL DEFAULT 0,
`input_tokens` BIGINT NOT NULL DEFAULT 0,
`effective_input_tokens` BIGINT NOT NULL DEFAULT 0,
`output_tokens` BIGINT NOT NULL DEFAULT 0,
`total_tokens` BIGINT NOT NULL DEFAULT 0,
`total_input_context` BIGINT NOT NULL DEFAULT 0,
`cache_creation_tokens` BIGINT NOT NULL DEFAULT 0,
`cache_creation_ephemeral_5m_tokens` BIGINT NOT NULL DEFAULT 0,
`cache_creation_ephemeral_1h_tokens` BIGINT NOT NULL DEFAULT 0,
`cache_read_tokens` BIGINT NOT NULL DEFAULT 0,
`total_cost` DOUBLE NOT NULL DEFAULT 0,
`actual_total_cost` DOUBLE NOT NULL DEFAULT 0,
`response_time_sum_ms` DOUBLE NOT NULL DEFAULT 0,
`response_time_samples` BIGINT NOT NULL DEFAULT 0,
`successful_response_time_sum_ms` DOUBLE NOT NULL DEFAULT 0,
`successful_response_time_samples` BIGINT NOT NULL DEFAULT 0,
`created_at` BIGINT NOT NULL,
`updated_at` BIGINT NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY uq_stats_user_daily_model (`user_id`, `date`, `model`),
KEY idx_stats_user_daily_model_date (`date`),
KEY idx_stats_user_daily_model_user_id (`user_id`)
);
CREATE TABLE IF NOT EXISTS stats_user_daily_provider (
`id` VARCHAR(64) NOT NULL,
`user_id` VARCHAR(64) NOT NULL,
`username` VARCHAR(255),
`date` BIGINT NOT NULL,
`provider_name` VARCHAR(255) NOT NULL,
`total_requests` BIGINT NOT NULL DEFAULT 0,
`success_requests` BIGINT NOT NULL DEFAULT 0,
`input_tokens` BIGINT NOT NULL DEFAULT 0,
`effective_input_tokens` BIGINT NOT NULL DEFAULT 0,
`output_tokens` BIGINT NOT NULL DEFAULT 0,
`total_tokens` BIGINT NOT NULL DEFAULT 0,
`total_input_context` BIGINT NOT NULL DEFAULT 0,
`cache_creation_tokens` BIGINT NOT NULL DEFAULT 0,
`cache_creation_ephemeral_5m_tokens` BIGINT NOT NULL DEFAULT 0,
`cache_creation_ephemeral_1h_tokens` BIGINT NOT NULL DEFAULT 0,
`cache_read_tokens` BIGINT NOT NULL DEFAULT 0,
`total_cost` DOUBLE NOT NULL DEFAULT 0,
`actual_total_cost` DOUBLE NOT NULL DEFAULT 0,
`response_time_sum_ms` DOUBLE NOT NULL DEFAULT 0,
`response_time_samples` BIGINT NOT NULL DEFAULT 0,
`successful_response_time_sum_ms` DOUBLE NOT NULL DEFAULT 0,
`successful_response_time_samples` BIGINT NOT NULL DEFAULT 0,
`created_at` BIGINT NOT NULL,
`updated_at` BIGINT NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY uq_stats_user_daily_provider (`user_id`, `date`, `provider_name`),
KEY idx_stats_user_daily_provider_date (`date`),
KEY idx_stats_user_daily_provider_user_id (`user_id`)
);
CREATE TABLE IF NOT EXISTS stats_user_daily_api_format (
`id` VARCHAR(64) NOT NULL,
`user_id` VARCHAR(64) NOT NULL,
`username` VARCHAR(255),
`date` BIGINT NOT NULL,
`api_format` VARCHAR(128) NOT NULL,
`total_requests` BIGINT NOT NULL DEFAULT 0,
`success_requests` BIGINT NOT NULL DEFAULT 0,
`input_tokens` BIGINT NOT NULL DEFAULT 0,
`effective_input_tokens` BIGINT NOT NULL DEFAULT 0,
`output_tokens` BIGINT NOT NULL DEFAULT 0,
`total_tokens` BIGINT NOT NULL DEFAULT 0,
`total_input_context` BIGINT NOT NULL DEFAULT 0,
`cache_creation_tokens` BIGINT NOT NULL DEFAULT 0,
`cache_creation_ephemeral_5m_tokens` BIGINT NOT NULL DEFAULT 0,
`cache_creation_ephemeral_1h_tokens` BIGINT NOT NULL DEFAULT 0,
`cache_read_tokens` BIGINT NOT NULL DEFAULT 0,
`total_cost` DOUBLE NOT NULL DEFAULT 0,
`actual_total_cost` DOUBLE NOT NULL DEFAULT 0,
`response_time_sum_ms` DOUBLE NOT NULL DEFAULT 0,
`response_time_samples` BIGINT NOT NULL DEFAULT 0,
`successful_response_time_sum_ms` DOUBLE NOT NULL DEFAULT 0,
`successful_response_time_samples` BIGINT NOT NULL DEFAULT 0,
`created_at` BIGINT NOT NULL,
`updated_at` BIGINT NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY uq_stats_user_daily_api_format (`user_id`, `date`, `api_format`),
KEY idx_stats_user_daily_api_format_date (`date`),
KEY idx_stats_user_daily_api_format_user_id (`user_id`)
);
CREATE TABLE IF NOT EXISTS stats_daily_model_provider (
`id` VARCHAR(64) NOT NULL,
`date` BIGINT NOT NULL,
`model` VARCHAR(255) NOT NULL,
`provider_name` VARCHAR(255) NOT NULL,
`total_requests` BIGINT NOT NULL DEFAULT 0,
`total_tokens` BIGINT NOT NULL DEFAULT 0,
`total_cost` DOUBLE NOT NULL DEFAULT 0,
`response_time_sum_ms` DOUBLE NOT NULL DEFAULT 0,
`response_time_samples` BIGINT NOT NULL DEFAULT 0,
`created_at` BIGINT NOT NULL,
`updated_at` BIGINT NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY uq_stats_daily_model_provider (`date`, `model`, `provider_name`),
KEY idx_stats_daily_model_provider_date (`date`)
);
CREATE TABLE IF NOT EXISTS stats_user_daily_model_provider (
`id` VARCHAR(64) NOT NULL,
`user_id` VARCHAR(64) NOT NULL,
`username` VARCHAR(255),
`date` BIGINT NOT NULL,
`model` VARCHAR(255) NOT NULL,
`provider_name` VARCHAR(255) NOT NULL,
`total_requests` BIGINT NOT NULL DEFAULT 0,
`total_tokens` BIGINT NOT NULL DEFAULT 0,
`total_cost` DOUBLE NOT NULL DEFAULT 0,
`response_time_sum_ms` DOUBLE NOT NULL DEFAULT 0,
`response_time_samples` BIGINT NOT NULL DEFAULT 0,
`created_at` BIGINT NOT NULL,
`updated_at` BIGINT NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY uq_stats_user_daily_model_provider (`user_id`, `date`, `model`, `provider_name`),
KEY idx_stats_user_daily_model_provider_date (`date`),
KEY idx_stats_user_daily_model_provider_user_date (`user_id`, `date`)
);
CREATE TABLE IF NOT EXISTS stats_daily_cost_savings (
`id` VARCHAR(64) NOT NULL,
`date` BIGINT NOT NULL,
`cache_read_tokens` BIGINT NOT NULL DEFAULT 0,
`cache_read_cost` DOUBLE NOT NULL DEFAULT 0,
`cache_creation_cost` DOUBLE NOT NULL DEFAULT 0,
`estimated_full_cost` DOUBLE NOT NULL DEFAULT 0,
`created_at` BIGINT NOT NULL,
`updated_at` BIGINT NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY uq_stats_daily_cost_savings_date (`date`)
);
CREATE TABLE IF NOT EXISTS stats_daily_cost_savings_provider (
`id` VARCHAR(64) NOT NULL,
`date` BIGINT NOT NULL,
`provider_name` VARCHAR(255) NOT NULL,
`cache_read_tokens` BIGINT NOT NULL DEFAULT 0,
`cache_read_cost` DOUBLE NOT NULL DEFAULT 0,
`cache_creation_cost` DOUBLE NOT NULL DEFAULT 0,
`estimated_full_cost` DOUBLE NOT NULL DEFAULT 0,
`created_at` BIGINT NOT NULL,
`updated_at` BIGINT NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY uq_stats_daily_cost_savings_provider (`date`, `provider_name`),
KEY idx_stats_daily_cost_savings_provider_date (`date`)
);
CREATE TABLE IF NOT EXISTS stats_daily_cost_savings_model (
`id` VARCHAR(64) NOT NULL,
`date` BIGINT NOT NULL,
`model` VARCHAR(255) NOT NULL,
`cache_read_tokens` BIGINT NOT NULL DEFAULT 0,
`cache_read_cost` DOUBLE NOT NULL DEFAULT 0,
`cache_creation_cost` DOUBLE NOT NULL DEFAULT 0,
`estimated_full_cost` DOUBLE NOT NULL DEFAULT 0,
`created_at` BIGINT NOT NULL,
`updated_at` BIGINT NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY uq_stats_daily_cost_savings_model (`date`, `model`),
KEY idx_stats_daily_cost_savings_model_date (`date`)
);
CREATE TABLE IF NOT EXISTS stats_daily_cost_savings_model_provider (
`id` VARCHAR(64) NOT NULL,
`date` BIGINT NOT NULL,
`model` VARCHAR(255) NOT NULL,
`provider_name` VARCHAR(255) NOT NULL,
`cache_read_tokens` BIGINT NOT NULL DEFAULT 0,
`cache_read_cost` DOUBLE NOT NULL DEFAULT 0,
`cache_creation_cost` DOUBLE NOT NULL DEFAULT 0,
`estimated_full_cost` DOUBLE NOT NULL DEFAULT 0,
`created_at` BIGINT NOT NULL,
`updated_at` BIGINT NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY uq_stats_daily_cost_savings_model_provider (`date`, `model`, `provider_name`),
KEY idx_stats_daily_cost_savings_model_provider_date (`date`)
);
CREATE TABLE IF NOT EXISTS stats_user_daily_cost_savings (
`id` VARCHAR(64) NOT NULL,
`user_id` VARCHAR(64) NOT NULL,
`username` VARCHAR(255),
`date` BIGINT NOT NULL,
`cache_read_tokens` BIGINT NOT NULL DEFAULT 0,
`cache_read_cost` DOUBLE NOT NULL DEFAULT 0,
`cache_creation_cost` DOUBLE NOT NULL DEFAULT 0,
`estimated_full_cost` DOUBLE NOT NULL DEFAULT 0,
`created_at` BIGINT NOT NULL,
`updated_at` BIGINT NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY uq_stats_user_daily_cost_savings (`user_id`, `date`),
KEY idx_stats_user_daily_cost_savings_date (`date`)
);
CREATE TABLE IF NOT EXISTS stats_user_daily_cost_savings_provider (
`id` VARCHAR(64) NOT NULL,
`user_id` VARCHAR(64) NOT NULL,
`username` VARCHAR(255),
`date` BIGINT NOT NULL,
`provider_name` VARCHAR(255) NOT NULL,
`cache_read_tokens` BIGINT NOT NULL DEFAULT 0,
`cache_read_cost` DOUBLE NOT NULL DEFAULT 0,
`cache_creation_cost` DOUBLE NOT NULL DEFAULT 0,
`estimated_full_cost` DOUBLE NOT NULL DEFAULT 0,
`created_at` BIGINT NOT NULL,
`updated_at` BIGINT NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY uq_stats_user_daily_cost_savings_provider (`user_id`, `date`, `provider_name`),
KEY idx_stats_user_daily_cost_savings_provider_date (`date`)
);
CREATE TABLE IF NOT EXISTS stats_user_daily_cost_savings_model (
`id` VARCHAR(64) NOT NULL,
`user_id` VARCHAR(64) NOT NULL,
`username` VARCHAR(255),
`date` BIGINT NOT NULL,
`model` VARCHAR(255) NOT NULL,
`cache_read_tokens` BIGINT NOT NULL DEFAULT 0,
`cache_read_cost` DOUBLE NOT NULL DEFAULT 0,
`cache_creation_cost` DOUBLE NOT NULL DEFAULT 0,
`estimated_full_cost` DOUBLE NOT NULL DEFAULT 0,
`created_at` BIGINT NOT NULL,
`updated_at` BIGINT NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY uq_stats_user_daily_cost_savings_model (`user_id`, `date`, `model`),
KEY idx_stats_user_daily_cost_savings_model_date (`date`)
);
CREATE TABLE IF NOT EXISTS stats_user_daily_cost_savings_model_provider (
`id` VARCHAR(64) NOT NULL,
`user_id` VARCHAR(64) NOT NULL,
`username` VARCHAR(255),
`date` BIGINT NOT NULL,
`model` VARCHAR(255) NOT NULL,
`provider_name` VARCHAR(255) NOT NULL,
`cache_read_tokens` BIGINT NOT NULL DEFAULT 0,
`cache_read_cost` DOUBLE NOT NULL DEFAULT 0,
`cache_creation_cost` DOUBLE NOT NULL DEFAULT 0,
`estimated_full_cost` DOUBLE NOT NULL DEFAULT 0,
`created_at` BIGINT NOT NULL,
`updated_at` BIGINT NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY uq_stats_user_daily_cost_savings_model_provider (`user_id`, `date`, `model`, `provider_name`),
KEY idx_stats_user_daily_cost_savings_model_provider_date (`date`)
);
crates/aether-data/runtime/schema/generated/postgres/baseline/006_usage.sql
+89
View File
@@ -112,6 +112,66 @@ CREATE INDEX IF NOT EXISTS usage_request_id_idx ON public.usage USING btree (req
CREATE INDEX IF NOT EXISTS usage_user_id_idx ON public.usage USING btree (user_id);
CREATE INDEX IF NOT EXISTS usage_wallet_id_idx ON public.usage USING btree (wallet_id);
CREATE TABLE IF NOT EXISTS public.usage_body_blobs (
body_ref character varying(160) NOT NULL,
request_id character varying(128) NOT NULL,
body_field character varying(50) NOT NULL,
payload_gzip bytea NOT NULL,
created_at timestamp with time zone NOT NULL,
updated_at timestamp with time zone NOT NULL
);
ALTER TABLE ONLY public.usage_body_blobs ADD CONSTRAINT usage_body_blobs_pkey PRIMARY KEY (body_ref);
ALTER TABLE ONLY public.usage_body_blobs ADD CONSTRAINT usage_body_blobs_request_id_field_key UNIQUE (request_id, body_field);
CREATE INDEX IF NOT EXISTS ix_usage_body_blobs_request_id ON public.usage_body_blobs USING btree (request_id);
ALTER TABLE ONLY public.usage_body_blobs ADD CONSTRAINT usage_body_blobs_request_id_fkey FOREIGN KEY (request_id) REFERENCES public.usage(request_id) ON DELETE CASCADE;
CREATE TABLE IF NOT EXISTS public.usage_http_audits (
request_id character varying(128) NOT NULL,
request_headers jsonb,
provider_request_headers jsonb,
response_headers jsonb,
client_response_headers jsonb,
request_body_ref character varying(160),
provider_request_body_ref character varying(160),
response_body_ref character varying(160),
client_response_body_ref character varying(160),
request_body_state character varying(32),
provider_request_body_state character varying(32),
response_body_state character varying(32),
client_response_body_state character varying(32),
body_capture_mode character varying(32) DEFAULT 'none' NOT NULL,
created_at timestamp with time zone NOT NULL,
updated_at timestamp with time zone NOT NULL
);
ALTER TABLE ONLY public.usage_http_audits ADD CONSTRAINT usage_http_audits_pkey PRIMARY KEY (request_id);
CREATE INDEX IF NOT EXISTS ix_usage_http_audits_updated_at ON public.usage_http_audits USING btree (updated_at);
ALTER TABLE ONLY public.usage_http_audits ADD CONSTRAINT usage_http_audits_request_id_fkey FOREIGN KEY (request_id) REFERENCES public.usage(request_id) ON DELETE CASCADE;
CREATE TABLE IF NOT EXISTS public.usage_routing_snapshots (
request_id character varying(128) NOT NULL,
candidate_id character varying(160),
candidate_index bigint,
key_name character varying(255),
planner_kind character varying(120),
route_family character varying(80),
route_kind character varying(80),
execution_path character varying(80),
local_execution_runtime_miss_reason character varying(255),
selected_provider_id character varying(100),
selected_endpoint_id character varying(100),
selected_provider_api_key_id character varying(100),
has_format_conversion boolean,
created_at timestamp with time zone NOT NULL,
updated_at timestamp with time zone NOT NULL
);
ALTER TABLE ONLY public.usage_routing_snapshots ADD CONSTRAINT usage_routing_snapshots_pkey PRIMARY KEY (request_id);
CREATE INDEX IF NOT EXISTS ix_usage_routing_snapshots_route_family_kind ON public.usage_routing_snapshots USING btree (route_family, route_kind);
CREATE INDEX IF NOT EXISTS ix_usage_routing_snapshots_candidate_id ON public.usage_routing_snapshots USING btree (candidate_id);
ALTER TABLE ONLY public.usage_routing_snapshots ADD CONSTRAINT usage_routing_snapshots_request_id_fkey FOREIGN KEY (request_id) REFERENCES public.usage(request_id) ON DELETE CASCADE;
CREATE TABLE IF NOT EXISTS public.usage_counter_deltas (
id character varying(36) NOT NULL,
request_id character varying(128) NOT NULL,
@@ -151,6 +211,33 @@ CREATE TABLE IF NOT EXISTS public.usage_settlement_snapshots (
wallet_gift_balance_before double precision,
wallet_gift_balance_after double precision,
provider_monthly_used_usd double precision,
billing_snapshot_schema_version character varying(20),
billing_snapshot_status character varying(20),
rate_multiplier double precision,
is_free_tier boolean,
input_price_per_1m double precision,
output_price_per_1m double precision,
cache_creation_price_per_1m double precision,
cache_read_price_per_1m double precision,
price_per_request double precision,
settlement_snapshot_schema_version character varying(20),
settlement_snapshot jsonb,
billing_dimensions jsonb,
billing_input_tokens bigint,
billing_effective_input_tokens bigint,
billing_output_tokens bigint,
billing_cache_creation_tokens bigint,
billing_cache_creation_5m_tokens bigint,
billing_cache_creation_1h_tokens bigint,
billing_cache_read_tokens bigint,
billing_total_input_context bigint,
billing_cache_creation_cost_usd double precision,
billing_cache_read_cost_usd double precision,
billing_total_cost_usd double precision,
billing_actual_total_cost_usd double precision,
billing_pricing_source character varying(50),
billing_rule_id character varying(100),
billing_rule_version character varying(50),
finalized_at bigint,
created_at bigint NOT NULL,
updated_at bigint NOT NULL
@@ -159,4 +246,6 @@ CREATE TABLE IF NOT EXISTS public.usage_settlement_snapshots (
ALTER TABLE ONLY public.usage_settlement_snapshots ADD CONSTRAINT usage_settlement_snapshots_pkey PRIMARY KEY (request_id);
CREATE INDEX IF NOT EXISTS usage_settlement_snapshots_billing_status_idx ON public.usage_settlement_snapshots USING btree (billing_status);
CREATE INDEX IF NOT EXISTS usage_settlement_snapshots_wallet_id_idx ON public.usage_settlement_snapshots USING btree (wallet_id);
CREATE INDEX IF NOT EXISTS ix_usage_settlement_snapshots_schema_version ON public.usage_settlement_snapshots USING btree (settlement_snapshot_schema_version);
CREATE INDEX IF NOT EXISTS ix_usage_settlement_snapshots_pricing_source ON public.usage_settlement_snapshots USING btree (billing_pricing_source);
crates/aether-data/runtime/schema/generated/postgres/baseline/007_stats.sql
+384 -7
View File
@@ -17,7 +17,27 @@ CREATE TABLE IF NOT EXISTS public.stats_hourly (
is_complete boolean DEFAULT false NOT NULL,
aggregated_at bigint,
created_at bigint NOT NULL,
updated_at bigint NOT NULL
updated_at bigint NOT NULL,
response_time_sum_ms double precision DEFAULT 0 NOT NULL,
response_time_samples bigint DEFAULT 0 NOT NULL,
cache_hit_total_requests bigint DEFAULT 0 NOT NULL,
cache_hit_requests bigint DEFAULT 0 NOT NULL,
completed_total_requests bigint DEFAULT 0 NOT NULL,
completed_cache_hit_requests bigint DEFAULT 0 NOT NULL,
completed_input_tokens bigint DEFAULT 0 NOT NULL,
completed_cache_creation_tokens bigint DEFAULT 0 NOT NULL,
completed_cache_read_tokens bigint DEFAULT 0 NOT NULL,
completed_total_input_context bigint DEFAULT 0 NOT NULL,
completed_cache_creation_cost double precision DEFAULT 0 NOT NULL,
completed_cache_read_cost double precision DEFAULT 0 NOT NULL,
settled_total_cost double precision DEFAULT 0 NOT NULL,
settled_total_requests bigint DEFAULT 0 NOT NULL,
settled_input_tokens bigint DEFAULT 0 NOT NULL,
settled_output_tokens bigint DEFAULT 0 NOT NULL,
settled_cache_creation_tokens bigint DEFAULT 0 NOT NULL,
settled_cache_read_tokens bigint DEFAULT 0 NOT NULL,
settled_first_finalized_at_unix_secs bigint,
settled_last_finalized_at_unix_secs bigint
);
ALTER TABLE ONLY public.stats_hourly ADD CONSTRAINT stats_hourly_pkey PRIMARY KEY (id);
@@ -56,7 +76,20 @@ CREATE TABLE IF NOT EXISTS public.stats_hourly_user (
output_tokens bigint DEFAULT 0 NOT NULL,
total_cost double precision DEFAULT 0 NOT NULL,
created_at bigint NOT NULL,
updated_at bigint NOT NULL
updated_at bigint NOT NULL,
cache_creation_tokens bigint DEFAULT 0 NOT NULL,
cache_read_tokens bigint DEFAULT 0 NOT NULL,
actual_total_cost double precision DEFAULT 0 NOT NULL,
response_time_sum_ms double precision DEFAULT 0 NOT NULL,
response_time_samples bigint DEFAULT 0 NOT NULL,
settled_total_cost double precision DEFAULT 0 NOT NULL,
settled_total_requests bigint DEFAULT 0 NOT NULL,
settled_input_tokens bigint DEFAULT 0 NOT NULL,
settled_output_tokens bigint DEFAULT 0 NOT NULL,
settled_cache_creation_tokens bigint DEFAULT 0 NOT NULL,
settled_cache_read_tokens bigint DEFAULT 0 NOT NULL,
settled_first_finalized_at_unix_secs bigint,
settled_last_finalized_at_unix_secs bigint
);
ALTER TABLE ONLY public.stats_hourly_user ADD CONSTRAINT stats_hourly_user_pkey PRIMARY KEY (id);
@@ -72,7 +105,9 @@ CREATE TABLE IF NOT EXISTS public.stats_hourly_user_model (
output_tokens bigint DEFAULT 0 NOT NULL,
total_cost double precision DEFAULT 0 NOT NULL,
created_at bigint NOT NULL,
updated_at bigint NOT NULL
updated_at bigint NOT NULL,
response_time_sum_ms double precision DEFAULT 0 NOT NULL,
response_time_samples bigint DEFAULT 0 NOT NULL
);
ALTER TABLE ONLY public.stats_hourly_user_model ADD CONSTRAINT stats_hourly_user_model_pkey PRIMARY KEY (id);
@@ -102,7 +137,9 @@ CREATE TABLE IF NOT EXISTS public.stats_hourly_model (
total_cost double precision DEFAULT 0 NOT NULL,
avg_response_time_ms double precision DEFAULT 0 NOT NULL,
created_at bigint NOT NULL,
updated_at bigint NOT NULL
updated_at bigint NOT NULL,
response_time_sum_ms double precision DEFAULT 0 NOT NULL,
response_time_samples bigint DEFAULT 0 NOT NULL
);
ALTER TABLE ONLY public.stats_hourly_model ADD CONSTRAINT stats_hourly_model_pkey PRIMARY KEY (id);
@@ -152,7 +189,31 @@ CREATE TABLE IF NOT EXISTS public.stats_daily (
p99_response_time_ms bigint,
p50_first_byte_time_ms bigint,
p90_first_byte_time_ms bigint,
p99_first_byte_time_ms bigint
p99_first_byte_time_ms bigint,
effective_input_tokens bigint DEFAULT 0 NOT NULL,
total_input_context bigint DEFAULT 0 NOT NULL,
response_time_sum_ms double precision DEFAULT 0 NOT NULL,
response_time_samples bigint DEFAULT 0 NOT NULL,
cache_creation_ephemeral_5m_tokens bigint DEFAULT 0 NOT NULL,
cache_creation_ephemeral_1h_tokens bigint DEFAULT 0 NOT NULL,
cache_hit_total_requests bigint DEFAULT 0 NOT NULL,
cache_hit_requests bigint DEFAULT 0 NOT NULL,
completed_total_requests bigint DEFAULT 0 NOT NULL,
completed_cache_hit_requests bigint DEFAULT 0 NOT NULL,
completed_input_tokens bigint DEFAULT 0 NOT NULL,
completed_cache_creation_tokens bigint DEFAULT 0 NOT NULL,
completed_cache_read_tokens bigint DEFAULT 0 NOT NULL,
completed_total_input_context bigint DEFAULT 0 NOT NULL,
completed_cache_creation_cost double precision DEFAULT 0 NOT NULL,
completed_cache_read_cost double precision DEFAULT 0 NOT NULL,
settled_total_cost double precision DEFAULT 0 NOT NULL,
settled_total_requests bigint DEFAULT 0 NOT NULL,
settled_input_tokens bigint DEFAULT 0 NOT NULL,
settled_output_tokens bigint DEFAULT 0 NOT NULL,
settled_cache_creation_tokens bigint DEFAULT 0 NOT NULL,
settled_cache_read_tokens bigint DEFAULT 0 NOT NULL,
settled_first_finalized_at_unix_secs bigint,
settled_last_finalized_at_unix_secs bigint
);
ALTER TABLE ONLY public.stats_daily ADD CONSTRAINT stats_daily_pkey PRIMARY KEY (id);
@@ -170,7 +231,11 @@ CREATE TABLE IF NOT EXISTS public.stats_daily_model (
total_cost double precision DEFAULT 0 NOT NULL,
avg_response_time_ms double precision DEFAULT 0 NOT NULL,
created_at bigint NOT NULL,
updated_at bigint NOT NULL
updated_at bigint NOT NULL,
response_time_sum_ms double precision DEFAULT 0 NOT NULL,
response_time_samples bigint DEFAULT 0 NOT NULL,
cache_creation_ephemeral_5m_tokens bigint DEFAULT 0 NOT NULL,
cache_creation_ephemeral_1h_tokens bigint DEFAULT 0 NOT NULL
);
ALTER TABLE ONLY public.stats_daily_model ADD CONSTRAINT stats_daily_model_pkey PRIMARY KEY (id);
@@ -241,9 +306,321 @@ CREATE TABLE IF NOT EXISTS public.stats_user_daily (
total_cost double precision DEFAULT 0 NOT NULL,
username character varying(255),
created_at bigint NOT NULL,
updated_at bigint NOT NULL
updated_at bigint NOT NULL,
actual_total_cost double precision DEFAULT 0 NOT NULL,
response_time_sum_ms double precision DEFAULT 0 NOT NULL,
response_time_samples bigint DEFAULT 0 NOT NULL,
effective_input_tokens bigint DEFAULT 0 NOT NULL,
total_input_context bigint DEFAULT 0 NOT NULL,
cache_creation_cost double precision DEFAULT 0 NOT NULL,
cache_read_cost double precision DEFAULT 0 NOT NULL,
cache_creation_ephemeral_5m_tokens bigint DEFAULT 0 NOT NULL,
cache_creation_ephemeral_1h_tokens bigint DEFAULT 0 NOT NULL,
settled_total_cost double precision DEFAULT 0 NOT NULL,
settled_total_requests bigint DEFAULT 0 NOT NULL,
settled_input_tokens bigint DEFAULT 0 NOT NULL,
settled_output_tokens bigint DEFAULT 0 NOT NULL,
settled_cache_creation_tokens bigint DEFAULT 0 NOT NULL,
settled_cache_read_tokens bigint DEFAULT 0 NOT NULL,
settled_first_finalized_at_unix_secs bigint,
settled_last_finalized_at_unix_secs bigint
);
ALTER TABLE ONLY public.stats_user_daily ADD CONSTRAINT stats_user_daily_pkey PRIMARY KEY (id);
ALTER TABLE ONLY public.stats_user_daily ADD CONSTRAINT uq_stats_user_daily UNIQUE (date, user_id);
CREATE TABLE IF NOT EXISTS public.stats_user_summary (
id character varying(64) NOT NULL,
user_id character varying(64) NOT NULL,
username character varying(255),
cutoff_date bigint NOT NULL,
all_time_requests bigint DEFAULT 0 NOT NULL,
all_time_success_requests bigint DEFAULT 0 NOT NULL,
all_time_error_requests bigint DEFAULT 0 NOT NULL,
all_time_input_tokens bigint DEFAULT 0 NOT NULL,
all_time_output_tokens bigint DEFAULT 0 NOT NULL,
all_time_cache_creation_tokens bigint DEFAULT 0 NOT NULL,
all_time_cache_read_tokens bigint DEFAULT 0 NOT NULL,
all_time_cost double precision DEFAULT 0 NOT NULL,
all_time_actual_cost double precision DEFAULT 0 NOT NULL,
active_days bigint DEFAULT 0 NOT NULL,
first_active_date bigint,
last_active_date bigint,
created_at bigint NOT NULL,
updated_at bigint NOT NULL
);
ALTER TABLE ONLY public.stats_user_summary ADD CONSTRAINT stats_user_summary_pkey PRIMARY KEY (id);
ALTER TABLE ONLY public.stats_user_summary ADD CONSTRAINT uq_stats_user_summary_user_id UNIQUE (user_id);
CREATE INDEX IF NOT EXISTS idx_stats_user_summary_cutoff_date ON public.stats_user_summary USING btree (cutoff_date);
CREATE TABLE IF NOT EXISTS public.stats_user_daily_model (
id character varying(64) NOT NULL,
user_id character varying(64) NOT NULL,
username character varying(255),
date bigint NOT NULL,
model character varying(255) NOT NULL,
total_requests bigint DEFAULT 0 NOT NULL,
success_requests bigint DEFAULT 0 NOT NULL,
input_tokens bigint DEFAULT 0 NOT NULL,
effective_input_tokens bigint DEFAULT 0 NOT NULL,
output_tokens bigint DEFAULT 0 NOT NULL,
total_tokens bigint DEFAULT 0 NOT NULL,
total_input_context bigint DEFAULT 0 NOT NULL,
cache_creation_tokens bigint DEFAULT 0 NOT NULL,
cache_creation_ephemeral_5m_tokens bigint DEFAULT 0 NOT NULL,
cache_creation_ephemeral_1h_tokens bigint DEFAULT 0 NOT NULL,
cache_read_tokens bigint DEFAULT 0 NOT NULL,
total_cost double precision DEFAULT 0 NOT NULL,
actual_total_cost double precision DEFAULT 0 NOT NULL,
response_time_sum_ms double precision DEFAULT 0 NOT NULL,
response_time_samples bigint DEFAULT 0 NOT NULL,
successful_response_time_sum_ms double precision DEFAULT 0 NOT NULL,
successful_response_time_samples bigint DEFAULT 0 NOT NULL,
created_at bigint NOT NULL,
updated_at bigint NOT NULL
);
ALTER TABLE ONLY public.stats_user_daily_model ADD CONSTRAINT stats_user_daily_model_pkey PRIMARY KEY (id);
ALTER TABLE ONLY public.stats_user_daily_model ADD CONSTRAINT uq_stats_user_daily_model UNIQUE (user_id, date, model);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_model_date ON public.stats_user_daily_model USING btree (date);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_model_user_id ON public.stats_user_daily_model USING btree (user_id);
CREATE TABLE IF NOT EXISTS public.stats_user_daily_provider (
id character varying(64) NOT NULL,
user_id character varying(64) NOT NULL,
username character varying(255),
date bigint NOT NULL,
provider_name character varying(255) NOT NULL,
total_requests bigint DEFAULT 0 NOT NULL,
success_requests bigint DEFAULT 0 NOT NULL,
input_tokens bigint DEFAULT 0 NOT NULL,
effective_input_tokens bigint DEFAULT 0 NOT NULL,
output_tokens bigint DEFAULT 0 NOT NULL,
total_tokens bigint DEFAULT 0 NOT NULL,
total_input_context bigint DEFAULT 0 NOT NULL,
cache_creation_tokens bigint DEFAULT 0 NOT NULL,
cache_creation_ephemeral_5m_tokens bigint DEFAULT 0 NOT NULL,
cache_creation_ephemeral_1h_tokens bigint DEFAULT 0 NOT NULL,
cache_read_tokens bigint DEFAULT 0 NOT NULL,
total_cost double precision DEFAULT 0 NOT NULL,
actual_total_cost double precision DEFAULT 0 NOT NULL,
response_time_sum_ms double precision DEFAULT 0 NOT NULL,
response_time_samples bigint DEFAULT 0 NOT NULL,
successful_response_time_sum_ms double precision DEFAULT 0 NOT NULL,
successful_response_time_samples bigint DEFAULT 0 NOT NULL,
created_at bigint NOT NULL,
updated_at bigint NOT NULL
);
ALTER TABLE ONLY public.stats_user_daily_provider ADD CONSTRAINT stats_user_daily_provider_pkey PRIMARY KEY (id);
ALTER TABLE ONLY public.stats_user_daily_provider ADD CONSTRAINT uq_stats_user_daily_provider UNIQUE (user_id, date, provider_name);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_provider_date ON public.stats_user_daily_provider USING btree (date);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_provider_user_id ON public.stats_user_daily_provider USING btree (user_id);
CREATE TABLE IF NOT EXISTS public.stats_user_daily_api_format (
id character varying(64) NOT NULL,
user_id character varying(64) NOT NULL,
username character varying(255),
date bigint NOT NULL,
api_format character varying(128) NOT NULL,
total_requests bigint DEFAULT 0 NOT NULL,
success_requests bigint DEFAULT 0 NOT NULL,
input_tokens bigint DEFAULT 0 NOT NULL,
effective_input_tokens bigint DEFAULT 0 NOT NULL,
output_tokens bigint DEFAULT 0 NOT NULL,
total_tokens bigint DEFAULT 0 NOT NULL,
total_input_context bigint DEFAULT 0 NOT NULL,
cache_creation_tokens bigint DEFAULT 0 NOT NULL,
cache_creation_ephemeral_5m_tokens bigint DEFAULT 0 NOT NULL,
cache_creation_ephemeral_1h_tokens bigint DEFAULT 0 NOT NULL,
cache_read_tokens bigint DEFAULT 0 NOT NULL,
total_cost double precision DEFAULT 0 NOT NULL,
actual_total_cost double precision DEFAULT 0 NOT NULL,
response_time_sum_ms double precision DEFAULT 0 NOT NULL,
response_time_samples bigint DEFAULT 0 NOT NULL,
successful_response_time_sum_ms double precision DEFAULT 0 NOT NULL,
successful_response_time_samples bigint DEFAULT 0 NOT NULL,
created_at bigint NOT NULL,
updated_at bigint NOT NULL
);
ALTER TABLE ONLY public.stats_user_daily_api_format ADD CONSTRAINT stats_user_daily_api_format_pkey PRIMARY KEY (id);
ALTER TABLE ONLY public.stats_user_daily_api_format ADD CONSTRAINT uq_stats_user_daily_api_format UNIQUE (user_id, date, api_format);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_api_format_date ON public.stats_user_daily_api_format USING btree (date);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_api_format_user_id ON public.stats_user_daily_api_format USING btree (user_id);
CREATE TABLE IF NOT EXISTS public.stats_daily_model_provider (
id character varying(64) NOT NULL,
date bigint NOT NULL,
model character varying(255) NOT NULL,
provider_name character varying(255) NOT NULL,
total_requests bigint DEFAULT 0 NOT NULL,
total_tokens bigint DEFAULT 0 NOT NULL,
total_cost double precision DEFAULT 0 NOT NULL,
response_time_sum_ms double precision DEFAULT 0 NOT NULL,
response_time_samples bigint DEFAULT 0 NOT NULL,
created_at bigint NOT NULL,
updated_at bigint NOT NULL
);
ALTER TABLE ONLY public.stats_daily_model_provider ADD CONSTRAINT stats_daily_model_provider_pkey PRIMARY KEY (id);
ALTER TABLE ONLY public.stats_daily_model_provider ADD CONSTRAINT uq_stats_daily_model_provider UNIQUE (date, model, provider_name);
CREATE INDEX IF NOT EXISTS idx_stats_daily_model_provider_date ON public.stats_daily_model_provider USING btree (date);
CREATE TABLE IF NOT EXISTS public.stats_user_daily_model_provider (
id character varying(64) NOT NULL,
user_id character varying(64) NOT NULL,
username character varying(255),
date bigint NOT NULL,
model character varying(255) NOT NULL,
provider_name character varying(255) NOT NULL,
total_requests bigint DEFAULT 0 NOT NULL,
total_tokens bigint DEFAULT 0 NOT NULL,
total_cost double precision DEFAULT 0 NOT NULL,
response_time_sum_ms double precision DEFAULT 0 NOT NULL,
response_time_samples bigint DEFAULT 0 NOT NULL,
created_at bigint NOT NULL,
updated_at bigint NOT NULL
);
ALTER TABLE ONLY public.stats_user_daily_model_provider ADD CONSTRAINT stats_user_daily_model_provider_pkey PRIMARY KEY (id);
ALTER TABLE ONLY public.stats_user_daily_model_provider ADD CONSTRAINT uq_stats_user_daily_model_provider UNIQUE (user_id, date, model, provider_name);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_model_provider_date ON public.stats_user_daily_model_provider USING btree (date);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_model_provider_user_date ON public.stats_user_daily_model_provider USING btree (user_id, date);
CREATE TABLE IF NOT EXISTS public.stats_daily_cost_savings (
id character varying(64) NOT NULL,
date bigint NOT NULL,
cache_read_tokens bigint DEFAULT 0 NOT NULL,
cache_read_cost double precision DEFAULT 0 NOT NULL,
cache_creation_cost double precision DEFAULT 0 NOT NULL,
estimated_full_cost double precision DEFAULT 0 NOT NULL,
created_at bigint NOT NULL,
updated_at bigint NOT NULL
);
ALTER TABLE ONLY public.stats_daily_cost_savings ADD CONSTRAINT stats_daily_cost_savings_pkey PRIMARY KEY (id);
ALTER TABLE ONLY public.stats_daily_cost_savings ADD CONSTRAINT uq_stats_daily_cost_savings_date UNIQUE (date);
CREATE TABLE IF NOT EXISTS public.stats_daily_cost_savings_provider (
id character varying(64) NOT NULL,
date bigint NOT NULL,
provider_name character varying(255) NOT NULL,
cache_read_tokens bigint DEFAULT 0 NOT NULL,
cache_read_cost double precision DEFAULT 0 NOT NULL,
cache_creation_cost double precision DEFAULT 0 NOT NULL,
estimated_full_cost double precision DEFAULT 0 NOT NULL,
created_at bigint NOT NULL,
updated_at bigint NOT NULL
);
ALTER TABLE ONLY public.stats_daily_cost_savings_provider ADD CONSTRAINT stats_daily_cost_savings_provider_pkey PRIMARY KEY (id);
ALTER TABLE ONLY public.stats_daily_cost_savings_provider ADD CONSTRAINT uq_stats_daily_cost_savings_provider UNIQUE (date, provider_name);
CREATE INDEX IF NOT EXISTS idx_stats_daily_cost_savings_provider_date ON public.stats_daily_cost_savings_provider USING btree (date);
CREATE TABLE IF NOT EXISTS public.stats_daily_cost_savings_model (
id character varying(64) NOT NULL,
date bigint NOT NULL,
model character varying(255) NOT NULL,
cache_read_tokens bigint DEFAULT 0 NOT NULL,
cache_read_cost double precision DEFAULT 0 NOT NULL,
cache_creation_cost double precision DEFAULT 0 NOT NULL,
estimated_full_cost double precision DEFAULT 0 NOT NULL,
created_at bigint NOT NULL,
updated_at bigint NOT NULL
);
ALTER TABLE ONLY public.stats_daily_cost_savings_model ADD CONSTRAINT stats_daily_cost_savings_model_pkey PRIMARY KEY (id);
ALTER TABLE ONLY public.stats_daily_cost_savings_model ADD CONSTRAINT uq_stats_daily_cost_savings_model UNIQUE (date, model);
CREATE INDEX IF NOT EXISTS idx_stats_daily_cost_savings_model_date ON public.stats_daily_cost_savings_model USING btree (date);
CREATE TABLE IF NOT EXISTS public.stats_daily_cost_savings_model_provider (
id character varying(64) NOT NULL,
date bigint NOT NULL,
model character varying(255) NOT NULL,
provider_name character varying(255) NOT NULL,
cache_read_tokens bigint DEFAULT 0 NOT NULL,
cache_read_cost double precision DEFAULT 0 NOT NULL,
cache_creation_cost double precision DEFAULT 0 NOT NULL,
estimated_full_cost double precision DEFAULT 0 NOT NULL,
created_at bigint NOT NULL,
updated_at bigint NOT NULL
);
ALTER TABLE ONLY public.stats_daily_cost_savings_model_provider ADD CONSTRAINT stats_daily_cost_savings_model_provider_pkey PRIMARY KEY (id);
ALTER TABLE ONLY public.stats_daily_cost_savings_model_provider ADD CONSTRAINT uq_stats_daily_cost_savings_model_provider UNIQUE (date, model, provider_name);
CREATE INDEX IF NOT EXISTS idx_stats_daily_cost_savings_model_provider_date ON public.stats_daily_cost_savings_model_provider USING btree (date);
CREATE TABLE IF NOT EXISTS public.stats_user_daily_cost_savings (
id character varying(64) NOT NULL,
user_id character varying(64) NOT NULL,
username character varying(255),
date bigint NOT NULL,
cache_read_tokens bigint DEFAULT 0 NOT NULL,
cache_read_cost double precision DEFAULT 0 NOT NULL,
cache_creation_cost double precision DEFAULT 0 NOT NULL,
estimated_full_cost double precision DEFAULT 0 NOT NULL,
created_at bigint NOT NULL,
updated_at bigint NOT NULL
);
ALTER TABLE ONLY public.stats_user_daily_cost_savings ADD CONSTRAINT stats_user_daily_cost_savings_pkey PRIMARY KEY (id);
ALTER TABLE ONLY public.stats_user_daily_cost_savings ADD CONSTRAINT uq_stats_user_daily_cost_savings UNIQUE (user_id, date);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_cost_savings_date ON public.stats_user_daily_cost_savings USING btree (date);
CREATE TABLE IF NOT EXISTS public.stats_user_daily_cost_savings_provider (
id character varying(64) NOT NULL,
user_id character varying(64) NOT NULL,
username character varying(255),
date bigint NOT NULL,
provider_name character varying(255) NOT NULL,
cache_read_tokens bigint DEFAULT 0 NOT NULL,
cache_read_cost double precision DEFAULT 0 NOT NULL,
cache_creation_cost double precision DEFAULT 0 NOT NULL,
estimated_full_cost double precision DEFAULT 0 NOT NULL,
created_at bigint NOT NULL,
updated_at bigint NOT NULL
);
ALTER TABLE ONLY public.stats_user_daily_cost_savings_provider ADD CONSTRAINT stats_user_daily_cost_savings_provider_pkey PRIMARY KEY (id);
ALTER TABLE ONLY public.stats_user_daily_cost_savings_provider ADD CONSTRAINT uq_stats_user_daily_cost_savings_provider UNIQUE (user_id, date, provider_name);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_cost_savings_provider_date ON public.stats_user_daily_cost_savings_provider USING btree (date);
CREATE TABLE IF NOT EXISTS public.stats_user_daily_cost_savings_model (
id character varying(64) NOT NULL,
user_id character varying(64) NOT NULL,
username character varying(255),
date bigint NOT NULL,
model character varying(255) NOT NULL,
cache_read_tokens bigint DEFAULT 0 NOT NULL,
cache_read_cost double precision DEFAULT 0 NOT NULL,
cache_creation_cost double precision DEFAULT 0 NOT NULL,
estimated_full_cost double precision DEFAULT 0 NOT NULL,
created_at bigint NOT NULL,
updated_at bigint NOT NULL
);
ALTER TABLE ONLY public.stats_user_daily_cost_savings_model ADD CONSTRAINT stats_user_daily_cost_savings_model_pkey PRIMARY KEY (id);
ALTER TABLE ONLY public.stats_user_daily_cost_savings_model ADD CONSTRAINT uq_stats_user_daily_cost_savings_model UNIQUE (user_id, date, model);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_cost_savings_model_date ON public.stats_user_daily_cost_savings_model USING btree (date);
CREATE TABLE IF NOT EXISTS public.stats_user_daily_cost_savings_model_provider (
id character varying(64) NOT NULL,
user_id character varying(64) NOT NULL,
username character varying(255),
date bigint NOT NULL,
model character varying(255) NOT NULL,
provider_name character varying(255) NOT NULL,
cache_read_tokens bigint DEFAULT 0 NOT NULL,
cache_read_cost double precision DEFAULT 0 NOT NULL,
cache_creation_cost double precision DEFAULT 0 NOT NULL,
estimated_full_cost double precision DEFAULT 0 NOT NULL,
created_at bigint NOT NULL,
updated_at bigint NOT NULL
);
ALTER TABLE ONLY public.stats_user_daily_cost_savings_model_provider ADD CONSTRAINT stats_user_daily_cost_savings_model_provider_pkey PRIMARY KEY (id);
ALTER TABLE ONLY public.stats_user_daily_cost_savings_model_provider ADD CONSTRAINT uq_stats_user_daily_cost_savings_model_provider UNIQUE (user_id, date, model, provider_name);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_cost_savings_model_provider_date ON public.stats_user_daily_cost_savings_model_provider USING btree (date);
crates/aether-data/runtime/schema/generated/sqlite/baseline/006_usage.sql
+83
View File
@@ -110,6 +110,60 @@ CREATE INDEX IF NOT EXISTS usage_request_id_idx ON "usage" (request_id);
CREATE INDEX IF NOT EXISTS usage_user_id_idx ON "usage" (user_id);
CREATE INDEX IF NOT EXISTS usage_wallet_id_idx ON "usage" (wallet_id);
CREATE TABLE IF NOT EXISTS usage_body_blobs (
body_ref TEXT PRIMARY KEY NOT NULL,
request_id TEXT NOT NULL,
body_field TEXT NOT NULL,
payload_gzip BLOB NOT NULL,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
UNIQUE (request_id, body_field),
CONSTRAINT usage_body_blobs_request_id_fkey FOREIGN KEY (request_id) REFERENCES usage (request_id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS ix_usage_body_blobs_request_id ON usage_body_blobs (request_id);
CREATE TABLE IF NOT EXISTS usage_http_audits (
request_id TEXT PRIMARY KEY NOT NULL,
request_headers TEXT,
provider_request_headers TEXT,
response_headers TEXT,
client_response_headers TEXT,
request_body_ref TEXT,
provider_request_body_ref TEXT,
response_body_ref TEXT,
client_response_body_ref TEXT,
request_body_state TEXT,
provider_request_body_state TEXT,
response_body_state TEXT,
client_response_body_state TEXT,
body_capture_mode TEXT NOT NULL DEFAULT 'none',
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
CONSTRAINT usage_http_audits_request_id_fkey FOREIGN KEY (request_id) REFERENCES usage (request_id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS ix_usage_http_audits_updated_at ON usage_http_audits (updated_at);
CREATE TABLE IF NOT EXISTS usage_routing_snapshots (
request_id TEXT PRIMARY KEY NOT NULL,
candidate_id TEXT,
candidate_index INTEGER,
key_name TEXT,
planner_kind TEXT,
route_family TEXT,
route_kind TEXT,
execution_path TEXT,
local_execution_runtime_miss_reason TEXT,
selected_provider_id TEXT,
selected_endpoint_id TEXT,
selected_provider_api_key_id TEXT,
has_format_conversion INTEGER,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
CONSTRAINT usage_routing_snapshots_request_id_fkey FOREIGN KEY (request_id) REFERENCES usage (request_id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS ix_usage_routing_snapshots_route_family_kind ON usage_routing_snapshots (route_family, route_kind);
CREATE INDEX IF NOT EXISTS ix_usage_routing_snapshots_candidate_id ON usage_routing_snapshots (candidate_id);
CREATE TABLE IF NOT EXISTS usage_counter_deltas (
id TEXT PRIMARY KEY NOT NULL,
request_id TEXT NOT NULL,
@@ -147,10 +201,39 @@ CREATE TABLE IF NOT EXISTS usage_settlement_snapshots (
wallet_gift_balance_before REAL,
wallet_gift_balance_after REAL,
provider_monthly_used_usd REAL,
billing_snapshot_schema_version TEXT,
billing_snapshot_status TEXT,
rate_multiplier REAL,
is_free_tier INTEGER,
input_price_per_1m REAL,
output_price_per_1m REAL,
cache_creation_price_per_1m REAL,
cache_read_price_per_1m REAL,
price_per_request REAL,
settlement_snapshot_schema_version TEXT,
settlement_snapshot TEXT,
billing_dimensions TEXT,
billing_input_tokens INTEGER,
billing_effective_input_tokens INTEGER,
billing_output_tokens INTEGER,
billing_cache_creation_tokens INTEGER,
billing_cache_creation_5m_tokens INTEGER,
billing_cache_creation_1h_tokens INTEGER,
billing_cache_read_tokens INTEGER,
billing_total_input_context INTEGER,
billing_cache_creation_cost_usd REAL,
billing_cache_read_cost_usd REAL,
billing_total_cost_usd REAL,
billing_actual_total_cost_usd REAL,
billing_pricing_source TEXT,
billing_rule_id TEXT,
billing_rule_version TEXT,
finalized_at INTEGER,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE INDEX IF NOT EXISTS usage_settlement_snapshots_billing_status_idx ON usage_settlement_snapshots (billing_status);
CREATE INDEX IF NOT EXISTS usage_settlement_snapshots_wallet_id_idx ON usage_settlement_snapshots (wallet_id);
CREATE INDEX IF NOT EXISTS ix_usage_settlement_snapshots_schema_version ON usage_settlement_snapshots (settlement_snapshot_schema_version);
CREATE INDEX IF NOT EXISTS ix_usage_settlement_snapshots_pricing_source ON usage_settlement_snapshots (billing_pricing_source);
crates/aether-data/runtime/schema/generated/sqlite/baseline/007_stats.sql
+349
View File
@@ -18,6 +18,26 @@ CREATE TABLE IF NOT EXISTS stats_hourly (
aggregated_at INTEGER,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
response_time_sum_ms REAL NOT NULL DEFAULT 0,
response_time_samples INTEGER NOT NULL DEFAULT 0,
cache_hit_total_requests INTEGER NOT NULL DEFAULT 0,
cache_hit_requests INTEGER NOT NULL DEFAULT 0,
completed_total_requests INTEGER NOT NULL DEFAULT 0,
completed_cache_hit_requests INTEGER NOT NULL DEFAULT 0,
completed_input_tokens INTEGER NOT NULL DEFAULT 0,
completed_cache_creation_tokens INTEGER NOT NULL DEFAULT 0,
completed_cache_read_tokens INTEGER NOT NULL DEFAULT 0,
completed_total_input_context INTEGER NOT NULL DEFAULT 0,
completed_cache_creation_cost REAL NOT NULL DEFAULT 0,
completed_cache_read_cost REAL NOT NULL DEFAULT 0,
settled_total_cost REAL NOT NULL DEFAULT 0,
settled_total_requests INTEGER NOT NULL DEFAULT 0,
settled_input_tokens INTEGER NOT NULL DEFAULT 0,
settled_output_tokens INTEGER NOT NULL DEFAULT 0,
settled_cache_creation_tokens INTEGER NOT NULL DEFAULT 0,
settled_cache_read_tokens INTEGER NOT NULL DEFAULT 0,
settled_first_finalized_at_unix_secs INTEGER,
settled_last_finalized_at_unix_secs INTEGER,
UNIQUE (hour_utc)
);
@@ -53,6 +73,19 @@ CREATE TABLE IF NOT EXISTS stats_hourly_user (
total_cost REAL NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
cache_creation_tokens INTEGER NOT NULL DEFAULT 0,
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
actual_total_cost REAL NOT NULL DEFAULT 0,
response_time_sum_ms REAL NOT NULL DEFAULT 0,
response_time_samples INTEGER NOT NULL DEFAULT 0,
settled_total_cost REAL NOT NULL DEFAULT 0,
settled_total_requests INTEGER NOT NULL DEFAULT 0,
settled_input_tokens INTEGER NOT NULL DEFAULT 0,
settled_output_tokens INTEGER NOT NULL DEFAULT 0,
settled_cache_creation_tokens INTEGER NOT NULL DEFAULT 0,
settled_cache_read_tokens INTEGER NOT NULL DEFAULT 0,
settled_first_finalized_at_unix_secs INTEGER,
settled_last_finalized_at_unix_secs INTEGER,
UNIQUE (hour_utc, user_id)
);
@@ -67,6 +100,8 @@ CREATE TABLE IF NOT EXISTS stats_hourly_user_model (
total_cost REAL NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
response_time_sum_ms REAL NOT NULL DEFAULT 0,
response_time_samples INTEGER NOT NULL DEFAULT 0,
UNIQUE (hour_utc, user_id, model)
);
@@ -93,6 +128,8 @@ CREATE TABLE IF NOT EXISTS stats_hourly_model (
avg_response_time_ms REAL NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
response_time_sum_ms REAL NOT NULL DEFAULT 0,
response_time_samples INTEGER NOT NULL DEFAULT 0,
UNIQUE (hour_utc, model)
);
@@ -139,6 +176,30 @@ CREATE TABLE IF NOT EXISTS stats_daily (
p50_first_byte_time_ms INTEGER,
p90_first_byte_time_ms INTEGER,
p99_first_byte_time_ms INTEGER,
effective_input_tokens INTEGER NOT NULL DEFAULT 0,
total_input_context INTEGER NOT NULL DEFAULT 0,
response_time_sum_ms REAL NOT NULL DEFAULT 0,
response_time_samples INTEGER NOT NULL DEFAULT 0,
cache_creation_ephemeral_5m_tokens INTEGER NOT NULL DEFAULT 0,
cache_creation_ephemeral_1h_tokens INTEGER NOT NULL DEFAULT 0,
cache_hit_total_requests INTEGER NOT NULL DEFAULT 0,
cache_hit_requests INTEGER NOT NULL DEFAULT 0,
completed_total_requests INTEGER NOT NULL DEFAULT 0,
completed_cache_hit_requests INTEGER NOT NULL DEFAULT 0,
completed_input_tokens INTEGER NOT NULL DEFAULT 0,
completed_cache_creation_tokens INTEGER NOT NULL DEFAULT 0,
completed_cache_read_tokens INTEGER NOT NULL DEFAULT 0,
completed_total_input_context INTEGER NOT NULL DEFAULT 0,
completed_cache_creation_cost REAL NOT NULL DEFAULT 0,
completed_cache_read_cost REAL NOT NULL DEFAULT 0,
settled_total_cost REAL NOT NULL DEFAULT 0,
settled_total_requests INTEGER NOT NULL DEFAULT 0,
settled_input_tokens INTEGER NOT NULL DEFAULT 0,
settled_output_tokens INTEGER NOT NULL DEFAULT 0,
settled_cache_creation_tokens INTEGER NOT NULL DEFAULT 0,
settled_cache_read_tokens INTEGER NOT NULL DEFAULT 0,
settled_first_finalized_at_unix_secs INTEGER,
settled_last_finalized_at_unix_secs INTEGER,
UNIQUE (date)
);
@@ -155,6 +216,10 @@ CREATE TABLE IF NOT EXISTS stats_daily_model (
avg_response_time_ms REAL NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
response_time_sum_ms REAL NOT NULL DEFAULT 0,
response_time_samples INTEGER NOT NULL DEFAULT 0,
cache_creation_ephemeral_5m_tokens INTEGER NOT NULL DEFAULT 0,
cache_creation_ephemeral_1h_tokens INTEGER NOT NULL DEFAULT 0,
UNIQUE (date, model)
);
@@ -218,6 +283,290 @@ CREATE TABLE IF NOT EXISTS stats_user_daily (
username TEXT,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
actual_total_cost REAL NOT NULL DEFAULT 0,
response_time_sum_ms REAL NOT NULL DEFAULT 0,
response_time_samples INTEGER NOT NULL DEFAULT 0,
effective_input_tokens INTEGER NOT NULL DEFAULT 0,
total_input_context INTEGER NOT NULL DEFAULT 0,
cache_creation_cost REAL NOT NULL DEFAULT 0,
cache_read_cost REAL NOT NULL DEFAULT 0,
cache_creation_ephemeral_5m_tokens INTEGER NOT NULL DEFAULT 0,
cache_creation_ephemeral_1h_tokens INTEGER NOT NULL DEFAULT 0,
settled_total_cost REAL NOT NULL DEFAULT 0,
settled_total_requests INTEGER NOT NULL DEFAULT 0,
settled_input_tokens INTEGER NOT NULL DEFAULT 0,
settled_output_tokens INTEGER NOT NULL DEFAULT 0,
settled_cache_creation_tokens INTEGER NOT NULL DEFAULT 0,
settled_cache_read_tokens INTEGER NOT NULL DEFAULT 0,
settled_first_finalized_at_unix_secs INTEGER,
settled_last_finalized_at_unix_secs INTEGER,
UNIQUE (date, user_id)
);
CREATE TABLE IF NOT EXISTS stats_user_summary (
id TEXT PRIMARY KEY NOT NULL,
user_id TEXT NOT NULL,
username TEXT,
cutoff_date INTEGER NOT NULL,
all_time_requests INTEGER NOT NULL DEFAULT 0,
all_time_success_requests INTEGER NOT NULL DEFAULT 0,
all_time_error_requests INTEGER NOT NULL DEFAULT 0,
all_time_input_tokens INTEGER NOT NULL DEFAULT 0,
all_time_output_tokens INTEGER NOT NULL DEFAULT 0,
all_time_cache_creation_tokens INTEGER NOT NULL DEFAULT 0,
all_time_cache_read_tokens INTEGER NOT NULL DEFAULT 0,
all_time_cost REAL NOT NULL DEFAULT 0,
all_time_actual_cost REAL NOT NULL DEFAULT 0,
active_days INTEGER NOT NULL DEFAULT 0,
first_active_date INTEGER,
last_active_date INTEGER,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
UNIQUE (user_id)
);
CREATE INDEX IF NOT EXISTS idx_stats_user_summary_cutoff_date ON stats_user_summary (cutoff_date);
CREATE TABLE IF NOT EXISTS stats_user_daily_model (
id TEXT PRIMARY KEY NOT NULL,
user_id TEXT NOT NULL,
username TEXT,
date INTEGER NOT NULL,
model TEXT NOT NULL,
total_requests INTEGER NOT NULL DEFAULT 0,
success_requests INTEGER NOT NULL DEFAULT 0,
input_tokens INTEGER NOT NULL DEFAULT 0,
effective_input_tokens INTEGER NOT NULL DEFAULT 0,
output_tokens INTEGER NOT NULL DEFAULT 0,
total_tokens INTEGER NOT NULL DEFAULT 0,
total_input_context INTEGER NOT NULL DEFAULT 0,
cache_creation_tokens INTEGER NOT NULL DEFAULT 0,
cache_creation_ephemeral_5m_tokens INTEGER NOT NULL DEFAULT 0,
cache_creation_ephemeral_1h_tokens INTEGER NOT NULL DEFAULT 0,
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
total_cost REAL NOT NULL DEFAULT 0,
actual_total_cost REAL NOT NULL DEFAULT 0,
response_time_sum_ms REAL NOT NULL DEFAULT 0,
response_time_samples INTEGER NOT NULL DEFAULT 0,
successful_response_time_sum_ms REAL NOT NULL DEFAULT 0,
successful_response_time_samples INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
UNIQUE (user_id, date, model)
);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_model_date ON stats_user_daily_model (date);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_model_user_id ON stats_user_daily_model (user_id);
CREATE TABLE IF NOT EXISTS stats_user_daily_provider (
id TEXT PRIMARY KEY NOT NULL,
user_id TEXT NOT NULL,
username TEXT,
date INTEGER NOT NULL,
provider_name TEXT NOT NULL,
total_requests INTEGER NOT NULL DEFAULT 0,
success_requests INTEGER NOT NULL DEFAULT 0,
input_tokens INTEGER NOT NULL DEFAULT 0,
effective_input_tokens INTEGER NOT NULL DEFAULT 0,
output_tokens INTEGER NOT NULL DEFAULT 0,
total_tokens INTEGER NOT NULL DEFAULT 0,
total_input_context INTEGER NOT NULL DEFAULT 0,
cache_creation_tokens INTEGER NOT NULL DEFAULT 0,
cache_creation_ephemeral_5m_tokens INTEGER NOT NULL DEFAULT 0,
cache_creation_ephemeral_1h_tokens INTEGER NOT NULL DEFAULT 0,
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
total_cost REAL NOT NULL DEFAULT 0,
actual_total_cost REAL NOT NULL DEFAULT 0,
response_time_sum_ms REAL NOT NULL DEFAULT 0,
response_time_samples INTEGER NOT NULL DEFAULT 0,
successful_response_time_sum_ms REAL NOT NULL DEFAULT 0,
successful_response_time_samples INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
UNIQUE (user_id, date, provider_name)
);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_provider_date ON stats_user_daily_provider (date);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_provider_user_id ON stats_user_daily_provider (user_id);
CREATE TABLE IF NOT EXISTS stats_user_daily_api_format (
id TEXT PRIMARY KEY NOT NULL,
user_id TEXT NOT NULL,
username TEXT,
date INTEGER NOT NULL,
api_format TEXT NOT NULL,
total_requests INTEGER NOT NULL DEFAULT 0,
success_requests INTEGER NOT NULL DEFAULT 0,
input_tokens INTEGER NOT NULL DEFAULT 0,
effective_input_tokens INTEGER NOT NULL DEFAULT 0,
output_tokens INTEGER NOT NULL DEFAULT 0,
total_tokens INTEGER NOT NULL DEFAULT 0,
total_input_context INTEGER NOT NULL DEFAULT 0,
cache_creation_tokens INTEGER NOT NULL DEFAULT 0,
cache_creation_ephemeral_5m_tokens INTEGER NOT NULL DEFAULT 0,
cache_creation_ephemeral_1h_tokens INTEGER NOT NULL DEFAULT 0,
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
total_cost REAL NOT NULL DEFAULT 0,
actual_total_cost REAL NOT NULL DEFAULT 0,
response_time_sum_ms REAL NOT NULL DEFAULT 0,
response_time_samples INTEGER NOT NULL DEFAULT 0,
successful_response_time_sum_ms REAL NOT NULL DEFAULT 0,
successful_response_time_samples INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
UNIQUE (user_id, date, api_format)
);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_api_format_date ON stats_user_daily_api_format (date);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_api_format_user_id ON stats_user_daily_api_format (user_id);
CREATE TABLE IF NOT EXISTS stats_daily_model_provider (
id TEXT PRIMARY KEY NOT NULL,
date INTEGER NOT NULL,
model TEXT NOT NULL,
provider_name TEXT NOT NULL,
total_requests INTEGER NOT NULL DEFAULT 0,
total_tokens INTEGER NOT NULL DEFAULT 0,
total_cost REAL NOT NULL DEFAULT 0,
response_time_sum_ms REAL NOT NULL DEFAULT 0,
response_time_samples INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
UNIQUE (date, model, provider_name)
);
CREATE INDEX IF NOT EXISTS idx_stats_daily_model_provider_date ON stats_daily_model_provider (date);
CREATE TABLE IF NOT EXISTS stats_user_daily_model_provider (
id TEXT PRIMARY KEY NOT NULL,
user_id TEXT NOT NULL,
username TEXT,
date INTEGER NOT NULL,
model TEXT NOT NULL,
provider_name TEXT NOT NULL,
total_requests INTEGER NOT NULL DEFAULT 0,
total_tokens INTEGER NOT NULL DEFAULT 0,
total_cost REAL NOT NULL DEFAULT 0,
response_time_sum_ms REAL NOT NULL DEFAULT 0,
response_time_samples INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
UNIQUE (user_id, date, model, provider_name)
);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_model_provider_date ON stats_user_daily_model_provider (date);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_model_provider_user_date ON stats_user_daily_model_provider (user_id, date);
CREATE TABLE IF NOT EXISTS stats_daily_cost_savings (
id TEXT PRIMARY KEY NOT NULL,
date INTEGER NOT NULL,
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
cache_read_cost REAL NOT NULL DEFAULT 0,
cache_creation_cost REAL NOT NULL DEFAULT 0,
estimated_full_cost REAL NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
UNIQUE (date)
);
CREATE TABLE IF NOT EXISTS stats_daily_cost_savings_provider (
id TEXT PRIMARY KEY NOT NULL,
date INTEGER NOT NULL,
provider_name TEXT NOT NULL,
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
cache_read_cost REAL NOT NULL DEFAULT 0,
cache_creation_cost REAL NOT NULL DEFAULT 0,
estimated_full_cost REAL NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
UNIQUE (date, provider_name)
);
CREATE INDEX IF NOT EXISTS idx_stats_daily_cost_savings_provider_date ON stats_daily_cost_savings_provider (date);
CREATE TABLE IF NOT EXISTS stats_daily_cost_savings_model (
id TEXT PRIMARY KEY NOT NULL,
date INTEGER NOT NULL,
model TEXT NOT NULL,
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
cache_read_cost REAL NOT NULL DEFAULT 0,
cache_creation_cost REAL NOT NULL DEFAULT 0,
estimated_full_cost REAL NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
UNIQUE (date, model)
);
CREATE INDEX IF NOT EXISTS idx_stats_daily_cost_savings_model_date ON stats_daily_cost_savings_model (date);
CREATE TABLE IF NOT EXISTS stats_daily_cost_savings_model_provider (
id TEXT PRIMARY KEY NOT NULL,
date INTEGER NOT NULL,
model TEXT NOT NULL,
provider_name TEXT NOT NULL,
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
cache_read_cost REAL NOT NULL DEFAULT 0,
cache_creation_cost REAL NOT NULL DEFAULT 0,
estimated_full_cost REAL NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
UNIQUE (date, model, provider_name)
);
CREATE INDEX IF NOT EXISTS idx_stats_daily_cost_savings_model_provider_date ON stats_daily_cost_savings_model_provider (date);
CREATE TABLE IF NOT EXISTS stats_user_daily_cost_savings (
id TEXT PRIMARY KEY NOT NULL,
user_id TEXT NOT NULL,
username TEXT,
date INTEGER NOT NULL,
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
cache_read_cost REAL NOT NULL DEFAULT 0,
cache_creation_cost REAL NOT NULL DEFAULT 0,
estimated_full_cost REAL NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
UNIQUE (user_id, date)
);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_cost_savings_date ON stats_user_daily_cost_savings (date);
CREATE TABLE IF NOT EXISTS stats_user_daily_cost_savings_provider (
id TEXT PRIMARY KEY NOT NULL,
user_id TEXT NOT NULL,
username TEXT,
date INTEGER NOT NULL,
provider_name TEXT NOT NULL,
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
cache_read_cost REAL NOT NULL DEFAULT 0,
cache_creation_cost REAL NOT NULL DEFAULT 0,
estimated_full_cost REAL NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
UNIQUE (user_id, date, provider_name)
);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_cost_savings_provider_date ON stats_user_daily_cost_savings_provider (date);
CREATE TABLE IF NOT EXISTS stats_user_daily_cost_savings_model (
id TEXT PRIMARY KEY NOT NULL,
user_id TEXT NOT NULL,
username TEXT,
date INTEGER NOT NULL,
model TEXT NOT NULL,
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
cache_read_cost REAL NOT NULL DEFAULT 0,
cache_creation_cost REAL NOT NULL DEFAULT 0,
estimated_full_cost REAL NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
UNIQUE (user_id, date, model)
);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_cost_savings_model_date ON stats_user_daily_cost_savings_model (date);
CREATE TABLE IF NOT EXISTS stats_user_daily_cost_savings_model_provider (
id TEXT PRIMARY KEY NOT NULL,
user_id TEXT NOT NULL,
username TEXT,
date INTEGER NOT NULL,
model TEXT NOT NULL,
provider_name TEXT NOT NULL,
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
cache_read_cost REAL NOT NULL DEFAULT 0,
cache_creation_cost REAL NOT NULL DEFAULT 0,
estimated_full_cost REAL NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
UNIQUE (user_id, date, model, provider_name)
);
CREATE INDEX IF NOT EXISTS idx_stats_user_daily_cost_savings_model_provider_date ON stats_user_daily_cost_savings_model_provider (date);
crates/aether-data/runtime/schema/logical/006_usage.toml
+402
View File
@@ -554,6 +554,259 @@ columns = ["user_id"]
name = "usage_wallet_id_idx"
columns = ["wallet_id"]
[table.usage_body_blobs]
domain = "usage"
order = 11
primary_key = ["body_ref"]
[[table.usage_body_blobs.columns]]
name = "body_ref"
type = "text"
length = 160
[[table.usage_body_blobs.columns]]
name = "request_id"
type = "text"
length = 128
[[table.usage_body_blobs.columns]]
name = "body_field"
type = "text"
length = 50
[[table.usage_body_blobs.columns]]
name = "payload_gzip"
type = "bytes"
[[table.usage_body_blobs.columns]]
name = "created_at"
type = "timestamp"
[[table.usage_body_blobs.columns]]
name = "updated_at"
type = "timestamp"
[[table.usage_body_blobs.uniques]]
name = "usage_body_blobs_request_id_field_key"
columns = ["request_id", "body_field"]
[[table.usage_body_blobs.indexes]]
name = "ix_usage_body_blobs_request_id"
columns = ["request_id"]
[[table.usage_body_blobs.foreign_keys]]
name = "usage_body_blobs_request_id_fkey"
columns = ["request_id"]
references_table = "usage"
references_columns = ["request_id"]
on_delete = "cascade"
[table.usage_http_audits]
domain = "usage"
order = 12
primary_key = ["request_id"]
[[table.usage_http_audits.columns]]
name = "request_id"
type = "text"
length = 128
[[table.usage_http_audits.columns]]
name = "request_headers"
type = "json"
nullable = true
[[table.usage_http_audits.columns]]
name = "provider_request_headers"
type = "json"
nullable = true
[[table.usage_http_audits.columns]]
name = "response_headers"
type = "json"
nullable = true
[[table.usage_http_audits.columns]]
name = "client_response_headers"
type = "json"
nullable = true
[[table.usage_http_audits.columns]]
name = "request_body_ref"
type = "text"
length = 160
nullable = true
[[table.usage_http_audits.columns]]
name = "provider_request_body_ref"
type = "text"
length = 160
nullable = true
[[table.usage_http_audits.columns]]
name = "response_body_ref"
type = "text"
length = 160
nullable = true
[[table.usage_http_audits.columns]]
name = "client_response_body_ref"
type = "text"
length = 160
nullable = true
[[table.usage_http_audits.columns]]
name = "request_body_state"
type = "text"
length = 32
nullable = true
[[table.usage_http_audits.columns]]
name = "provider_request_body_state"
type = "text"
length = 32
nullable = true
[[table.usage_http_audits.columns]]
name = "response_body_state"
type = "text"
length = 32
nullable = true
[[table.usage_http_audits.columns]]
name = "client_response_body_state"
type = "text"
length = 32
nullable = true
[[table.usage_http_audits.columns]]
name = "body_capture_mode"
type = "text"
length = 32
default = "none"
[[table.usage_http_audits.columns]]
name = "created_at"
type = "timestamp"
[[table.usage_http_audits.columns]]
name = "updated_at"
type = "timestamp"
[[table.usage_http_audits.indexes]]
name = "ix_usage_http_audits_updated_at"
columns = ["updated_at"]
[[table.usage_http_audits.foreign_keys]]
name = "usage_http_audits_request_id_fkey"
columns = ["request_id"]
references_table = "usage"
references_columns = ["request_id"]
on_delete = "cascade"
[table.usage_routing_snapshots]
domain = "usage"
order = 13
primary_key = ["request_id"]
[[table.usage_routing_snapshots.columns]]
name = "request_id"
type = "text"
length = 128
[[table.usage_routing_snapshots.columns]]
name = "candidate_id"
type = "text"
length = 160
nullable = true
[[table.usage_routing_snapshots.columns]]
name = "candidate_index"
type = "int64"
nullable = true
[[table.usage_routing_snapshots.columns]]
name = "key_name"
type = "text"
length = 255
nullable = true
[[table.usage_routing_snapshots.columns]]
name = "planner_kind"
type = "text"
length = 120
nullable = true
[[table.usage_routing_snapshots.columns]]
name = "route_family"
type = "text"
length = 80
nullable = true
[[table.usage_routing_snapshots.columns]]
name = "route_kind"
type = "text"
length = 80
nullable = true
[[table.usage_routing_snapshots.columns]]
name = "execution_path"
type = "text"
length = 80
nullable = true
[[table.usage_routing_snapshots.columns]]
name = "local_execution_runtime_miss_reason"
type = "text"
length = 255
nullable = true
[[table.usage_routing_snapshots.columns]]
name = "selected_provider_id"
type = "text_id"
length = 100
nullable = true
[[table.usage_routing_snapshots.columns]]
name = "selected_endpoint_id"
type = "text_id"
length = 100
nullable = true
[[table.usage_routing_snapshots.columns]]
name = "selected_provider_api_key_id"
type = "text_id"
length = 100
nullable = true
[[table.usage_routing_snapshots.columns]]
name = "has_format_conversion"
type = "bool"
nullable = true
[[table.usage_routing_snapshots.columns]]
name = "created_at"
type = "timestamp"
[[table.usage_routing_snapshots.columns]]
name = "updated_at"
type = "timestamp"
[[table.usage_routing_snapshots.indexes]]
name = "ix_usage_routing_snapshots_route_family_kind"
columns = ["route_family", "route_kind"]
[[table.usage_routing_snapshots.indexes]]
name = "ix_usage_routing_snapshots_candidate_id"
columns = ["candidate_id"]
[[table.usage_routing_snapshots.foreign_keys]]
name = "usage_routing_snapshots_request_id_fkey"
columns = ["request_id"]
references_table = "usage"
references_columns = ["request_id"]
on_delete = "cascade"
[table.usage_counter_deltas]
domain = "usage"
order = 15
@@ -725,6 +978,147 @@ name = "provider_monthly_used_usd"
type = "float64"
nullable = true
[[table.usage_settlement_snapshots.columns]]
name = "billing_snapshot_schema_version"
type = "text"
length = 20
nullable = true
[[table.usage_settlement_snapshots.columns]]
name = "billing_snapshot_status"
type = "text"
length = 20
nullable = true
[[table.usage_settlement_snapshots.columns]]
name = "rate_multiplier"
type = "float64"
nullable = true
[[table.usage_settlement_snapshots.columns]]
name = "is_free_tier"
type = "bool"
nullable = true
[[table.usage_settlement_snapshots.columns]]
name = "input_price_per_1m"
type = "float64"
nullable = true
[[table.usage_settlement_snapshots.columns]]
name = "output_price_per_1m"
type = "float64"
nullable = true
[[table.usage_settlement_snapshots.columns]]
name = "cache_creation_price_per_1m"
type = "float64"
nullable = true
[[table.usage_settlement_snapshots.columns]]
name = "cache_read_price_per_1m"
type = "float64"
nullable = true
[[table.usage_settlement_snapshots.columns]]
name = "price_per_request"
type = "float64"
nullable = true
[[table.usage_settlement_snapshots.columns]]
name = "settlement_snapshot_schema_version"
type = "text"
length = 20
nullable = true
[[table.usage_settlement_snapshots.columns]]
name = "settlement_snapshot"
type = "json"
nullable = true
[[table.usage_settlement_snapshots.columns]]
name = "billing_dimensions"
type = "json"
nullable = true
[[table.usage_settlement_snapshots.columns]]
name = "billing_input_tokens"
type = "int64"
nullable = true
[[table.usage_settlement_snapshots.columns]]
name = "billing_effective_input_tokens"
type = "int64"
nullable = true
[[table.usage_settlement_snapshots.columns]]
name = "billing_output_tokens"
type = "int64"
nullable = true
[[table.usage_settlement_snapshots.columns]]
name = "billing_cache_creation_tokens"
type = "int64"
nullable = true
[[table.usage_settlement_snapshots.columns]]
name = "billing_cache_creation_5m_tokens"
type = "int64"
nullable = true
[[table.usage_settlement_snapshots.columns]]
name = "billing_cache_creation_1h_tokens"
type = "int64"
nullable = true
[[table.usage_settlement_snapshots.columns]]
name = "billing_cache_read_tokens"
type = "int64"
nullable = true
[[table.usage_settlement_snapshots.columns]]
name = "billing_total_input_context"
type = "int64"
nullable = true
[[table.usage_settlement_snapshots.columns]]
name = "billing_cache_creation_cost_usd"
type = "float64"
nullable = true
[[table.usage_settlement_snapshots.columns]]
name = "billing_cache_read_cost_usd"
type = "float64"
nullable = true
[[table.usage_settlement_snapshots.columns]]
name = "billing_total_cost_usd"
type = "float64"
nullable = true
[[table.usage_settlement_snapshots.columns]]
name = "billing_actual_total_cost_usd"
type = "float64"
nullable = true
[[table.usage_settlement_snapshots.columns]]
name = "billing_pricing_source"
type = "text"
length = 50
nullable = true
[[table.usage_settlement_snapshots.columns]]
name = "billing_rule_id"
type = "text"
length = 100
nullable = true
[[table.usage_settlement_snapshots.columns]]
name = "billing_rule_version"
type = "text"
length = 50
nullable = true
[[table.usage_settlement_snapshots.columns]]
name = "finalized_at"
type = "unix_seconds"
@@ -745,3 +1139,11 @@ columns = ["billing_status"]
[[table.usage_settlement_snapshots.indexes]]
name = "usage_settlement_snapshots_wallet_id_idx"
columns = ["wallet_id"]
[[table.usage_settlement_snapshots.indexes]]
name = "ix_usage_settlement_snapshots_schema_version"
columns = ["settlement_snapshot_schema_version"]
[[table.usage_settlement_snapshots.indexes]]
name = "ix_usage_settlement_snapshots_pricing_source"
columns = ["billing_pricing_source"]
crates/aether-data/runtime/schema/logical/007_stats.toml
+1542
View File
File diff suppressed because it is too large Load Diff
crates/aether-data/runtime/src/backend/maintenance/mysql.rs
+42 -3
View File
@@ -14,11 +14,11 @@ impl MysqlBackend {
let table_name = maintenance_identifier(table_name)?;
summary.attempted += 1;
let statement = format!("ANALYZE TABLE `{table_name}`");
if sqlx::raw_sql(&statement)
.execute(self.pool())
if sqlx::query_as::<_, (String, String, String, String)>(&statement)
.fetch_all(self.pool())
.await
.map_sql_err()
.is_ok()
.is_ok_and(|rows| mysql_analyze_succeeded(&rows))
{
summary.succeeded += 1;
}
@@ -26,3 +26,42 @@ impl MysqlBackend {
Ok(summary)
}
}
fn mysql_analyze_succeeded(rows: &[(String, String, String, String)]) -> bool {
!rows.is_empty()
&& rows.iter().any(|(_, _, message_type, message)| {
message_type.eq_ignore_ascii_case("status") && message.eq_ignore_ascii_case("ok")
})
&& rows
.iter()
.all(|(_, _, message_type, _)| !message_type.eq_ignore_ascii_case("error"))
}
#[cfg(test)]
mod tests {
use super::mysql_analyze_succeeded;
fn row(message_type: &str, message: &str) -> (String, String, String, String) {
(
"aether.usage".to_string(),
"analyze".to_string(),
message_type.to_string(),
message.to_string(),
)
}
#[test]
fn analyze_requires_an_explicit_ok_status() {
assert!(mysql_analyze_succeeded(&[row("status", "OK")]));
assert!(!mysql_analyze_succeeded(&[]));
assert!(!mysql_analyze_succeeded(&[row("note", "skipped")]));
}
#[test]
fn analyze_rejects_error_rows_even_when_an_ok_row_is_present() {
assert!(!mysql_analyze_succeeded(&[
row("Error", "Table does not exist"),
row("status", "OK"),
]));
}
}
crates/aether-data/runtime/src/backend/mysql.rs
+68 -11
View File
@@ -538,6 +538,20 @@ WHERE wallet_id = ?
for sql in [
"DELETE FROM stats_daily WHERE `date` = 0",
"DELETE FROM stats_hourly WHERE hour_utc = 3600",
"DELETE FROM stats_user_summary WHERE user_id LIKE 'user-%'",
"DELETE FROM stats_user_daily_model WHERE `date` = 0",
"DELETE FROM stats_user_daily_provider WHERE `date` = 0",
"DELETE FROM stats_user_daily_api_format WHERE `date` = 0",
"DELETE FROM stats_daily_model_provider WHERE `date` = 0",
"DELETE FROM stats_user_daily_model_provider WHERE `date` = 0",
"DELETE FROM stats_daily_cost_savings WHERE `date` = 0",
"DELETE FROM stats_daily_cost_savings_provider WHERE `date` = 0",
"DELETE FROM stats_daily_cost_savings_model WHERE `date` = 0",
"DELETE FROM stats_daily_cost_savings_model_provider WHERE `date` = 0",
"DELETE FROM stats_user_daily_cost_savings WHERE `date` = 0",
"DELETE FROM stats_user_daily_cost_savings_provider WHERE `date` = 0",
"DELETE FROM stats_user_daily_cost_savings_model WHERE `date` = 0",
"DELETE FROM stats_user_daily_cost_savings_model_provider WHERE `date` = 0",
"DELETE FROM usage_settlement_snapshots WHERE request_id LIKE 'request-daily-%' OR request_id LIKE 'stats-%'",
"DELETE FROM `usage` WHERE request_id LIKE 'request-%' OR request_id LIKE 'export-request-%' OR request_id LIKE 'stats-%'",
] {
@@ -550,19 +564,21 @@ WHERE wallet_id = ?
sqlx::query(
r#"
INSERT INTO `usage` (
request_id, user_id, api_key_id, provider_name, model, status, billing_status,
request_id, user_id, api_key_id, provider_name, model, api_format, status, billing_status,
status_code, error_category, input_tokens, output_tokens,
cache_creation_input_tokens, cache_read_input_tokens, total_cost_usd,
actual_total_cost_usd, response_time_ms, created_at_unix_ms, updated_at_unix_secs
actual_total_cost_usd, cache_creation_cost_usd, cache_read_cost_usd,
input_price_per_1m, response_time_ms, first_byte_time_ms,
created_at_unix_ms, updated_at_unix_secs
) VALUES
('stats-1', 'user-1', 'key-1', 'provider-a', 'model-a', 'completed', 'settled',
200, NULL, 10, 20, 1, 2, 0.30, 0.25, 100, 3600000, 3600),
('stats-2', 'user-2', 'key-2', 'provider-b', 'model-b', 'failed', 'void',
500, 'upstream_error', 5, 7, 0, 1, 0.20, 0.20, 300, 3610000, 3610),
('stats-pending', 'user-3', 'key-3', 'provider-a', 'model-a', 'pending', 'pending',
NULL, NULL, 100, 100, 0, 0, 9.99, 9.99, 50, 3620000, 3620),
('stats-unknown-provider', 'user-4', 'key-4', 'unknown', 'model-a', 'completed', 'settled',
200, NULL, 100, 100, 0, 0, 9.99, 9.99, 50, 3630000, 3630)
('stats-1', 'user-1', 'key-1', 'provider-a', 'model-a', 'openai', 'completed', 'settled',
200, NULL, 10, 20, 1, 2, 0.30, 0.25, 0.01, 0.02, 10.0, 100, 50, 3600, 3600),
('stats-2', 'user-2', 'key-2', 'provider-b', 'model-b', 'claude', 'failed', 'void',
500, 'upstream_error', 5, 7, 0, 1, 0.20, 0.20, 0.00, 0.01, 20.0, 300, 200, 3610, 3610),
('stats-pending', 'user-3', 'key-3', 'provider-a', 'model-a', 'openai', 'pending', 'pending',
NULL, NULL, 100, 100, 0, 0, 9.99, 9.99, 0.00, 0.00, 0.0, 50, 25, 3620, 3620),
('stats-unknown-provider', 'user-4', 'key-4', 'unknown', 'model-a', 'openai', 'completed', 'settled',
200, NULL, 100, 100, 0, 0, 9.99, 9.99, 0.00, 0.00, 0.0, 50, 25, 3630, 3630)
"#,
)
.execute(backend.pool())
@@ -627,7 +643,7 @@ WHERE hour_utc = 3600
assert_eq!(daily.total_requests, 2);
assert_eq!(daily.model_rows, 2);
assert_eq!(daily.provider_rows, 2);
assert_eq!(daily.api_key_rows, 2);
assert_eq!(daily.api_key_rows, 4);
assert_eq!(daily.error_rows, 1);
assert_eq!(daily.user_rows, 2);
@@ -642,5 +658,46 @@ WHERE `date` = 0
.await
.expect("daily stats row should load");
assert_eq!(daily_row, (2, 1, 1, 2));
let enriched_daily = sqlx::query_as::<_, (i64, i64, i64, i64, i64, Option<i64>)>(
r#"
SELECT effective_input_tokens, total_input_context, cache_hit_total_requests,
completed_total_requests, settled_total_requests, p50_response_time_ms
FROM stats_daily
WHERE `date` = 0
"#,
)
.fetch_one(backend.pool())
.await
.expect("mysql enriched daily stats row should load");
assert_eq!(enriched_daily, (13, 17, 4, 2, 2, None));
for table in [
"stats_user_summary",
"stats_user_daily_model",
"stats_user_daily_provider",
"stats_user_daily_api_format",
"stats_daily_model_provider",
"stats_user_daily_model_provider",
"stats_daily_cost_savings",
"stats_daily_cost_savings_provider",
"stats_daily_cost_savings_model",
"stats_daily_cost_savings_model_provider",
"stats_user_daily_cost_savings",
"stats_user_daily_cost_savings_provider",
"stats_user_daily_cost_savings_model",
"stats_user_daily_cost_savings_model_provider",
] {
let sql = if table == "stats_user_summary" {
format!("SELECT COUNT(*) FROM {table} WHERE user_id IN ('user-1', 'user-2')")
} else {
format!("SELECT COUNT(*) FROM {table} WHERE `date` = 0")
};
let count: i64 = sqlx::query_scalar(&sql)
.fetch_one(backend.pool())
.await
.expect("mysql advanced stats count should load");
assert!(count > 0, "{table} should be populated");
}
}
}
crates/aether-data/runtime/src/backend/sqlite.rs
+250 -30
View File
@@ -586,22 +586,6 @@ VALUES ('target-key-1', 'target-user-1', 'hash-target-key', 'target key', 1, 1)
.await
.expect("sqlite migrations should run");
for (column, ty) in [
("request_body", "TEXT"),
("response_body", "TEXT"),
("provider_request_body", "TEXT"),
("client_response_body", "TEXT"),
("request_body_compressed", "BLOB"),
("response_body_compressed", "BLOB"),
("provider_request_body_compressed", "BLOB"),
("client_response_body_compressed", "BLOB"),
] {
sqlx::query(&format!(r#"ALTER TABLE "usage" ADD COLUMN {column} {ty}"#))
.execute(backend.pool())
.await
.expect("legacy body column should be added");
}
sqlx::query(
r#"
INSERT INTO "usage" (
@@ -842,25 +826,40 @@ WHERE billing_date = '2026-05-03'
sqlx::query(
r#"
INSERT INTO "usage" (
request_id, user_id, api_key_id, provider_name, model, status, billing_status,
request_id, user_id, api_key_id, provider_name, model, api_format, status, billing_status,
status_code, error_category, input_tokens, output_tokens,
cache_creation_input_tokens, cache_read_input_tokens, total_cost_usd,
actual_total_cost_usd, response_time_ms, created_at_unix_ms, updated_at_unix_secs
actual_total_cost_usd, cache_creation_cost_usd, cache_read_cost_usd,
input_price_per_1m, response_time_ms, first_byte_time_ms,
created_at_unix_ms, updated_at_unix_secs
) VALUES
('stats-1', 'user-1', 'key-1', 'provider-a', 'model-a', 'completed', 'settled',
200, NULL, 10, 20, 1, 2, 0.30, 0.25, 100, 3600000, 3600),
('stats-2', 'user-2', 'key-2', 'provider-b', 'model-b', 'failed', 'void',
500, 'upstream_error', 5, 7, 0, 1, 0.20, 0.20, 300, 3610000, 3610),
('stats-pending', 'user-3', 'key-3', 'provider-a', 'model-a', 'pending', 'pending',
NULL, NULL, 100, 100, 0, 0, 9.99, 9.99, 50, 3620000, 3620),
('stats-unknown-provider', 'user-4', 'key-4', 'unknown', 'model-a', 'completed', 'settled',
200, NULL, 100, 100, 0, 0, 9.99, 9.99, 50, 3630000, 3630)
('stats-1', 'user-1', 'key-1', 'provider-a', 'model-a', 'openai', 'completed', 'settled',
200, NULL, 10, 20, 1, 2, 0.30, 0.25, 0.01, 0.02, 10.0, 100, 50, 3600, 3600),
('stats-2', 'user-2', 'key-2', 'provider-b', 'model-b', 'claude', 'failed', 'void',
500, 'upstream_error', 5, 7, 0, 1, 0.20, 0.20, 0.00, 0.01, 20.0, 300, 200, 3610, 3610),
('stats-pending', 'user-3', 'key-3', 'provider-a', 'model-a', 'openai', 'pending', 'pending',
NULL, NULL, 100, 100, 0, 0, 9.99, 9.99, 0.00, 0.00, 0.0, 50, 25, 3620, 3620),
('stats-unknown-provider', 'user-4', 'key-4', 'unknown', 'model-a', 'openai', 'completed', 'settled',
200, NULL, 100, 100, 0, 0, 9.99, 9.99, 0.00, 0.00, 0.0, 50, 25, 3630, 3630)
"#,
)
.execute(backend.pool())
.await
.expect("usage stats rows should seed");
sqlx::query(
r#"
INSERT INTO request_candidates (
id, request_id, candidate_index, retry_index, status, created_at
) VALUES
('stats-candidate-1', 'stats-fallback', 0, 0, 'failed', 3600000),
('stats-candidate-2', 'stats-fallback', 1, 0, 'success', 3610000)
"#,
)
.execute(backend.pool())
.await
.expect("fallback candidates should seed");
let target_hour = chrono::DateTime::<chrono::Utc>::from_timestamp(3600, 0)
.expect("target hour should be valid");
let aggregated_at = chrono::DateTime::<chrono::Utc>::from_timestamp(7200, 0)
@@ -896,6 +895,66 @@ WHERE hour_utc = 3600
assert_eq!(hourly_row.3, 15);
assert!((hourly_row.4 - 0.50).abs() < f64::EPSILON);
let enriched_hourly: (f64, i64, i64, i64, i64, i64) = sqlx::query_as(
r#"
SELECT response_time_sum_ms, response_time_samples, cache_hit_total_requests,
cache_hit_requests, completed_total_requests, settled_total_requests
FROM stats_hourly
WHERE hour_utc = 3600
"#,
)
.fetch_one(backend.pool())
.await
.expect("enriched hourly stats row should load");
assert!((enriched_hourly.0 - 400.0).abs() < f64::EPSILON);
assert_eq!(enriched_hourly.1, 2);
assert_eq!(enriched_hourly.2, 4);
assert_eq!(enriched_hourly.3, 2);
assert_eq!(enriched_hourly.4, 2);
assert_eq!(enriched_hourly.5, 2);
assert_eq!(sqlite_count(backend.pool(), "stats_hourly_user").await, 2);
assert_eq!(
sqlite_count(backend.pool(), "stats_hourly_user_model").await,
2
);
assert_eq!(sqlite_count(backend.pool(), "stats_hourly_model").await, 2);
assert_eq!(
sqlite_count(backend.pool(), "stats_hourly_provider").await,
2
);
let hourly_user = sqlx::query_as::<_, (i64, i64, i64, i64, i64, f64)>(
r#"
SELECT total_requests, success_requests, error_requests, input_tokens, output_tokens, total_cost
FROM stats_hourly_user
WHERE hour_utc = 3600 AND user_id = 'user-2'
"#,
)
.fetch_one(backend.pool())
.await
.expect("hourly user stats row should load");
assert_eq!(hourly_user.0, 1);
assert_eq!(hourly_user.1, 0);
assert_eq!(hourly_user.2, 1);
assert_eq!(hourly_user.3, 5);
assert_eq!(hourly_user.4, 7);
assert!((hourly_user.5 - 0.20).abs() < f64::EPSILON);
let hourly_model = sqlx::query_as::<_, (i64, i64, i64, f64, f64)>(
r#"
SELECT total_requests, input_tokens, output_tokens, total_cost, avg_response_time_ms
FROM stats_hourly_model
WHERE hour_utc = 3600 AND model = 'model-a'
"#,
)
.fetch_one(backend.pool())
.await
.expect("hourly model stats row should load");
assert_eq!(hourly_model.0, 1);
assert_eq!(hourly_model.1, 10);
assert_eq!(hourly_model.2, 20);
assert!((hourly_model.3 - 0.30).abs() < f64::EPSILON);
assert!((hourly_model.4 - 100.0).abs() < f64::EPSILON);
let second_hourly = backend
.aggregate_stats_hourly(&StatsHourlyAggregationInput {
target_hour_utc: target_hour,
@@ -919,13 +978,13 @@ WHERE hour_utc = 3600
assert_eq!(daily.total_requests, 2);
assert_eq!(daily.model_rows, 2);
assert_eq!(daily.provider_rows, 2);
assert_eq!(daily.api_key_rows, 2);
assert_eq!(daily.api_key_rows, 4);
assert_eq!(daily.error_rows, 1);
assert_eq!(daily.user_rows, 2);
let daily_row = sqlx::query_as::<_, (i64, i64, i64, i64)>(
let daily_row = sqlx::query_as::<_, (i64, i64, i64, i64, i64)>(
r#"
SELECT total_requests, success_requests, error_requests, unique_models
SELECT total_requests, success_requests, error_requests, unique_models, fallback_count
FROM stats_daily
WHERE "date" = 0
"#,
@@ -933,6 +992,167 @@ WHERE "date" = 0
.fetch_one(backend.pool())
.await
.expect("daily stats row should load");
assert_eq!(daily_row, (2, 1, 1, 2));
assert_eq!(daily_row, (2, 1, 1, 2, 1));
assert_eq!(sqlite_count(backend.pool(), "stats_daily_model").await, 2);
assert_eq!(
sqlite_count(backend.pool(), "stats_daily_provider").await,
2
);
assert_eq!(sqlite_count(backend.pool(), "stats_daily_api_key").await, 4);
assert_eq!(sqlite_count(backend.pool(), "stats_daily_error").await, 1);
assert_eq!(sqlite_count(backend.pool(), "stats_user_daily").await, 2);
let daily_model = sqlx::query_as::<_, (i64, i64, i64, i64, i64, f64, f64)>(
r#"
SELECT total_requests, input_tokens, output_tokens, cache_creation_tokens,
cache_read_tokens, total_cost, avg_response_time_ms
FROM stats_daily_model
WHERE "date" = 0 AND model = 'model-a'
"#,
)
.fetch_one(backend.pool())
.await
.expect("daily model stats row should load");
assert_eq!(daily_model.0, 1);
assert_eq!(daily_model.1, 10);
assert_eq!(daily_model.2, 20);
assert_eq!(daily_model.3, 1);
assert_eq!(daily_model.4, 2);
assert!((daily_model.5 - 0.30).abs() < f64::EPSILON);
assert!((daily_model.6 - 100.0).abs() < f64::EPSILON);
let daily_error = sqlx::query_as::<_, (String, Option<String>, Option<String>, i64)>(
r#"
SELECT error_category, provider_name, model, count
FROM stats_daily_error
WHERE "date" = 0
"#,
)
.fetch_one(backend.pool())
.await
.expect("daily error stats row should load");
assert_eq!(
daily_error,
(
"upstream_error".to_string(),
Some("provider-b".to_string()),
Some("model-b".to_string()),
1,
)
);
let daily_user = sqlx::query_as::<_, (i64, i64, i64, i64, i64, f64)>(
r#"
SELECT total_requests, success_requests, error_requests, input_tokens, output_tokens, total_cost
FROM stats_user_daily
WHERE "date" = 0 AND user_id = 'user-2'
"#,
)
.fetch_one(backend.pool())
.await
.expect("daily user stats row should load");
assert_eq!(daily_user.0, 1);
assert_eq!(daily_user.1, 0);
assert_eq!(daily_user.2, 1);
assert_eq!(daily_user.3, 5);
assert_eq!(daily_user.4, 7);
assert!((daily_user.5 - 0.20).abs() < f64::EPSILON);
let enriched_daily =
sqlx::query_as::<_, (i64, i64, f64, i64, i64, i64, i64, i64, i64, Option<i64>)>(
r#"
SELECT effective_input_tokens, total_input_context, response_time_sum_ms,
response_time_samples, cache_hit_total_requests, cache_hit_requests,
completed_total_requests, completed_cache_hit_requests,
settled_total_requests, p50_response_time_ms
FROM stats_daily
WHERE "date" = 0
"#,
)
.fetch_one(backend.pool())
.await
.expect("enriched daily stats row should load");
assert_eq!(enriched_daily.0, 13);
assert_eq!(enriched_daily.1, 17);
assert!((enriched_daily.2 - 400.0).abs() < f64::EPSILON);
assert_eq!(enriched_daily.3, 2);
assert_eq!(enriched_daily.4, 4);
assert_eq!(enriched_daily.5, 2);
assert_eq!(enriched_daily.6, 2);
assert_eq!(enriched_daily.7, 1);
assert_eq!(enriched_daily.8, 2);
assert_eq!(enriched_daily.9, None);
for (table, expected) in [
("stats_user_summary", 2),
("stats_user_daily_model", 2),
("stats_user_daily_provider", 2),
("stats_user_daily_api_format", 2),
("stats_daily_model_provider", 2),
("stats_user_daily_model_provider", 2),
("stats_daily_cost_savings", 1),
("stats_daily_cost_savings_provider", 3),
("stats_daily_cost_savings_model", 2),
("stats_daily_cost_savings_model_provider", 3),
("stats_user_daily_cost_savings", 4),
("stats_user_daily_cost_savings_provider", 4),
("stats_user_daily_cost_savings_model", 4),
("stats_user_daily_cost_savings_model_provider", 4),
] {
assert_eq!(
sqlite_count(backend.pool(), table).await,
expected,
"{table}"
);
}
let model_rollup: (i64, i64, i64, f64, i64) = sqlx::query_as(
r#"
SELECT total_requests, effective_input_tokens, total_tokens,
response_time_sum_ms, successful_response_time_samples
FROM stats_user_daily_model
WHERE user_id = 'user-1' AND "date" = 0 AND model = 'model-a'
"#,
)
.fetch_one(backend.pool())
.await
.expect("advanced user model row should load");
assert_eq!(model_rollup.0, 1);
assert_eq!(model_rollup.1, 8);
assert_eq!(model_rollup.2, 31);
assert!((model_rollup.3 - 100.0).abs() < f64::EPSILON);
assert_eq!(model_rollup.4, 1);
let savings: (i64, f64, f64, f64) = sqlx::query_as(
r#"
SELECT cache_read_tokens, cache_read_cost, cache_creation_cost, estimated_full_cost
FROM stats_daily_cost_savings
WHERE "date" = 0
"#,
)
.fetch_one(backend.pool())
.await
.expect("daily cost savings row should load");
assert_eq!(savings.0, 3);
assert!((savings.1 - 0.03).abs() < 1e-12);
assert!((savings.2 - 0.01).abs() < 1e-12);
assert!((savings.3 - 0.00004).abs() < 1e-12);
let summary: (i64, i64, i64) = sqlx::query_as(
r#"
SELECT all_time_requests, all_time_input_tokens, active_days
FROM stats_user_summary
WHERE user_id = 'user-1'
"#,
)
.fetch_one(backend.pool())
.await
.expect("user summary row should load");
assert_eq!(summary, (1, 10, 1));
let global_summary: (i64, i64) =
sqlx::query_as("SELECT all_time_requests, all_time_input_tokens FROM stats_summary")
.fetch_one(backend.pool())
.await
.expect("global stats summary should load");
assert_eq!(global_summary, (2, 15));
}
}
crates/aether-data/runtime/src/backend/stats/mysql.rs
+524 -64
View File
@@ -1,7 +1,7 @@
use chrono::{DateTime, Utc};
use sqlx::Row;
use crate::backend::stats_common::{stats_id, unix_ms, unix_secs, utc_from_unix_secs};
use crate::backend::stats_common::{stats_id, unix_secs, utc_from_unix_secs};
use crate::backend::MysqlBackend;
use crate::driver::mysql::MysqlPool;
use crate::error::SqlResultExt;
@@ -10,6 +10,8 @@ use crate::{
StatsHourlyAggregationInput, StatsHourlyAggregationSummary,
};
mod advanced;
impl MysqlBackend {
pub async fn aggregate_stats_hourly(
&self,
@@ -56,7 +58,7 @@ async fn next_mysql_stats_hourly_bucket(
}
let next_bucket: Option<i64> = sqlx::query_scalar(
r#"
SELECT CAST(MIN(FLOOR(created_at_unix_ms / 3600000) * 3600) AS SIGNED)
SELECT CAST(MIN(FLOOR(created_at_unix_ms / 3600) * 3600) AS SIGNED)
FROM `usage`
WHERE created_at_unix_ms >= ?
AND created_at_unix_ms < ?
@@ -64,8 +66,8 @@ WHERE created_at_unix_ms >= ?
AND provider_name NOT IN ('unknown', 'pending')
"#,
)
.bind(unix_ms(search_from)?)
.bind(unix_ms(search_until)?)
.bind(search_from)
.bind(search_until)
.fetch_one(pool)
.await
.map_sql_err()?;
@@ -88,7 +90,7 @@ async fn next_mysql_stats_daily_bucket(
}
let next_bucket: Option<i64> = sqlx::query_scalar(
r#"
SELECT CAST(MIN(FLOOR(created_at_unix_ms / 86400000) * 86400) AS SIGNED)
SELECT CAST(MIN(FLOOR(created_at_unix_ms / 86400) * 86400) AS SIGNED)
FROM `usage`
WHERE created_at_unix_ms >= ?
AND created_at_unix_ms < ?
@@ -96,8 +98,8 @@ WHERE created_at_unix_ms >= ?
AND provider_name NOT IN ('unknown', 'pending')
"#,
)
.bind(unix_ms(search_from)?)
.bind(unix_ms(search_until)?)
.bind(search_from)
.bind(search_until)
.fetch_one(pool)
.await
.map_sql_err()?;
@@ -110,7 +112,7 @@ SELECT
CAST(COALESCE(SUM(CASE
WHEN status = 'failed'
OR status_code >= 400
OR (error_category IS NOT NULL AND error_category <> '')
OR error_message IS NOT NULL
THEN 1 ELSE 0 END), 0) AS SIGNED) AS error_requests,
CAST(COALESCE(SUM(input_tokens), 0) AS SIGNED) AS input_tokens,
CAST(COALESCE(SUM(output_tokens), 0) AS SIGNED) AS output_tokens,
@@ -131,13 +133,13 @@ async fn perform_mysql_stats_hourly_aggregation(
hour_utc_unix_secs: i64,
aggregated_at: DateTime<Utc>,
) -> Result<StatsHourlyAggregationSummary, DataLayerError> {
let start_ms = unix_ms(hour_utc_unix_secs)?;
let end_ms = unix_ms(hour_utc_unix_secs + 3600)?;
let start_unix_secs = hour_utc_unix_secs;
let end_unix_secs = hour_utc_unix_secs + 3600;
let aggregated_at_unix_secs = unix_secs(aggregated_at);
let mut tx = pool.begin().await.map_sql_err()?;
let row = sqlx::query(MYSQL_STATS_AGGREGATE_SQL)
.bind(start_ms)
.bind(end_ms)
.bind(start_unix_secs)
.bind(end_unix_secs)
.fetch_one(&mut *tx)
.await
.map_sql_err()?;
@@ -193,10 +195,39 @@ ON DUPLICATE KEY UPDATE
.await
.map_sql_err()?;
let user_rows = mysql_group_count(&mut tx, "user_id", start_ms, end_ms).await?;
let user_model_rows = mysql_group_count(&mut tx, "user_id, model", start_ms, end_ms).await?;
let model_rows = mysql_group_count(&mut tx, "model", start_ms, end_ms).await?;
let provider_rows = mysql_group_count(&mut tx, "provider_name", start_ms, end_ms).await?;
let user_rows = upsert_mysql_stats_hourly_user_rows(
&mut tx,
hour_utc_unix_secs,
start_unix_secs,
end_unix_secs,
aggregated_at_unix_secs,
)
.await?;
let user_model_rows = upsert_mysql_stats_hourly_user_model_rows(
&mut tx,
hour_utc_unix_secs,
start_unix_secs,
end_unix_secs,
aggregated_at_unix_secs,
)
.await?;
let model_rows = upsert_mysql_stats_hourly_model_rows(
&mut tx,
hour_utc_unix_secs,
start_unix_secs,
end_unix_secs,
aggregated_at_unix_secs,
)
.await?;
let provider_rows = upsert_mysql_stats_hourly_provider_rows(
&mut tx,
hour_utc_unix_secs,
start_unix_secs,
end_unix_secs,
aggregated_at_unix_secs,
)
.await?;
advanced::refresh_hourly(&mut tx, hour_utc_unix_secs, start_unix_secs, end_unix_secs).await?;
tx.commit().await.map_sql_err()?;
Ok(StatsHourlyAggregationSummary {
@@ -214,21 +245,24 @@ async fn perform_mysql_stats_daily_aggregation(
day_start_unix_secs: i64,
aggregated_at: DateTime<Utc>,
) -> Result<StatsDailyAggregationSummary, DataLayerError> {
let start_ms = unix_ms(day_start_unix_secs)?;
let end_ms = unix_ms(day_start_unix_secs + 86_400)?;
let start_unix_secs = day_start_unix_secs;
let end_unix_secs = day_start_unix_secs + 86_400;
let aggregated_at_unix_secs = unix_secs(aggregated_at);
let mut tx = pool.begin().await.map_sql_err()?;
let row = sqlx::query(MYSQL_STATS_AGGREGATE_SQL)
.bind(start_ms)
.bind(end_ms)
.bind(start_unix_secs)
.bind(end_unix_secs)
.fetch_one(&mut *tx)
.await
.map_sql_err()?;
let total_requests: i64 = row.try_get("total_requests").map_sql_err()?;
let error_requests: i64 = row.try_get("error_requests").map_sql_err()?;
let unique_models = mysql_group_count(&mut tx, "model", start_ms, end_ms).await? as i64;
let unique_models =
mysql_group_count(&mut tx, "model", start_unix_secs, end_unix_secs).await? as i64;
let unique_providers =
mysql_group_count(&mut tx, "provider_name", start_ms, end_ms).await? as i64;
mysql_group_count(&mut tx, "provider_name", start_unix_secs, end_unix_secs).await? as i64;
let fallback_count =
mysql_daily_fallback_count(&mut tx, start_unix_secs, end_unix_secs).await?;
sqlx::query(
r#"
@@ -237,7 +271,7 @@ INSERT INTO stats_daily (
input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens,
total_cost, actual_total_cost, avg_response_time_ms, fallback_count,
unique_models, unique_providers, is_complete, aggregated_at, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?, TRUE, ?, ?, ?)
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, TRUE, ?, ?, ?)
ON DUPLICATE KEY UPDATE
total_requests = VALUES(total_requests),
success_requests = VALUES(success_requests),
@@ -275,6 +309,7 @@ ON DUPLICATE KEY UPDATE
row.try_get::<f64, _>("avg_response_time_ms")
.map_sql_err()?,
)
.bind(fallback_count)
.bind(unique_models)
.bind(unique_providers)
.bind(aggregated_at_unix_secs)
@@ -284,11 +319,54 @@ ON DUPLICATE KEY UPDATE
.await
.map_sql_err()?;
let model_rows = usize::try_from(unique_models).unwrap_or(usize::MAX);
let provider_rows = usize::try_from(unique_providers).unwrap_or(usize::MAX);
let api_key_rows = mysql_group_count(&mut tx, "api_key_id", start_ms, end_ms).await?;
let error_rows = mysql_error_group_count(&mut tx, start_ms, end_ms).await?;
let user_rows = mysql_group_count(&mut tx, "user_id", start_ms, end_ms).await?;
let model_rows = upsert_mysql_stats_daily_model_rows(
&mut tx,
day_start_unix_secs,
start_unix_secs,
end_unix_secs,
aggregated_at_unix_secs,
)
.await?;
let provider_rows = upsert_mysql_stats_daily_provider_rows(
&mut tx,
day_start_unix_secs,
start_unix_secs,
end_unix_secs,
aggregated_at_unix_secs,
)
.await?;
let api_key_rows = upsert_mysql_stats_daily_api_key_rows(
&mut tx,
day_start_unix_secs,
start_unix_secs,
end_unix_secs,
aggregated_at_unix_secs,
)
.await?;
let error_rows = refresh_mysql_stats_daily_error_rows(
&mut tx,
day_start_unix_secs,
start_unix_secs,
end_unix_secs,
aggregated_at_unix_secs,
)
.await?;
let user_rows = upsert_mysql_stats_user_daily_rows(
&mut tx,
day_start_unix_secs,
start_unix_secs,
end_unix_secs,
aggregated_at_unix_secs,
)
.await?;
advanced::refresh_daily(
&mut tx,
day_start_unix_secs,
start_unix_secs,
end_unix_secs,
aggregated_at_unix_secs,
)
.await?;
tx.commit().await.map_sql_err()?;
Ok(StatsDailyAggregationSummary {
@@ -302,11 +380,425 @@ ON DUPLICATE KEY UPDATE
})
}
async fn upsert_mysql_stats_hourly_user_rows(
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
hour_utc: i64,
start_unix_secs: i64,
end_unix_secs: i64,
now_unix_secs: i64,
) -> Result<usize, DataLayerError> {
let result = sqlx::query(
r#"
INSERT INTO stats_hourly_user (
id, hour_utc, user_id, total_requests, success_requests, error_requests,
input_tokens, output_tokens, total_cost, created_at, updated_at
)
SELECT
SHA2(UUID(), 256), ?, user_id, COUNT(*),
COUNT(*) - COALESCE(SUM(CASE
WHEN status = 'failed' OR status_code >= 400 OR error_message IS NOT NULL
THEN 1 ELSE 0 END), 0),
COALESCE(SUM(CASE
WHEN status = 'failed' OR status_code >= 400 OR error_message IS NOT NULL
THEN 1 ELSE 0 END), 0),
COALESCE(SUM(input_tokens), 0), COALESCE(SUM(output_tokens), 0),
COALESCE(SUM(total_cost_usd), 0), ?, ?
FROM `usage`
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
AND user_id IS NOT NULL AND user_id <> ''
AND status NOT IN ('pending', 'streaming')
AND provider_name NOT IN ('unknown', 'pending')
GROUP BY user_id
ON DUPLICATE KEY UPDATE
total_requests = VALUES(total_requests), success_requests = VALUES(success_requests),
error_requests = VALUES(error_requests), input_tokens = VALUES(input_tokens),
output_tokens = VALUES(output_tokens), total_cost = VALUES(total_cost),
updated_at = VALUES(updated_at)
"#,
)
.bind(hour_utc)
.bind(now_unix_secs)
.bind(now_unix_secs)
.bind(start_unix_secs)
.bind(end_unix_secs)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
}
async fn upsert_mysql_stats_hourly_user_model_rows(
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
hour_utc: i64,
start_unix_secs: i64,
end_unix_secs: i64,
now_unix_secs: i64,
) -> Result<usize, DataLayerError> {
let result = sqlx::query(
r#"
INSERT INTO stats_hourly_user_model (
id, hour_utc, user_id, model, total_requests, input_tokens, output_tokens,
total_cost, created_at, updated_at
)
SELECT SHA2(UUID(), 256), ?, user_id, model, COUNT(*),
COALESCE(SUM(input_tokens), 0), COALESCE(SUM(output_tokens), 0),
COALESCE(SUM(total_cost_usd), 0), ?, ?
FROM `usage`
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
AND user_id IS NOT NULL AND user_id <> '' AND model IS NOT NULL AND model <> ''
AND status NOT IN ('pending', 'streaming')
AND provider_name NOT IN ('unknown', 'pending')
GROUP BY user_id, model
ON DUPLICATE KEY UPDATE
total_requests = VALUES(total_requests), input_tokens = VALUES(input_tokens),
output_tokens = VALUES(output_tokens), total_cost = VALUES(total_cost),
updated_at = VALUES(updated_at)
"#,
)
.bind(hour_utc)
.bind(now_unix_secs)
.bind(now_unix_secs)
.bind(start_unix_secs)
.bind(end_unix_secs)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
}
async fn upsert_mysql_stats_hourly_model_rows(
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
hour_utc: i64,
start_unix_secs: i64,
end_unix_secs: i64,
now_unix_secs: i64,
) -> Result<usize, DataLayerError> {
let result = sqlx::query(
r#"
INSERT INTO stats_hourly_model (
id, hour_utc, model, total_requests, input_tokens, output_tokens, total_cost,
avg_response_time_ms, created_at, updated_at
)
SELECT SHA2(UUID(), 256), ?, model, COUNT(*),
COALESCE(SUM(input_tokens), 0), COALESCE(SUM(output_tokens), 0),
COALESCE(SUM(total_cost_usd), 0), COALESCE(AVG(response_time_ms), 0), ?, ?
FROM `usage`
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
AND model IS NOT NULL AND model <> ''
AND status NOT IN ('pending', 'streaming')
AND provider_name NOT IN ('unknown', 'pending')
GROUP BY model
ON DUPLICATE KEY UPDATE
total_requests = VALUES(total_requests), input_tokens = VALUES(input_tokens),
output_tokens = VALUES(output_tokens), total_cost = VALUES(total_cost),
avg_response_time_ms = VALUES(avg_response_time_ms), updated_at = VALUES(updated_at)
"#,
)
.bind(hour_utc)
.bind(now_unix_secs)
.bind(now_unix_secs)
.bind(start_unix_secs)
.bind(end_unix_secs)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
}
async fn upsert_mysql_stats_hourly_provider_rows(
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
hour_utc: i64,
start_unix_secs: i64,
end_unix_secs: i64,
now_unix_secs: i64,
) -> Result<usize, DataLayerError> {
let result = sqlx::query(
r#"
INSERT INTO stats_hourly_provider (
id, hour_utc, provider_name, total_requests, input_tokens, output_tokens,
total_cost, created_at, updated_at
)
SELECT SHA2(UUID(), 256), ?, provider_name, COUNT(*),
COALESCE(SUM(input_tokens), 0), COALESCE(SUM(output_tokens), 0),
COALESCE(SUM(total_cost_usd), 0), ?, ?
FROM `usage`
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
AND status NOT IN ('pending', 'streaming')
AND provider_name NOT IN ('unknown', 'pending')
GROUP BY provider_name
ON DUPLICATE KEY UPDATE
total_requests = VALUES(total_requests), input_tokens = VALUES(input_tokens),
output_tokens = VALUES(output_tokens), total_cost = VALUES(total_cost),
updated_at = VALUES(updated_at)
"#,
)
.bind(hour_utc)
.bind(now_unix_secs)
.bind(now_unix_secs)
.bind(start_unix_secs)
.bind(end_unix_secs)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
}
async fn upsert_mysql_stats_daily_model_rows(
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
day_start: i64,
start_unix_secs: i64,
end_unix_secs: i64,
now_unix_secs: i64,
) -> Result<usize, DataLayerError> {
let result = sqlx::query(
r#"
INSERT INTO stats_daily_model (
id, `date`, model, total_requests, input_tokens, output_tokens,
cache_creation_tokens, cache_read_tokens, total_cost, avg_response_time_ms,
created_at, updated_at
)
SELECT SHA2(UUID(), 256), ?, model, COUNT(*),
COALESCE(SUM(input_tokens), 0), COALESCE(SUM(output_tokens), 0),
COALESCE(SUM(cache_creation_input_tokens), 0),
COALESCE(SUM(cache_read_input_tokens), 0), COALESCE(SUM(total_cost_usd), 0),
COALESCE(AVG(response_time_ms), 0), ?, ?
FROM `usage`
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
AND model IS NOT NULL AND model <> ''
AND status NOT IN ('pending', 'streaming')
AND provider_name NOT IN ('unknown', 'pending')
GROUP BY model
ON DUPLICATE KEY UPDATE
total_requests = VALUES(total_requests), input_tokens = VALUES(input_tokens),
output_tokens = VALUES(output_tokens), cache_creation_tokens = VALUES(cache_creation_tokens),
cache_read_tokens = VALUES(cache_read_tokens), total_cost = VALUES(total_cost),
avg_response_time_ms = VALUES(avg_response_time_ms), updated_at = VALUES(updated_at)
"#,
)
.bind(day_start)
.bind(now_unix_secs)
.bind(now_unix_secs)
.bind(start_unix_secs)
.bind(end_unix_secs)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
}
async fn upsert_mysql_stats_daily_provider_rows(
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
day_start: i64,
start_unix_secs: i64,
end_unix_secs: i64,
now_unix_secs: i64,
) -> Result<usize, DataLayerError> {
let result = sqlx::query(
r#"
INSERT INTO stats_daily_provider (
id, `date`, provider_name, total_requests, input_tokens, output_tokens,
cache_creation_tokens, cache_read_tokens, total_cost, created_at, updated_at
)
SELECT SHA2(UUID(), 256), ?, provider_name, COUNT(*),
COALESCE(SUM(input_tokens), 0), COALESCE(SUM(output_tokens), 0),
COALESCE(SUM(cache_creation_input_tokens), 0),
COALESCE(SUM(cache_read_input_tokens), 0), COALESCE(SUM(total_cost_usd), 0), ?, ?
FROM `usage`
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
AND status NOT IN ('pending', 'streaming')
AND provider_name NOT IN ('unknown', 'pending')
GROUP BY provider_name
ON DUPLICATE KEY UPDATE
total_requests = VALUES(total_requests), input_tokens = VALUES(input_tokens),
output_tokens = VALUES(output_tokens), cache_creation_tokens = VALUES(cache_creation_tokens),
cache_read_tokens = VALUES(cache_read_tokens), total_cost = VALUES(total_cost),
updated_at = VALUES(updated_at)
"#,
)
.bind(day_start)
.bind(now_unix_secs)
.bind(now_unix_secs)
.bind(start_unix_secs)
.bind(end_unix_secs)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
}
async fn upsert_mysql_stats_daily_api_key_rows(
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
day_start: i64,
start_unix_secs: i64,
end_unix_secs: i64,
now_unix_secs: i64,
) -> Result<usize, DataLayerError> {
let result = sqlx::query(
r#"
INSERT INTO stats_daily_api_key (
id, api_key_id, `date`, total_requests, success_requests, error_requests,
input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens,
total_cost, api_key_name, created_at, updated_at
)
SELECT SHA2(UUID(), 256), usage.api_key_id, ?, COUNT(*),
COUNT(*) - COALESCE(SUM(CASE
WHEN usage.status = 'failed' OR usage.status_code >= 400
OR usage.error_message IS NOT NULL THEN 1 ELSE 0 END), 0),
COALESCE(SUM(CASE
WHEN usage.status = 'failed' OR usage.status_code >= 400
OR usage.error_message IS NOT NULL THEN 1 ELSE 0 END), 0),
COALESCE(SUM(usage.input_tokens), 0), COALESCE(SUM(usage.output_tokens), 0),
COALESCE(SUM(usage.cache_creation_input_tokens), 0),
COALESCE(SUM(usage.cache_read_input_tokens), 0),
COALESCE(SUM(usage.total_cost_usd), 0), MAX(api_keys.name), ?, ?
FROM `usage` AS usage
LEFT JOIN api_keys ON api_keys.id = usage.api_key_id
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
AND usage.api_key_id IS NOT NULL AND usage.api_key_id <> ''
GROUP BY usage.api_key_id
ON DUPLICATE KEY UPDATE
total_requests = VALUES(total_requests), success_requests = VALUES(success_requests),
error_requests = VALUES(error_requests), input_tokens = VALUES(input_tokens),
output_tokens = VALUES(output_tokens), cache_creation_tokens = VALUES(cache_creation_tokens),
cache_read_tokens = VALUES(cache_read_tokens), total_cost = VALUES(total_cost),
api_key_name = COALESCE(VALUES(api_key_name), stats_daily_api_key.api_key_name),
updated_at = VALUES(updated_at)
"#,
)
.bind(day_start)
.bind(now_unix_secs)
.bind(now_unix_secs)
.bind(start_unix_secs)
.bind(end_unix_secs)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
}
async fn refresh_mysql_stats_daily_error_rows(
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
day_start: i64,
start_unix_secs: i64,
end_unix_secs: i64,
now_unix_secs: i64,
) -> Result<usize, DataLayerError> {
sqlx::query("DELETE FROM stats_daily_error WHERE `date` = ?")
.bind(day_start)
.execute(&mut **tx)
.await
.map_sql_err()?;
let result = sqlx::query(
r#"
INSERT INTO stats_daily_error (
id, `date`, error_category, provider_name, model, count, created_at, updated_at
)
SELECT SHA2(UUID(), 256), ?, error_category, provider_name, model, COUNT(*), ?, ?
FROM `usage`
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
AND error_category IS NOT NULL AND error_category <> ''
GROUP BY error_category, provider_name, model
"#,
)
.bind(day_start)
.bind(now_unix_secs)
.bind(now_unix_secs)
.bind(start_unix_secs)
.bind(end_unix_secs)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
}
async fn upsert_mysql_stats_user_daily_rows(
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
day_start: i64,
start_unix_secs: i64,
end_unix_secs: i64,
now_unix_secs: i64,
) -> Result<usize, DataLayerError> {
let result = sqlx::query(
r#"
INSERT INTO stats_user_daily (
id, user_id, `date`, total_requests, success_requests, error_requests,
input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens,
total_cost, username, created_at, updated_at
)
SELECT SHA2(UUID(), 256), usage.user_id, ?, COUNT(*),
COUNT(*) - COALESCE(SUM(CASE
WHEN usage.status = 'failed' OR usage.status_code >= 400
OR usage.error_message IS NOT NULL THEN 1 ELSE 0 END), 0),
COALESCE(SUM(CASE
WHEN usage.status = 'failed' OR usage.status_code >= 400
OR usage.error_message IS NOT NULL THEN 1 ELSE 0 END), 0),
COALESCE(SUM(usage.input_tokens), 0), COALESCE(SUM(usage.output_tokens), 0),
COALESCE(SUM(usage.cache_creation_input_tokens), 0),
COALESCE(SUM(usage.cache_read_input_tokens), 0),
COALESCE(SUM(usage.total_cost_usd), 0), MAX(users.username), ?, ?
FROM `usage` AS usage
LEFT JOIN users ON users.id = usage.user_id
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
AND usage.user_id IS NOT NULL AND usage.user_id <> ''
AND usage.status NOT IN ('pending', 'streaming')
AND usage.provider_name NOT IN ('unknown', 'pending')
GROUP BY usage.user_id
ON DUPLICATE KEY UPDATE
total_requests = VALUES(total_requests), success_requests = VALUES(success_requests),
error_requests = VALUES(error_requests), input_tokens = VALUES(input_tokens),
output_tokens = VALUES(output_tokens), cache_creation_tokens = VALUES(cache_creation_tokens),
cache_read_tokens = VALUES(cache_read_tokens), total_cost = VALUES(total_cost),
username = COALESCE(VALUES(username), stats_user_daily.username),
updated_at = VALUES(updated_at)
"#,
)
.bind(day_start)
.bind(now_unix_secs)
.bind(now_unix_secs)
.bind(start_unix_secs)
.bind(end_unix_secs)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
}
async fn mysql_daily_fallback_count(
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
start_unix_secs: i64,
end_unix_secs: i64,
) -> Result<i64, DataLayerError> {
let start_unix_ms = start_unix_secs.checked_mul(1000).ok_or_else(|| {
DataLayerError::InvalidInput("stats fallback window start overflows milliseconds".into())
})?;
let end_unix_ms = end_unix_secs.checked_mul(1000).ok_or_else(|| {
DataLayerError::InvalidInput("stats fallback window end overflows milliseconds".into())
})?;
sqlx::query_scalar(
r#"
SELECT COUNT(*)
FROM (
SELECT request_id
FROM request_candidates
WHERE created_at >= ? AND created_at < ?
AND status IN ('success', 'failed')
GROUP BY request_id
HAVING COUNT(id) > 1
) AS fallback_requests
"#,
)
.bind(start_unix_ms)
.bind(end_unix_ms)
.fetch_one(&mut **tx)
.await
.map_sql_err()
}
async fn mysql_group_count(
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
group_columns: &str,
start_ms: i64,
end_ms: i64,
start_unix_secs: i64,
end_unix_secs: i64,
) -> Result<usize, DataLayerError> {
let not_empty = group_columns
.split(',')
@@ -330,42 +822,10 @@ FROM (
"#
);
let count: i64 = sqlx::query_scalar(&sql)
.bind(start_ms)
.bind(end_ms)
.bind(start_unix_secs)
.bind(end_unix_secs)
.fetch_one(&mut **tx)
.await
.map_sql_err()?;
Ok(usize::try_from(count.max(0)).unwrap_or(usize::MAX))
}
async fn mysql_error_group_count(
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
start_ms: i64,
end_ms: i64,
) -> Result<usize, DataLayerError> {
let count: i64 = sqlx::query_scalar(
r#"
SELECT COUNT(*)
FROM (
SELECT 1
FROM `usage`
WHERE created_at_unix_ms >= ?
AND created_at_unix_ms < ?
AND status NOT IN ('pending', 'streaming')
AND provider_name NOT IN ('unknown', 'pending')
AND (
status = 'failed'
OR status_code >= 400
OR (error_category IS NOT NULL AND error_category <> '')
)
GROUP BY COALESCE(NULLIF(error_category, ''), 'unknown_error'), provider_name, model
) AS grouped
"#,
)
.bind(start_ms)
.bind(end_ms)
.fetch_one(&mut **tx)
.await
.map_sql_err()?;
Ok(usize::try_from(count.max(0)).unwrap_or(usize::MAX))
}
crates/aether-data/runtime/src/backend/stats/mysql/advanced.rs
+973
View File
@@ -0,0 +1,973 @@
use sqlx::MySql;
use crate::backend::stats_common::stats_id;
use crate::error::SqlResultExt;
use crate::DataLayerError;
const CACHE_5M: &str = r#"GREATEST(
COALESCE(usage.cache_creation_input_tokens_5m, 0),
COALESCE(usage.cache_creation_ephemeral_5m_input_tokens, 0)
)"#;
const CACHE_1H: &str = r#"GREATEST(
COALESCE(usage.cache_creation_input_tokens_1h, 0),
COALESCE(usage.cache_creation_ephemeral_1h_input_tokens, 0)
)"#;
const CACHE_CREATION: &str = r#"CASE
WHEN COALESCE(usage.cache_creation_input_tokens, 0) = 0
AND ({cache_5m} + {cache_1h}) > 0
THEN {cache_5m} + {cache_1h}
ELSE GREATEST(COALESCE(usage.cache_creation_input_tokens, 0), 0)
END"#;
const EFFECTIVE_INPUT: &str = r#"CASE
WHEN SUBSTRING_INDEX(
LOWER(COALESCE(usage.endpoint_api_format, usage.api_format, '')), ':', 1
) IN ('openai', 'gemini', 'google')
AND COALESCE(usage.input_tokens, 0) > 0
AND COALESCE(usage.cache_read_input_tokens, 0) > 0
THEN GREATEST(COALESCE(usage.input_tokens, 0) - COALESCE(usage.cache_read_input_tokens, 0), 0)
ELSE GREATEST(COALESCE(usage.input_tokens, 0), 0)
END"#;
const SUCCESS: &str = r#"CASE
WHEN usage.status <> 'failed'
AND (usage.status_code IS NULL OR usage.status_code < 400)
AND usage.error_message IS NULL
THEN 1 ELSE 0
END"#;
const AGGREGATABLE: &str = r#"usage.status NOT IN ('pending', 'streaming')
AND usage.provider_name NOT IN ('unknown', 'pending')"#;
const SETTLED: &str = r#"COALESCE(settlement.billing_status, usage.billing_status) = 'settled'
AND COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0) > 0"#;
fn cache_creation_expr() -> String {
CACHE_CREATION
.replace("{cache_5m}", CACHE_5M)
.replace("{cache_1h}", CACHE_1H)
}
fn total_input_context_expr() -> String {
format!(
"({EFFECTIVE_INPUT}) + ({}) + GREATEST(COALESCE(usage.cache_read_input_tokens, 0), 0)",
cache_creation_expr()
)
}
fn total_tokens_expr() -> String {
format!(
r#"COALESCE(
NULLIF(GREATEST(COALESCE(usage.total_tokens, 0), 0), 0),
({EFFECTIVE_INPUT})
+ GREATEST(COALESCE(usage.output_tokens, 0), 0)
+ ({})
+ GREATEST(COALESCE(usage.cache_read_input_tokens, 0), 0),
0
)"#,
cache_creation_expr()
)
}
fn percentile_cont(sorted: &[i64], percentile: f64) -> Option<i64> {
if sorted.is_empty() {
return None;
}
let position = percentile * (sorted.len().saturating_sub(1) as f64);
let lower = position.floor() as usize;
let upper = position.ceil() as usize;
let fraction = position - lower as f64;
let value = sorted[lower] as f64 + (sorted[upper] - sorted[lower]) as f64 * fraction;
Some(value.round() as i64)
}
async fn load_percentiles(
tx: &mut sqlx::Transaction<'_, MySql>,
column: &str,
start_unix_secs: i64,
end_unix_secs: i64,
) -> Result<(Option<i64>, Option<i64>, Option<i64>), DataLayerError> {
let sql = format!(
r#"
SELECT {column}
FROM `usage`
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
AND status = 'completed'
AND provider_name NOT IN ('unknown', 'pending')
AND {column} IS NOT NULL
ORDER BY {column}
"#
);
let values: Vec<i64> = sqlx::query_scalar(&sql)
.bind(start_unix_secs)
.bind(end_unix_secs)
.fetch_all(&mut **tx)
.await
.map_sql_err()?;
if values.len() < 10 {
return Ok((None, None, None));
}
Ok((
percentile_cont(&values, 0.50),
percentile_cont(&values, 0.90),
percentile_cont(&values, 0.99),
))
}
pub(super) async fn refresh_hourly(
tx: &mut sqlx::Transaction<'_, MySql>,
hour_utc: i64,
start_unix_secs: i64,
end_unix_secs: i64,
) -> Result<(), DataLayerError> {
let cache_creation = cache_creation_expr();
let total_context = total_input_context_expr();
let sql = format!(
r#"
UPDATE stats_hourly AS target
JOIN (
SELECT
COUNT(*) AS cache_hit_total_requests,
COALESCE(SUM(CASE WHEN COALESCE(usage.cache_read_input_tokens, 0) > 0 THEN 1 ELSE 0 END), 0) AS cache_hit_requests,
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN 1 ELSE 0 END), 0) AS completed_total_requests,
COALESCE(SUM(CASE WHEN usage.status = 'completed' AND COALESCE(usage.cache_read_input_tokens, 0) > 0 THEN 1 ELSE 0 END), 0) AS completed_cache_hit_requests,
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN GREATEST(COALESCE(usage.input_tokens, 0), 0) ELSE 0 END), 0) AS completed_input_tokens,
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN ({cache_creation}) ELSE 0 END), 0) AS completed_cache_creation_tokens,
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN GREATEST(COALESCE(usage.cache_read_input_tokens, 0), 0) ELSE 0 END), 0) AS completed_cache_read_tokens,
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN ({total_context}) ELSE 0 END), 0) AS completed_total_input_context,
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN COALESCE(usage.cache_creation_cost_usd, 0) ELSE 0 END), 0) AS completed_cache_creation_cost,
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN COALESCE(usage.cache_read_cost_usd, 0) ELSE 0 END), 0) AS completed_cache_read_cost,
COALESCE(SUM(CASE WHEN {SETTLED} THEN COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0) ELSE 0 END), 0) AS settled_total_cost,
COALESCE(SUM(CASE WHEN {SETTLED} THEN 1 ELSE 0 END), 0) AS settled_total_requests,
COALESCE(SUM(CASE WHEN {SETTLED} THEN GREATEST(COALESCE(usage.input_tokens, 0), 0) ELSE 0 END), 0) AS settled_input_tokens,
COALESCE(SUM(CASE WHEN {SETTLED} THEN GREATEST(COALESCE(usage.output_tokens, 0), 0) ELSE 0 END), 0) AS settled_output_tokens,
COALESCE(SUM(CASE WHEN {SETTLED} THEN ({cache_creation}) ELSE 0 END), 0) AS settled_cache_creation_tokens,
COALESCE(SUM(CASE WHEN {SETTLED} THEN GREATEST(COALESCE(usage.cache_read_input_tokens, 0), 0) ELSE 0 END), 0) AS settled_cache_read_tokens,
MIN(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_first_finalized_at_unix_secs,
MAX(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_last_finalized_at_unix_secs,
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL AND {AGGREGATABLE} THEN GREATEST(usage.response_time_ms, 0) ELSE 0 END), 0) AS response_time_sum_ms,
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL AND {AGGREGATABLE} THEN 1 ELSE 0 END), 0) AS response_time_samples
FROM `usage` AS usage
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
) AS aggregated
SET
target.cache_hit_total_requests = aggregated.cache_hit_total_requests,
target.cache_hit_requests = aggregated.cache_hit_requests,
target.completed_total_requests = aggregated.completed_total_requests,
target.completed_cache_hit_requests = aggregated.completed_cache_hit_requests,
target.completed_input_tokens = aggregated.completed_input_tokens,
target.completed_cache_creation_tokens = aggregated.completed_cache_creation_tokens,
target.completed_cache_read_tokens = aggregated.completed_cache_read_tokens,
target.completed_total_input_context = aggregated.completed_total_input_context,
target.completed_cache_creation_cost = aggregated.completed_cache_creation_cost,
target.completed_cache_read_cost = aggregated.completed_cache_read_cost,
target.settled_total_cost = aggregated.settled_total_cost,
target.settled_total_requests = aggregated.settled_total_requests,
target.settled_input_tokens = aggregated.settled_input_tokens,
target.settled_output_tokens = aggregated.settled_output_tokens,
target.settled_cache_creation_tokens = aggregated.settled_cache_creation_tokens,
target.settled_cache_read_tokens = aggregated.settled_cache_read_tokens,
target.settled_first_finalized_at_unix_secs = aggregated.settled_first_finalized_at_unix_secs,
target.settled_last_finalized_at_unix_secs = aggregated.settled_last_finalized_at_unix_secs,
target.response_time_sum_ms = aggregated.response_time_sum_ms,
target.response_time_samples = aggregated.response_time_samples
WHERE target.hour_utc = ?
"#
);
sqlx::query(&sql)
.bind(start_unix_secs)
.bind(end_unix_secs)
.bind(hour_utc)
.execute(&mut **tx)
.await
.map_sql_err()?;
refresh_hourly_user(tx, hour_utc, start_unix_secs, end_unix_secs).await?;
refresh_hourly_response_dimensions(tx, hour_utc, start_unix_secs, end_unix_secs).await
}
async fn refresh_hourly_user(
tx: &mut sqlx::Transaction<'_, MySql>,
hour_utc: i64,
start_unix_secs: i64,
end_unix_secs: i64,
) -> Result<(), DataLayerError> {
let cache_creation = cache_creation_expr();
let sql = format!(
r#"
UPDATE stats_hourly_user AS target
JOIN (
SELECT usage.user_id,
COALESCE(SUM({cache_creation}), 0) AS cache_creation_tokens,
COALESCE(SUM(GREATEST(COALESCE(usage.cache_read_input_tokens, 0), 0)), 0) AS cache_read_tokens,
COALESCE(SUM(COALESCE(settlement.billing_actual_total_cost_usd, usage.actual_total_cost_usd, 0)), 0) AS actual_total_cost,
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN GREATEST(usage.response_time_ms, 0) ELSE 0 END), 0) AS response_time_sum_ms,
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0) AS response_time_samples,
COALESCE(SUM(CASE WHEN {SETTLED} THEN COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0) ELSE 0 END), 0) AS settled_total_cost,
COALESCE(SUM(CASE WHEN {SETTLED} THEN 1 ELSE 0 END), 0) AS settled_total_requests,
COALESCE(SUM(CASE WHEN {SETTLED} THEN GREATEST(COALESCE(usage.input_tokens, 0), 0) ELSE 0 END), 0) AS settled_input_tokens,
COALESCE(SUM(CASE WHEN {SETTLED} THEN GREATEST(COALESCE(usage.output_tokens, 0), 0) ELSE 0 END), 0) AS settled_output_tokens,
COALESCE(SUM(CASE WHEN {SETTLED} THEN ({cache_creation}) ELSE 0 END), 0) AS settled_cache_creation_tokens,
COALESCE(SUM(CASE WHEN {SETTLED} THEN GREATEST(COALESCE(usage.cache_read_input_tokens, 0), 0) ELSE 0 END), 0) AS settled_cache_read_tokens,
MIN(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_first_finalized_at_unix_secs,
MAX(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_last_finalized_at_unix_secs
FROM `usage` AS usage
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
AND usage.user_id IS NOT NULL AND usage.user_id <> '' AND {AGGREGATABLE}
GROUP BY usage.user_id
) AS aggregated ON target.user_id = aggregated.user_id
SET target.cache_creation_tokens = aggregated.cache_creation_tokens,
target.cache_read_tokens = aggregated.cache_read_tokens,
target.actual_total_cost = aggregated.actual_total_cost,
target.response_time_sum_ms = aggregated.response_time_sum_ms,
target.response_time_samples = aggregated.response_time_samples,
target.settled_total_cost = aggregated.settled_total_cost,
target.settled_total_requests = aggregated.settled_total_requests,
target.settled_input_tokens = aggregated.settled_input_tokens,
target.settled_output_tokens = aggregated.settled_output_tokens,
target.settled_cache_creation_tokens = aggregated.settled_cache_creation_tokens,
target.settled_cache_read_tokens = aggregated.settled_cache_read_tokens,
target.settled_first_finalized_at_unix_secs = aggregated.settled_first_finalized_at_unix_secs,
target.settled_last_finalized_at_unix_secs = aggregated.settled_last_finalized_at_unix_secs
WHERE target.hour_utc = ?
"#
);
sqlx::query(&sql)
.bind(start_unix_secs)
.bind(end_unix_secs)
.bind(hour_utc)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(())
}
async fn refresh_hourly_response_dimensions(
tx: &mut sqlx::Transaction<'_, MySql>,
hour_utc: i64,
start_unix_secs: i64,
end_unix_secs: i64,
) -> Result<(), DataLayerError> {
for (table, select_dimensions, group_by, join) in [
(
"stats_hourly_model",
"usage.model AS model",
"usage.model",
"target.model = aggregated.model",
),
(
"stats_hourly_user_model",
"usage.user_id AS user_id, usage.model AS model",
"usage.user_id, usage.model",
"target.user_id = aggregated.user_id AND target.model = aggregated.model",
),
] {
let sql = format!(
r#"
UPDATE {table} AS target
JOIN (
SELECT {select_dimensions},
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN GREATEST(usage.response_time_ms, 0) ELSE 0 END), 0) AS response_time_sum_ms,
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0) AS response_time_samples
FROM `usage` AS usage
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ? AND {AGGREGATABLE}
GROUP BY {group_by}
) AS aggregated ON {join}
SET target.response_time_sum_ms = aggregated.response_time_sum_ms,
target.response_time_samples = aggregated.response_time_samples
WHERE target.hour_utc = ?
"#
);
sqlx::query(&sql)
.bind(start_unix_secs)
.bind(end_unix_secs)
.bind(hour_utc)
.execute(&mut **tx)
.await
.map_sql_err()?;
}
Ok(())
}
pub(super) async fn refresh_daily(
tx: &mut sqlx::Transaction<'_, MySql>,
day_start: i64,
start_unix_secs: i64,
end_unix_secs: i64,
now_unix_secs: i64,
) -> Result<(), DataLayerError> {
let response = load_percentiles(tx, "response_time_ms", start_unix_secs, end_unix_secs).await?;
let first_byte =
load_percentiles(tx, "first_byte_time_ms", start_unix_secs, end_unix_secs).await?;
refresh_daily_root(
tx,
day_start,
start_unix_secs,
end_unix_secs,
response,
first_byte,
)
.await?;
refresh_daily_existing_dimensions(tx, day_start, start_unix_secs, end_unix_secs).await?;
upsert_user_dimension(
tx,
"stats_user_daily_model",
"model",
"usage.model",
"usage.model IS NOT NULL AND usage.model <> ''",
day_start,
start_unix_secs,
end_unix_secs,
now_unix_secs,
)
.await?;
upsert_user_dimension(
tx,
"stats_user_daily_provider",
"provider_name",
"usage.provider_name",
"usage.provider_name IS NOT NULL AND usage.provider_name <> ''",
day_start,
start_unix_secs,
end_unix_secs,
now_unix_secs,
)
.await?;
upsert_user_dimension(
tx,
"stats_user_daily_api_format",
"api_format",
"LOWER(COALESCE(usage.endpoint_api_format, usage.api_format, ''))",
"COALESCE(usage.endpoint_api_format, usage.api_format, '') <> ''",
day_start,
start_unix_secs,
end_unix_secs,
now_unix_secs,
)
.await?;
upsert_model_provider_rows(tx, day_start, start_unix_secs, end_unix_secs, now_unix_secs)
.await?;
upsert_cost_savings_rows(tx, day_start, start_unix_secs, end_unix_secs, now_unix_secs).await?;
refresh_user_summary(tx, end_unix_secs, now_unix_secs).await
}
async fn refresh_daily_root(
tx: &mut sqlx::Transaction<'_, MySql>,
day_start: i64,
start_unix_secs: i64,
end_unix_secs: i64,
response: (Option<i64>, Option<i64>, Option<i64>),
first_byte: (Option<i64>, Option<i64>, Option<i64>),
) -> Result<(), DataLayerError> {
let cache_creation = cache_creation_expr();
let total_context = total_input_context_expr();
let sql = format!(
r#"
UPDATE stats_daily AS target
JOIN (
SELECT
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN ({EFFECTIVE_INPUT}) ELSE 0 END), 0) AS effective_input_tokens,
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN ({total_context}) ELSE 0 END), 0) AS total_input_context,
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL AND {AGGREGATABLE} THEN GREATEST(usage.response_time_ms, 0) ELSE 0 END), 0) AS response_time_sum_ms,
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL AND {AGGREGATABLE} THEN 1 ELSE 0 END), 0) AS response_time_samples,
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN {CACHE_5M} ELSE 0 END), 0) AS cache_creation_ephemeral_5m_tokens,
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN {CACHE_1H} ELSE 0 END), 0) AS cache_creation_ephemeral_1h_tokens,
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN COALESCE(usage.input_cost_usd, 0) ELSE 0 END), 0) AS input_cost,
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN COALESCE(usage.output_cost_usd, 0) ELSE 0 END), 0) AS output_cost,
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN COALESCE(usage.cache_creation_cost_usd, 0) ELSE 0 END), 0) AS cache_creation_cost,
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN COALESCE(usage.cache_read_cost_usd, 0) ELSE 0 END), 0) AS cache_read_cost,
COUNT(*) AS cache_hit_total_requests,
COALESCE(SUM(CASE WHEN COALESCE(usage.cache_read_input_tokens, 0) > 0 THEN 1 ELSE 0 END), 0) AS cache_hit_requests,
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN 1 ELSE 0 END), 0) AS completed_total_requests,
COALESCE(SUM(CASE WHEN usage.status = 'completed' AND COALESCE(usage.cache_read_input_tokens, 0) > 0 THEN 1 ELSE 0 END), 0) AS completed_cache_hit_requests,
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN GREATEST(COALESCE(usage.input_tokens, 0), 0) ELSE 0 END), 0) AS completed_input_tokens,
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN ({cache_creation}) ELSE 0 END), 0) AS completed_cache_creation_tokens,
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN GREATEST(COALESCE(usage.cache_read_input_tokens, 0), 0) ELSE 0 END), 0) AS completed_cache_read_tokens,
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN ({total_context}) ELSE 0 END), 0) AS completed_total_input_context,
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN COALESCE(usage.cache_creation_cost_usd, 0) ELSE 0 END), 0) AS completed_cache_creation_cost,
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN COALESCE(usage.cache_read_cost_usd, 0) ELSE 0 END), 0) AS completed_cache_read_cost,
COALESCE(SUM(CASE WHEN {SETTLED} THEN COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0) ELSE 0 END), 0) AS settled_total_cost,
COALESCE(SUM(CASE WHEN {SETTLED} THEN 1 ELSE 0 END), 0) AS settled_total_requests,
COALESCE(SUM(CASE WHEN {SETTLED} THEN GREATEST(COALESCE(usage.input_tokens, 0), 0) ELSE 0 END), 0) AS settled_input_tokens,
COALESCE(SUM(CASE WHEN {SETTLED} THEN GREATEST(COALESCE(usage.output_tokens, 0), 0) ELSE 0 END), 0) AS settled_output_tokens,
COALESCE(SUM(CASE WHEN {SETTLED} THEN ({cache_creation}) ELSE 0 END), 0) AS settled_cache_creation_tokens,
COALESCE(SUM(CASE WHEN {SETTLED} THEN GREATEST(COALESCE(usage.cache_read_input_tokens, 0), 0) ELSE 0 END), 0) AS settled_cache_read_tokens,
MIN(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_first_finalized_at_unix_secs,
MAX(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_last_finalized_at_unix_secs
FROM `usage` AS usage
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
) AS aggregated
SET target.effective_input_tokens = aggregated.effective_input_tokens,
target.total_input_context = aggregated.total_input_context,
target.response_time_sum_ms = aggregated.response_time_sum_ms,
target.response_time_samples = aggregated.response_time_samples,
target.cache_creation_ephemeral_5m_tokens = aggregated.cache_creation_ephemeral_5m_tokens,
target.cache_creation_ephemeral_1h_tokens = aggregated.cache_creation_ephemeral_1h_tokens,
target.input_cost = aggregated.input_cost,
target.output_cost = aggregated.output_cost,
target.cache_creation_cost = aggregated.cache_creation_cost,
target.cache_read_cost = aggregated.cache_read_cost,
target.cache_hit_total_requests = aggregated.cache_hit_total_requests,
target.cache_hit_requests = aggregated.cache_hit_requests,
target.completed_total_requests = aggregated.completed_total_requests,
target.completed_cache_hit_requests = aggregated.completed_cache_hit_requests,
target.completed_input_tokens = aggregated.completed_input_tokens,
target.completed_cache_creation_tokens = aggregated.completed_cache_creation_tokens,
target.completed_cache_read_tokens = aggregated.completed_cache_read_tokens,
target.completed_total_input_context = aggregated.completed_total_input_context,
target.completed_cache_creation_cost = aggregated.completed_cache_creation_cost,
target.completed_cache_read_cost = aggregated.completed_cache_read_cost,
target.settled_total_cost = aggregated.settled_total_cost,
target.settled_total_requests = aggregated.settled_total_requests,
target.settled_input_tokens = aggregated.settled_input_tokens,
target.settled_output_tokens = aggregated.settled_output_tokens,
target.settled_cache_creation_tokens = aggregated.settled_cache_creation_tokens,
target.settled_cache_read_tokens = aggregated.settled_cache_read_tokens,
target.settled_first_finalized_at_unix_secs = aggregated.settled_first_finalized_at_unix_secs,
target.settled_last_finalized_at_unix_secs = aggregated.settled_last_finalized_at_unix_secs,
target.p50_response_time_ms = ?, target.p90_response_time_ms = ?, target.p99_response_time_ms = ?,
target.p50_first_byte_time_ms = ?, target.p90_first_byte_time_ms = ?, target.p99_first_byte_time_ms = ?
WHERE target.`date` = ?
"#
);
sqlx::query(&sql)
.bind(start_unix_secs)
.bind(end_unix_secs)
.bind(response.0)
.bind(response.1)
.bind(response.2)
.bind(first_byte.0)
.bind(first_byte.1)
.bind(first_byte.2)
.bind(day_start)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(())
}
async fn refresh_daily_existing_dimensions(
tx: &mut sqlx::Transaction<'_, MySql>,
day_start: i64,
start_unix_secs: i64,
end_unix_secs: i64,
) -> Result<(), DataLayerError> {
let cache_creation = cache_creation_expr();
let total_context = total_input_context_expr();
let model_sql = format!(
r#"
UPDATE stats_daily_model AS target
JOIN (
SELECT usage.model,
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN GREATEST(usage.response_time_ms, 0) ELSE 0 END), 0) AS response_time_sum_ms,
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0) AS response_time_samples,
COALESCE(SUM({CACHE_5M}), 0) AS cache_creation_ephemeral_5m_tokens,
COALESCE(SUM({CACHE_1H}), 0) AS cache_creation_ephemeral_1h_tokens
FROM `usage` AS usage
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
AND {AGGREGATABLE} AND usage.model IS NOT NULL AND usage.model <> ''
GROUP BY usage.model
) AS aggregated ON target.model = aggregated.model
SET target.response_time_sum_ms = aggregated.response_time_sum_ms,
target.response_time_samples = aggregated.response_time_samples,
target.cache_creation_ephemeral_5m_tokens = aggregated.cache_creation_ephemeral_5m_tokens,
target.cache_creation_ephemeral_1h_tokens = aggregated.cache_creation_ephemeral_1h_tokens
WHERE target.`date` = ?
"#
);
sqlx::query(&model_sql)
.bind(start_unix_secs)
.bind(end_unix_secs)
.bind(day_start)
.execute(&mut **tx)
.await
.map_sql_err()?;
let user_sql = format!(
r#"
UPDATE stats_user_daily AS target
JOIN (
SELECT usage.user_id,
COALESCE(SUM({EFFECTIVE_INPUT}), 0) AS effective_input_tokens,
COALESCE(SUM({total_context}), 0) AS total_input_context,
COALESCE(SUM(COALESCE(usage.cache_creation_cost_usd, 0)), 0) AS cache_creation_cost,
COALESCE(SUM(COALESCE(usage.cache_read_cost_usd, 0)), 0) AS cache_read_cost,
COALESCE(SUM(COALESCE(settlement.billing_actual_total_cost_usd, usage.actual_total_cost_usd, 0)), 0) AS actual_total_cost,
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN GREATEST(usage.response_time_ms, 0) ELSE 0 END), 0) AS response_time_sum_ms,
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0) AS response_time_samples,
COALESCE(SUM({CACHE_5M}), 0) AS cache_creation_ephemeral_5m_tokens,
COALESCE(SUM({CACHE_1H}), 0) AS cache_creation_ephemeral_1h_tokens,
COALESCE(SUM(CASE WHEN {SETTLED} THEN COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0) ELSE 0 END), 0) AS settled_total_cost,
COALESCE(SUM(CASE WHEN {SETTLED} THEN 1 ELSE 0 END), 0) AS settled_total_requests,
COALESCE(SUM(CASE WHEN {SETTLED} THEN GREATEST(COALESCE(usage.input_tokens, 0), 0) ELSE 0 END), 0) AS settled_input_tokens,
COALESCE(SUM(CASE WHEN {SETTLED} THEN GREATEST(COALESCE(usage.output_tokens, 0), 0) ELSE 0 END), 0) AS settled_output_tokens,
COALESCE(SUM(CASE WHEN {SETTLED} THEN ({cache_creation}) ELSE 0 END), 0) AS settled_cache_creation_tokens,
COALESCE(SUM(CASE WHEN {SETTLED} THEN GREATEST(COALESCE(usage.cache_read_input_tokens, 0), 0) ELSE 0 END), 0) AS settled_cache_read_tokens,
MIN(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_first_finalized_at_unix_secs,
MAX(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_last_finalized_at_unix_secs
FROM `usage` AS usage
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
AND usage.user_id IS NOT NULL AND usage.user_id <> '' AND {AGGREGATABLE}
GROUP BY usage.user_id
) AS aggregated ON target.user_id = aggregated.user_id
SET target.effective_input_tokens = aggregated.effective_input_tokens,
target.total_input_context = aggregated.total_input_context,
target.cache_creation_cost = aggregated.cache_creation_cost,
target.cache_read_cost = aggregated.cache_read_cost,
target.actual_total_cost = aggregated.actual_total_cost,
target.response_time_sum_ms = aggregated.response_time_sum_ms,
target.response_time_samples = aggregated.response_time_samples,
target.cache_creation_ephemeral_5m_tokens = aggregated.cache_creation_ephemeral_5m_tokens,
target.cache_creation_ephemeral_1h_tokens = aggregated.cache_creation_ephemeral_1h_tokens,
target.settled_total_cost = aggregated.settled_total_cost,
target.settled_total_requests = aggregated.settled_total_requests,
target.settled_input_tokens = aggregated.settled_input_tokens,
target.settled_output_tokens = aggregated.settled_output_tokens,
target.settled_cache_creation_tokens = aggregated.settled_cache_creation_tokens,
target.settled_cache_read_tokens = aggregated.settled_cache_read_tokens,
target.settled_first_finalized_at_unix_secs = aggregated.settled_first_finalized_at_unix_secs,
target.settled_last_finalized_at_unix_secs = aggregated.settled_last_finalized_at_unix_secs
WHERE target.`date` = ?
"#
);
sqlx::query(&user_sql)
.bind(start_unix_secs)
.bind(end_unix_secs)
.bind(day_start)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(())
}
#[allow(clippy::too_many_arguments)]
async fn upsert_user_dimension(
tx: &mut sqlx::Transaction<'_, MySql>,
table: &str,
dimension_column: &str,
dimension_expr: &str,
dimension_filter: &str,
day_start: i64,
start_unix_secs: i64,
end_unix_secs: i64,
now_unix_secs: i64,
) -> Result<(), DataLayerError> {
let cache_creation = cache_creation_expr();
let total_context = total_input_context_expr();
let total_tokens = total_tokens_expr();
let sql = format!(
r#"
INSERT INTO {table} (
id, user_id, username, `date`, {dimension_column}, total_requests, success_requests,
input_tokens, effective_input_tokens, output_tokens, total_tokens, total_input_context,
cache_creation_tokens, cache_creation_ephemeral_5m_tokens,
cache_creation_ephemeral_1h_tokens, cache_read_tokens, total_cost, actual_total_cost,
response_time_sum_ms, response_time_samples, successful_response_time_sum_ms,
successful_response_time_samples, created_at, updated_at
)
SELECT SHA2(UUID(), 256), usage.user_id,
MAX(COALESCE(usage.username, users.username)), ?, {dimension_expr}, COUNT(*),
COALESCE(SUM({SUCCESS}), 0),
COALESCE(SUM(GREATEST(COALESCE(usage.input_tokens, 0), 0)), 0),
COALESCE(SUM({EFFECTIVE_INPUT}), 0),
COALESCE(SUM(GREATEST(COALESCE(usage.output_tokens, 0), 0)), 0),
COALESCE(SUM({total_tokens}), 0), COALESCE(SUM({total_context}), 0),
COALESCE(SUM({cache_creation}), 0), COALESCE(SUM({CACHE_5M}), 0),
COALESCE(SUM({CACHE_1H}), 0),
COALESCE(SUM(GREATEST(COALESCE(usage.cache_read_input_tokens, 0), 0)), 0),
COALESCE(SUM(COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0)), 0),
COALESCE(SUM(COALESCE(settlement.billing_actual_total_cost_usd, usage.actual_total_cost_usd, 0)), 0),
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN GREATEST(usage.response_time_ms, 0) ELSE 0 END), 0),
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0),
COALESCE(SUM(CASE WHEN ({SUCCESS}) = 1 AND usage.response_time_ms IS NOT NULL THEN GREATEST(usage.response_time_ms, 0) ELSE 0 END), 0),
COALESCE(SUM(CASE WHEN ({SUCCESS}) = 1 AND usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0),
?, ?
FROM `usage` AS usage
LEFT JOIN users ON users.id = usage.user_id
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
AND usage.user_id IS NOT NULL AND usage.user_id <> ''
AND {dimension_filter} AND {AGGREGATABLE}
GROUP BY usage.user_id, {dimension_expr}
ON DUPLICATE KEY UPDATE
username = COALESCE(VALUES(username), {table}.username),
total_requests = VALUES(total_requests), success_requests = VALUES(success_requests),
input_tokens = VALUES(input_tokens), effective_input_tokens = VALUES(effective_input_tokens),
output_tokens = VALUES(output_tokens), total_tokens = VALUES(total_tokens),
total_input_context = VALUES(total_input_context),
cache_creation_tokens = VALUES(cache_creation_tokens),
cache_creation_ephemeral_5m_tokens = VALUES(cache_creation_ephemeral_5m_tokens),
cache_creation_ephemeral_1h_tokens = VALUES(cache_creation_ephemeral_1h_tokens),
cache_read_tokens = VALUES(cache_read_tokens), total_cost = VALUES(total_cost),
actual_total_cost = VALUES(actual_total_cost),
response_time_sum_ms = VALUES(response_time_sum_ms),
response_time_samples = VALUES(response_time_samples),
successful_response_time_sum_ms = VALUES(successful_response_time_sum_ms),
successful_response_time_samples = VALUES(successful_response_time_samples),
updated_at = VALUES(updated_at)
"#
);
sqlx::query(&sql)
.bind(day_start)
.bind(now_unix_secs)
.bind(now_unix_secs)
.bind(start_unix_secs)
.bind(end_unix_secs)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(())
}
async fn upsert_model_provider_rows(
tx: &mut sqlx::Transaction<'_, MySql>,
day_start: i64,
start_unix_secs: i64,
end_unix_secs: i64,
now_unix_secs: i64,
) -> Result<(), DataLayerError> {
let total_tokens = total_tokens_expr();
let model_provider_sql = format!(
r#"
INSERT INTO stats_daily_model_provider (
id, `date`, model, provider_name, total_requests, total_tokens, total_cost,
response_time_sum_ms, response_time_samples, created_at, updated_at
)
SELECT SHA2(UUID(), 256), ?, usage.model, usage.provider_name, COUNT(*),
COALESCE(SUM({total_tokens}), 0),
COALESCE(SUM(COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0)), 0),
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN GREATEST(usage.response_time_ms, 0) ELSE 0 END), 0),
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0), ?, ?
FROM `usage` AS usage
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
AND usage.model IS NOT NULL AND usage.model <> '' AND {AGGREGATABLE}
GROUP BY usage.model, usage.provider_name
ON DUPLICATE KEY UPDATE
total_requests = VALUES(total_requests), total_tokens = VALUES(total_tokens),
total_cost = VALUES(total_cost), response_time_sum_ms = VALUES(response_time_sum_ms),
response_time_samples = VALUES(response_time_samples), updated_at = VALUES(updated_at)
"#
);
sqlx::query(&model_provider_sql)
.bind(day_start)
.bind(now_unix_secs)
.bind(now_unix_secs)
.bind(start_unix_secs)
.bind(end_unix_secs)
.execute(&mut **tx)
.await
.map_sql_err()?;
let user_model_provider_sql = format!(
r#"
INSERT INTO stats_user_daily_model_provider (
id, user_id, username, `date`, model, provider_name, total_requests, total_tokens,
total_cost, response_time_sum_ms, response_time_samples, created_at, updated_at
)
SELECT SHA2(UUID(), 256), usage.user_id, MAX(COALESCE(usage.username, users.username)),
?, usage.model, usage.provider_name, COUNT(*), COALESCE(SUM({total_tokens}), 0),
COALESCE(SUM(COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0)), 0),
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN GREATEST(usage.response_time_ms, 0) ELSE 0 END), 0),
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0), ?, ?
FROM `usage` AS usage
LEFT JOIN users ON users.id = usage.user_id
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
AND usage.user_id IS NOT NULL AND usage.user_id <> ''
AND usage.model IS NOT NULL AND usage.model <> '' AND {AGGREGATABLE}
GROUP BY usage.user_id, usage.model, usage.provider_name
ON DUPLICATE KEY UPDATE
username = COALESCE(VALUES(username), stats_user_daily_model_provider.username),
total_requests = VALUES(total_requests), total_tokens = VALUES(total_tokens),
total_cost = VALUES(total_cost), response_time_sum_ms = VALUES(response_time_sum_ms),
response_time_samples = VALUES(response_time_samples), updated_at = VALUES(updated_at)
"#
);
sqlx::query(&user_model_provider_sql)
.bind(day_start)
.bind(now_unix_secs)
.bind(now_unix_secs)
.bind(start_unix_secs)
.bind(end_unix_secs)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(())
}
async fn upsert_cost_savings_rows(
tx: &mut sqlx::Transaction<'_, MySql>,
day_start: i64,
start_unix_secs: i64,
end_unix_secs: i64,
now_unix_secs: i64,
) -> Result<(), DataLayerError> {
for (table, dimensions) in [
("stats_daily_cost_savings", Vec::new()),
(
"stats_daily_cost_savings_provider",
vec![("provider_name", "COALESCE(usage.provider_name, '')")],
),
(
"stats_daily_cost_savings_model",
vec![("model", "COALESCE(usage.model, '')")],
),
(
"stats_daily_cost_savings_model_provider",
vec![
("model", "COALESCE(usage.model, '')"),
("provider_name", "COALESCE(usage.provider_name, '')"),
],
),
] {
upsert_cost_savings_dimension(
tx,
table,
false,
&dimensions,
day_start,
start_unix_secs,
end_unix_secs,
now_unix_secs,
)
.await?;
}
for (table, dimensions) in [
("stats_user_daily_cost_savings", Vec::new()),
(
"stats_user_daily_cost_savings_provider",
vec![("provider_name", "COALESCE(usage.provider_name, '')")],
),
(
"stats_user_daily_cost_savings_model",
vec![("model", "COALESCE(usage.model, '')")],
),
(
"stats_user_daily_cost_savings_model_provider",
vec![
("model", "COALESCE(usage.model, '')"),
("provider_name", "COALESCE(usage.provider_name, '')"),
],
),
] {
upsert_cost_savings_dimension(
tx,
table,
true,
&dimensions,
day_start,
start_unix_secs,
end_unix_secs,
now_unix_secs,
)
.await?;
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
async fn upsert_cost_savings_dimension(
tx: &mut sqlx::Transaction<'_, MySql>,
table: &str,
per_user: bool,
dimensions: &[(&str, &str)],
day_start: i64,
start_unix_secs: i64,
end_unix_secs: i64,
now_unix_secs: i64,
) -> Result<(), DataLayerError> {
let dimension_columns = dimensions
.iter()
.map(|(column, _)| *column)
.collect::<Vec<_>>();
let dimension_exprs = dimensions
.iter()
.map(|(_, expression)| *expression)
.collect::<Vec<_>>();
let user_columns = if per_user { "user_id, username, " } else { "" };
let user_select = if per_user {
"usage.user_id, MAX(COALESCE(usage.username, users.username)), "
} else {
""
};
let user_join = if per_user {
"LEFT JOIN users ON users.id = usage.user_id"
} else {
""
};
let user_filter = if per_user {
"AND usage.user_id IS NOT NULL AND usage.user_id <> ''"
} else {
""
};
let mut group_by = Vec::new();
if per_user {
group_by.push("usage.user_id");
}
group_by.extend(dimension_exprs.iter().copied());
let dimension_columns_sql = if dimension_columns.is_empty() {
String::new()
} else {
format!("{}, ", dimension_columns.join(", "))
};
let dimension_select_sql = if dimension_exprs.is_empty() {
String::new()
} else {
format!("{}, ", dimension_exprs.join(", "))
};
let group_by_sql = if group_by.is_empty() {
String::new()
} else {
format!("GROUP BY {}", group_by.join(", "))
};
let sql = format!(
r#"
INSERT INTO {table} (
id, {user_columns}`date`, {dimension_columns_sql}cache_read_tokens,
cache_read_cost, cache_creation_cost, estimated_full_cost, created_at, updated_at
)
SELECT SHA2(UUID(), 256), {user_select}?, {dimension_select_sql}
COALESCE(SUM(GREATEST(COALESCE(usage.cache_read_input_tokens, 0), 0)), 0),
COALESCE(SUM(COALESCE(usage.cache_read_cost_usd, 0)), 0),
COALESCE(SUM(COALESCE(usage.cache_creation_cost_usd, 0)), 0),
COALESCE(SUM(
COALESCE(settlement.input_price_per_1m, usage.input_price_per_1m, 0)
* GREATEST(COALESCE(usage.cache_read_input_tokens, 0), 0) / 1000000.0
), 0), ?, ?
FROM `usage` AS usage
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
{user_join}
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ? {user_filter}
{group_by_sql}
ON DUPLICATE KEY UPDATE
{}cache_read_tokens = VALUES(cache_read_tokens),
cache_read_cost = VALUES(cache_read_cost),
cache_creation_cost = VALUES(cache_creation_cost),
estimated_full_cost = VALUES(estimated_full_cost), updated_at = VALUES(updated_at)
"#,
if per_user {
format!("username = COALESCE(VALUES(username), {table}.username), ")
} else {
String::new()
}
);
sqlx::query(&sql)
.bind(day_start)
.bind(now_unix_secs)
.bind(now_unix_secs)
.bind(start_unix_secs)
.bind(end_unix_secs)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(())
}
async fn refresh_user_summary(
tx: &mut sqlx::Transaction<'_, MySql>,
cutoff_date: i64,
now_unix_secs: i64,
) -> Result<(), DataLayerError> {
sqlx::query(
r#"
INSERT INTO stats_user_summary (
id, user_id, username, cutoff_date, all_time_requests, all_time_success_requests,
all_time_error_requests, all_time_input_tokens, all_time_output_tokens,
all_time_cache_creation_tokens, all_time_cache_read_tokens, all_time_cost,
all_time_actual_cost, active_days, first_active_date, last_active_date,
created_at, updated_at
)
SELECT SHA2(UUID(), 256), user_id, MAX(username), ?,
COALESCE(SUM(total_requests), 0), COALESCE(SUM(success_requests), 0),
COALESCE(SUM(error_requests), 0), COALESCE(SUM(input_tokens), 0),
COALESCE(SUM(output_tokens), 0), COALESCE(SUM(cache_creation_tokens), 0),
COALESCE(SUM(cache_read_tokens), 0), COALESCE(SUM(total_cost), 0),
COALESCE(SUM(actual_total_cost), 0),
COALESCE(SUM(CASE WHEN total_requests > 0 THEN 1 ELSE 0 END), 0),
MIN(CASE WHEN total_requests > 0 THEN `date` END),
MAX(CASE WHEN total_requests > 0 THEN `date` END), ?, ?
FROM stats_user_daily
WHERE `date` < ?
GROUP BY user_id
ON DUPLICATE KEY UPDATE
username = COALESCE(VALUES(username), stats_user_summary.username),
cutoff_date = VALUES(cutoff_date), all_time_requests = VALUES(all_time_requests),
all_time_success_requests = VALUES(all_time_success_requests),
all_time_error_requests = VALUES(all_time_error_requests),
all_time_input_tokens = VALUES(all_time_input_tokens),
all_time_output_tokens = VALUES(all_time_output_tokens),
all_time_cache_creation_tokens = VALUES(all_time_cache_creation_tokens),
all_time_cache_read_tokens = VALUES(all_time_cache_read_tokens),
all_time_cost = VALUES(all_time_cost), all_time_actual_cost = VALUES(all_time_actual_cost),
active_days = VALUES(active_days), first_active_date = VALUES(first_active_date),
last_active_date = VALUES(last_active_date), updated_at = VALUES(updated_at)
"#,
)
.bind(cutoff_date)
.bind(now_unix_secs)
.bind(now_unix_secs)
.bind(cutoff_date)
.execute(&mut **tx)
.await
.map_sql_err()?;
refresh_global_summary(tx, cutoff_date, now_unix_secs).await?;
Ok(())
}
async fn refresh_global_summary(
tx: &mut sqlx::Transaction<'_, MySql>,
cutoff_date: i64,
now_unix_secs: i64,
) -> Result<(), DataLayerError> {
let existing_id: Option<String> =
sqlx::query_scalar("SELECT id FROM stats_summary ORDER BY created_at, id LIMIT 1")
.fetch_optional(&mut **tx)
.await
.map_sql_err()?;
let summary_id = existing_id.unwrap_or_else(|| stats_id("stats-summary"));
sqlx::query(
r#"
INSERT INTO stats_summary (
id, cutoff_date, all_time_requests, all_time_success_requests,
all_time_error_requests, all_time_input_tokens, all_time_output_tokens,
all_time_cache_creation_tokens, all_time_cache_read_tokens, all_time_cost,
all_time_actual_cost, total_users, active_users, total_api_keys,
active_api_keys, created_at, updated_at
)
SELECT ?, ?, COALESCE(SUM(total_requests), 0), COALESCE(SUM(success_requests), 0),
COALESCE(SUM(error_requests), 0), COALESCE(SUM(input_tokens), 0),
COALESCE(SUM(output_tokens), 0), COALESCE(SUM(cache_creation_tokens), 0),
COALESCE(SUM(cache_read_tokens), 0), COALESCE(SUM(total_cost), 0),
COALESCE(SUM(actual_total_cost), 0),
(SELECT COUNT(*) FROM users),
(SELECT COUNT(*) FROM users WHERE is_active <> 0),
(SELECT COUNT(*) FROM api_keys),
(SELECT COUNT(*) FROM api_keys WHERE is_active <> 0), ?, ?
FROM stats_daily
WHERE `date` < ?
ON DUPLICATE KEY UPDATE
cutoff_date = VALUES(cutoff_date),
all_time_requests = VALUES(all_time_requests),
all_time_success_requests = VALUES(all_time_success_requests),
all_time_error_requests = VALUES(all_time_error_requests),
all_time_input_tokens = VALUES(all_time_input_tokens),
all_time_output_tokens = VALUES(all_time_output_tokens),
all_time_cache_creation_tokens = VALUES(all_time_cache_creation_tokens),
all_time_cache_read_tokens = VALUES(all_time_cache_read_tokens),
all_time_cost = VALUES(all_time_cost),
all_time_actual_cost = VALUES(all_time_actual_cost),
total_users = VALUES(total_users), active_users = VALUES(active_users),
total_api_keys = VALUES(total_api_keys), active_api_keys = VALUES(active_api_keys),
updated_at = VALUES(updated_at)
"#,
)
.bind(summary_id)
.bind(cutoff_date)
.bind(now_unix_secs)
.bind(now_unix_secs)
.bind(cutoff_date)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(())
}
crates/aether-data/runtime/src/backend/stats/sqlite.rs
+562 -64
View File
@@ -1,7 +1,7 @@
use chrono::{DateTime, Utc};
use sqlx::Row;
use crate::backend::stats_common::{stats_id, unix_ms, unix_secs, utc_from_unix_secs};
use crate::backend::stats_common::{stats_id, unix_secs, utc_from_unix_secs};
use crate::backend::SqliteBackend;
use crate::driver::sqlite::{sqlite_real, SqlitePool};
use crate::error::SqlResultExt;
@@ -10,6 +10,8 @@ use crate::{
StatsHourlyAggregationInput, StatsHourlyAggregationSummary,
};
mod advanced;
impl SqliteBackend {
pub async fn aggregate_stats_hourly(
&self,
@@ -64,7 +66,7 @@ async fn next_sqlite_stats_hourly_bucket(
}
let next_bucket: Option<i64> = sqlx::query_scalar(
r#"
SELECT MIN(CAST(created_at_unix_ms / 3600000 AS INTEGER) * 3600)
SELECT MIN(CAST(created_at_unix_ms / 3600 AS INTEGER) * 3600)
FROM "usage"
WHERE created_at_unix_ms >= ?
AND created_at_unix_ms < ?
@@ -72,8 +74,8 @@ WHERE created_at_unix_ms >= ?
AND provider_name NOT IN ('unknown', 'pending')
"#,
)
.bind(unix_ms(search_from)?)
.bind(unix_ms(search_until)?)
.bind(search_from)
.bind(search_until)
.fetch_one(pool)
.await
.map_sql_err()?;
@@ -96,7 +98,7 @@ async fn next_sqlite_stats_daily_bucket(
}
let next_bucket: Option<i64> = sqlx::query_scalar(
r#"
SELECT MIN(CAST(created_at_unix_ms / 86400000 AS INTEGER) * 86400)
SELECT MIN(CAST(created_at_unix_ms / 86400 AS INTEGER) * 86400)
FROM "usage"
WHERE created_at_unix_ms >= ?
AND created_at_unix_ms < ?
@@ -104,8 +106,8 @@ WHERE created_at_unix_ms >= ?
AND provider_name NOT IN ('unknown', 'pending')
"#,
)
.bind(unix_ms(search_from)?)
.bind(unix_ms(search_until)?)
.bind(search_from)
.bind(search_until)
.fetch_one(pool)
.await
.map_sql_err()?;
@@ -118,7 +120,7 @@ SELECT
COALESCE(SUM(CASE
WHEN status = 'failed'
OR status_code >= 400
OR (error_category IS NOT NULL AND error_category <> '')
OR error_message IS NOT NULL
THEN 1 ELSE 0 END), 0) AS error_requests,
COALESCE(SUM(input_tokens), 0) AS input_tokens,
COALESCE(SUM(output_tokens), 0) AS output_tokens,
@@ -139,13 +141,13 @@ async fn perform_sqlite_stats_hourly_aggregation(
hour_utc_unix_secs: i64,
aggregated_at: DateTime<Utc>,
) -> Result<StatsHourlyAggregationSummary, DataLayerError> {
let start_ms = unix_ms(hour_utc_unix_secs)?;
let end_ms = unix_ms(hour_utc_unix_secs + 3600)?;
let start_unix_secs = hour_utc_unix_secs;
let end_unix_secs = hour_utc_unix_secs + 3600;
let aggregated_at_unix_secs = unix_secs(aggregated_at);
let mut tx = pool.begin().await.map_sql_err()?;
let row = sqlx::query(SQLITE_STATS_AGGREGATE_SQL)
.bind(start_ms)
.bind(end_ms)
.bind(start_unix_secs)
.bind(end_unix_secs)
.fetch_one(&mut *tx)
.await
.map_sql_err()?;
@@ -198,10 +200,39 @@ ON CONFLICT (hour_utc) DO UPDATE SET
.await
.map_sql_err()?;
let user_rows = sqlite_group_count(&mut tx, "user_id", start_ms, end_ms).await?;
let user_model_rows = sqlite_group_count(&mut tx, "user_id, model", start_ms, end_ms).await?;
let model_rows = sqlite_group_count(&mut tx, "model", start_ms, end_ms).await?;
let provider_rows = sqlite_group_count(&mut tx, "provider_name", start_ms, end_ms).await?;
let user_rows = upsert_sqlite_stats_hourly_user_rows(
&mut tx,
hour_utc_unix_secs,
start_unix_secs,
end_unix_secs,
aggregated_at_unix_secs,
)
.await?;
let user_model_rows = upsert_sqlite_stats_hourly_user_model_rows(
&mut tx,
hour_utc_unix_secs,
start_unix_secs,
end_unix_secs,
aggregated_at_unix_secs,
)
.await?;
let model_rows = upsert_sqlite_stats_hourly_model_rows(
&mut tx,
hour_utc_unix_secs,
start_unix_secs,
end_unix_secs,
aggregated_at_unix_secs,
)
.await?;
let provider_rows = upsert_sqlite_stats_hourly_provider_rows(
&mut tx,
hour_utc_unix_secs,
start_unix_secs,
end_unix_secs,
aggregated_at_unix_secs,
)
.await?;
advanced::refresh_hourly(&mut tx, hour_utc_unix_secs, start_unix_secs, end_unix_secs).await?;
tx.commit().await.map_sql_err()?;
Ok(StatsHourlyAggregationSummary {
@@ -219,21 +250,24 @@ async fn perform_sqlite_stats_daily_aggregation(
day_start_unix_secs: i64,
aggregated_at: DateTime<Utc>,
) -> Result<StatsDailyAggregationSummary, DataLayerError> {
let start_ms = unix_ms(day_start_unix_secs)?;
let end_ms = unix_ms(day_start_unix_secs + 86_400)?;
let start_unix_secs = day_start_unix_secs;
let end_unix_secs = day_start_unix_secs + 86_400;
let aggregated_at_unix_secs = unix_secs(aggregated_at);
let mut tx = pool.begin().await.map_sql_err()?;
let row = sqlx::query(SQLITE_STATS_AGGREGATE_SQL)
.bind(start_ms)
.bind(end_ms)
.bind(start_unix_secs)
.bind(end_unix_secs)
.fetch_one(&mut *tx)
.await
.map_sql_err()?;
let total_requests: i64 = row.try_get("total_requests").map_sql_err()?;
let error_requests: i64 = row.try_get("error_requests").map_sql_err()?;
let unique_models = sqlite_group_count(&mut tx, "model", start_ms, end_ms).await? as i64;
let unique_models =
sqlite_group_count(&mut tx, "model", start_unix_secs, end_unix_secs).await? as i64;
let unique_providers =
sqlite_group_count(&mut tx, "provider_name", start_ms, end_ms).await? as i64;
sqlite_group_count(&mut tx, "provider_name", start_unix_secs, end_unix_secs).await? as i64;
let fallback_count =
sqlite_daily_fallback_count(&mut tx, start_unix_secs, end_unix_secs).await?;
sqlx::query(
r#"
@@ -242,7 +276,7 @@ INSERT INTO stats_daily (
input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens,
total_cost, actual_total_cost, avg_response_time_ms, fallback_count,
unique_models, unique_providers, is_complete, aggregated_at, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?, 1, ?, ?, ?)
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?)
ON CONFLICT ("date") DO UPDATE SET
total_requests = excluded.total_requests,
success_requests = excluded.success_requests,
@@ -277,6 +311,7 @@ ON CONFLICT ("date") DO UPDATE SET
.bind(sqlite_real(&row, "total_cost")?)
.bind(sqlite_real(&row, "actual_total_cost")?)
.bind(sqlite_real(&row, "avg_response_time_ms")?)
.bind(fallback_count)
.bind(unique_models)
.bind(unique_providers)
.bind(aggregated_at_unix_secs)
@@ -286,11 +321,54 @@ ON CONFLICT ("date") DO UPDATE SET
.await
.map_sql_err()?;
let model_rows = usize::try_from(unique_models).unwrap_or(usize::MAX);
let provider_rows = usize::try_from(unique_providers).unwrap_or(usize::MAX);
let api_key_rows = sqlite_group_count(&mut tx, "api_key_id", start_ms, end_ms).await?;
let error_rows = sqlite_error_group_count(&mut tx, start_ms, end_ms).await?;
let user_rows = sqlite_group_count(&mut tx, "user_id", start_ms, end_ms).await?;
let model_rows = upsert_sqlite_stats_daily_model_rows(
&mut tx,
day_start_unix_secs,
start_unix_secs,
end_unix_secs,
aggregated_at_unix_secs,
)
.await?;
let provider_rows = upsert_sqlite_stats_daily_provider_rows(
&mut tx,
day_start_unix_secs,
start_unix_secs,
end_unix_secs,
aggregated_at_unix_secs,
)
.await?;
let api_key_rows = upsert_sqlite_stats_daily_api_key_rows(
&mut tx,
day_start_unix_secs,
start_unix_secs,
end_unix_secs,
aggregated_at_unix_secs,
)
.await?;
let error_rows = refresh_sqlite_stats_daily_error_rows(
&mut tx,
day_start_unix_secs,
start_unix_secs,
end_unix_secs,
aggregated_at_unix_secs,
)
.await?;
let user_rows = upsert_sqlite_stats_user_daily_rows(
&mut tx,
day_start_unix_secs,
start_unix_secs,
end_unix_secs,
aggregated_at_unix_secs,
)
.await?;
advanced::refresh_daily(
&mut tx,
day_start_unix_secs,
start_unix_secs,
end_unix_secs,
aggregated_at_unix_secs,
)
.await?;
tx.commit().await.map_sql_err()?;
Ok(StatsDailyAggregationSummary {
@@ -304,11 +382,463 @@ ON CONFLICT ("date") DO UPDATE SET
})
}
async fn upsert_sqlite_stats_hourly_user_rows(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
hour_utc: i64,
start_unix_secs: i64,
end_unix_secs: i64,
now_unix_secs: i64,
) -> Result<usize, DataLayerError> {
let result = sqlx::query(
r#"
INSERT INTO stats_hourly_user (
id, hour_utc, user_id, total_requests, success_requests, error_requests,
input_tokens, output_tokens, total_cost, created_at, updated_at
)
SELECT
lower(hex(randomblob(32))), ?, user_id, COUNT(*),
COUNT(*) - COALESCE(SUM(CASE
WHEN status = 'failed' OR status_code >= 400 OR error_message IS NOT NULL
THEN 1 ELSE 0 END), 0),
COALESCE(SUM(CASE
WHEN status = 'failed' OR status_code >= 400 OR error_message IS NOT NULL
THEN 1 ELSE 0 END), 0),
COALESCE(SUM(input_tokens), 0), COALESCE(SUM(output_tokens), 0),
CAST(COALESCE(SUM(total_cost_usd), 0) AS REAL), ?, ?
FROM "usage"
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
AND user_id IS NOT NULL AND user_id <> ''
AND status NOT IN ('pending', 'streaming')
AND provider_name NOT IN ('unknown', 'pending')
GROUP BY user_id
ON CONFLICT (hour_utc, user_id) DO UPDATE SET
total_requests = excluded.total_requests,
success_requests = excluded.success_requests,
error_requests = excluded.error_requests,
input_tokens = excluded.input_tokens,
output_tokens = excluded.output_tokens,
total_cost = excluded.total_cost,
updated_at = excluded.updated_at
"#,
)
.bind(hour_utc)
.bind(now_unix_secs)
.bind(now_unix_secs)
.bind(start_unix_secs)
.bind(end_unix_secs)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
}
async fn upsert_sqlite_stats_hourly_user_model_rows(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
hour_utc: i64,
start_unix_secs: i64,
end_unix_secs: i64,
now_unix_secs: i64,
) -> Result<usize, DataLayerError> {
let result = sqlx::query(
r#"
INSERT INTO stats_hourly_user_model (
id, hour_utc, user_id, model, total_requests, input_tokens, output_tokens,
total_cost, created_at, updated_at
)
SELECT
lower(hex(randomblob(32))), ?, user_id, model, COUNT(*),
COALESCE(SUM(input_tokens), 0), COALESCE(SUM(output_tokens), 0),
CAST(COALESCE(SUM(total_cost_usd), 0) AS REAL), ?, ?
FROM "usage"
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
AND user_id IS NOT NULL AND user_id <> ''
AND model IS NOT NULL AND model <> ''
AND status NOT IN ('pending', 'streaming')
AND provider_name NOT IN ('unknown', 'pending')
GROUP BY user_id, model
ON CONFLICT (hour_utc, user_id, model) DO UPDATE SET
total_requests = excluded.total_requests,
input_tokens = excluded.input_tokens,
output_tokens = excluded.output_tokens,
total_cost = excluded.total_cost,
updated_at = excluded.updated_at
"#,
)
.bind(hour_utc)
.bind(now_unix_secs)
.bind(now_unix_secs)
.bind(start_unix_secs)
.bind(end_unix_secs)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
}
async fn upsert_sqlite_stats_hourly_model_rows(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
hour_utc: i64,
start_unix_secs: i64,
end_unix_secs: i64,
now_unix_secs: i64,
) -> Result<usize, DataLayerError> {
let result = sqlx::query(
r#"
INSERT INTO stats_hourly_model (
id, hour_utc, model, total_requests, input_tokens, output_tokens, total_cost,
avg_response_time_ms, created_at, updated_at
)
SELECT
lower(hex(randomblob(32))), ?, model, COUNT(*),
COALESCE(SUM(input_tokens), 0), COALESCE(SUM(output_tokens), 0),
CAST(COALESCE(SUM(total_cost_usd), 0) AS REAL),
CAST(COALESCE(AVG(response_time_ms), 0) AS REAL), ?, ?
FROM "usage"
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
AND model IS NOT NULL AND model <> ''
AND status NOT IN ('pending', 'streaming')
AND provider_name NOT IN ('unknown', 'pending')
GROUP BY model
ON CONFLICT (hour_utc, model) DO UPDATE SET
total_requests = excluded.total_requests,
input_tokens = excluded.input_tokens,
output_tokens = excluded.output_tokens,
total_cost = excluded.total_cost,
avg_response_time_ms = excluded.avg_response_time_ms,
updated_at = excluded.updated_at
"#,
)
.bind(hour_utc)
.bind(now_unix_secs)
.bind(now_unix_secs)
.bind(start_unix_secs)
.bind(end_unix_secs)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
}
async fn upsert_sqlite_stats_hourly_provider_rows(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
hour_utc: i64,
start_unix_secs: i64,
end_unix_secs: i64,
now_unix_secs: i64,
) -> Result<usize, DataLayerError> {
let result = sqlx::query(
r#"
INSERT INTO stats_hourly_provider (
id, hour_utc, provider_name, total_requests, input_tokens, output_tokens,
total_cost, created_at, updated_at
)
SELECT
lower(hex(randomblob(32))), ?, provider_name, COUNT(*),
COALESCE(SUM(input_tokens), 0), COALESCE(SUM(output_tokens), 0),
CAST(COALESCE(SUM(total_cost_usd), 0) AS REAL), ?, ?
FROM "usage"
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
AND status NOT IN ('pending', 'streaming')
AND provider_name NOT IN ('unknown', 'pending')
GROUP BY provider_name
ON CONFLICT (hour_utc, provider_name) DO UPDATE SET
total_requests = excluded.total_requests,
input_tokens = excluded.input_tokens,
output_tokens = excluded.output_tokens,
total_cost = excluded.total_cost,
updated_at = excluded.updated_at
"#,
)
.bind(hour_utc)
.bind(now_unix_secs)
.bind(now_unix_secs)
.bind(start_unix_secs)
.bind(end_unix_secs)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
}
async fn upsert_sqlite_stats_daily_model_rows(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
day_start: i64,
start_unix_secs: i64,
end_unix_secs: i64,
now_unix_secs: i64,
) -> Result<usize, DataLayerError> {
let result = sqlx::query(
r#"
INSERT INTO stats_daily_model (
id, "date", model, total_requests, input_tokens, output_tokens,
cache_creation_tokens, cache_read_tokens, total_cost, avg_response_time_ms,
created_at, updated_at
)
SELECT
lower(hex(randomblob(32))), ?, model, COUNT(*),
COALESCE(SUM(input_tokens), 0), COALESCE(SUM(output_tokens), 0),
COALESCE(SUM(cache_creation_input_tokens), 0),
COALESCE(SUM(cache_read_input_tokens), 0),
CAST(COALESCE(SUM(total_cost_usd), 0) AS REAL),
CAST(COALESCE(AVG(response_time_ms), 0) AS REAL), ?, ?
FROM "usage"
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
AND model IS NOT NULL AND model <> ''
AND status NOT IN ('pending', 'streaming')
AND provider_name NOT IN ('unknown', 'pending')
GROUP BY model
ON CONFLICT ("date", model) DO UPDATE SET
total_requests = excluded.total_requests,
input_tokens = excluded.input_tokens,
output_tokens = excluded.output_tokens,
cache_creation_tokens = excluded.cache_creation_tokens,
cache_read_tokens = excluded.cache_read_tokens,
total_cost = excluded.total_cost,
avg_response_time_ms = excluded.avg_response_time_ms,
updated_at = excluded.updated_at
"#,
)
.bind(day_start)
.bind(now_unix_secs)
.bind(now_unix_secs)
.bind(start_unix_secs)
.bind(end_unix_secs)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
}
async fn upsert_sqlite_stats_daily_provider_rows(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
day_start: i64,
start_unix_secs: i64,
end_unix_secs: i64,
now_unix_secs: i64,
) -> Result<usize, DataLayerError> {
let result = sqlx::query(
r#"
INSERT INTO stats_daily_provider (
id, "date", provider_name, total_requests, input_tokens, output_tokens,
cache_creation_tokens, cache_read_tokens, total_cost, created_at, updated_at
)
SELECT
lower(hex(randomblob(32))), ?, provider_name, COUNT(*),
COALESCE(SUM(input_tokens), 0), COALESCE(SUM(output_tokens), 0),
COALESCE(SUM(cache_creation_input_tokens), 0),
COALESCE(SUM(cache_read_input_tokens), 0),
CAST(COALESCE(SUM(total_cost_usd), 0) AS REAL), ?, ?
FROM "usage"
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
AND status NOT IN ('pending', 'streaming')
AND provider_name NOT IN ('unknown', 'pending')
GROUP BY provider_name
ON CONFLICT ("date", provider_name) DO UPDATE SET
total_requests = excluded.total_requests,
input_tokens = excluded.input_tokens,
output_tokens = excluded.output_tokens,
cache_creation_tokens = excluded.cache_creation_tokens,
cache_read_tokens = excluded.cache_read_tokens,
total_cost = excluded.total_cost,
updated_at = excluded.updated_at
"#,
)
.bind(day_start)
.bind(now_unix_secs)
.bind(now_unix_secs)
.bind(start_unix_secs)
.bind(end_unix_secs)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
}
async fn upsert_sqlite_stats_daily_api_key_rows(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
day_start: i64,
start_unix_secs: i64,
end_unix_secs: i64,
now_unix_secs: i64,
) -> Result<usize, DataLayerError> {
let result = sqlx::query(
r#"
INSERT INTO stats_daily_api_key (
id, api_key_id, "date", total_requests, success_requests, error_requests,
input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens,
total_cost, api_key_name, created_at, updated_at
)
SELECT
lower(hex(randomblob(32))), usage.api_key_id, ?, COUNT(*),
COUNT(*) - COALESCE(SUM(CASE
WHEN usage.status = 'failed' OR usage.status_code >= 400
OR usage.error_message IS NOT NULL THEN 1 ELSE 0 END), 0),
COALESCE(SUM(CASE
WHEN usage.status = 'failed' OR usage.status_code >= 400
OR usage.error_message IS NOT NULL THEN 1 ELSE 0 END), 0),
COALESCE(SUM(usage.input_tokens), 0), COALESCE(SUM(usage.output_tokens), 0),
COALESCE(SUM(usage.cache_creation_input_tokens), 0),
COALESCE(SUM(usage.cache_read_input_tokens), 0),
CAST(COALESCE(SUM(usage.total_cost_usd), 0) AS REAL), MAX(api_keys.name), ?, ?
FROM "usage" AS usage
LEFT JOIN api_keys ON api_keys.id = usage.api_key_id
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
AND usage.api_key_id IS NOT NULL AND usage.api_key_id <> ''
GROUP BY usage.api_key_id
ON CONFLICT ("date", api_key_id) DO UPDATE SET
total_requests = excluded.total_requests,
success_requests = excluded.success_requests,
error_requests = excluded.error_requests,
input_tokens = excluded.input_tokens,
output_tokens = excluded.output_tokens,
cache_creation_tokens = excluded.cache_creation_tokens,
cache_read_tokens = excluded.cache_read_tokens,
total_cost = excluded.total_cost,
api_key_name = COALESCE(excluded.api_key_name, stats_daily_api_key.api_key_name),
updated_at = excluded.updated_at
"#,
)
.bind(day_start)
.bind(now_unix_secs)
.bind(now_unix_secs)
.bind(start_unix_secs)
.bind(end_unix_secs)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
}
async fn refresh_sqlite_stats_daily_error_rows(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
day_start: i64,
start_unix_secs: i64,
end_unix_secs: i64,
now_unix_secs: i64,
) -> Result<usize, DataLayerError> {
sqlx::query(r#"DELETE FROM stats_daily_error WHERE "date" = ?"#)
.bind(day_start)
.execute(&mut **tx)
.await
.map_sql_err()?;
let result = sqlx::query(
r#"
INSERT INTO stats_daily_error (
id, "date", error_category, provider_name, model, count, created_at, updated_at
)
SELECT
lower(hex(randomblob(32))), ?, error_category, provider_name, model,
COUNT(*), ?, ?
FROM "usage"
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
AND error_category IS NOT NULL AND error_category <> ''
GROUP BY error_category, provider_name, model
"#,
)
.bind(day_start)
.bind(now_unix_secs)
.bind(now_unix_secs)
.bind(start_unix_secs)
.bind(end_unix_secs)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
}
async fn upsert_sqlite_stats_user_daily_rows(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
day_start: i64,
start_unix_secs: i64,
end_unix_secs: i64,
now_unix_secs: i64,
) -> Result<usize, DataLayerError> {
let result = sqlx::query(
r#"
INSERT INTO stats_user_daily (
id, user_id, "date", total_requests, success_requests, error_requests,
input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens,
total_cost, username, created_at, updated_at
)
SELECT
lower(hex(randomblob(32))), usage.user_id, ?, COUNT(*),
COUNT(*) - COALESCE(SUM(CASE
WHEN usage.status = 'failed' OR usage.status_code >= 400
OR usage.error_message IS NOT NULL THEN 1 ELSE 0 END), 0),
COALESCE(SUM(CASE
WHEN usage.status = 'failed' OR usage.status_code >= 400
OR usage.error_message IS NOT NULL THEN 1 ELSE 0 END), 0),
COALESCE(SUM(usage.input_tokens), 0), COALESCE(SUM(usage.output_tokens), 0),
COALESCE(SUM(usage.cache_creation_input_tokens), 0),
COALESCE(SUM(usage.cache_read_input_tokens), 0),
CAST(COALESCE(SUM(usage.total_cost_usd), 0) AS REAL), MAX(users.username), ?, ?
FROM "usage" AS usage
LEFT JOIN users ON users.id = usage.user_id
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
AND usage.user_id IS NOT NULL AND usage.user_id <> ''
AND usage.status NOT IN ('pending', 'streaming')
AND usage.provider_name NOT IN ('unknown', 'pending')
GROUP BY usage.user_id
ON CONFLICT ("date", user_id) DO UPDATE SET
total_requests = excluded.total_requests,
success_requests = excluded.success_requests,
error_requests = excluded.error_requests,
input_tokens = excluded.input_tokens,
output_tokens = excluded.output_tokens,
cache_creation_tokens = excluded.cache_creation_tokens,
cache_read_tokens = excluded.cache_read_tokens,
total_cost = excluded.total_cost,
username = COALESCE(excluded.username, stats_user_daily.username),
updated_at = excluded.updated_at
"#,
)
.bind(day_start)
.bind(now_unix_secs)
.bind(now_unix_secs)
.bind(start_unix_secs)
.bind(end_unix_secs)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
}
async fn sqlite_daily_fallback_count(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
start_unix_secs: i64,
end_unix_secs: i64,
) -> Result<i64, DataLayerError> {
let start_unix_ms = start_unix_secs.checked_mul(1000).ok_or_else(|| {
DataLayerError::InvalidInput("stats fallback window start overflows milliseconds".into())
})?;
let end_unix_ms = end_unix_secs.checked_mul(1000).ok_or_else(|| {
DataLayerError::InvalidInput("stats fallback window end overflows milliseconds".into())
})?;
sqlx::query_scalar(
r#"
SELECT COUNT(*)
FROM (
SELECT request_id
FROM request_candidates
WHERE created_at >= ? AND created_at < ?
AND status IN ('success', 'failed')
GROUP BY request_id
HAVING COUNT(id) > 1
)
"#,
)
.bind(start_unix_ms)
.bind(end_unix_ms)
.fetch_one(&mut **tx)
.await
.map_sql_err()
}
async fn sqlite_group_count(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
group_columns: &str,
start_ms: i64,
end_ms: i64,
start_unix_secs: i64,
end_unix_secs: i64,
) -> Result<usize, DataLayerError> {
let not_empty = group_columns
.split(',')
@@ -332,42 +862,10 @@ FROM (
"#
);
let count: i64 = sqlx::query_scalar(&sql)
.bind(start_ms)
.bind(end_ms)
.bind(start_unix_secs)
.bind(end_unix_secs)
.fetch_one(&mut **tx)
.await
.map_sql_err()?;
Ok(usize::try_from(count.max(0)).unwrap_or(usize::MAX))
}
async fn sqlite_error_group_count(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
start_ms: i64,
end_ms: i64,
) -> Result<usize, DataLayerError> {
let count: i64 = sqlx::query_scalar(
r#"
SELECT COUNT(*)
FROM (
SELECT 1
FROM "usage"
WHERE created_at_unix_ms >= ?
AND created_at_unix_ms < ?
AND status NOT IN ('pending', 'streaming')
AND provider_name NOT IN ('unknown', 'pending')
AND (
status = 'failed'
OR status_code >= 400
OR (error_category IS NOT NULL AND error_category <> '')
)
GROUP BY COALESCE(NULLIF(error_category, ''), 'unknown_error'), provider_name, model
)
"#,
)
.bind(start_ms)
.bind(end_ms)
.fetch_one(&mut **tx)
.await
.map_sql_err()?;
Ok(usize::try_from(count.max(0)).unwrap_or(usize::MAX))
}
crates/aether-data/runtime/src/backend/stats/sqlite/advanced.rs
+986
View File
@@ -0,0 +1,986 @@
use sqlx::Sqlite;
use crate::backend::stats_common::stats_id;
use crate::error::SqlResultExt;
use crate::DataLayerError;
const CACHE_5M: &str = r#"MAX(
COALESCE(usage.cache_creation_input_tokens_5m, 0),
COALESCE(usage.cache_creation_ephemeral_5m_input_tokens, 0)
)"#;
const CACHE_1H: &str = r#"MAX(
COALESCE(usage.cache_creation_input_tokens_1h, 0),
COALESCE(usage.cache_creation_ephemeral_1h_input_tokens, 0)
)"#;
const CACHE_CREATION: &str = r#"CASE
WHEN COALESCE(usage.cache_creation_input_tokens, 0) = 0
AND ({cache_5m} + {cache_1h}) > 0
THEN {cache_5m} + {cache_1h}
ELSE MAX(COALESCE(usage.cache_creation_input_tokens, 0), 0)
END"#;
const EFFECTIVE_INPUT: &str = r#"CASE
WHEN (
LOWER(COALESCE(usage.endpoint_api_format, usage.api_format, '')) IN ('openai', 'gemini', 'google')
OR LOWER(COALESCE(usage.endpoint_api_format, usage.api_format, '')) LIKE 'openai:%'
OR LOWER(COALESCE(usage.endpoint_api_format, usage.api_format, '')) LIKE 'gemini:%'
OR LOWER(COALESCE(usage.endpoint_api_format, usage.api_format, '')) LIKE 'google:%'
)
AND COALESCE(usage.input_tokens, 0) > 0
AND COALESCE(usage.cache_read_input_tokens, 0) > 0
THEN MAX(COALESCE(usage.input_tokens, 0) - COALESCE(usage.cache_read_input_tokens, 0), 0)
ELSE MAX(COALESCE(usage.input_tokens, 0), 0)
END"#;
const SUCCESS: &str = r#"CASE
WHEN usage.status <> 'failed'
AND (usage.status_code IS NULL OR usage.status_code < 400)
AND usage.error_message IS NULL
THEN 1 ELSE 0
END"#;
const AGGREGATABLE: &str = r#"usage.status NOT IN ('pending', 'streaming')
AND usage.provider_name NOT IN ('unknown', 'pending')"#;
const SETTLED: &str = r#"COALESCE(settlement.billing_status, usage.billing_status) = 'settled'
AND COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0) > 0"#;
fn cache_creation_expr() -> String {
CACHE_CREATION
.replace("{cache_5m}", CACHE_5M)
.replace("{cache_1h}", CACHE_1H)
}
fn total_input_context_expr() -> String {
format!(
"({EFFECTIVE_INPUT}) + ({}) + MAX(COALESCE(usage.cache_read_input_tokens, 0), 0)",
cache_creation_expr()
)
}
fn total_tokens_expr() -> String {
format!(
r#"COALESCE(
NULLIF(MAX(COALESCE(usage.total_tokens, 0), 0), 0),
({EFFECTIVE_INPUT})
+ MAX(COALESCE(usage.output_tokens, 0), 0)
+ ({})
+ MAX(COALESCE(usage.cache_read_input_tokens, 0), 0),
0
)"#,
cache_creation_expr()
)
}
fn percentile_cont(sorted: &[i64], percentile: f64) -> Option<i64> {
if sorted.is_empty() {
return None;
}
let position = percentile * (sorted.len().saturating_sub(1) as f64);
let lower = position.floor() as usize;
let upper = position.ceil() as usize;
let fraction = position - lower as f64;
let value = sorted[lower] as f64 + (sorted[upper] - sorted[lower]) as f64 * fraction;
Some(value.round() as i64)
}
async fn load_percentiles(
tx: &mut sqlx::Transaction<'_, Sqlite>,
column: &str,
start_unix_secs: i64,
end_unix_secs: i64,
) -> Result<(Option<i64>, Option<i64>, Option<i64>), DataLayerError> {
let sql = format!(
r#"
SELECT {column}
FROM "usage"
WHERE created_at_unix_ms >= ? AND created_at_unix_ms < ?
AND status = 'completed'
AND provider_name NOT IN ('unknown', 'pending')
AND {column} IS NOT NULL
ORDER BY {column}
"#
);
let values: Vec<i64> = sqlx::query_scalar(&sql)
.bind(start_unix_secs)
.bind(end_unix_secs)
.fetch_all(&mut **tx)
.await
.map_sql_err()?;
if values.len() < 10 {
return Ok((None, None, None));
}
Ok((
percentile_cont(&values, 0.50),
percentile_cont(&values, 0.90),
percentile_cont(&values, 0.99),
))
}
pub(super) async fn refresh_hourly(
tx: &mut sqlx::Transaction<'_, Sqlite>,
hour_utc: i64,
start_unix_secs: i64,
end_unix_secs: i64,
) -> Result<(), DataLayerError> {
let cache_creation = cache_creation_expr();
let total_context = total_input_context_expr();
let sql = format!(
r#"
UPDATE stats_hourly AS target
SET
cache_hit_total_requests = aggregated.cache_hit_total_requests,
cache_hit_requests = aggregated.cache_hit_requests,
completed_total_requests = aggregated.completed_total_requests,
completed_cache_hit_requests = aggregated.completed_cache_hit_requests,
completed_input_tokens = aggregated.completed_input_tokens,
completed_cache_creation_tokens = aggregated.completed_cache_creation_tokens,
completed_cache_read_tokens = aggregated.completed_cache_read_tokens,
completed_total_input_context = aggregated.completed_total_input_context,
completed_cache_creation_cost = aggregated.completed_cache_creation_cost,
completed_cache_read_cost = aggregated.completed_cache_read_cost,
settled_total_cost = aggregated.settled_total_cost,
settled_total_requests = aggregated.settled_total_requests,
settled_input_tokens = aggregated.settled_input_tokens,
settled_output_tokens = aggregated.settled_output_tokens,
settled_cache_creation_tokens = aggregated.settled_cache_creation_tokens,
settled_cache_read_tokens = aggregated.settled_cache_read_tokens,
settled_first_finalized_at_unix_secs = aggregated.settled_first_finalized_at_unix_secs,
settled_last_finalized_at_unix_secs = aggregated.settled_last_finalized_at_unix_secs,
response_time_sum_ms = aggregated.response_time_sum_ms,
response_time_samples = aggregated.response_time_samples
FROM (
SELECT
COUNT(*) AS cache_hit_total_requests,
COALESCE(SUM(CASE WHEN COALESCE(usage.cache_read_input_tokens, 0) > 0 THEN 1 ELSE 0 END), 0) AS cache_hit_requests,
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN 1 ELSE 0 END), 0) AS completed_total_requests,
COALESCE(SUM(CASE WHEN usage.status = 'completed' AND COALESCE(usage.cache_read_input_tokens, 0) > 0 THEN 1 ELSE 0 END), 0) AS completed_cache_hit_requests,
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN MAX(COALESCE(usage.input_tokens, 0), 0) ELSE 0 END), 0) AS completed_input_tokens,
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN ({cache_creation}) ELSE 0 END), 0) AS completed_cache_creation_tokens,
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN MAX(COALESCE(usage.cache_read_input_tokens, 0), 0) ELSE 0 END), 0) AS completed_cache_read_tokens,
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN ({total_context}) ELSE 0 END), 0) AS completed_total_input_context,
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN COALESCE(usage.cache_creation_cost_usd, 0) ELSE 0 END), 0) AS completed_cache_creation_cost,
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN COALESCE(usage.cache_read_cost_usd, 0) ELSE 0 END), 0) AS completed_cache_read_cost,
COALESCE(SUM(CASE WHEN {SETTLED} THEN COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0) ELSE 0 END), 0) AS settled_total_cost,
COALESCE(SUM(CASE WHEN {SETTLED} THEN 1 ELSE 0 END), 0) AS settled_total_requests,
COALESCE(SUM(CASE WHEN {SETTLED} THEN MAX(COALESCE(usage.input_tokens, 0), 0) ELSE 0 END), 0) AS settled_input_tokens,
COALESCE(SUM(CASE WHEN {SETTLED} THEN MAX(COALESCE(usage.output_tokens, 0), 0) ELSE 0 END), 0) AS settled_output_tokens,
COALESCE(SUM(CASE WHEN {SETTLED} THEN ({cache_creation}) ELSE 0 END), 0) AS settled_cache_creation_tokens,
COALESCE(SUM(CASE WHEN {SETTLED} THEN MAX(COALESCE(usage.cache_read_input_tokens, 0), 0) ELSE 0 END), 0) AS settled_cache_read_tokens,
MIN(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_first_finalized_at_unix_secs,
MAX(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_last_finalized_at_unix_secs,
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL AND {AGGREGATABLE} THEN MAX(usage.response_time_ms, 0) ELSE 0 END), 0) AS response_time_sum_ms,
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL AND {AGGREGATABLE} THEN 1 ELSE 0 END), 0) AS response_time_samples
FROM "usage" AS usage
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
) AS aggregated
WHERE target.hour_utc = ?
"#
);
sqlx::query(&sql)
.bind(start_unix_secs)
.bind(end_unix_secs)
.bind(hour_utc)
.execute(&mut **tx)
.await
.map_sql_err()?;
refresh_hourly_user(tx, hour_utc, start_unix_secs, end_unix_secs).await?;
refresh_hourly_response_dimensions(tx, hour_utc, start_unix_secs, end_unix_secs).await
}
async fn refresh_hourly_user(
tx: &mut sqlx::Transaction<'_, Sqlite>,
hour_utc: i64,
start_unix_secs: i64,
end_unix_secs: i64,
) -> Result<(), DataLayerError> {
let cache_creation = cache_creation_expr();
let sql = format!(
r#"
UPDATE stats_hourly_user AS target
SET
cache_creation_tokens = aggregated.cache_creation_tokens,
cache_read_tokens = aggregated.cache_read_tokens,
actual_total_cost = aggregated.actual_total_cost,
response_time_sum_ms = aggregated.response_time_sum_ms,
response_time_samples = aggregated.response_time_samples,
settled_total_cost = aggregated.settled_total_cost,
settled_total_requests = aggregated.settled_total_requests,
settled_input_tokens = aggregated.settled_input_tokens,
settled_output_tokens = aggregated.settled_output_tokens,
settled_cache_creation_tokens = aggregated.settled_cache_creation_tokens,
settled_cache_read_tokens = aggregated.settled_cache_read_tokens,
settled_first_finalized_at_unix_secs = aggregated.settled_first_finalized_at_unix_secs,
settled_last_finalized_at_unix_secs = aggregated.settled_last_finalized_at_unix_secs
FROM (
SELECT
usage.user_id,
COALESCE(SUM({cache_creation}), 0) AS cache_creation_tokens,
COALESCE(SUM(MAX(COALESCE(usage.cache_read_input_tokens, 0), 0)), 0) AS cache_read_tokens,
COALESCE(SUM(COALESCE(settlement.billing_actual_total_cost_usd, usage.actual_total_cost_usd, 0)), 0) AS actual_total_cost,
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN MAX(usage.response_time_ms, 0) ELSE 0 END), 0) AS response_time_sum_ms,
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0) AS response_time_samples,
COALESCE(SUM(CASE WHEN {SETTLED} THEN COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0) ELSE 0 END), 0) AS settled_total_cost,
COALESCE(SUM(CASE WHEN {SETTLED} THEN 1 ELSE 0 END), 0) AS settled_total_requests,
COALESCE(SUM(CASE WHEN {SETTLED} THEN MAX(COALESCE(usage.input_tokens, 0), 0) ELSE 0 END), 0) AS settled_input_tokens,
COALESCE(SUM(CASE WHEN {SETTLED} THEN MAX(COALESCE(usage.output_tokens, 0), 0) ELSE 0 END), 0) AS settled_output_tokens,
COALESCE(SUM(CASE WHEN {SETTLED} THEN ({cache_creation}) ELSE 0 END), 0) AS settled_cache_creation_tokens,
COALESCE(SUM(CASE WHEN {SETTLED} THEN MAX(COALESCE(usage.cache_read_input_tokens, 0), 0) ELSE 0 END), 0) AS settled_cache_read_tokens,
MIN(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_first_finalized_at_unix_secs,
MAX(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_last_finalized_at_unix_secs
FROM "usage" AS usage
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
AND usage.user_id IS NOT NULL AND usage.user_id <> '' AND {AGGREGATABLE}
GROUP BY usage.user_id
) AS aggregated
WHERE target.hour_utc = ? AND target.user_id = aggregated.user_id
"#
);
sqlx::query(&sql)
.bind(start_unix_secs)
.bind(end_unix_secs)
.bind(hour_utc)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(())
}
async fn refresh_hourly_response_dimensions(
tx: &mut sqlx::Transaction<'_, Sqlite>,
hour_utc: i64,
start_unix_secs: i64,
end_unix_secs: i64,
) -> Result<(), DataLayerError> {
for (table, dimensions, group_by, target_match) in [
(
"stats_hourly_model",
"usage.model AS model",
"usage.model",
"target.model = aggregated.model",
),
(
"stats_hourly_user_model",
"usage.user_id AS user_id, usage.model AS model",
"usage.user_id, usage.model",
"target.user_id = aggregated.user_id AND target.model = aggregated.model",
),
] {
let sql = format!(
r#"
UPDATE {table} AS target
SET response_time_sum_ms = aggregated.response_time_sum_ms,
response_time_samples = aggregated.response_time_samples
FROM (
SELECT {dimensions},
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN MAX(usage.response_time_ms, 0) ELSE 0 END), 0) AS response_time_sum_ms,
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0) AS response_time_samples
FROM "usage" AS usage
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
AND {AGGREGATABLE}
GROUP BY {group_by}
) AS aggregated
WHERE target.hour_utc = ? AND {target_match}
"#
);
sqlx::query(&sql)
.bind(start_unix_secs)
.bind(end_unix_secs)
.bind(hour_utc)
.execute(&mut **tx)
.await
.map_sql_err()?;
}
Ok(())
}
pub(super) async fn refresh_daily(
tx: &mut sqlx::Transaction<'_, Sqlite>,
day_start: i64,
start_unix_secs: i64,
end_unix_secs: i64,
now_unix_secs: i64,
) -> Result<(), DataLayerError> {
let response = load_percentiles(tx, "response_time_ms", start_unix_secs, end_unix_secs).await?;
let first_byte =
load_percentiles(tx, "first_byte_time_ms", start_unix_secs, end_unix_secs).await?;
refresh_daily_root(
tx,
day_start,
start_unix_secs,
end_unix_secs,
response,
first_byte,
)
.await?;
refresh_daily_existing_dimensions(tx, day_start, start_unix_secs, end_unix_secs).await?;
upsert_user_dimension(
tx,
"stats_user_daily_model",
"model",
"usage.model",
"usage.model IS NOT NULL AND usage.model <> ''",
day_start,
start_unix_secs,
end_unix_secs,
now_unix_secs,
)
.await?;
upsert_user_dimension(
tx,
"stats_user_daily_provider",
"provider_name",
"usage.provider_name",
"usage.provider_name IS NOT NULL AND usage.provider_name <> ''",
day_start,
start_unix_secs,
end_unix_secs,
now_unix_secs,
)
.await?;
upsert_user_dimension(
tx,
"stats_user_daily_api_format",
"api_format",
"LOWER(COALESCE(usage.endpoint_api_format, usage.api_format, ''))",
"COALESCE(usage.endpoint_api_format, usage.api_format, '') <> ''",
day_start,
start_unix_secs,
end_unix_secs,
now_unix_secs,
)
.await?;
upsert_model_provider_rows(tx, day_start, start_unix_secs, end_unix_secs, now_unix_secs)
.await?;
upsert_cost_savings_rows(tx, day_start, start_unix_secs, end_unix_secs, now_unix_secs).await?;
refresh_user_summary(tx, end_unix_secs, now_unix_secs).await
}
async fn refresh_daily_root(
tx: &mut sqlx::Transaction<'_, Sqlite>,
day_start: i64,
start_unix_secs: i64,
end_unix_secs: i64,
response: (Option<i64>, Option<i64>, Option<i64>),
first_byte: (Option<i64>, Option<i64>, Option<i64>),
) -> Result<(), DataLayerError> {
let cache_creation = cache_creation_expr();
let total_context = total_input_context_expr();
let sql = format!(
r#"
UPDATE stats_daily AS target
SET
effective_input_tokens = aggregated.effective_input_tokens,
total_input_context = aggregated.total_input_context,
response_time_sum_ms = aggregated.response_time_sum_ms,
response_time_samples = aggregated.response_time_samples,
cache_creation_ephemeral_5m_tokens = aggregated.cache_creation_ephemeral_5m_tokens,
cache_creation_ephemeral_1h_tokens = aggregated.cache_creation_ephemeral_1h_tokens,
input_cost = aggregated.input_cost,
output_cost = aggregated.output_cost,
cache_creation_cost = aggregated.cache_creation_cost,
cache_read_cost = aggregated.cache_read_cost,
cache_hit_total_requests = aggregated.cache_hit_total_requests,
cache_hit_requests = aggregated.cache_hit_requests,
completed_total_requests = aggregated.completed_total_requests,
completed_cache_hit_requests = aggregated.completed_cache_hit_requests,
completed_input_tokens = aggregated.completed_input_tokens,
completed_cache_creation_tokens = aggregated.completed_cache_creation_tokens,
completed_cache_read_tokens = aggregated.completed_cache_read_tokens,
completed_total_input_context = aggregated.completed_total_input_context,
completed_cache_creation_cost = aggregated.completed_cache_creation_cost,
completed_cache_read_cost = aggregated.completed_cache_read_cost,
settled_total_cost = aggregated.settled_total_cost,
settled_total_requests = aggregated.settled_total_requests,
settled_input_tokens = aggregated.settled_input_tokens,
settled_output_tokens = aggregated.settled_output_tokens,
settled_cache_creation_tokens = aggregated.settled_cache_creation_tokens,
settled_cache_read_tokens = aggregated.settled_cache_read_tokens,
settled_first_finalized_at_unix_secs = aggregated.settled_first_finalized_at_unix_secs,
settled_last_finalized_at_unix_secs = aggregated.settled_last_finalized_at_unix_secs,
p50_response_time_ms = ?, p90_response_time_ms = ?, p99_response_time_ms = ?,
p50_first_byte_time_ms = ?, p90_first_byte_time_ms = ?, p99_first_byte_time_ms = ?
FROM (
SELECT
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN ({EFFECTIVE_INPUT}) ELSE 0 END), 0) AS effective_input_tokens,
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN ({total_context}) ELSE 0 END), 0) AS total_input_context,
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL AND {AGGREGATABLE} THEN MAX(usage.response_time_ms, 0) ELSE 0 END), 0) AS response_time_sum_ms,
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL AND {AGGREGATABLE} THEN 1 ELSE 0 END), 0) AS response_time_samples,
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN {CACHE_5M} ELSE 0 END), 0) AS cache_creation_ephemeral_5m_tokens,
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN {CACHE_1H} ELSE 0 END), 0) AS cache_creation_ephemeral_1h_tokens,
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN COALESCE(usage.input_cost_usd, 0) ELSE 0 END), 0) AS input_cost,
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN COALESCE(usage.output_cost_usd, 0) ELSE 0 END), 0) AS output_cost,
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN COALESCE(usage.cache_creation_cost_usd, 0) ELSE 0 END), 0) AS cache_creation_cost,
COALESCE(SUM(CASE WHEN {AGGREGATABLE} THEN COALESCE(usage.cache_read_cost_usd, 0) ELSE 0 END), 0) AS cache_read_cost,
COUNT(*) AS cache_hit_total_requests,
COALESCE(SUM(CASE WHEN COALESCE(usage.cache_read_input_tokens, 0) > 0 THEN 1 ELSE 0 END), 0) AS cache_hit_requests,
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN 1 ELSE 0 END), 0) AS completed_total_requests,
COALESCE(SUM(CASE WHEN usage.status = 'completed' AND COALESCE(usage.cache_read_input_tokens, 0) > 0 THEN 1 ELSE 0 END), 0) AS completed_cache_hit_requests,
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN MAX(COALESCE(usage.input_tokens, 0), 0) ELSE 0 END), 0) AS completed_input_tokens,
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN ({cache_creation}) ELSE 0 END), 0) AS completed_cache_creation_tokens,
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN MAX(COALESCE(usage.cache_read_input_tokens, 0), 0) ELSE 0 END), 0) AS completed_cache_read_tokens,
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN ({total_context}) ELSE 0 END), 0) AS completed_total_input_context,
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN COALESCE(usage.cache_creation_cost_usd, 0) ELSE 0 END), 0) AS completed_cache_creation_cost,
COALESCE(SUM(CASE WHEN usage.status = 'completed' THEN COALESCE(usage.cache_read_cost_usd, 0) ELSE 0 END), 0) AS completed_cache_read_cost,
COALESCE(SUM(CASE WHEN {SETTLED} THEN COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0) ELSE 0 END), 0) AS settled_total_cost,
COALESCE(SUM(CASE WHEN {SETTLED} THEN 1 ELSE 0 END), 0) AS settled_total_requests,
COALESCE(SUM(CASE WHEN {SETTLED} THEN MAX(COALESCE(usage.input_tokens, 0), 0) ELSE 0 END), 0) AS settled_input_tokens,
COALESCE(SUM(CASE WHEN {SETTLED} THEN MAX(COALESCE(usage.output_tokens, 0), 0) ELSE 0 END), 0) AS settled_output_tokens,
COALESCE(SUM(CASE WHEN {SETTLED} THEN ({cache_creation}) ELSE 0 END), 0) AS settled_cache_creation_tokens,
COALESCE(SUM(CASE WHEN {SETTLED} THEN MAX(COALESCE(usage.cache_read_input_tokens, 0), 0) ELSE 0 END), 0) AS settled_cache_read_tokens,
MIN(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_first_finalized_at_unix_secs,
MAX(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_last_finalized_at_unix_secs
FROM "usage" AS usage
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
) AS aggregated
WHERE target."date" = ?
"#
);
sqlx::query(&sql)
.bind(response.0)
.bind(response.1)
.bind(response.2)
.bind(first_byte.0)
.bind(first_byte.1)
.bind(first_byte.2)
.bind(start_unix_secs)
.bind(end_unix_secs)
.bind(day_start)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(())
}
async fn refresh_daily_existing_dimensions(
tx: &mut sqlx::Transaction<'_, Sqlite>,
day_start: i64,
start_unix_secs: i64,
end_unix_secs: i64,
) -> Result<(), DataLayerError> {
let cache_creation = cache_creation_expr();
let total_context = total_input_context_expr();
let daily_model_sql = format!(
r#"
UPDATE stats_daily_model AS target
SET response_time_sum_ms = aggregated.response_time_sum_ms,
response_time_samples = aggregated.response_time_samples,
cache_creation_ephemeral_5m_tokens = aggregated.cache_creation_ephemeral_5m_tokens,
cache_creation_ephemeral_1h_tokens = aggregated.cache_creation_ephemeral_1h_tokens
FROM (
SELECT usage.model,
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN MAX(usage.response_time_ms, 0) ELSE 0 END), 0) AS response_time_sum_ms,
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0) AS response_time_samples,
COALESCE(SUM({CACHE_5M}), 0) AS cache_creation_ephemeral_5m_tokens,
COALESCE(SUM({CACHE_1H}), 0) AS cache_creation_ephemeral_1h_tokens
FROM "usage" AS usage
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
AND {AGGREGATABLE} AND usage.model IS NOT NULL AND usage.model <> ''
GROUP BY usage.model
) AS aggregated
WHERE target."date" = ? AND target.model = aggregated.model
"#
);
sqlx::query(&daily_model_sql)
.bind(start_unix_secs)
.bind(end_unix_secs)
.bind(day_start)
.execute(&mut **tx)
.await
.map_sql_err()?;
let user_sql = format!(
r#"
UPDATE stats_user_daily AS target
SET effective_input_tokens = aggregated.effective_input_tokens,
total_input_context = aggregated.total_input_context,
cache_creation_cost = aggregated.cache_creation_cost,
cache_read_cost = aggregated.cache_read_cost,
actual_total_cost = aggregated.actual_total_cost,
response_time_sum_ms = aggregated.response_time_sum_ms,
response_time_samples = aggregated.response_time_samples,
cache_creation_ephemeral_5m_tokens = aggregated.cache_creation_ephemeral_5m_tokens,
cache_creation_ephemeral_1h_tokens = aggregated.cache_creation_ephemeral_1h_tokens,
settled_total_cost = aggregated.settled_total_cost,
settled_total_requests = aggregated.settled_total_requests,
settled_input_tokens = aggregated.settled_input_tokens,
settled_output_tokens = aggregated.settled_output_tokens,
settled_cache_creation_tokens = aggregated.settled_cache_creation_tokens,
settled_cache_read_tokens = aggregated.settled_cache_read_tokens,
settled_first_finalized_at_unix_secs = aggregated.settled_first_finalized_at_unix_secs,
settled_last_finalized_at_unix_secs = aggregated.settled_last_finalized_at_unix_secs
FROM (
SELECT usage.user_id,
COALESCE(SUM({EFFECTIVE_INPUT}), 0) AS effective_input_tokens,
COALESCE(SUM({total_context}), 0) AS total_input_context,
COALESCE(SUM(COALESCE(usage.cache_creation_cost_usd, 0)), 0) AS cache_creation_cost,
COALESCE(SUM(COALESCE(usage.cache_read_cost_usd, 0)), 0) AS cache_read_cost,
COALESCE(SUM(COALESCE(settlement.billing_actual_total_cost_usd, usage.actual_total_cost_usd, 0)), 0) AS actual_total_cost,
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN MAX(usage.response_time_ms, 0) ELSE 0 END), 0) AS response_time_sum_ms,
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0) AS response_time_samples,
COALESCE(SUM({CACHE_5M}), 0) AS cache_creation_ephemeral_5m_tokens,
COALESCE(SUM({CACHE_1H}), 0) AS cache_creation_ephemeral_1h_tokens,
COALESCE(SUM(CASE WHEN {SETTLED} THEN COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0) ELSE 0 END), 0) AS settled_total_cost,
COALESCE(SUM(CASE WHEN {SETTLED} THEN 1 ELSE 0 END), 0) AS settled_total_requests,
COALESCE(SUM(CASE WHEN {SETTLED} THEN MAX(COALESCE(usage.input_tokens, 0), 0) ELSE 0 END), 0) AS settled_input_tokens,
COALESCE(SUM(CASE WHEN {SETTLED} THEN MAX(COALESCE(usage.output_tokens, 0), 0) ELSE 0 END), 0) AS settled_output_tokens,
COALESCE(SUM(CASE WHEN {SETTLED} THEN ({cache_creation}) ELSE 0 END), 0) AS settled_cache_creation_tokens,
COALESCE(SUM(CASE WHEN {SETTLED} THEN MAX(COALESCE(usage.cache_read_input_tokens, 0), 0) ELSE 0 END), 0) AS settled_cache_read_tokens,
MIN(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_first_finalized_at_unix_secs,
MAX(CASE WHEN {SETTLED} THEN COALESCE(settlement.finalized_at, usage.finalized_at) END) AS settled_last_finalized_at_unix_secs
FROM "usage" AS usage
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
AND usage.user_id IS NOT NULL AND usage.user_id <> '' AND {AGGREGATABLE}
GROUP BY usage.user_id
) AS aggregated
WHERE target."date" = ? AND target.user_id = aggregated.user_id
"#
);
sqlx::query(&user_sql)
.bind(start_unix_secs)
.bind(end_unix_secs)
.bind(day_start)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(())
}
#[allow(clippy::too_many_arguments)]
async fn upsert_user_dimension(
tx: &mut sqlx::Transaction<'_, Sqlite>,
table: &str,
dimension_column: &str,
dimension_expr: &str,
dimension_filter: &str,
day_start: i64,
start_unix_secs: i64,
end_unix_secs: i64,
now_unix_secs: i64,
) -> Result<(), DataLayerError> {
let cache_creation = cache_creation_expr();
let total_context = total_input_context_expr();
let total_tokens = total_tokens_expr();
let sql = format!(
r#"
INSERT INTO {table} (
id, user_id, username, "date", {dimension_column}, total_requests, success_requests,
input_tokens, effective_input_tokens, output_tokens, total_tokens, total_input_context,
cache_creation_tokens, cache_creation_ephemeral_5m_tokens,
cache_creation_ephemeral_1h_tokens, cache_read_tokens, total_cost, actual_total_cost,
response_time_sum_ms, response_time_samples, successful_response_time_sum_ms,
successful_response_time_samples, created_at, updated_at
)
SELECT lower(hex(randomblob(32))), usage.user_id,
MAX(COALESCE(usage.username, users.username)), ?, {dimension_expr}, COUNT(*),
COALESCE(SUM({SUCCESS}), 0),
COALESCE(SUM(MAX(COALESCE(usage.input_tokens, 0), 0)), 0),
COALESCE(SUM({EFFECTIVE_INPUT}), 0),
COALESCE(SUM(MAX(COALESCE(usage.output_tokens, 0), 0)), 0),
COALESCE(SUM({total_tokens}), 0), COALESCE(SUM({total_context}), 0),
COALESCE(SUM({cache_creation}), 0), COALESCE(SUM({CACHE_5M}), 0),
COALESCE(SUM({CACHE_1H}), 0),
COALESCE(SUM(MAX(COALESCE(usage.cache_read_input_tokens, 0), 0)), 0),
COALESCE(SUM(COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0)), 0),
COALESCE(SUM(COALESCE(settlement.billing_actual_total_cost_usd, usage.actual_total_cost_usd, 0)), 0),
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN MAX(usage.response_time_ms, 0) ELSE 0 END), 0),
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0),
COALESCE(SUM(CASE WHEN ({SUCCESS}) = 1 AND usage.response_time_ms IS NOT NULL THEN MAX(usage.response_time_ms, 0) ELSE 0 END), 0),
COALESCE(SUM(CASE WHEN ({SUCCESS}) = 1 AND usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0),
?, ?
FROM "usage" AS usage
LEFT JOIN users ON users.id = usage.user_id
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
AND usage.user_id IS NOT NULL AND usage.user_id <> ''
AND {dimension_filter} AND {AGGREGATABLE}
GROUP BY usage.user_id, {dimension_expr}
ON CONFLICT (user_id, "date", {dimension_column}) DO UPDATE SET
username = COALESCE(excluded.username, {table}.username),
total_requests = excluded.total_requests, success_requests = excluded.success_requests,
input_tokens = excluded.input_tokens, effective_input_tokens = excluded.effective_input_tokens,
output_tokens = excluded.output_tokens, total_tokens = excluded.total_tokens,
total_input_context = excluded.total_input_context,
cache_creation_tokens = excluded.cache_creation_tokens,
cache_creation_ephemeral_5m_tokens = excluded.cache_creation_ephemeral_5m_tokens,
cache_creation_ephemeral_1h_tokens = excluded.cache_creation_ephemeral_1h_tokens,
cache_read_tokens = excluded.cache_read_tokens, total_cost = excluded.total_cost,
actual_total_cost = excluded.actual_total_cost,
response_time_sum_ms = excluded.response_time_sum_ms,
response_time_samples = excluded.response_time_samples,
successful_response_time_sum_ms = excluded.successful_response_time_sum_ms,
successful_response_time_samples = excluded.successful_response_time_samples,
updated_at = excluded.updated_at
"#
);
sqlx::query(&sql)
.bind(day_start)
.bind(now_unix_secs)
.bind(now_unix_secs)
.bind(start_unix_secs)
.bind(end_unix_secs)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(())
}
async fn upsert_model_provider_rows(
tx: &mut sqlx::Transaction<'_, Sqlite>,
day_start: i64,
start_unix_secs: i64,
end_unix_secs: i64,
now_unix_secs: i64,
) -> Result<(), DataLayerError> {
let total_tokens = total_tokens_expr();
let model_provider_sql = format!(
r#"
INSERT INTO stats_daily_model_provider (
id, "date", model, provider_name, total_requests, total_tokens, total_cost,
response_time_sum_ms, response_time_samples, created_at, updated_at
)
SELECT lower(hex(randomblob(32))), ?, usage.model, usage.provider_name, COUNT(*),
COALESCE(SUM({total_tokens}), 0),
COALESCE(SUM(COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0)), 0),
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN MAX(usage.response_time_ms, 0) ELSE 0 END), 0),
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0), ?, ?
FROM "usage" AS usage
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
AND usage.model IS NOT NULL AND usage.model <> '' AND {AGGREGATABLE}
GROUP BY usage.model, usage.provider_name
ON CONFLICT ("date", model, provider_name) DO UPDATE SET
total_requests = excluded.total_requests, total_tokens = excluded.total_tokens,
total_cost = excluded.total_cost, response_time_sum_ms = excluded.response_time_sum_ms,
response_time_samples = excluded.response_time_samples, updated_at = excluded.updated_at
"#
);
sqlx::query(&model_provider_sql)
.bind(day_start)
.bind(now_unix_secs)
.bind(now_unix_secs)
.bind(start_unix_secs)
.bind(end_unix_secs)
.execute(&mut **tx)
.await
.map_sql_err()?;
let user_model_provider_sql = format!(
r#"
INSERT INTO stats_user_daily_model_provider (
id, user_id, username, "date", model, provider_name, total_requests, total_tokens,
total_cost, response_time_sum_ms, response_time_samples, created_at, updated_at
)
SELECT lower(hex(randomblob(32))), usage.user_id, MAX(COALESCE(usage.username, users.username)),
?, usage.model, usage.provider_name, COUNT(*), COALESCE(SUM({total_tokens}), 0),
COALESCE(SUM(COALESCE(settlement.billing_total_cost_usd, usage.total_cost_usd, 0)), 0),
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN MAX(usage.response_time_ms, 0) ELSE 0 END), 0),
COALESCE(SUM(CASE WHEN usage.response_time_ms IS NOT NULL THEN 1 ELSE 0 END), 0), ?, ?
FROM "usage" AS usage
LEFT JOIN users ON users.id = usage.user_id
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ?
AND usage.user_id IS NOT NULL AND usage.user_id <> ''
AND usage.model IS NOT NULL AND usage.model <> '' AND {AGGREGATABLE}
GROUP BY usage.user_id, usage.model, usage.provider_name
ON CONFLICT (user_id, "date", model, provider_name) DO UPDATE SET
username = COALESCE(excluded.username, stats_user_daily_model_provider.username),
total_requests = excluded.total_requests, total_tokens = excluded.total_tokens,
total_cost = excluded.total_cost, response_time_sum_ms = excluded.response_time_sum_ms,
response_time_samples = excluded.response_time_samples, updated_at = excluded.updated_at
"#
);
sqlx::query(&user_model_provider_sql)
.bind(day_start)
.bind(now_unix_secs)
.bind(now_unix_secs)
.bind(start_unix_secs)
.bind(end_unix_secs)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(())
}
async fn upsert_cost_savings_rows(
tx: &mut sqlx::Transaction<'_, Sqlite>,
day_start: i64,
start_unix_secs: i64,
end_unix_secs: i64,
now_unix_secs: i64,
) -> Result<(), DataLayerError> {
for (table, dimensions) in [
("stats_daily_cost_savings", Vec::new()),
(
"stats_daily_cost_savings_provider",
vec![("provider_name", "COALESCE(usage.provider_name, '')")],
),
(
"stats_daily_cost_savings_model",
vec![("model", "COALESCE(usage.model, '')")],
),
(
"stats_daily_cost_savings_model_provider",
vec![
("model", "COALESCE(usage.model, '')"),
("provider_name", "COALESCE(usage.provider_name, '')"),
],
),
] {
upsert_cost_savings_dimension(
tx,
table,
false,
&dimensions,
day_start,
start_unix_secs,
end_unix_secs,
now_unix_secs,
)
.await?;
}
for (table, dimensions) in [
("stats_user_daily_cost_savings", Vec::new()),
(
"stats_user_daily_cost_savings_provider",
vec![("provider_name", "COALESCE(usage.provider_name, '')")],
),
(
"stats_user_daily_cost_savings_model",
vec![("model", "COALESCE(usage.model, '')")],
),
(
"stats_user_daily_cost_savings_model_provider",
vec![
("model", "COALESCE(usage.model, '')"),
("provider_name", "COALESCE(usage.provider_name, '')"),
],
),
] {
upsert_cost_savings_dimension(
tx,
table,
true,
&dimensions,
day_start,
start_unix_secs,
end_unix_secs,
now_unix_secs,
)
.await?;
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
async fn upsert_cost_savings_dimension(
tx: &mut sqlx::Transaction<'_, Sqlite>,
table: &str,
per_user: bool,
dimensions: &[(&str, &str)],
day_start: i64,
start_unix_secs: i64,
end_unix_secs: i64,
now_unix_secs: i64,
) -> Result<(), DataLayerError> {
let dimension_columns = dimensions
.iter()
.map(|(column, _)| *column)
.collect::<Vec<_>>();
let dimension_exprs = dimensions
.iter()
.map(|(_, expression)| *expression)
.collect::<Vec<_>>();
let user_columns = if per_user { "user_id, username, " } else { "" };
let user_select = if per_user {
"usage.user_id, MAX(COALESCE(usage.username, users.username)), "
} else {
""
};
let user_join = if per_user {
"LEFT JOIN users ON users.id = usage.user_id"
} else {
""
};
let user_filter = if per_user {
"AND usage.user_id IS NOT NULL AND usage.user_id <> ''"
} else {
""
};
let mut conflict_columns = vec!["\"date\""];
let mut group_by = Vec::new();
if per_user {
conflict_columns.insert(0, "user_id");
group_by.push("usage.user_id");
}
conflict_columns.extend(dimension_columns.iter().copied());
group_by.extend(dimension_exprs.iter().copied());
let dimension_columns_sql = if dimension_columns.is_empty() {
String::new()
} else {
format!("{}, ", dimension_columns.join(", "))
};
let dimension_select_sql = if dimension_exprs.is_empty() {
String::new()
} else {
format!("{}, ", dimension_exprs.join(", "))
};
let group_by_sql = if group_by.is_empty() {
String::new()
} else {
format!("GROUP BY {}", group_by.join(", "))
};
let sql = format!(
r#"
INSERT INTO {table} (
id, {user_columns}"date", {dimension_columns_sql}cache_read_tokens,
cache_read_cost, cache_creation_cost, estimated_full_cost, created_at, updated_at
)
SELECT lower(hex(randomblob(32))), {user_select}?, {dimension_select_sql}
COALESCE(SUM(MAX(COALESCE(usage.cache_read_input_tokens, 0), 0)), 0),
COALESCE(SUM(COALESCE(usage.cache_read_cost_usd, 0)), 0),
COALESCE(SUM(COALESCE(usage.cache_creation_cost_usd, 0)), 0),
COALESCE(SUM(
COALESCE(settlement.input_price_per_1m, usage.input_price_per_1m, 0)
* MAX(COALESCE(usage.cache_read_input_tokens, 0), 0) / 1000000.0
), 0), ?, ?
FROM "usage" AS usage
LEFT JOIN usage_settlement_snapshots AS settlement ON settlement.request_id = usage.request_id
{user_join}
WHERE usage.created_at_unix_ms >= ? AND usage.created_at_unix_ms < ? {user_filter}
{group_by_sql}
ON CONFLICT ({}) DO UPDATE SET
{}cache_read_tokens = excluded.cache_read_tokens,
cache_read_cost = excluded.cache_read_cost,
cache_creation_cost = excluded.cache_creation_cost,
estimated_full_cost = excluded.estimated_full_cost,
updated_at = excluded.updated_at
"#,
conflict_columns.join(", "),
if per_user {
format!("username = COALESCE(excluded.username, {table}.username), ")
} else {
String::new()
}
);
sqlx::query(&sql)
.bind(day_start)
.bind(now_unix_secs)
.bind(now_unix_secs)
.bind(start_unix_secs)
.bind(end_unix_secs)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(())
}
async fn refresh_user_summary(
tx: &mut sqlx::Transaction<'_, Sqlite>,
cutoff_date: i64,
now_unix_secs: i64,
) -> Result<(), DataLayerError> {
sqlx::query(
r#"
INSERT INTO stats_user_summary (
id, user_id, username, cutoff_date, all_time_requests, all_time_success_requests,
all_time_error_requests, all_time_input_tokens, all_time_output_tokens,
all_time_cache_creation_tokens, all_time_cache_read_tokens, all_time_cost,
all_time_actual_cost, active_days, first_active_date, last_active_date,
created_at, updated_at
)
SELECT lower(hex(randomblob(32))), user_id, MAX(username), ?,
COALESCE(SUM(total_requests), 0), COALESCE(SUM(success_requests), 0),
COALESCE(SUM(error_requests), 0), COALESCE(SUM(input_tokens), 0),
COALESCE(SUM(output_tokens), 0), COALESCE(SUM(cache_creation_tokens), 0),
COALESCE(SUM(cache_read_tokens), 0), COALESCE(SUM(total_cost), 0),
COALESCE(SUM(actual_total_cost), 0),
COALESCE(SUM(CASE WHEN total_requests > 0 THEN 1 ELSE 0 END), 0),
MIN(CASE WHEN total_requests > 0 THEN "date" END),
MAX(CASE WHEN total_requests > 0 THEN "date" END), ?, ?
FROM stats_user_daily
WHERE "date" < ?
GROUP BY user_id
ON CONFLICT (user_id) DO UPDATE SET
username = COALESCE(excluded.username, stats_user_summary.username),
cutoff_date = excluded.cutoff_date, all_time_requests = excluded.all_time_requests,
all_time_success_requests = excluded.all_time_success_requests,
all_time_error_requests = excluded.all_time_error_requests,
all_time_input_tokens = excluded.all_time_input_tokens,
all_time_output_tokens = excluded.all_time_output_tokens,
all_time_cache_creation_tokens = excluded.all_time_cache_creation_tokens,
all_time_cache_read_tokens = excluded.all_time_cache_read_tokens,
all_time_cost = excluded.all_time_cost, all_time_actual_cost = excluded.all_time_actual_cost,
active_days = excluded.active_days, first_active_date = excluded.first_active_date,
last_active_date = excluded.last_active_date, updated_at = excluded.updated_at
"#,
)
.bind(cutoff_date)
.bind(now_unix_secs)
.bind(now_unix_secs)
.bind(cutoff_date)
.execute(&mut **tx)
.await
.map_sql_err()?;
refresh_global_summary(tx, cutoff_date, now_unix_secs).await?;
Ok(())
}
async fn refresh_global_summary(
tx: &mut sqlx::Transaction<'_, Sqlite>,
cutoff_date: i64,
now_unix_secs: i64,
) -> Result<(), DataLayerError> {
let existing_id: Option<String> =
sqlx::query_scalar("SELECT id FROM stats_summary ORDER BY created_at, id LIMIT 1")
.fetch_optional(&mut **tx)
.await
.map_sql_err()?;
let summary_id = existing_id.unwrap_or_else(|| stats_id("stats-summary"));
sqlx::query(
r#"
INSERT INTO stats_summary (
id, cutoff_date, all_time_requests, all_time_success_requests,
all_time_error_requests, all_time_input_tokens, all_time_output_tokens,
all_time_cache_creation_tokens, all_time_cache_read_tokens, all_time_cost,
all_time_actual_cost, total_users, active_users, total_api_keys,
active_api_keys, created_at, updated_at
)
SELECT ?, ?, COALESCE(SUM(total_requests), 0), COALESCE(SUM(success_requests), 0),
COALESCE(SUM(error_requests), 0), COALESCE(SUM(input_tokens), 0),
COALESCE(SUM(output_tokens), 0), COALESCE(SUM(cache_creation_tokens), 0),
COALESCE(SUM(cache_read_tokens), 0), COALESCE(SUM(total_cost), 0),
COALESCE(SUM(actual_total_cost), 0),
(SELECT COUNT(*) FROM users),
(SELECT COUNT(*) FROM users WHERE is_active <> 0),
(SELECT COUNT(*) FROM api_keys),
(SELECT COUNT(*) FROM api_keys WHERE is_active <> 0), ?, ?
FROM stats_daily
WHERE "date" < ?
ON CONFLICT (id) DO UPDATE SET
cutoff_date = excluded.cutoff_date,
all_time_requests = excluded.all_time_requests,
all_time_success_requests = excluded.all_time_success_requests,
all_time_error_requests = excluded.all_time_error_requests,
all_time_input_tokens = excluded.all_time_input_tokens,
all_time_output_tokens = excluded.all_time_output_tokens,
all_time_cache_creation_tokens = excluded.all_time_cache_creation_tokens,
all_time_cache_read_tokens = excluded.all_time_cache_read_tokens,
all_time_cost = excluded.all_time_cost,
all_time_actual_cost = excluded.all_time_actual_cost,
total_users = excluded.total_users, active_users = excluded.active_users,
total_api_keys = excluded.total_api_keys, active_api_keys = excluded.active_api_keys,
updated_at = excluded.updated_at
"#,
)
.bind(summary_id)
.bind(cutoff_date)
.bind(now_unix_secs)
.bind(now_unix_secs)
.bind(cutoff_date)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(())
}
crates/aether-data/runtime/src/backend/stats_common.rs
-6
View File
@@ -7,12 +7,6 @@ pub(crate) fn unix_secs(value: DateTime<Utc>) -> i64 {
value.timestamp().max(0)
}
pub(crate) fn unix_ms(value: i64) -> Result<i64, DataLayerError> {
value.checked_mul(1000).ok_or_else(|| {
DataLayerError::InvalidInput(format!("timestamp overflow while converting {value} to ms"))
})
}
pub(crate) fn utc_from_unix_secs(
value: i64,
field_name: &str,
crates/aether-data/runtime/src/backend/system.rs
+2
View File
@@ -70,10 +70,12 @@ const ADMIN_STATS_PURGE_TABLES: &[&str] = &[
];
const ADMIN_USAGE_CHILD_TABLES: &[&str] = &[
"usage_counter_deltas",
"usage_body_blobs",
"usage_http_audits",
"usage_routing_snapshots",
"usage_settlement_snapshots",
"user_model_usage_counts",
];
const USAGE_BODY_FIELD_COLUMNS: &[&str] = &[
crates/aether-data/runtime/src/backend/system/mysql.rs
+24
View File
@@ -855,6 +855,22 @@ SET provider_id = NULL,
WHERE provider_id IS NOT NULL
OR provider_endpoint_id IS NOT NULL
OR provider_api_key_id IS NOT NULL
"#,
summary,
)
.await?;
mysql_execute_if_table(
tx,
"usage_routing_snapshots",
"usage_routing_provider_refs_cleared",
r#"
UPDATE usage_routing_snapshots
SET selected_provider_id = NULL,
selected_endpoint_id = NULL,
selected_provider_api_key_id = NULL
WHERE selected_provider_id IS NOT NULL
OR selected_endpoint_id IS NOT NULL
OR selected_provider_api_key_id IS NOT NULL
"#,
summary,
)
@@ -952,6 +968,14 @@ WHERE request_count <> 0
summary,
)
.await?;
mysql_execute_if_table(
tx,
"global_models",
"global_model_usage_stats_reset",
"UPDATE global_models SET usage_count = 0 WHERE usage_count <> 0",
summary,
)
.await?;
}
AdminSystemPurgeTarget::AuditLogs => {
mysql_delete_table(tx, "audit_logs", summary).await?;
crates/aether-data/runtime/src/backend/system/postgres.rs
+24
View File
@@ -460,6 +460,22 @@ SET provider_id = NULL,
WHERE provider_id IS NOT NULL
OR provider_endpoint_id IS NOT NULL
OR provider_api_key_id IS NOT NULL
"#,
summary,
)
.await?;
pg_execute_if_table(
tx,
"usage_routing_snapshots",
"usage_routing_provider_refs_cleared",
r#"
UPDATE public.usage_routing_snapshots
SET selected_provider_id = NULL,
selected_endpoint_id = NULL,
selected_provider_api_key_id = NULL
WHERE selected_provider_id IS NOT NULL
OR selected_endpoint_id IS NOT NULL
OR selected_provider_api_key_id IS NOT NULL
"#,
summary,
)
@@ -559,6 +575,14 @@ WHERE request_count <> 0
summary,
)
.await?;
pg_execute_if_table(
tx,
"global_models",
"global_model_usage_stats_reset",
"UPDATE public.global_models SET usage_count = 0 WHERE usage_count <> 0",
summary,
)
.await?;
}
AdminSystemPurgeTarget::AuditLogs => {
pg_delete_table(tx, "audit_logs", summary).await?;
crates/aether-data/runtime/src/backend/system/sqlite.rs
+153
View File
@@ -601,6 +601,22 @@ SET provider_id = NULL,
WHERE provider_id IS NOT NULL
OR provider_endpoint_id IS NOT NULL
OR provider_api_key_id IS NOT NULL
"#,
summary,
)
.await?;
sqlite_execute_if_table(
tx,
"usage_routing_snapshots",
"usage_routing_provider_refs_cleared",
r#"
UPDATE usage_routing_snapshots
SET selected_provider_id = NULL,
selected_endpoint_id = NULL,
selected_provider_api_key_id = NULL
WHERE selected_provider_id IS NOT NULL
OR selected_endpoint_id IS NOT NULL
OR selected_provider_api_key_id IS NOT NULL
"#,
summary,
)
@@ -698,6 +714,14 @@ WHERE request_count <> 0
summary,
)
.await?;
sqlite_execute_if_table(
tx,
"global_models",
"global_model_usage_stats_reset",
"UPDATE global_models SET usage_count = 0 WHERE usage_count <> 0",
summary,
)
.await?;
}
AdminSystemPurgeTarget::AuditLogs => {
sqlite_delete_table(tx, "audit_logs", summary).await?;
@@ -1250,3 +1274,132 @@ pub(super) fn map_admin_system_stats(
.max(0) as u64,
})
}
#[cfg(test)]
mod tests {
use super::{purge_sqlite_admin_system_data, AdminSystemPurgeSummary, AdminSystemPurgeTarget};
#[tokio::test]
async fn usage_purge_removes_pending_counters_and_resets_model_usage() {
let pool = sqlx::sqlite::SqlitePoolOptions::new()
.max_connections(1)
.connect("sqlite::memory:")
.await
.expect("sqlite pool should connect");
crate::lifecycle::migrate::run_sqlite_migrations(&pool)
.await
.expect("sqlite migrations should run");
sqlx::raw_sql(
r#"
INSERT INTO users (id, email, username, created_at, updated_at)
VALUES ('purge-user', 'purge@example.com', 'purge-user', 1, 1);
INSERT INTO global_models (id, name, usage_count, created_at, updated_at)
VALUES ('purge-model', 'purge-model', 7, 1, 1);
INSERT INTO "usage" (
request_id, user_id, provider_name, model, status, billing_status,
created_at_unix_ms, updated_at_unix_secs
) VALUES ('purge-request', 'purge-user', 'provider', 'purge-model', 'completed', 'settled', 1, 1);
INSERT INTO usage_counter_deltas (
id, request_id, kind, target_id, request_count_delta, created_at
) VALUES ('purge-delta', 'purge-request', 'model', 'purge-model', 1, 1);
INSERT INTO user_model_usage_counts (
id, user_id, model, usage_count, created_at, updated_at
) VALUES ('purge-user-model', 'purge-user', 'purge-model', 7, 1, 1);
"#,
)
.execute(&pool)
.await
.expect("usage purge fixtures should insert");
let mut tx = pool.begin().await.expect("purge transaction should begin");
let mut summary = AdminSystemPurgeSummary::default();
purge_sqlite_admin_system_data(&mut tx, AdminSystemPurgeTarget::Usage, &mut summary)
.await
.expect("usage purge should succeed");
tx.commit().await.expect("purge transaction should commit");
let usage_count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM \"usage\"")
.fetch_one(&pool)
.await
.expect("usage count should load");
let delta_count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM usage_counter_deltas")
.fetch_one(&pool)
.await
.expect("counter delta count should load");
let user_model_count: i64 =
sqlx::query_scalar("SELECT COUNT(*) FROM user_model_usage_counts")
.fetch_one(&pool)
.await
.expect("user model count should load");
let model_usage_count: i64 =
sqlx::query_scalar("SELECT usage_count FROM global_models WHERE id = 'purge-model'")
.fetch_one(&pool)
.await
.expect("global model usage count should load");
assert_eq!(usage_count, 0);
assert_eq!(delta_count, 0);
assert_eq!(user_model_count, 0);
assert_eq!(model_usage_count, 0);
assert_eq!(summary.affected.get("usage_counter_deltas"), Some(&1));
assert_eq!(summary.affected.get("user_model_usage_counts"), Some(&1));
assert_eq!(
summary.affected.get("global_model_usage_stats_reset"),
Some(&1)
);
}
#[tokio::test]
async fn config_purge_clears_canonical_routing_provider_refs() {
let pool = sqlx::sqlite::SqlitePoolOptions::new()
.max_connections(1)
.connect("sqlite::memory:")
.await
.expect("sqlite pool should connect");
crate::lifecycle::migrate::run_sqlite_migrations(&pool)
.await
.expect("sqlite migrations should run");
sqlx::raw_sql(
r#"
INSERT INTO "usage" (
request_id, provider_name, model, status, billing_status,
created_at_unix_ms, updated_at_unix_secs
) VALUES ('routing-purge-request', 'provider', 'model', 'completed', 'settled', 1, 1);
INSERT INTO usage_routing_snapshots (
request_id, selected_provider_id, selected_endpoint_id,
selected_provider_api_key_id, created_at, updated_at
) VALUES (
'routing-purge-request', 'provider-1', 'endpoint-1', 'key-1', 1, 1
);
"#,
)
.execute(&pool)
.await
.expect("routing purge fixture should insert");
let mut tx = pool.begin().await.expect("purge transaction should begin");
let mut summary = AdminSystemPurgeSummary::default();
purge_sqlite_admin_system_data(&mut tx, AdminSystemPurgeTarget::Config, &mut summary)
.await
.expect("config purge should succeed");
tx.commit().await.expect("purge transaction should commit");
let refs = sqlx::query_as::<_, (Option<String>, Option<String>, Option<String>)>(
r#"
SELECT selected_provider_id, selected_endpoint_id, selected_provider_api_key_id
FROM usage_routing_snapshots
WHERE request_id = 'routing-purge-request'
"#,
)
.fetch_one(&pool)
.await
.expect("routing refs should load");
assert_eq!(refs, (None, None, None));
assert_eq!(
summary.affected.get("usage_routing_provider_refs_cleared"),
Some(&1)
);
}
}
crates/aether-data/runtime/src/lifecycle/backfill/mysql.rs
+244 -7
View File
@@ -1,16 +1,253 @@
use sqlx::migrate::MigrateError;
use tracing::info;
use std::collections::{HashMap, HashSet};
use sqlx::{
migrate::{Migrate, MigrateError, Migrator},
query, Connection, MySqlConnection, Row,
};
use tracing::{error, info, warn};
use super::types::PendingBackfillInfo;
use crate::driver::mysql::MysqlPool;
pub async fn run_backfills(_pool: &MysqlPool) -> Result<(), MigrateError> {
info!("mysql database backfills are up to date");
static BACKFILL_MIGRATOR: Migrator = sqlx::migrate!("./backfills/mysql");
const ENSURE_SCHEMA_BACKFILLS_TABLE_SQL: &str = r#"
CREATE TABLE IF NOT EXISTS schema_backfills (
version BIGINT NOT NULL,
description TEXT NOT NULL,
success BOOLEAN NOT NULL DEFAULT TRUE,
checksum BLOB NOT NULL,
execution_time BIGINT NOT NULL DEFAULT 0,
applied_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
PRIMARY KEY (version)
)
"#;
const LIST_APPLIED_BACKFILLS_SQL: &str = r#"
SELECT version, checksum
FROM schema_backfills
WHERE success IS TRUE
ORDER BY version ASC
"#;
const INSERT_APPLIED_BACKFILL_SQL: &str = r#"
INSERT INTO schema_backfills (
version,
description,
success,
checksum,
execution_time,
applied_at
) VALUES (
?,
?,
TRUE,
?,
?,
CURRENT_TIMESTAMP(6)
)
ON DUPLICATE KEY UPDATE version = schema_backfills.version
"#;
#[derive(Debug, Clone, PartialEq, Eq)]
struct AppliedBackfill {
version: i64,
checksum: Vec<u8>,
}
pub async fn run_backfills(pool: &MysqlPool) -> Result<(), MigrateError> {
let mut conn = pool.acquire().await?;
if BACKFILL_MIGRATOR.locking {
conn.lock().await?;
}
let result = run_backfills_locked(&mut conn).await;
if BACKFILL_MIGRATOR.locking {
match conn.unlock().await {
Ok(()) => {}
Err(unlock_error) if result.is_ok() => return Err(unlock_error),
Err(unlock_error) => {
warn!(
error = %unlock_error,
"mysql database backfill lock release failed after backfill error"
);
}
}
}
result
}
pub async fn pending_backfills(pool: &MysqlPool) -> Result<Vec<PendingBackfillInfo>, MigrateError> {
let mut conn = pool.acquire().await?;
pending_backfills_locked(&mut conn).await
}
async fn run_backfills_locked(conn: &mut MySqlConnection) -> Result<(), MigrateError> {
ensure_schema_backfills_table(conn).await?;
let applied_backfills = list_applied_backfills(conn).await?;
validate_applied_backfills(&applied_backfills)?;
let applied_by_version: HashMap<_, _> = applied_backfills
.iter()
.map(|backfill| (backfill.version, backfill))
.collect();
let pending_backfills: Vec<_> = BACKFILL_MIGRATOR
.iter()
.filter(|backfill| backfill.migration_type.is_up_migration())
.filter(|backfill| !applied_by_version.contains_key(&backfill.version))
.collect();
if pending_backfills.is_empty() {
info!(
driver = "mysql",
pending_backfills = 0,
"database backfills already up to date"
);
return Ok(());
}
info!(
driver = "mysql",
pending_backfills = pending_backfills.len(),
"database backfills pending"
);
for (index, backfill) in pending_backfills.iter().enumerate() {
let current = index + 1;
let total = pending_backfills.len();
info!(
driver = "mysql",
current,
total,
version = backfill.version,
description = %backfill.description,
"applying database backfill"
);
let mut tx = conn.begin().await?;
let started_at = std::time::Instant::now();
sqlx::raw_sql(&backfill.sql).execute(&mut *tx).await?;
let elapsed_ms = i64::try_from(started_at.elapsed().as_millis()).unwrap_or(i64::MAX);
query(INSERT_APPLIED_BACKFILL_SQL)
.bind(backfill.version)
.bind(backfill.description.as_ref())
.bind(backfill.checksum.as_ref())
.bind(elapsed_ms)
.execute(&mut *tx)
.await?;
tx.commit().await?;
info!(
driver = "mysql",
current,
total,
version = backfill.version,
description = %backfill.description,
elapsed_ms,
"applied database backfill"
);
}
info!(
driver = "mysql",
pending_backfills = 0,
"database backfills complete"
);
Ok(())
}
pub async fn pending_backfills(
_pool: &MysqlPool,
async fn pending_backfills_locked(
conn: &mut MySqlConnection,
) -> Result<Vec<PendingBackfillInfo>, MigrateError> {
Ok(Vec::new())
ensure_schema_backfills_table(conn).await?;
let applied_backfills = list_applied_backfills(conn).await?;
validate_applied_backfills(&applied_backfills)?;
Ok(pending_backfills_from_applied(&applied_backfills))
}
async fn ensure_schema_backfills_table(conn: &mut MySqlConnection) -> Result<(), MigrateError> {
query(ENSURE_SCHEMA_BACKFILLS_TABLE_SQL)
.execute(&mut *conn)
.await?;
Ok(())
}
async fn list_applied_backfills(
conn: &mut MySqlConnection,
) -> Result<Vec<AppliedBackfill>, MigrateError> {
let rows = query(LIST_APPLIED_BACKFILLS_SQL)
.fetch_all(&mut *conn)
.await?;
rows.into_iter()
.map(|row| {
Ok(AppliedBackfill {
version: row.try_get("version")?,
checksum: row.try_get("checksum")?,
})
})
.collect::<Result<Vec<_>, sqlx::Error>>()
.map_err(MigrateError::from)
}
fn validate_applied_backfills(applied_backfills: &[AppliedBackfill]) -> Result<(), MigrateError> {
if BACKFILL_MIGRATOR.ignore_missing {
return Ok(());
}
let known_versions: HashSet<_> = BACKFILL_MIGRATOR
.iter()
.map(|backfill| backfill.version)
.collect();
for applied_backfill in applied_backfills {
if !known_versions.contains(&applied_backfill.version) {
error!(
driver = "mysql",
version = applied_backfill.version,
"applied database backfill is missing from embedded backfills"
);
return Err(MigrateError::VersionMissing(applied_backfill.version));
}
}
for backfill in BACKFILL_MIGRATOR
.iter()
.filter(|backfill| backfill.migration_type.is_up_migration())
{
let Some(applied) = applied_backfills
.iter()
.find(|applied| applied.version == backfill.version)
else {
continue;
};
if backfill.checksum != applied.checksum {
warn!(
driver = "mysql",
version = backfill.version,
description = %backfill.description,
"applied database backfill checksum differs from embedded backfill; skipping strict enforcement"
);
}
}
Ok(())
}
fn pending_backfills_from_applied(
applied_backfills: &[AppliedBackfill],
) -> Vec<PendingBackfillInfo> {
let applied_versions: HashSet<_> = applied_backfills
.iter()
.map(|backfill| backfill.version)
.collect();
BACKFILL_MIGRATOR
.iter()
.filter(|backfill| backfill.migration_type.is_up_migration())
.filter(|backfill| !applied_versions.contains(&backfill.version))
.map(|backfill| PendingBackfillInfo {
version: backfill.version,
description: backfill.description.to_string(),
})
.collect()
}
crates/aether-data/runtime/src/lifecycle/backfill/sqlite.rs
+245 -7
View File
@@ -1,16 +1,254 @@
use sqlx::migrate::MigrateError;
use tracing::info;
use std::collections::{HashMap, HashSet};
use sqlx::{
migrate::{Migrate, MigrateError, Migrator},
query, Connection, Row, SqliteConnection,
};
use tracing::{error, info, warn};
use super::types::PendingBackfillInfo;
use crate::driver::sqlite::SqlitePool;
pub async fn run_backfills(_pool: &SqlitePool) -> Result<(), MigrateError> {
info!("sqlite database backfills are up to date");
Ok(())
static BACKFILL_MIGRATOR: Migrator = sqlx::migrate!("./backfills/sqlite");
const ENSURE_SCHEMA_BACKFILLS_TABLE_SQL: &str = r#"
CREATE TABLE IF NOT EXISTS schema_backfills (
version INTEGER NOT NULL PRIMARY KEY,
description TEXT NOT NULL,
success INTEGER NOT NULL DEFAULT 1,
checksum BLOB NOT NULL,
execution_time INTEGER NOT NULL DEFAULT 0,
applied_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s', 'now') AS INTEGER))
)
"#;
const LIST_APPLIED_BACKFILLS_SQL: &str = r#"
SELECT version, checksum
FROM schema_backfills
WHERE success = 1
ORDER BY version ASC
"#;
const INSERT_APPLIED_BACKFILL_SQL: &str = r#"
INSERT INTO schema_backfills (
version,
description,
success,
checksum,
execution_time,
applied_at
) VALUES (
?,
?,
1,
?,
?,
CAST(strftime('%s', 'now') AS INTEGER)
)
ON CONFLICT(version) DO NOTHING
"#;
#[derive(Debug, Clone, PartialEq, Eq)]
struct AppliedBackfill {
version: i64,
checksum: Vec<u8>,
}
pub async fn run_backfills(pool: &SqlitePool) -> Result<(), MigrateError> {
let mut conn = pool.acquire().await?;
if BACKFILL_MIGRATOR.locking {
conn.lock().await?;
}
let result = run_backfills_locked(&mut conn).await;
if BACKFILL_MIGRATOR.locking {
match conn.unlock().await {
Ok(()) => {}
Err(unlock_error) if result.is_ok() => return Err(unlock_error),
Err(unlock_error) => {
warn!(
error = %unlock_error,
"sqlite database backfill lock release failed after backfill error"
);
}
}
}
result
}
pub async fn pending_backfills(
_pool: &SqlitePool,
pool: &SqlitePool,
) -> Result<Vec<PendingBackfillInfo>, MigrateError> {
Ok(Vec::new())
let mut conn = pool.acquire().await?;
pending_backfills_locked(&mut conn).await
}
async fn run_backfills_locked(conn: &mut SqliteConnection) -> Result<(), MigrateError> {
ensure_schema_backfills_table(conn).await?;
let applied_backfills = list_applied_backfills(conn).await?;
validate_applied_backfills(&applied_backfills)?;
let applied_by_version: HashMap<_, _> = applied_backfills
.iter()
.map(|backfill| (backfill.version, backfill))
.collect();
let pending_backfills: Vec<_> = BACKFILL_MIGRATOR
.iter()
.filter(|backfill| backfill.migration_type.is_up_migration())
.filter(|backfill| !applied_by_version.contains_key(&backfill.version))
.collect();
if pending_backfills.is_empty() {
info!(
driver = "sqlite",
pending_backfills = 0,
"database backfills already up to date"
);
return Ok(());
}
info!(
driver = "sqlite",
pending_backfills = pending_backfills.len(),
"database backfills pending"
);
for (index, backfill) in pending_backfills.iter().enumerate() {
let current = index + 1;
let total = pending_backfills.len();
info!(
driver = "sqlite",
current,
total,
version = backfill.version,
description = %backfill.description,
"applying database backfill"
);
let mut tx = conn.begin().await?;
let started_at = std::time::Instant::now();
sqlx::raw_sql(&backfill.sql).execute(&mut *tx).await?;
let elapsed_ms = i64::try_from(started_at.elapsed().as_millis()).unwrap_or(i64::MAX);
query(INSERT_APPLIED_BACKFILL_SQL)
.bind(backfill.version)
.bind(backfill.description.as_ref())
.bind(backfill.checksum.as_ref())
.bind(elapsed_ms)
.execute(&mut *tx)
.await?;
tx.commit().await?;
info!(
driver = "sqlite",
current,
total,
version = backfill.version,
description = %backfill.description,
elapsed_ms,
"applied database backfill"
);
}
info!(
driver = "sqlite",
pending_backfills = 0,
"database backfills complete"
);
Ok(())
}
async fn pending_backfills_locked(
conn: &mut SqliteConnection,
) -> Result<Vec<PendingBackfillInfo>, MigrateError> {
ensure_schema_backfills_table(conn).await?;
let applied_backfills = list_applied_backfills(conn).await?;
validate_applied_backfills(&applied_backfills)?;
Ok(pending_backfills_from_applied(&applied_backfills))
}
async fn ensure_schema_backfills_table(conn: &mut SqliteConnection) -> Result<(), MigrateError> {
query(ENSURE_SCHEMA_BACKFILLS_TABLE_SQL)
.execute(&mut *conn)
.await?;
Ok(())
}
async fn list_applied_backfills(
conn: &mut SqliteConnection,
) -> Result<Vec<AppliedBackfill>, MigrateError> {
let rows = query(LIST_APPLIED_BACKFILLS_SQL)
.fetch_all(&mut *conn)
.await?;
rows.into_iter()
.map(|row| {
Ok(AppliedBackfill {
version: row.try_get("version")?,
checksum: row.try_get("checksum")?,
})
})
.collect::<Result<Vec<_>, sqlx::Error>>()
.map_err(MigrateError::from)
}
fn validate_applied_backfills(applied_backfills: &[AppliedBackfill]) -> Result<(), MigrateError> {
if BACKFILL_MIGRATOR.ignore_missing {
return Ok(());
}
let known_versions: HashSet<_> = BACKFILL_MIGRATOR
.iter()
.map(|backfill| backfill.version)
.collect();
for applied_backfill in applied_backfills {
if !known_versions.contains(&applied_backfill.version) {
error!(
driver = "sqlite",
version = applied_backfill.version,
"applied database backfill is missing from embedded backfills"
);
return Err(MigrateError::VersionMissing(applied_backfill.version));
}
}
for backfill in BACKFILL_MIGRATOR
.iter()
.filter(|backfill| backfill.migration_type.is_up_migration())
{
let Some(applied) = applied_backfills
.iter()
.find(|applied| applied.version == backfill.version)
else {
continue;
};
if backfill.checksum != applied.checksum {
warn!(
driver = "sqlite",
version = backfill.version,
description = %backfill.description,
"applied database backfill checksum differs from embedded backfill; skipping strict enforcement"
);
}
}
Ok(())
}
fn pending_backfills_from_applied(
applied_backfills: &[AppliedBackfill],
) -> Vec<PendingBackfillInfo> {
let applied_versions: HashSet<_> = applied_backfills
.iter()
.map(|backfill| backfill.version)
.collect();
BACKFILL_MIGRATOR
.iter()
.filter(|backfill| backfill.migration_type.is_up_migration())
.filter(|backfill| !applied_versions.contains(&backfill.version))
.map(|backfill| PendingBackfillInfo {
version: backfill.version,
description: backfill.description.to_string(),
})
.collect()
}
crates/aether-data/runtime/src/lifecycle/backfill/tests.rs
+541 -28
View File
@@ -4,15 +4,14 @@ use std::{
time::{Duration, Instant},
};
use sqlx::{query, query_scalar, Connection, PgConnection, PgPool};
use sqlx::{query, query_as, query_scalar, Connection, PgConnection, PgPool};
use super::{
pending_backfills, pending_backfills_from_applied, pending_mysql_backfills,
pending_sqlite_backfills, run_backfills, run_mysql_backfills, run_sqlite_backfills,
AppliedBackfill,
};
use crate::lifecycle::migrate::prepare_database_for_startup;
use crate::{DatabaseDriver, SqlDatabaseConfig, SqlPoolConfig};
use crate::lifecycle::migrate::{prepare_database_for_startup, run_sqlite_migrations};
const LEGACY_SYNC_ENABLED_ACTIVE_FLAGS_VERSION: i64 = 20260517012000;
const LEGACY_SYNC_ENABLED_ACTIVE_FLAGS_SQL: &str =
@@ -88,44 +87,558 @@ fn corrected_legacy_backfill_is_not_requeued_after_application() {
}
#[tokio::test]
async fn mysql_backfills_are_empty_until_driver_specific_backfills_exist() {
let pool = sqlx::mysql::MySqlPoolOptions::new().connect_lazy_with(
"mysql://user:pass@localhost:3306/aether"
.parse()
.expect("mysql options should parse"),
async fn mysql_backfills_apply_portable_repairs_when_url_is_set() {
let Some(database_url) = std::env::var("AETHER_TEST_MYSQL_URL")
.ok()
.filter(|value| !value.trim().is_empty())
else {
eprintln!("skipping mysql backfill test because AETHER_TEST_MYSQL_URL is unset");
return;
};
let pool = sqlx::mysql::MySqlPoolOptions::new()
.max_connections(1)
.connect(&database_url)
.await
.expect("mysql backfill test pool should connect");
let mut conn = pool
.acquire()
.await
.expect("mysql backfill test connection should acquire");
sqlx::raw_sql(
r#"
CREATE TEMPORARY TABLE schema_backfills (
version BIGINT PRIMARY KEY,
description TEXT NOT NULL,
success BOOLEAN NOT NULL,
checksum BLOB NOT NULL,
execution_time BIGINT NOT NULL,
applied_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6)
);
CREATE TEMPORARY TABLE api_keys (
id VARCHAR(64) PRIMARY KEY,
total_requests BIGINT NOT NULL DEFAULT 0,
total_tokens BIGINT NOT NULL DEFAULT 0,
total_cost_usd DOUBLE NOT NULL DEFAULT 0,
last_used_at BIGINT
);
CREATE TEMPORARY TABLE provider_api_keys (
id VARCHAR(64) PRIMARY KEY,
total_tokens BIGINT NOT NULL DEFAULT 0
);
CREATE TEMPORARY TABLE global_models (
id VARCHAR(64) PRIMARY KEY,
name VARCHAR(255) NOT NULL,
usage_count BIGINT NOT NULL DEFAULT 0,
updated_at BIGINT NOT NULL
);
CREATE TEMPORARY TABLE providers (
id VARCHAR(64) PRIMARY KEY,
enabled BOOLEAN NOT NULL,
is_active BOOLEAN NOT NULL
);
CREATE TEMPORARY TABLE provider_endpoints (
id VARCHAR(64) PRIMARY KEY,
enabled BOOLEAN NOT NULL,
is_active BOOLEAN NOT NULL
);
CREATE TEMPORARY TABLE models (
id VARCHAR(64) PRIMARY KEY,
enabled BOOLEAN NOT NULL,
is_active BOOLEAN NOT NULL
);
CREATE TEMPORARY TABLE `usage` (
request_id VARCHAR(128) PRIMARY KEY,
api_key_id VARCHAR(64),
provider_api_key_id VARCHAR(64),
model VARCHAR(255),
status VARCHAR(64) NOT NULL,
total_tokens BIGINT NOT NULL DEFAULT 0,
input_tokens BIGINT NOT NULL DEFAULT 0,
output_tokens BIGINT NOT NULL DEFAULT 0,
cache_creation_input_tokens BIGINT NOT NULL DEFAULT 0,
cache_creation_input_tokens_5m BIGINT NOT NULL DEFAULT 0,
cache_creation_input_tokens_1h BIGINT NOT NULL DEFAULT 0,
cache_creation_ephemeral_5m_input_tokens BIGINT NOT NULL DEFAULT 0,
cache_creation_ephemeral_1h_input_tokens BIGINT NOT NULL DEFAULT 0,
cache_read_input_tokens BIGINT NOT NULL DEFAULT 0,
endpoint_api_format VARCHAR(64),
api_format VARCHAR(64),
total_cost_usd DOUBLE NOT NULL DEFAULT 0,
created_at BIGINT,
created_at_unix_ms BIGINT NOT NULL DEFAULT 0,
updated_at_unix_secs BIGINT NOT NULL DEFAULT 0
);
CREATE TEMPORARY TABLE usage_settlement_snapshots (
request_id VARCHAR(128) PRIMARY KEY,
billing_effective_input_tokens BIGINT,
billing_output_tokens BIGINT,
billing_cache_creation_tokens BIGINT,
billing_cache_creation_5m_tokens BIGINT,
billing_cache_creation_1h_tokens BIGINT,
billing_cache_read_tokens BIGINT,
billing_total_input_context BIGINT
);
INSERT INTO api_keys (id, total_requests, total_tokens, total_cost_usd)
VALUES ('mysql-backfill-api-key', 77, 7777, 77.0);
INSERT INTO provider_api_keys (id, total_tokens)
VALUES ('mysql-backfill-provider-key', 7777);
INSERT INTO global_models (id, name, usage_count, updated_at)
VALUES ('mysql-backfill-model', 'gpt-portable', 77, 1);
INSERT INTO providers (id, enabled, is_active)
VALUES ('mysql-backfill-provider', TRUE, FALSE);
INSERT INTO provider_endpoints (id, enabled, is_active)
VALUES ('mysql-backfill-endpoint', TRUE, FALSE);
INSERT INTO models (id, enabled, is_active)
VALUES ('mysql-backfill-provider-model', TRUE, FALSE);
INSERT INTO `usage` (
request_id,
api_key_id,
provider_api_key_id,
model,
status,
total_tokens,
input_tokens,
output_tokens,
cache_read_input_tokens,
api_format,
total_cost_usd,
created_at,
created_at_unix_ms,
updated_at_unix_secs
) VALUES
(
'mysql-backfill-completed',
'mysql-backfill-api-key',
'mysql-backfill-provider-key',
'gpt-portable',
'completed',
0,
120,
30,
20,
'openai',
1.25,
1714979289,
1714979289,
1714979289
),
(
'mysql-backfill-pending',
'mysql-backfill-api-key',
'mysql-backfill-provider-key',
'gpt-portable',
'pending',
777,
700,
77,
0,
'openai',
0.25,
1714979349,
1714979349,
1714979349
);
INSERT INTO usage_settlement_snapshots (
request_id,
billing_effective_input_tokens,
billing_output_tokens,
billing_cache_creation_tokens,
billing_cache_read_tokens
) VALUES ('mysql-backfill-completed', 100, 30, 10, 20);
"#,
)
.execute(&mut *conn)
.await
.expect("mysql temporary backfill schema should initialize");
drop(conn);
let pending_versions = pending_mysql_backfills(&pool)
.await
.expect("mysql pending backfills should load")
.into_iter()
.map(|item| item.version)
.collect::<Vec<_>>();
assert_eq!(
pending_mysql_backfills(&pool)
.await
.expect("mysql pending backfills should load"),
Vec::new()
pending_versions,
vec![
20260422120000,
20260505120000,
20260517012000,
20260716010000
]
);
run_mysql_backfills(&pool)
.await
.expect("mysql backfills should no-op");
.expect("mysql backfills should apply");
assert!(pending_mysql_backfills(&pool)
.await
.expect("mysql pending backfills should reload")
.is_empty());
let api_key_stats: (i64, i64, f64, Option<i64>) = query_as(
"SELECT total_requests, total_tokens, total_cost_usd, last_used_at FROM api_keys WHERE id = 'mysql-backfill-api-key'",
)
.fetch_one(&pool)
.await
.expect("mysql api key backfill result should load");
assert_eq!(api_key_stats, (2, 160, 1.5, Some(1714979349)));
let provider_total_tokens: i64 = query_scalar(
"SELECT total_tokens FROM provider_api_keys WHERE id = 'mysql-backfill-provider-key'",
)
.fetch_one(&pool)
.await
.expect("mysql provider key total should load");
assert_eq!(provider_total_tokens, 160);
let global_usage_count: i64 =
query_scalar("SELECT usage_count FROM global_models WHERE id = 'mysql-backfill-model'")
.fetch_one(&pool)
.await
.expect("mysql global model count should load");
assert_eq!(global_usage_count, 1);
for table in ["providers", "provider_endpoints", "models"] {
let enabled: bool = query_scalar(&format!(
"SELECT enabled FROM {table} WHERE is_active = FALSE"
))
.fetch_one(&pool)
.await
.unwrap_or_else(|error| panic!("mysql {table} legacy flag should load: {error}"));
assert!(!enabled, "mysql {table}.enabled should follow is_active");
}
}
#[tokio::test]
async fn sqlite_backfills_are_empty_until_driver_specific_backfills_exist() {
let config = SqlDatabaseConfig::new(
DatabaseDriver::Sqlite,
"sqlite::memory:",
SqlPoolConfig::default(),
async fn sqlite_backfills_apply_portable_repairs_and_record_versions() {
let pool = sqlx::sqlite::SqlitePoolOptions::new()
.max_connections(1)
.connect("sqlite::memory:")
.await
.expect("sqlite backfill test pool should connect");
run_sqlite_migrations(&pool)
.await
.expect("sqlite schema should migrate");
query(
r#"
INSERT INTO api_keys (
id, user_id, key_hash, total_requests, total_tokens, total_cost_usd, created_at, updated_at
) VALUES (
'sqlite-backfill-api-key', 'sqlite-backfill-user', 'sqlite-backfill-hash',
77, 7777, 77.0, 1, 1
)
"#,
)
.expect("sqlite config should build");
let pool = crate::driver::sqlite::SqlitePoolFactory::new(config)
.expect("sqlite factory should build")
.connect_lazy()
.expect("sqlite pool should build");
.execute(&pool)
.await
.expect("sqlite api key fixture should insert");
query(
r#"
INSERT INTO provider_api_keys (
id, provider_id, name, total_tokens, created_at, updated_at
) VALUES (
'sqlite-backfill-provider-key', 'sqlite-backfill-provider', 'Portable key', 7777, 1, 1
)
"#,
)
.execute(&pool)
.await
.expect("sqlite provider key fixture should insert");
query(
r#"
INSERT INTO global_models (
id, name, display_name, usage_count, created_at, updated_at
) VALUES (
'sqlite-backfill-model', 'gpt-portable', 'GPT Portable', 77, 1, 1
)
"#,
)
.execute(&pool)
.await
.expect("sqlite global model fixture should insert");
query(
r#"
INSERT INTO providers (
id, name, provider_type, enabled, is_active, created_at, updated_at
) VALUES (
'sqlite-backfill-provider', 'SQLite Backfill Provider', 'openai', 1, 0, 1, 1
)
"#,
)
.execute(&pool)
.await
.expect("sqlite provider flag fixture should insert");
query(
r#"
INSERT INTO provider_endpoints (
id, provider_id, name, base_url, enabled, is_active, created_at, updated_at
) VALUES (
'sqlite-backfill-endpoint', 'sqlite-backfill-provider', 'Default',
'https://example.invalid', 1, 0, 1, 1
)
"#,
)
.execute(&pool)
.await
.expect("sqlite provider endpoint flag fixture should insert");
query(
r#"
INSERT INTO models (
id, provider_id, provider_model_name, enabled, is_active, created_at, updated_at
) VALUES (
'sqlite-backfill-provider-model', 'sqlite-backfill-provider', 'gpt-portable',
1, 0, 1, 1
)
"#,
)
.execute(&pool)
.await
.expect("sqlite model flag fixture should insert");
query(
r#"
INSERT INTO "usage" (
request_id,
api_key_id,
provider_api_key_id,
model,
status,
total_tokens,
input_tokens,
output_tokens,
cache_read_input_tokens,
api_format,
total_cost_usd,
created_at,
created_at_unix_ms,
updated_at_unix_secs
) VALUES
(
'sqlite-backfill-completed',
'sqlite-backfill-api-key',
'sqlite-backfill-provider-key',
'gpt-portable',
'completed',
0,
120,
30,
20,
'openai',
1.25,
1714979289,
1714979289,
1714979289
),
(
'sqlite-backfill-pending',
'sqlite-backfill-api-key',
'sqlite-backfill-provider-key',
'gpt-portable',
'pending',
777,
700,
77,
0,
'openai',
0.25,
1714979349,
1714979349,
1714979349
)
"#,
)
.execute(&pool)
.await
.expect("sqlite usage fixtures should insert");
query(
r#"
INSERT INTO usage_settlement_snapshots (
request_id,
billing_status,
billing_effective_input_tokens,
billing_output_tokens,
billing_cache_creation_tokens,
billing_cache_read_tokens,
created_at,
updated_at
) VALUES (
'sqlite-backfill-completed', 'settled', 100, 30, 10, 20, 1, 1
)
"#,
)
.execute(&pool)
.await
.expect("sqlite settlement fixture should insert");
let pending_versions = pending_sqlite_backfills(&pool)
.await
.expect("sqlite pending backfills should load")
.into_iter()
.map(|item| item.version)
.collect::<Vec<_>>();
assert_eq!(
pending_sqlite_backfills(&pool)
.await
.expect("sqlite pending backfills should load"),
Vec::new()
pending_versions,
vec![
20260422120000,
20260505120000,
20260517012000,
20260716010000
]
);
run_sqlite_backfills(&pool)
.await
.expect("sqlite backfills should no-op");
.expect("sqlite backfills should apply");
assert!(pending_sqlite_backfills(&pool)
.await
.expect("sqlite pending backfills should reload")
.is_empty());
let applied_versions: Vec<i64> =
query_scalar("SELECT version FROM schema_backfills ORDER BY version")
.fetch_all(&pool)
.await
.expect("sqlite applied backfill versions should load");
assert_eq!(
applied_versions,
vec![
20260422120000,
20260505120000,
20260517012000,
20260716010000
]
);
let api_key_stats: (i64, i64, f64, Option<i64>) = query_as(
"SELECT total_requests, total_tokens, total_cost_usd, last_used_at FROM api_keys WHERE id = 'sqlite-backfill-api-key'",
)
.fetch_one(&pool)
.await
.expect("sqlite api key backfill result should load");
assert_eq!(api_key_stats, (2, 160, 1.5, Some(1714979349)));
let provider_total_tokens: i64 = query_scalar(
"SELECT total_tokens FROM provider_api_keys WHERE id = 'sqlite-backfill-provider-key'",
)
.fetch_one(&pool)
.await
.expect("sqlite provider key total should load");
assert_eq!(provider_total_tokens, 160);
let global_usage_count: i64 =
query_scalar("SELECT usage_count FROM global_models WHERE id = 'sqlite-backfill-model'")
.fetch_one(&pool)
.await
.expect("sqlite global model count should load");
assert_eq!(global_usage_count, 1);
for table in ["providers", "provider_endpoints", "models"] {
let enabled: i64 =
query_scalar(&format!("SELECT enabled FROM {table} WHERE is_active = 0"))
.fetch_one(&pool)
.await
.unwrap_or_else(|error| panic!("sqlite {table} legacy flag should load: {error}"));
assert_eq!(enabled, 0, "sqlite {table}.enabled should follow is_active");
}
run_sqlite_backfills(&pool)
.await
.expect("sqlite backfills should be idempotent");
let applied_count: i64 = query_scalar("SELECT COUNT(*) FROM schema_backfills")
.fetch_one(&pool)
.await
.expect("sqlite applied backfill count should load");
assert_eq!(applied_count, 4);
query("UPDATE schema_backfills SET checksum = X'00' WHERE version = 20260422120000")
.execute(&pool)
.await
.expect("sqlite checksum compatibility fixture should update");
assert!(pending_sqlite_backfills(&pool)
.await
.expect("checksum drift should retain the postgres compatibility policy")
.is_empty());
query(
r#"
INSERT INTO schema_backfills (
version, description, success, checksum, execution_time
) VALUES (
99999999999999, 'missing embedded backfill', 1, X'', 0
)
"#,
)
.execute(&pool)
.await
.expect("unknown sqlite backfill fixture should insert");
let error = pending_sqlite_backfills(&pool)
.await
.expect_err("unknown applied sqlite backfill should fail validation");
assert!(matches!(
error,
sqlx::migrate::MigrateError::VersionMissing(99999999999999)
));
}
#[tokio::test]
async fn sqlite_backfill_sql_and_version_record_commit_atomically() {
let pool = sqlx::sqlite::SqlitePoolOptions::new()
.max_connections(1)
.connect("sqlite::memory:")
.await
.expect("sqlite backfill transaction test pool should connect");
run_sqlite_migrations(&pool)
.await
.expect("sqlite schema should migrate");
query(
r#"
INSERT INTO global_models (
id, name, display_name, usage_count, created_at, updated_at
) VALUES (
'sqlite-backfill-rollback-model', 'rollback-model', 'Rollback Model', 77, 1, 1
)
"#,
)
.execute(&pool)
.await
.expect("sqlite rollback global model fixture should insert");
query(
r#"
CREATE TRIGGER reject_global_model_backfill
BEFORE UPDATE OF usage_count ON global_models
BEGIN
SELECT RAISE(ABORT, 'forced global model backfill failure');
END
"#,
)
.execute(&pool)
.await
.expect("sqlite rollback trigger should create");
run_sqlite_backfills(&pool)
.await
.expect_err("forced sqlite backfill failure should propagate");
let applied_versions: Vec<i64> =
query_scalar("SELECT version FROM schema_backfills ORDER BY version")
.fetch_all(&pool)
.await
.expect("sqlite partial applied versions should load");
assert_eq!(applied_versions, vec![20260422120000]);
let usage_count: i64 = query_scalar(
"SELECT usage_count FROM global_models WHERE id = 'sqlite-backfill-rollback-model'",
)
.fetch_one(&pool)
.await
.expect("sqlite rolled back global model should load");
assert_eq!(usage_count, 77);
query("DROP TRIGGER reject_global_model_backfill")
.execute(&pool)
.await
.expect("sqlite rollback trigger should drop");
run_sqlite_backfills(&pool)
.await
.expect("sqlite backfills should resume after the failed transaction");
let applied_count: i64 = query_scalar("SELECT COUNT(*) FROM schema_backfills")
.fetch_one(&pool)
.await
.expect("sqlite resumed applied backfill count should load");
assert_eq!(applied_count, 4);
}
#[derive(Debug)]
crates/aether-data/runtime/src/lifecycle/export.rs
+581 -57
View File
@@ -3,6 +3,8 @@ use std::collections::{BTreeMap, BTreeSet};
#[cfg(all(feature = "postgres", feature = "sqlite"))]
use futures_util::TryStreamExt;
use serde_json::Value;
#[cfg(all(feature = "postgres", feature = "sqlite"))]
use sqlx::Acquire;
use sqlx::Row;
#[cfg(any(feature = "mysql", feature = "sqlite"))]
use sqlx::{Column, TypeInfo, ValueRef};
@@ -41,7 +43,8 @@ use postgres::{
#[cfg(all(test, feature = "postgres", feature = "mysql", feature = "sqlite"))]
use postgres::normalize_postgres_import_payload;
pub const EXPORT_FORMAT_VERSION: u32 = 1;
pub const EXPORT_FORMAT_VERSION: u32 = 2;
const MIN_SUPPORTED_EXPORT_FORMAT_VERSION: u32 = 1;
#[derive(
Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize,
@@ -65,6 +68,7 @@ pub enum ExportDomain {
Wallets,
Usage,
Billing,
Auxiliary,
}
impl ExportDomain {
@@ -87,10 +91,296 @@ impl ExportDomain {
Self::Wallets => "wallets",
Self::Usage => "usage",
Self::Billing => "billing",
Self::Auxiliary => "auxiliary",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct AuxiliaryTable {
name: &'static str,
primary_key: &'static [&'static str],
}
const AUXILIARY_TABLES: &[AuxiliaryTable] = &[
AuxiliaryTable {
name: "audit_logs",
primary_key: &["id"],
},
AuxiliaryTable {
name: "announcements",
primary_key: &["id"],
},
AuxiliaryTable {
name: "announcement_reads",
primary_key: &["id"],
},
AuxiliaryTable {
name: "management_tokens",
primary_key: &["id"],
},
AuxiliaryTable {
name: "user_preferences",
primary_key: &["id"],
},
AuxiliaryTable {
name: "user_sessions",
primary_key: &["id"],
},
AuxiliaryTable {
name: "ldap_configs",
primary_key: &["id"],
},
AuxiliaryTable {
name: "pool_member_scores",
primary_key: &["id"],
},
AuxiliaryTable {
name: "api_key_provider_mappings",
primary_key: &["id"],
},
AuxiliaryTable {
name: "provider_usage_tracking",
primary_key: &["id"],
},
AuxiliaryTable {
name: "gemini_file_mappings",
primary_key: &["id"],
},
AuxiliaryTable {
name: "routing_groups",
primary_key: &["id"],
},
AuxiliaryTable {
name: "routing_group_versions",
primary_key: &["id"],
},
AuxiliaryTable {
name: "routing_group_bindings",
primary_key: &["id"],
},
AuxiliaryTable {
name: "proxy_node_events",
primary_key: &["id"],
},
AuxiliaryTable {
name: "proxy_node_metrics_1m",
primary_key: &["node_id", "bucket_start_unix_secs"],
},
AuxiliaryTable {
name: "proxy_node_metrics_1h",
primary_key: &["node_id", "bucket_start_unix_secs"],
},
AuxiliaryTable {
name: "user_invite_codes",
primary_key: &["user_id"],
},
AuxiliaryTable {
name: "user_referrals",
primary_key: &["id"],
},
AuxiliaryTable {
name: "referral_rewards",
primary_key: &["id"],
},
AuxiliaryTable {
name: "payment_gateway_configs",
primary_key: &["provider"],
},
AuxiliaryTable {
name: "billing_plans",
primary_key: &["id"],
},
AuxiliaryTable {
name: "user_plan_entitlements",
primary_key: &["id"],
},
AuxiliaryTable {
name: "entitlement_usage_ledgers",
primary_key: &["id"],
},
AuxiliaryTable {
name: "request_candidates",
primary_key: &["id"],
},
AuxiliaryTable {
name: "video_tasks",
primary_key: &["id"],
},
AuxiliaryTable {
name: "usage_body_blobs",
primary_key: &["body_ref"],
},
AuxiliaryTable {
name: "usage_http_audits",
primary_key: &["request_id"],
},
AuxiliaryTable {
name: "usage_routing_snapshots",
primary_key: &["request_id"],
},
AuxiliaryTable {
name: "usage_counter_deltas",
primary_key: &["id"],
},
AuxiliaryTable {
name: "background_task_runs",
primary_key: &["id"],
},
AuxiliaryTable {
name: "background_task_events",
primary_key: &["id"],
},
AuxiliaryTable {
name: "stats_hourly",
primary_key: &["id"],
},
AuxiliaryTable {
name: "stats_summary",
primary_key: &["id"],
},
AuxiliaryTable {
name: "stats_hourly_user",
primary_key: &["id"],
},
AuxiliaryTable {
name: "stats_hourly_user_model",
primary_key: &["id"],
},
AuxiliaryTable {
name: "user_model_usage_counts",
primary_key: &["id"],
},
AuxiliaryTable {
name: "stats_hourly_model",
primary_key: &["id"],
},
AuxiliaryTable {
name: "stats_hourly_provider",
primary_key: &["id"],
},
AuxiliaryTable {
name: "stats_daily",
primary_key: &["id"],
},
AuxiliaryTable {
name: "stats_daily_model",
primary_key: &["id"],
},
AuxiliaryTable {
name: "stats_daily_provider",
primary_key: &["id"],
},
AuxiliaryTable {
name: "stats_daily_api_key",
primary_key: &["id"],
},
AuxiliaryTable {
name: "stats_daily_error",
primary_key: &["id"],
},
AuxiliaryTable {
name: "stats_user_daily",
primary_key: &["id"],
},
AuxiliaryTable {
name: "stats_user_summary",
primary_key: &["id"],
},
AuxiliaryTable {
name: "stats_user_daily_model",
primary_key: &["id"],
},
AuxiliaryTable {
name: "stats_user_daily_provider",
primary_key: &["id"],
},
AuxiliaryTable {
name: "stats_user_daily_api_format",
primary_key: &["id"],
},
AuxiliaryTable {
name: "stats_daily_model_provider",
primary_key: &["id"],
},
AuxiliaryTable {
name: "stats_user_daily_model_provider",
primary_key: &["id"],
},
AuxiliaryTable {
name: "stats_daily_cost_savings",
primary_key: &["id"],
},
AuxiliaryTable {
name: "stats_daily_cost_savings_provider",
primary_key: &["id"],
},
AuxiliaryTable {
name: "stats_daily_cost_savings_model",
primary_key: &["id"],
},
AuxiliaryTable {
name: "stats_daily_cost_savings_model_provider",
primary_key: &["id"],
},
AuxiliaryTable {
name: "stats_user_daily_cost_savings",
primary_key: &["id"],
},
AuxiliaryTable {
name: "stats_user_daily_cost_savings_provider",
primary_key: &["id"],
},
AuxiliaryTable {
name: "stats_user_daily_cost_savings_model",
primary_key: &["id"],
},
AuxiliaryTable {
name: "stats_user_daily_cost_savings_model_provider",
primary_key: &["id"],
},
];
fn auxiliary_table(table_name: &str) -> Result<AuxiliaryTable, DataLayerError> {
AUXILIARY_TABLES
.iter()
.copied()
.find(|table| table.name == table_name)
.ok_or_else(|| {
DataLayerError::InvalidInput(format!(
"unsupported auxiliary export table '{table_name}'"
))
})
}
fn auxiliary_row_id(table: AuxiliaryTable, payload: &Value) -> Result<String, DataLayerError> {
let object = payload.as_object().ok_or_else(|| {
DataLayerError::UnexpectedValue(format!(
"auxiliary export row in table '{}' is not a JSON object",
table.name
))
})?;
let key = table
.primary_key
.iter()
.map(|column| {
object
.get(*column)
.filter(|value| !value.is_null())
.cloned()
.ok_or_else(|| {
DataLayerError::UnexpectedValue(format!(
"auxiliary export row in table '{}' has null or missing primary key column '{}'",
table.name, column
))
})
})
.collect::<Result<Vec<_>, _>>()?;
let encoded = serde_json::to_string(&key)
.map_err(|err| DataLayerError::UnexpectedValue(err.to_string()))?;
Ok(format!("{}:{encoded}", table.name))
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct DataExportManifest {
pub format_version: u32,
@@ -229,11 +519,167 @@ const USAGE_REQUEST_BODY_DETAIL_COLUMNS: &[&str] = &[
"client_response_body_compressed",
];
const USAGE_HTTP_BODY_DETAIL_COLUMNS: &[&str] = &[
"request_body_ref",
"provider_request_body_ref",
"response_body_ref",
"client_response_body_ref",
"request_body_state",
"provider_request_body_state",
"response_body_state",
"client_response_body_state",
"body_capture_mode",
];
#[cfg(all(feature = "postgres", feature = "sqlite"))]
const REQUEST_BODY_DETAIL_TABLES: &[&str] = &["usage_body_blobs", "usage_http_audits"];
const REQUEST_BODY_DETAIL_TABLES: &[&str] = &["usage_body_blobs"];
#[cfg(all(feature = "postgres", feature = "sqlite"))]
const LIFECYCLE_TABLES: &[&str] = &["_sqlx_migrations", "schema_backfills"];
fn import_column_stores_timestamp(column_name: &str) -> bool {
column_name.ends_with("_at")
|| column_name.ends_with("_unix_secs")
|| column_name.ends_with("_unix_ms")
|| column_name.ends_with("_date")
|| matches!(
column_name,
"start_time" | "end_time" | "window_start" | "window_end" | "hour_utc" | "date"
)
}
fn import_timestamp_uses_millis(table_name: &str, column_name: &str) -> bool {
if !column_name.ends_with("_unix_ms") {
return false;
}
// This legacy field is named `_unix_ms`, but every repository and API path
// has always stored and consumed it as Unix seconds.
let relation_name = table_name
.rsplit('.')
.next()
.unwrap_or(table_name)
.trim_matches(['"', '`']);
!(relation_name == "usage" && column_name == "created_at_unix_ms")
}
fn normalize_imported_integer_timestamp(
driver_name: &str,
table_name: &str,
column_name: &str,
value: &Value,
) -> Result<Option<i64>, DataLayerError> {
let invalid = || {
DataLayerError::InvalidInput(format!(
"{driver_name} import timestamp column '{column_name}' must contain an integer or supported datetime"
))
};
let timestamp = match value {
Value::Null => return Ok(None),
Value::Number(value) => value
.as_i64()
.or_else(|| value.as_u64().and_then(|value| i64::try_from(value).ok()))
.ok_or_else(invalid)?,
Value::String(value) => {
if let Ok(timestamp) = value.trim().parse::<i64>() {
timestamp
} else {
let datetime = parse_imported_datetime(value).ok_or_else(invalid)?;
if import_timestamp_uses_millis(table_name, column_name) {
datetime.timestamp_millis()
} else {
datetime.timestamp()
}
}
}
Value::Bool(_) | Value::Array(_) | Value::Object(_) => return Err(invalid()),
};
Ok(Some(timestamp))
}
fn parse_imported_datetime(value: &str) -> Option<chrono::DateTime<chrono::Utc>> {
let value = value.trim();
if let Ok(datetime) = chrono::DateTime::parse_from_rfc3339(value) {
return Some(datetime.with_timezone(&chrono::Utc));
}
if let Ok(datetime) = chrono::DateTime::parse_from_str(value, "%Y-%m-%d %H:%M:%S%.f%:z") {
return Some(datetime.with_timezone(&chrono::Utc));
}
for format in ["%Y-%m-%d %H:%M:%S%.f", "%Y-%m-%dT%H:%M:%S%.f"] {
if let Ok(datetime) = chrono::NaiveDateTime::parse_from_str(value, format) {
return Some(datetime.and_utc());
}
}
chrono::NaiveDate::parse_from_str(value, "%Y-%m-%d")
.ok()
.and_then(|date| date.and_hms_opt(0, 0, 0))
.map(|datetime| datetime.and_utc())
}
#[cfg(any(feature = "mysql", feature = "postgres", feature = "sqlite"))]
fn normalize_imported_binary(
driver_name: &str,
column_name: &str,
value: &Value,
) -> Result<Option<Vec<u8>>, DataLayerError> {
let invalid = |detail: &str| {
DataLayerError::InvalidInput(format!(
"{driver_name} import binary column '{column_name}' {detail}"
))
};
match value {
Value::Null => Ok(None),
Value::Array(values) => values
.iter()
.map(|value| {
value
.as_u64()
.and_then(|value| u8::try_from(value).ok())
.ok_or_else(|| invalid("contains a non-byte array value"))
})
.collect::<Result<Vec<_>, _>>()
.map(Some),
Value::String(value) => {
let encoded = value
.trim()
.strip_prefix("\\x")
.ok_or_else(|| invalid("must use PostgreSQL \\x hex encoding"))?;
if !encoded.len().is_multiple_of(2) {
return Err(invalid("contains odd-length hex data"));
}
let mut bytes = Vec::with_capacity(encoded.len() / 2);
for index in (0..encoded.len()).step_by(2) {
let byte = u8::from_str_radix(&encoded[index..index + 2], 16).map_err(|err| {
invalid(&format!(
"contains invalid hex data at byte {}: {err}",
index / 2
))
})?;
bytes.push(byte);
}
Ok(Some(bytes))
}
Value::Bool(_) | Value::Number(_) | Value::Object(_) => {
Err(invalid("must contain a byte array or PostgreSQL hex value"))
}
}
}
#[cfg(feature = "postgres")]
fn postgres_bytea_json_value(column_name: &str, value: &Value) -> Result<Value, DataLayerError> {
let Some(bytes) = normalize_imported_binary("postgres", column_name, value)? else {
return Ok(Value::Null);
};
let mut encoded = String::with_capacity(2 + bytes.len() * 2);
encoded.push_str("\\x");
for byte in bytes {
use std::fmt::Write as _;
write!(&mut encoded, "{byte:02x}")
.map_err(|err| DataLayerError::UnexpectedValue(err.to_string()))?;
}
Ok(Value::String(encoded))
}
pub fn encode_jsonl(records: &[DataExportRecord]) -> Result<String, DataLayerError> {
validate_export_records(records)?;
@@ -300,10 +746,12 @@ pub fn validate_export_records(records: &[DataExportRecord]) -> Result<(), DataL
"export JSONL must start with a manifest record".to_string(),
));
};
if manifest.format_version != EXPORT_FORMAT_VERSION {
if !(MIN_SUPPORTED_EXPORT_FORMAT_VERSION..=EXPORT_FORMAT_VERSION)
.contains(&manifest.format_version)
{
return Err(DataLayerError::InvalidInput(format!(
"unsupported export format version {}; expected {}",
manifest.format_version, EXPORT_FORMAT_VERSION
"unsupported export format version {}; supported versions are {} through {}",
manifest.format_version, MIN_SUPPORTED_EXPORT_FORMAT_VERSION, EXPORT_FORMAT_VERSION
)));
}
@@ -369,6 +817,7 @@ pub fn sqlite_core_export_domains() -> Vec<ExportDomain> {
ExportDomain::Wallets,
ExportDomain::Usage,
ExportDomain::Billing,
ExportDomain::Auxiliary,
]
}
@@ -490,23 +939,39 @@ pub async fn copy_database_records(
import_database_jsonl(target, &encode_jsonl(&records)?).await
}
fn omit_request_body_details_from_records(records: &mut [DataExportRecord]) {
for record in records {
fn omit_request_body_details_from_records(records: &mut Vec<DataExportRecord>) {
records.retain_mut(|record| {
let DataExportRecord::Row {
domain: ExportDomain::Usage,
payload,
..
domain, payload, ..
} = record
else {
continue;
return true;
};
if let Some(object) = payload.as_object_mut() {
for column_name in USAGE_REQUEST_BODY_DETAIL_COLUMNS {
object.remove(*column_name);
let Some(object) = payload.as_object_mut() else {
return true;
};
match *domain {
ExportDomain::Usage => {
for column_name in USAGE_REQUEST_BODY_DETAIL_COLUMNS {
object.remove(*column_name);
}
}
ExportDomain::Auxiliary
if object.get("__table").and_then(Value::as_str) == Some("usage_body_blobs") =>
{
return false;
}
ExportDomain::Auxiliary
if object.get("__table").and_then(Value::as_str) == Some("usage_http_audits") =>
{
for column_name in USAGE_HTTP_BODY_DETAIL_COLUMNS {
object.remove(*column_name);
}
}
_ => {}
}
}
true
});
}
#[cfg(all(feature = "postgres", feature = "sqlite"))]
@@ -522,24 +987,24 @@ async fn copy_postgres_to_sqlite_from_target_schema(
crate::driver::postgres::PostgresPoolFactory::new(source.to_postgres_config()?)?
.connect_lazy()?;
let sqlite_pool = crate::driver::sqlite::SqlitePoolFactory::new(target)?.connect_lazy()?;
let mut postgres_tx = postgres_pool.begin().await.map_sql_err()?;
sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY")
.execute(&mut *postgres_tx)
.await
.map_sql_err()?;
let source_tables = load_postgres_public_table_names(&postgres_pool).await?;
let source_tables = load_postgres_public_table_names(&mut postgres_tx).await?;
let target_tables = load_sqlite_copy_table_names(&sqlite_pool).await?;
ensure_no_nonempty_source_tables_outside_target_schema(
&postgres_pool,
&mut postgres_tx,
&source_tables,
&target_tables,
options,
)
.await?;
let mut imported = 0usize;
sqlx::raw_sql("PRAGMA foreign_keys = OFF")
.execute(&sqlite_pool)
.await
.map_sql_err()?;
let mut table_plans = Vec::new();
for table_name in target_tables {
if copy_table_is_lifecycle(&table_name)
|| copy_table_is_sqlite_internal(&table_name)
@@ -550,7 +1015,7 @@ async fn copy_postgres_to_sqlite_from_target_schema(
}
let table_plan = build_postgres_sqlite_copy_table_plan(
&postgres_pool,
&mut postgres_tx,
&sqlite_pool,
&table_name,
options,
@@ -559,22 +1024,39 @@ async fn copy_postgres_to_sqlite_from_target_schema(
if table_plan.columns.is_empty() {
continue;
}
imported = imported.saturating_add(
copy_postgres_sqlite_table(&postgres_pool, &sqlite_pool, &table_plan).await?,
);
table_plans.push(table_plan);
}
sqlx::raw_sql("PRAGMA foreign_keys = ON")
.execute(&sqlite_pool)
let mut connection = sqlite_pool.acquire().await.map_sql_err()?;
sqlx::raw_sql("PRAGMA foreign_keys = OFF")
.execute(&mut *connection)
.await
.map_sql_err()?;
ensure_sqlite_foreign_key_check_passes(&sqlite_pool).await?;
let copy_result = async {
let mut tx = connection.begin().await.map_sql_err()?;
let mut imported = 0usize;
for table_plan in &table_plans {
imported = imported.saturating_add(
copy_postgres_sqlite_table(&mut postgres_tx, &mut tx, table_plan).await?,
);
}
ensure_sqlite_foreign_key_check_passes(&mut tx).await?;
tx.commit().await.map_sql_err()?;
Ok::<_, DataLayerError>(imported)
}
.await;
sqlx::raw_sql("PRAGMA foreign_keys = ON")
.execute(&mut *connection)
.await
.map_sql_err()?;
let imported = copy_result?;
postgres_tx.commit().await.map_sql_err()?;
Ok(imported)
}
#[cfg(all(feature = "postgres", feature = "sqlite"))]
async fn ensure_no_nonempty_source_tables_outside_target_schema(
postgres_pool: &crate::driver::postgres::PostgresPool,
postgres_tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
source_tables: &BTreeSet<String>,
target_tables: &BTreeSet<String>,
options: DataCopyOptions,
@@ -587,7 +1069,7 @@ async fn ensure_no_nonempty_source_tables_outside_target_schema(
{
continue;
}
if postgres_public_table_has_rows(postgres_pool, table_name).await? {
if postgres_public_table_has_rows(postgres_tx, table_name).await? {
missing.push(table_name.clone());
}
}
@@ -603,15 +1085,15 @@ async fn ensure_no_nonempty_source_tables_outside_target_schema(
#[cfg(all(feature = "postgres", feature = "sqlite"))]
async fn build_postgres_sqlite_copy_table_plan(
postgres_pool: &crate::driver::postgres::PostgresPool,
postgres_tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
sqlite_pool: &crate::driver::sqlite::SqlitePool,
table_name: &str,
options: DataCopyOptions,
) -> Result<SchemaCopyTable, DataLayerError> {
let sqlite_columns = load_sqlite_copy_columns(sqlite_pool, table_name).await?;
let postgres_columns =
load_postgres_import_columns(postgres_pool, &format!("public.{table_name}")).await?;
let source_has_rows = postgres_public_table_has_rows(postgres_pool, table_name).await?;
load_postgres_import_columns(&mut **postgres_tx, &format!("public.{table_name}")).await?;
let source_has_rows = postgres_public_table_has_rows(postgres_tx, table_name).await?;
let mut columns = Vec::new();
for sqlite_column in sqlite_columns {
@@ -621,6 +1103,12 @@ async fn build_postgres_sqlite_copy_table_plan(
{
continue;
}
if options.omit_request_body_details
&& table_name == "usage_http_audits"
&& USAGE_HTTP_BODY_DETAIL_COLUMNS.contains(&sqlite_column.name.as_str())
{
continue;
}
if let Some(postgres_column) = postgres_columns.get(&sqlite_column.name) {
columns.push(SchemaCopyColumn {
@@ -652,13 +1140,13 @@ async fn build_postgres_sqlite_copy_table_plan(
#[cfg(all(feature = "postgres", feature = "sqlite"))]
async fn copy_postgres_sqlite_table(
postgres_pool: &crate::driver::postgres::PostgresPool,
sqlite_pool: &crate::driver::sqlite::SqlitePool,
postgres_tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
sqlite_tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
table: &SchemaCopyTable,
) -> Result<usize, DataLayerError> {
let source_sql = postgres_schema_copy_select_sql(table)?;
let target_sql = sqlite_schema_copy_insert_sql(table)?;
let mut rows = sqlx::query(&source_sql).fetch(postgres_pool);
let mut rows = sqlx::query(&source_sql).fetch(&mut **postgres_tx);
let mut imported = 0usize;
while let Some(row) = rows.try_next().await.map_sql_err()? {
@@ -679,7 +1167,7 @@ async fn copy_postgres_sqlite_table(
})?;
query = bind_sqlite_copy_value(query, value, &column.sqlite)?;
}
query.execute(sqlite_pool).await.map_sql_err()?;
query.execute(&mut **sqlite_tx).await.map_sql_err()?;
imported = imported.saturating_add(1);
}
@@ -694,7 +1182,7 @@ fn postgres_schema_copy_select_sql(table: &SchemaCopyTable) -> Result<String, Da
);
let mut payload_parts = Vec::new();
for column in &table.columns {
if let Some(expr) = postgres_schema_copy_override_expr(column)? {
if let Some(expr) = postgres_schema_copy_override_expr(&table.table_name, column)? {
payload_parts.push(sql_string_literal(&column.sqlite.name));
payload_parts.push(expr);
}
@@ -729,6 +1217,7 @@ fn postgres_schema_copy_select_sql(table: &SchemaCopyTable) -> Result<String, Da
#[cfg(all(feature = "postgres", feature = "sqlite"))]
fn postgres_schema_copy_override_expr(
table_name: &str,
column: &SchemaCopyColumn,
) -> Result<Option<String>, DataLayerError> {
let column_sql = format!("t.{}", postgres_quote_identifier(&column.sqlite.name)?);
@@ -755,7 +1244,7 @@ fn postgres_schema_copy_override_expr(
} else {
column_sql.clone()
};
let multiplier = if sqlite_copy_column_stores_unix_millis(&column.sqlite.name) {
let multiplier = if import_timestamp_uses_millis(table_name, &column.sqlite.name) {
" * 1000"
} else {
""
@@ -778,14 +1267,46 @@ fn sqlite_schema_copy_insert_sql(table: &SchemaCopyTable) -> Result<String, Data
.collect::<Result<Vec<_>, _>>()?
.join(", ");
let placeholder_sql = vec!["?"; table.columns.len()].join(", ");
let mut primary_key = table
.columns
.iter()
.filter(|column| column.sqlite.primary_key_position > 0)
.collect::<Vec<_>>();
primary_key.sort_by_key(|column| column.sqlite.primary_key_position);
if primary_key.is_empty() {
return Ok(format!(
"INSERT INTO {table_sql} ({column_sql}) VALUES ({placeholder_sql})"
));
}
let conflict_columns = primary_key
.iter()
.map(|column| sqlite_quote_identifier(&column.sqlite.name))
.collect::<Result<Vec<_>, _>>()?
.join(", ");
let update_sql = table
.columns
.iter()
.filter(|column| column.sqlite.primary_key_position == 0)
.map(|column| {
let quoted = sqlite_quote_identifier(&column.sqlite.name)?;
Ok(format!("{quoted} = excluded.{quoted}"))
})
.collect::<Result<Vec<_>, DataLayerError>>()?
.join(", ");
let conflict_sql = if update_sql.is_empty() {
format!("ON CONFLICT ({conflict_columns}) DO NOTHING")
} else {
format!("ON CONFLICT ({conflict_columns}) DO UPDATE SET {update_sql}")
};
Ok(format!(
"INSERT OR REPLACE INTO {table_sql} ({column_sql}) VALUES ({placeholder_sql})"
"INSERT INTO {table_sql} ({column_sql}) VALUES ({placeholder_sql}) {conflict_sql}"
))
}
#[cfg(all(feature = "postgres", feature = "sqlite"))]
async fn load_postgres_public_table_names(
pool: &crate::driver::postgres::PostgresPool,
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
) -> Result<BTreeSet<String>, DataLayerError> {
let rows = sqlx::query(
r#"
@@ -796,7 +1317,7 @@ WHERE table_schema = 'public'
ORDER BY table_name
"#,
)
.fetch_all(pool)
.fetch_all(&mut **tx)
.await
.map_sql_err()?;
@@ -872,24 +1393,24 @@ async fn load_sqlite_copy_columns(
#[cfg(all(feature = "postgres", feature = "sqlite"))]
async fn postgres_public_table_has_rows(
pool: &crate::driver::postgres::PostgresPool,
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
table_name: &str,
) -> Result<bool, DataLayerError> {
let table_sql = format!("public.{}", postgres_quote_identifier(table_name)?);
sqlx::query_scalar::<_, bool>(&format!(
"SELECT EXISTS (SELECT 1 FROM {table_sql} LIMIT 1)"
))
.fetch_one(pool)
.fetch_one(&mut **tx)
.await
.map_sql_err()
}
#[cfg(all(feature = "postgres", feature = "sqlite"))]
async fn ensure_sqlite_foreign_key_check_passes(
pool: &crate::driver::sqlite::SqlitePool,
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
) -> Result<(), DataLayerError> {
let rows = sqlx::query("PRAGMA foreign_key_check")
.fetch_all(pool)
.fetch_all(&mut **tx)
.await
.map_sql_err()?;
if rows.is_empty() {
@@ -935,11 +1456,6 @@ fn sqlite_copy_column_is_required(column: &SqliteCopyColumn) -> bool {
(column.not_null || column.primary_key_position > 0) && !column.has_default
}
#[cfg(all(feature = "postgres", feature = "sqlite"))]
fn sqlite_copy_column_stores_unix_millis(column_name: &str) -> bool {
column_name.ends_with("_unix_ms")
}
#[cfg(all(feature = "postgres", feature = "sqlite"))]
fn sqlite_copy_affinity(column: &SqliteCopyColumn) -> SqliteCopyAffinity {
let declared_type = column.declared_type.to_ascii_uppercase();
@@ -962,7 +1478,7 @@ fn sqlite_copy_affinity(column: &SqliteCopyColumn) -> SqliteCopyAffinity {
}
}
#[cfg(all(feature = "postgres", feature = "sqlite"))]
#[cfg(feature = "postgres")]
fn is_postgres_bytea_column(column: &PostgresImportColumn) -> bool {
column.data_type == "bytea" || column.udt_name == "bytea"
}
@@ -1194,7 +1710,17 @@ fn filter_import_payload(
for (column_name, value) in object {
if target_columns.contains(column_name) {
filtered.insert(column_name.clone(), value.clone());
continue;
}
if value.is_null() {
continue;
}
return Err(DataLayerError::InvalidInput(format!(
"{} export row '{}' contains column '{}' that does not exist in {driver_name} table '{table_name}'",
domain.as_str(),
row.id,
column_name
)));
}
if filtered.is_empty() {
@@ -1208,7 +1734,6 @@ fn filter_import_payload(
Ok(filtered)
}
#[cfg(any(feature = "mysql", feature = "sqlite"))]
fn payload_with_table(payload: Value, table_name: &str) -> Result<Value, DataLayerError> {
let mut object = payload.as_object().cloned().ok_or_else(|| {
DataLayerError::UnexpectedValue("export row payload must be a JSON object".to_string())
@@ -1218,7 +1743,6 @@ fn payload_with_table(payload: Value, table_name: &str) -> Result<Value, DataLay
Ok(Value::Object(object))
}
#[cfg(any(feature = "mysql", feature = "sqlite"))]
fn normalize_billing_payload(
table_name: &str,
object: &mut serde_json::Map<String, Value>,
crates/aether-data/runtime/src/lifecycle/export/mysql.rs
+300 -50
View File
@@ -1,4 +1,12 @@
use super::*;
use sqlx::Acquire;
#[derive(Debug, Clone, Default, PartialEq, Eq)]
struct MysqlImportColumns {
names: ImportColumnNames,
data_types: BTreeMap<String, String>,
primary_key: Vec<String>,
}
pub async fn export_mysql_core_jsonl(
pool: &crate::driver::mysql::MysqlPool,
@@ -12,6 +20,12 @@ pub async fn export_mysql_jsonl(
domains: Vec<ExportDomain>,
created_at_unix_secs: u64,
) -> Result<String, DataLayerError> {
let mut connection = pool.acquire().await.map_sql_err()?;
sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ")
.execute(&mut *connection)
.await
.map_sql_err()?;
let mut tx = connection.begin().await.map_sql_err()?;
let manifest = DataExportManifest::new(
created_at_unix_secs,
Some(DatabaseDriver::Mysql),
@@ -20,24 +34,29 @@ pub async fn export_mysql_jsonl(
let mut records = vec![DataExportRecord::manifest(manifest)];
for domain in domains {
if domain == ExportDomain::Auxiliary {
export_mysql_auxiliary_records(&mut tx, &mut records).await?;
continue;
}
if domain == ExportDomain::Billing {
export_mysql_billing_records(pool, &mut records).await?;
export_mysql_billing_records(&mut tx, &mut records).await?;
continue;
}
if domain == ExportDomain::Wallets {
export_mysql_wallet_records(pool, &mut records).await?;
export_mysql_wallet_records(&mut tx, &mut records).await?;
continue;
}
let (table_name, id_column) = mysql_domain_table(domain)?;
let order_by = export_order_by(domain, id_column);
let sql = format!("SELECT * FROM {table_name} ORDER BY {order_by}");
let rows = sqlx::query(&sql).fetch_all(pool).await.map_sql_err()?;
let rows = sqlx::query(&sql).fetch_all(&mut *tx).await.map_sql_err()?;
for row in rows {
let id = mysql_export_row_id(domain, &row, id_column)?;
records.push(DataExportRecord::row(domain, id, mysql_row_payload(&row)?));
}
}
tx.commit().await.map_sql_err()?;
encode_jsonl(&records)
}
@@ -53,31 +72,40 @@ pub async fn import_mysql_plan(
pool: &crate::driver::mysql::MysqlPool,
plan: &DataImportPlan,
) -> Result<usize, DataLayerError> {
let mut tx = pool.begin().await.map_sql_err()?;
let mut imported = 0usize;
let mut column_cache = BTreeMap::<String, ImportColumnNames>::new();
let mut column_cache = BTreeMap::<String, MysqlImportColumns>::new();
for domain in &plan.manifest.domains {
if *domain == ExportDomain::Auxiliary {
for row in plan.rows(*domain) {
import_mysql_auxiliary_row(&mut tx, row, &mut column_cache).await?;
imported = imported.saturating_add(1);
}
continue;
}
if *domain == ExportDomain::Billing {
for row in plan.rows(*domain) {
import_mysql_billing_row(pool, row, &mut column_cache).await?;
import_mysql_billing_row(&mut tx, row, &mut column_cache).await?;
imported = imported.saturating_add(1);
}
continue;
}
if *domain == ExportDomain::Wallets {
for row in plan.rows(*domain) {
import_mysql_wallet_row(pool, row, &mut column_cache).await?;
import_mysql_wallet_row(&mut tx, row, &mut column_cache).await?;
imported = imported.saturating_add(1);
}
continue;
}
let (table_name, _id_column) = mysql_domain_table(*domain)?;
let target_columns =
mysql_import_columns_cached(pool, &mut column_cache, table_name).await?;
mysql_import_columns_cached(&mut tx, &mut column_cache, table_name).await?;
for row in plan.rows(*domain) {
import_mysql_row(pool, table_name, *domain, row, &target_columns).await?;
import_mysql_row(&mut tx, table_name, *domain, row, &target_columns).await?;
imported = imported.saturating_add(1);
}
}
tx.commit().await.map_sql_err()?;
Ok(imported)
}
@@ -106,9 +134,42 @@ fn mysql_domain_table(
ExportDomain::Billing => Err(DataLayerError::InvalidInput(
"mysql billing export uses multiple tables and must be handled as a domain".to_string(),
)),
ExportDomain::Auxiliary => Err(DataLayerError::InvalidInput(
"mysql auxiliary export uses multiple tables and must be handled as a domain"
.to_string(),
)),
}
}
async fn export_mysql_auxiliary_records(
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
records: &mut Vec<DataExportRecord>,
) -> Result<(), DataLayerError> {
for table in AUXILIARY_TABLES {
let table_sql = mysql_quote_identifier(table.name)?;
let order_sql = table
.primary_key
.iter()
.map(|column| mysql_quote_identifier(column).map(|column| format!("{column} ASC")))
.collect::<Result<Vec<_>, _>>()?
.join(", ");
let rows = sqlx::query(&format!("SELECT * FROM {table_sql} ORDER BY {order_sql}"))
.fetch_all(&mut **tx)
.await
.map_sql_err()?;
for row in rows {
let payload = mysql_row_payload(&row)?;
let id = auxiliary_row_id(*table, &payload)?;
records.push(DataExportRecord::row(
ExportDomain::Auxiliary,
id,
payload_with_table(payload, table.name)?,
));
}
}
Ok(())
}
fn mysql_export_row_id(
domain: ExportDomain,
row: &sqlx::mysql::MySqlRow,
@@ -139,7 +200,7 @@ fn mysql_required_export_text(
}
async fn export_mysql_billing_records(
pool: &crate::driver::mysql::MysqlPool,
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
records: &mut Vec<DataExportRecord>,
) -> Result<(), DataLayerError> {
for (table_name, id_column) in [
@@ -148,7 +209,7 @@ async fn export_mysql_billing_records(
("usage_settlement_snapshots", "request_id"),
] {
let sql = format!("SELECT * FROM {table_name} ORDER BY {id_column} ASC");
let rows = sqlx::query(&sql).fetch_all(pool).await.map_sql_err()?;
let rows = sqlx::query(&sql).fetch_all(&mut **tx).await.map_sql_err()?;
for row in rows {
let id = row
.try_get::<Option<String>, _>(id_column)
@@ -169,12 +230,12 @@ async fn export_mysql_billing_records(
}
async fn export_mysql_wallet_records(
pool: &crate::driver::mysql::MysqlPool,
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
records: &mut Vec<DataExportRecord>,
) -> Result<(), DataLayerError> {
for (table_name, id_column) in mysql_wallet_tables() {
let sql = format!("SELECT * FROM {table_name} ORDER BY {id_column} ASC");
let rows = sqlx::query(&sql).fetch_all(pool).await.map_sql_err()?;
let rows = sqlx::query(&sql).fetch_all(&mut **tx).await.map_sql_err()?;
for row in rows {
let id = row
.try_get::<Option<String>, _>(id_column)
@@ -195,53 +256,115 @@ async fn export_mysql_wallet_records(
}
async fn import_mysql_row(
pool: &crate::driver::mysql::MysqlPool,
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
table_name: &str,
domain: ExportDomain,
row: &ExportRow,
target_columns: &ImportColumnNames,
target_columns: &MysqlImportColumns,
) -> Result<(), DataLayerError> {
let object = filter_import_payload("mysql", table_name, domain, row, target_columns)?;
let object = filter_import_payload("mysql", table_name, domain, row, &target_columns.names)?;
let columns = object.keys().map(String::as_str).collect::<Vec<_>>();
for primary_key in &target_columns.primary_key {
if object.get(primary_key).is_none_or(Value::is_null) {
return Err(DataLayerError::InvalidInput(format!(
"{} export row '{}' is missing non-null primary key column '{}' for mysql table '{}'",
domain.as_str(),
row.id,
primary_key,
table_name
)));
}
}
let primary_key_predicate = target_columns
.primary_key
.iter()
.map(|column| mysql_quote_identifier(column).map(|column| format!("{column} = ?")))
.collect::<Result<Vec<_>, _>>()?
.join(" AND ");
let lock_sql =
format!("SELECT 1 FROM {table_name} WHERE {primary_key_predicate} LIMIT 1 FOR UPDATE");
let mut lock_query = sqlx::query(&lock_sql);
for column in &target_columns.primary_key {
lock_query =
bind_mysql_import_column(lock_query, &object, target_columns, table_name, column)?;
}
let exists = lock_query
.fetch_optional(&mut **tx)
.await
.map_sql_err()?
.is_some();
if exists {
let update_columns = columns
.iter()
.copied()
.filter(|column| !target_columns.primary_key.iter().any(|key| key == column))
.collect::<Vec<_>>();
if update_columns.is_empty() {
return Ok(());
}
let update_sql = update_columns
.iter()
.map(|column| mysql_quote_identifier(column).map(|column| format!("{column} = ?")))
.collect::<Result<Vec<_>, _>>()?
.join(", ");
let sql = format!("UPDATE {table_name} SET {update_sql} WHERE {primary_key_predicate}");
let mut query = sqlx::query(&sql);
for column in update_columns {
query = bind_mysql_import_column(query, &object, target_columns, table_name, column)?;
}
for column in &target_columns.primary_key {
query = bind_mysql_import_column(query, &object, target_columns, table_name, column)?;
}
query.execute(&mut **tx).await.map_sql_err()?;
return Ok(());
}
let column_sql = columns
.iter()
.map(|column| mysql_quote_identifier(column))
.collect::<Result<Vec<_>, _>>()?
.join(", ");
let placeholder_sql = vec!["?"; columns.len()].join(", ");
let update_sql = columns
.iter()
.map(|column| {
let quoted = mysql_quote_identifier(column)?;
Ok(format!("{quoted} = VALUES({quoted})"))
})
.collect::<Result<Vec<_>, DataLayerError>>()?
.join(", ");
let sql = format!(
"INSERT INTO {table_name} ({column_sql}) VALUES ({placeholder_sql}) ON DUPLICATE KEY UPDATE {update_sql}"
);
let sql = format!("INSERT INTO {table_name} ({column_sql}) VALUES ({placeholder_sql})");
let mut query = sqlx::query(&sql);
for column in columns {
let value = object
.get(column)
.expect("column name came from payload object keys");
query = bind_mysql_json_value(query, value)?;
query = bind_mysql_import_column(query, &object, target_columns, table_name, column)?;
}
query.execute(pool).await.map_sql_err()?;
query.execute(&mut **tx).await.map_sql_err()?;
Ok(())
}
fn bind_mysql_import_column<'q>(
query: sqlx::query::Query<'q, sqlx::MySql, sqlx::mysql::MySqlArguments>,
object: &'q serde_json::Map<String, Value>,
target_columns: &MysqlImportColumns,
table_name: &str,
column: &str,
) -> Result<sqlx::query::Query<'q, sqlx::MySql, sqlx::mysql::MySqlArguments>, DataLayerError> {
let value = object
.get(column)
.expect("column name came from payload object keys");
let data_type = target_columns
.data_types
.get(column)
.map(String::as_str)
.unwrap_or_default();
bind_mysql_import_value(query, value, table_name, column, data_type)
}
async fn import_mysql_billing_row(
pool: &crate::driver::mysql::MysqlPool,
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
row: &ExportRow,
column_cache: &mut BTreeMap<String, ImportColumnNames>,
column_cache: &mut BTreeMap<String, MysqlImportColumns>,
) -> Result<(), DataLayerError> {
let (table_name, payload) = billing_payload_table(row)?;
let table_name = mysql_billing_table_name(&table_name)?;
let target_columns = mysql_import_columns_cached(pool, column_cache, table_name).await?;
let target_columns = mysql_import_columns_cached(tx, column_cache, table_name).await?;
import_mysql_row(
pool,
tx,
table_name,
ExportDomain::Billing,
&ExportRow {
@@ -253,6 +376,27 @@ async fn import_mysql_billing_row(
.await
}
async fn import_mysql_auxiliary_row(
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
row: &ExportRow,
column_cache: &mut BTreeMap<String, MysqlImportColumns>,
) -> Result<(), DataLayerError> {
let (table_name, payload) = domain_payload_table(row, "auxiliary", None)?;
let table = auxiliary_table(&table_name)?;
let target_columns = mysql_import_columns_cached(tx, column_cache, table.name).await?;
import_mysql_row(
tx,
table.name,
ExportDomain::Auxiliary,
&ExportRow {
id: row.id.clone(),
payload,
},
&target_columns,
)
.await
}
fn mysql_billing_table_name(table_name: &str) -> Result<&'static str, DataLayerError> {
match table_name {
"billing_rules" => Ok("billing_rules"),
@@ -265,15 +409,15 @@ fn mysql_billing_table_name(table_name: &str) -> Result<&'static str, DataLayerE
}
async fn import_mysql_wallet_row(
pool: &crate::driver::mysql::MysqlPool,
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
row: &ExportRow,
column_cache: &mut BTreeMap<String, ImportColumnNames>,
column_cache: &mut BTreeMap<String, MysqlImportColumns>,
) -> Result<(), DataLayerError> {
let (table_name, payload) = domain_payload_table(row, "wallet", Some("wallets"))?;
let table_name = mysql_wallet_table_name(&table_name)?;
let target_columns = mysql_import_columns_cached(pool, column_cache, table_name).await?;
let target_columns = mysql_import_columns_cached(tx, column_cache, table_name).await?;
import_mysql_row(
pool,
tx,
table_name,
ExportDomain::Wallets,
&ExportRow {
@@ -311,51 +455,130 @@ fn mysql_wallet_table_name(table_name: &str) -> Result<&'static str, DataLayerEr
}
async fn mysql_import_columns_cached(
pool: &crate::driver::mysql::MysqlPool,
cache: &mut BTreeMap<String, ImportColumnNames>,
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
cache: &mut BTreeMap<String, MysqlImportColumns>,
table_name: &str,
) -> Result<ImportColumnNames, DataLayerError> {
) -> Result<MysqlImportColumns, DataLayerError> {
if let Some(columns) = cache.get(table_name) {
return Ok(columns.clone());
}
let columns = load_mysql_import_columns(pool, table_name).await?;
let columns = load_mysql_import_columns(tx, table_name).await?;
cache.insert(table_name.to_string(), columns.clone());
Ok(columns)
}
async fn load_mysql_import_columns(
pool: &crate::driver::mysql::MysqlPool,
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
table_name: &str,
) -> Result<ImportColumnNames, DataLayerError> {
) -> Result<MysqlImportColumns, DataLayerError> {
let relation_name = table_name.trim_matches('`');
let rows = sqlx::query(
r#"
SELECT COLUMN_NAME AS column_name
SELECT
COLUMN_NAME AS column_name,
DATA_TYPE AS data_type,
COLUMN_KEY AS column_key,
ORDINAL_POSITION AS ordinal_position
FROM information_schema.columns
WHERE table_schema = DATABASE()
AND table_name = ?
"#,
)
.bind(relation_name)
.fetch_all(pool)
.fetch_all(&mut **tx)
.await
.map_sql_err()?;
let mut columns = ImportColumnNames::new();
let mut columns = MysqlImportColumns::default();
let mut primary_key = BTreeMap::new();
for row in rows {
columns.insert(row.try_get::<String, _>("column_name").map_sql_err()?);
let name = row.try_get::<String, _>("column_name").map_sql_err()?;
let data_type = row
.try_get::<String, _>("data_type")
.map_sql_err()?
.to_ascii_lowercase();
columns.names.insert(name.clone());
columns.data_types.insert(name.clone(), data_type);
if row
.try_get::<String, _>("column_key")
.map_sql_err()?
.eq_ignore_ascii_case("PRI")
{
primary_key.insert(
row.try_get::<i64, _>("ordinal_position").map_sql_err()?,
name,
);
}
}
if columns.is_empty() {
if columns.names.is_empty() {
return Err(DataLayerError::UnexpectedValue(format!(
"mysql import target table '{table_name}' has no visible columns"
)));
}
if primary_key.is_empty() {
return Err(DataLayerError::UnexpectedValue(format!(
"mysql import target table '{table_name}' has no primary key"
)));
}
columns.primary_key = primary_key.into_values().collect();
Ok(columns)
}
fn bind_mysql_import_value<'q>(
query: sqlx::query::Query<'q, sqlx::MySql, sqlx::mysql::MySqlArguments>,
json_value: &'q Value,
table_name: &str,
column_name: &str,
data_type: &str,
) -> Result<sqlx::query::Query<'q, sqlx::MySql, sqlx::mysql::MySqlArguments>, DataLayerError> {
if matches!(
data_type,
"binary" | "varbinary" | "blob" | "tinyblob" | "mediumblob" | "longblob"
) {
return match normalize_imported_binary("mysql", column_name, json_value)? {
Some(bytes) => Ok(query.bind(bytes)),
None => Ok(query.bind(Option::<Vec<u8>>::None)),
};
}
if matches!(data_type, "decimal" | "numeric") {
return match normalize_mysql_decimal_value(column_name, json_value)? {
Some(value) => Ok(query.bind(value)),
None => Ok(query.bind(Option::<String>::None)),
};
}
let has_integer_type = matches!(
data_type,
"tinyint" | "smallint" | "mediumint" | "int" | "integer" | "bigint"
);
if !has_integer_type || !import_column_stores_timestamp(column_name) {
return bind_mysql_json_value(query, json_value);
}
match normalize_imported_integer_timestamp("mysql", table_name, column_name, json_value)? {
Some(timestamp) => Ok(query.bind(timestamp)),
None => Ok(query.bind(Option::<i64>::None)),
}
}
fn normalize_mysql_decimal_value(
column_name: &str,
value: &Value,
) -> Result<Option<String>, DataLayerError> {
match value {
Value::Null => Ok(None),
Value::Number(value) => Ok(Some(value.to_string())),
Value::String(value) => Ok(Some(value.clone())),
Value::Bool(_) | Value::Array(_) | Value::Object(_) => {
Err(DataLayerError::InvalidInput(format!(
"mysql decimal import column '{column_name}' must contain a number or numeric string"
)))
}
}
}
fn mysql_quote_identifier(identifier: &str) -> Result<String, DataLayerError> {
if identifier.trim().is_empty() {
return Err(DataLayerError::InvalidInput(
@@ -454,3 +677,30 @@ fn mysql_value_to_json(row: &sqlx::mysql::MySqlRow, index: usize) -> Result<Valu
))),
}
}
#[cfg(test)]
mod tests {
use super::normalize_mysql_decimal_value;
use serde_json::json;
#[test]
fn decimal_import_binds_numbers_and_strings_as_decimal_text() {
let value = json!(12345.12345678);
assert_eq!(
normalize_mysql_decimal_value("billing_total_cost_usd", &value)
.expect("decimal value should normalize")
.as_deref(),
Some("12345.12345678")
);
assert_eq!(
normalize_mysql_decimal_value(
"billing_total_cost_usd",
&json!("123456789012.12345678")
)
.expect("decimal string should normalize")
.as_deref(),
Some("123456789012.12345678")
);
assert!(normalize_mysql_decimal_value("billing_total_cost_usd", &json!(true)).is_err());
}
}
crates/aether-data/runtime/src/lifecycle/export/postgres.rs
+173 -33
View File
@@ -12,6 +12,11 @@ pub async fn export_postgres_jsonl(
domains: Vec<ExportDomain>,
created_at_unix_secs: u64,
) -> Result<String, DataLayerError> {
let mut tx = pool.begin().await.map_sql_err()?;
sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY")
.execute(&mut *tx)
.await
.map_sql_err()?;
let manifest = DataExportManifest::new(
created_at_unix_secs,
Some(DatabaseDriver::Postgres),
@@ -20,12 +25,16 @@ pub async fn export_postgres_jsonl(
let mut records = vec![DataExportRecord::manifest(manifest)];
for domain in domains {
if domain == ExportDomain::Auxiliary {
export_postgres_auxiliary_records(&mut tx, &mut records).await?;
continue;
}
if domain == ExportDomain::Billing {
export_postgres_billing_records(pool, &mut records).await?;
export_postgres_billing_records(&mut tx, &mut records).await?;
continue;
}
if domain == ExportDomain::Wallets {
export_postgres_wallet_records(pool, &mut records).await?;
export_postgres_wallet_records(&mut tx, &mut records).await?;
continue;
}
let (table_name, id_column) = postgres_domain_table(domain)?;
@@ -34,7 +43,7 @@ pub async fn export_postgres_jsonl(
let sql = format!(
"SELECT {export_id_sql} AS export_id, to_jsonb(t) AS payload FROM {table_name} AS t ORDER BY {order_by}"
);
let rows = sqlx::query(&sql).fetch_all(pool).await.map_sql_err()?;
let rows = sqlx::query(&sql).fetch_all(&mut *tx).await.map_sql_err()?;
for row in rows {
let id = row.try_get::<String, _>("export_id").map_sql_err()?;
let payload = row.try_get::<Value, _>("payload").map_sql_err()?;
@@ -42,6 +51,7 @@ pub async fn export_postgres_jsonl(
}
}
tx.commit().await.map_sql_err()?;
encode_jsonl(&records)
}
@@ -57,19 +67,27 @@ pub async fn import_postgres_plan(
pool: &crate::driver::postgres::PostgresPool,
plan: &DataImportPlan,
) -> Result<usize, DataLayerError> {
let mut tx = pool.begin().await.map_sql_err()?;
let mut imported = 0usize;
let mut column_cache = BTreeMap::<String, PostgresImportColumns>::new();
for domain in &plan.manifest.domains {
if *domain == ExportDomain::Auxiliary {
for row in plan.rows(*domain) {
import_postgres_auxiliary_row(&mut tx, row, &mut column_cache).await?;
imported = imported.saturating_add(1);
}
continue;
}
if *domain == ExportDomain::Billing {
for row in plan.rows(*domain) {
import_postgres_billing_row(pool, row, &mut column_cache).await?;
import_postgres_billing_row(&mut tx, row, &mut column_cache).await?;
imported = imported.saturating_add(1);
}
continue;
}
if *domain == ExportDomain::Wallets {
for row in plan.rows(*domain) {
import_postgres_wallet_row(pool, row, &mut column_cache).await?;
import_postgres_wallet_row(&mut tx, row, &mut column_cache).await?;
imported = imported.saturating_add(1);
}
continue;
@@ -81,10 +99,10 @@ pub async fn import_postgres_plan(
continue;
}
let target_columns =
postgres_import_columns_cached(pool, &mut column_cache, table_name).await?;
postgres_import_columns_cached(&mut tx, &mut column_cache, table_name).await?;
for row in rows {
import_postgres_row(
pool,
&mut tx,
table_name,
&conflict_columns,
*domain,
@@ -95,9 +113,52 @@ pub async fn import_postgres_plan(
imported = imported.saturating_add(1);
}
}
if !plan.rows(ExportDomain::Auxiliary).is_empty() {
reset_postgres_auxiliary_sequences(&mut tx).await?;
}
tx.commit().await.map_sql_err()?;
Ok(imported)
}
async fn reset_postgres_auxiliary_sequences(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
) -> Result<(), DataLayerError> {
for table in AUXILIARY_TABLES {
let [primary_key] = table.primary_key else {
continue;
};
let relation_name = format!("public.{}", table.name);
let sequence =
sqlx::query_scalar::<_, Option<String>>("SELECT pg_get_serial_sequence($1, $2)")
.bind(&relation_name)
.bind(*primary_key)
.fetch_one(&mut **tx)
.await
.map_sql_err()?;
let Some(sequence) = sequence else {
continue;
};
let table_sql = postgres_quote_identifier(table.name)?;
let primary_key_sql = postgres_quote_identifier(primary_key)?;
let maximum = sqlx::query_scalar::<_, Option<i64>>(&format!(
"SELECT MAX({primary_key_sql})::bigint FROM public.{table_sql}"
))
.fetch_one(&mut **tx)
.await
.map_sql_err()?;
let (value, is_called) = maximum.map_or((1_i64, false), |value| (value, true));
sqlx::query("SELECT setval($1::regclass, $2, $3)")
.bind(sequence)
.bind(value)
.bind(is_called)
.execute(&mut **tx)
.await
.map_sql_err()?;
}
Ok(())
}
fn postgres_domain_table(
domain: ExportDomain,
) -> Result<(&'static str, &'static str), DataLayerError> {
@@ -125,9 +186,44 @@ fn postgres_domain_table(
"postgres billing export uses multiple tables and must be handled as a domain"
.to_string(),
)),
ExportDomain::Auxiliary => Err(DataLayerError::InvalidInput(
"postgres auxiliary export uses multiple tables and must be handled as a domain"
.to_string(),
)),
}
}
async fn export_postgres_auxiliary_records(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
records: &mut Vec<DataExportRecord>,
) -> Result<(), DataLayerError> {
for table in AUXILIARY_TABLES {
let table_sql = postgres_quote_identifier(table.name)?;
let order_sql = table
.primary_key
.iter()
.map(|column| postgres_quote_identifier(column).map(|column| format!("{column} ASC")))
.collect::<Result<Vec<_>, _>>()?
.join(", ");
let rows = sqlx::query(&format!(
"SELECT to_jsonb(t) AS payload FROM public.{table_sql} AS t ORDER BY {order_sql}"
))
.fetch_all(&mut **tx)
.await
.map_sql_err()?;
for row in rows {
let payload = row.try_get::<Value, _>("payload").map_sql_err()?;
let id = auxiliary_row_id(*table, &payload)?;
records.push(DataExportRecord::row(
ExportDomain::Auxiliary,
id,
payload_with_table(payload, table.name)?,
));
}
}
Ok(())
}
fn postgres_export_id_sql(domain: ExportDomain, id_column: &str) -> String {
if domain == ExportDomain::UserGroupMembers {
"group_id::text || ':' || user_id::text".to_string()
@@ -145,7 +241,7 @@ fn postgres_conflict_columns(domain: ExportDomain, id_column: &str) -> Vec<&str>
}
async fn postgres_import_columns_cached(
pool: &crate::driver::postgres::PostgresPool,
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
cache: &mut BTreeMap<String, PostgresImportColumns>,
table_name: &str,
) -> Result<PostgresImportColumns, DataLayerError> {
@@ -153,13 +249,13 @@ async fn postgres_import_columns_cached(
return Ok(columns.clone());
}
let columns = load_postgres_import_columns(pool, table_name).await?;
let columns = load_postgres_import_columns(&mut **tx, table_name).await?;
cache.insert(table_name.to_string(), columns.clone());
Ok(columns)
}
pub(super) async fn load_postgres_import_columns(
pool: &crate::driver::postgres::PostgresPool,
pub(super) async fn load_postgres_import_columns<'e>(
executor: impl sqlx::Executor<'e, Database = sqlx::Postgres>,
table_name: &str,
) -> Result<PostgresImportColumns, DataLayerError> {
let (schema_name, relation_name) = postgres_table_parts(table_name)?;
@@ -173,7 +269,7 @@ WHERE table_schema = $1
)
.bind(schema_name)
.bind(relation_name)
.fetch_all(pool)
.fetch_all(executor)
.await
.map_sql_err()?;
@@ -223,7 +319,7 @@ fn postgres_table_parts(table_name: &str) -> Result<(&str, &str), DataLayerError
}
async fn export_postgres_billing_records(
pool: &crate::driver::postgres::PostgresPool,
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
records: &mut Vec<DataExportRecord>,
) -> Result<(), DataLayerError> {
for (table_name, export_table, id_column) in [
@@ -238,7 +334,7 @@ async fn export_postgres_billing_records(
let sql = format!(
"SELECT {id_column}::text AS export_id, to_jsonb(t) || jsonb_build_object('__table', '{export_table}') AS payload FROM {table_name} AS t ORDER BY {id_column} ASC"
);
let rows = sqlx::query(&sql).fetch_all(pool).await.map_sql_err()?;
let rows = sqlx::query(&sql).fetch_all(&mut **tx).await.map_sql_err()?;
for row in rows {
let id = row.try_get::<String, _>("export_id").map_sql_err()?;
let payload = row.try_get::<Value, _>("payload").map_sql_err()?;
@@ -253,14 +349,14 @@ async fn export_postgres_billing_records(
}
async fn export_postgres_wallet_records(
pool: &crate::driver::postgres::PostgresPool,
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
records: &mut Vec<DataExportRecord>,
) -> Result<(), DataLayerError> {
for (table_name, export_table, id_column) in postgres_wallet_tables() {
let sql = format!(
"SELECT {id_column}::text AS export_id, to_jsonb(t) || jsonb_build_object('__table', '{export_table}') AS payload FROM {table_name} AS t ORDER BY {id_column} ASC"
);
let rows = sqlx::query(&sql).fetch_all(pool).await.map_sql_err()?;
let rows = sqlx::query(&sql).fetch_all(&mut **tx).await.map_sql_err()?;
for row in rows {
let id = row.try_get::<String, _>("export_id").map_sql_err()?;
let payload = row.try_get::<Value, _>("payload").map_sql_err()?;
@@ -275,7 +371,7 @@ async fn export_postgres_wallet_records(
}
async fn import_postgres_row(
pool: &crate::driver::postgres::PostgresPool,
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
table_name: &str,
conflict_columns: &[&str],
domain: ExportDomain,
@@ -316,7 +412,7 @@ async fn import_postgres_row(
sqlx::query(&sql)
.bind(&payload)
.execute(pool)
.execute(&mut **tx)
.await
.map_sql_err()?;
Ok(())
@@ -351,7 +447,7 @@ pub(super) fn normalize_postgres_import_payload(
}
normalized.insert(
column_name.clone(),
normalize_postgres_import_value(column_name, target_column, value)?,
normalize_postgres_import_value(table_name, column_name, target_column, value)?,
);
continue;
}
@@ -380,6 +476,7 @@ pub(super) fn normalize_postgres_import_payload(
}
fn normalize_postgres_import_value(
table_name: &str,
column_name: &str,
target_column: &PostgresImportColumn,
value: &Value,
@@ -392,7 +489,10 @@ fn normalize_postgres_import_value(
return normalize_postgres_boolean_value(column_name, value);
}
if is_postgres_timestamp_column(target_column) {
return normalize_postgres_timestamp_value(column_name, value);
return normalize_postgres_timestamp_value(table_name, column_name, value);
}
if is_postgres_bytea_column(target_column) {
return postgres_bytea_json_value(column_name, value);
}
if is_postgres_json_column(target_column) {
return normalize_postgres_json_value(value);
@@ -450,6 +550,7 @@ fn normalize_postgres_boolean_value(
}
fn normalize_postgres_timestamp_value(
table_name: &str,
column_name: &str,
value: &Value,
) -> Result<Value, DataLayerError> {
@@ -465,7 +566,7 @@ fn normalize_postgres_timestamp_value(
)));
};
let datetime = if column_name.ends_with("_unix_ms")
let datetime = if import_timestamp_uses_millis(table_name, column_name)
|| timestamp >= 100_000_000_000
|| timestamp <= -100_000_000_000
{
@@ -497,17 +598,17 @@ fn normalize_postgres_json_value(value: &Value) -> Result<Value, DataLayerError>
}
async fn import_postgres_billing_row(
pool: &crate::driver::postgres::PostgresPool,
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
row: &ExportRow,
column_cache: &mut BTreeMap<String, PostgresImportColumns>,
) -> Result<(), DataLayerError> {
let (export_table_name, payload) = billing_payload_table(row)?;
let table_name = postgres_billing_table_name(&export_table_name)?;
let target_columns = postgres_import_columns_cached(pool, column_cache, table_name).await?;
let (table_name, conflict_column) = postgres_billing_table_name(&export_table_name)?;
let target_columns = postgres_import_columns_cached(tx, column_cache, table_name).await?;
import_postgres_row(
pool,
tx,
table_name,
&["id"],
&[conflict_column],
ExportDomain::Billing,
&ExportRow {
id: row.id.clone(),
@@ -518,27 +619,66 @@ async fn import_postgres_billing_row(
.await
}
fn postgres_billing_table_name(table_name: &str) -> Result<&'static str, DataLayerError> {
async fn import_postgres_auxiliary_row(
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
row: &ExportRow,
column_cache: &mut BTreeMap<String, PostgresImportColumns>,
) -> Result<(), DataLayerError> {
let (table_name, payload) = domain_payload_table(row, "auxiliary", None)?;
let table = auxiliary_table(&table_name)?;
let target_table = format!("public.{}", postgres_quote_identifier(table.name)?);
let target_columns = postgres_import_columns_cached(tx, column_cache, &target_table).await?;
import_postgres_row(
tx,
&target_table,
table.primary_key,
ExportDomain::Auxiliary,
&ExportRow {
id: row.id.clone(),
payload,
},
&target_columns,
)
.await
}
fn postgres_billing_table_name(
table_name: &str,
) -> Result<(&'static str, &'static str), DataLayerError> {
match table_name {
"billing_rules" => Ok("public.billing_rules"),
"dimension_collectors" => Ok("public.dimension_collectors"),
"usage_settlement_snapshots" => Ok("public.usage_settlement_snapshots"),
"billing_rules" => Ok(("public.billing_rules", "id")),
"dimension_collectors" => Ok(("public.dimension_collectors", "id")),
"usage_settlement_snapshots" => Ok(("public.usage_settlement_snapshots", "request_id")),
other => Err(DataLayerError::InvalidInput(format!(
"unsupported postgres billing export table '{other}'"
))),
}
}
#[cfg(test)]
mod billing_table_tests {
use super::postgres_billing_table_name;
#[test]
fn settlement_snapshot_import_uses_request_id_conflict_key() {
assert_eq!(
postgres_billing_table_name("usage_settlement_snapshots")
.expect("settlement snapshot table should be supported"),
("public.usage_settlement_snapshots", "request_id")
);
}
}
async fn import_postgres_wallet_row(
pool: &crate::driver::postgres::PostgresPool,
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
row: &ExportRow,
column_cache: &mut BTreeMap<String, PostgresImportColumns>,
) -> Result<(), DataLayerError> {
let (export_table_name, payload) = domain_payload_table(row, "wallet", Some("wallets"))?;
let (table_name, id_column) = postgres_wallet_table_name(&export_table_name)?;
let target_columns = postgres_import_columns_cached(pool, column_cache, table_name).await?;
let target_columns = postgres_import_columns_cached(tx, column_cache, table_name).await?;
import_postgres_row(
pool,
tx,
table_name,
&[id_column],
ExportDomain::Wallets,
crates/aether-data/runtime/src/lifecycle/export/sqlite.rs
+181 -37
View File
@@ -1,5 +1,12 @@
use super::*;
#[derive(Debug, Clone, Default, PartialEq, Eq)]
struct SqliteImportColumns {
names: ImportColumnNames,
declared_types: BTreeMap<String, String>,
primary_key: Vec<String>,
}
pub async fn export_sqlite_core_jsonl(
pool: &crate::driver::sqlite::SqlitePool,
created_at_unix_secs: u64,
@@ -12,6 +19,7 @@ pub async fn export_sqlite_jsonl(
domains: Vec<ExportDomain>,
created_at_unix_secs: u64,
) -> Result<String, DataLayerError> {
let mut tx = pool.begin().await.map_sql_err()?;
let manifest = DataExportManifest::new(
created_at_unix_secs,
Some(DatabaseDriver::Sqlite),
@@ -20,24 +28,29 @@ pub async fn export_sqlite_jsonl(
let mut records = vec![DataExportRecord::manifest(manifest)];
for domain in domains {
if domain == ExportDomain::Auxiliary {
export_sqlite_auxiliary_records(&mut tx, &mut records).await?;
continue;
}
if domain == ExportDomain::Billing {
export_sqlite_billing_records(pool, &mut records).await?;
export_sqlite_billing_records(&mut tx, &mut records).await?;
continue;
}
if domain == ExportDomain::Wallets {
export_sqlite_wallet_records(pool, &mut records).await?;
export_sqlite_wallet_records(&mut tx, &mut records).await?;
continue;
}
let (table_name, id_column) = sqlite_domain_table(domain)?;
let order_by = export_order_by(domain, id_column);
let sql = format!("SELECT * FROM {table_name} ORDER BY {order_by}");
let rows = sqlx::query(&sql).fetch_all(pool).await.map_sql_err()?;
let rows = sqlx::query(&sql).fetch_all(&mut *tx).await.map_sql_err()?;
for row in rows {
let id = sqlite_export_row_id(domain, &row, id_column)?;
records.push(DataExportRecord::row(domain, id, sqlite_row_payload(&row)?));
}
}
tx.commit().await.map_sql_err()?;
encode_jsonl(&records)
}
@@ -53,31 +66,40 @@ pub async fn import_sqlite_plan(
pool: &crate::driver::sqlite::SqlitePool,
plan: &DataImportPlan,
) -> Result<usize, DataLayerError> {
let mut tx = pool.begin().await.map_sql_err()?;
let mut imported = 0usize;
let mut column_cache = BTreeMap::<String, ImportColumnNames>::new();
let mut column_cache = BTreeMap::<String, SqliteImportColumns>::new();
for domain in &plan.manifest.domains {
if *domain == ExportDomain::Auxiliary {
for row in plan.rows(*domain) {
import_sqlite_auxiliary_row(&mut tx, row, &mut column_cache).await?;
imported = imported.saturating_add(1);
}
continue;
}
if *domain == ExportDomain::Billing {
for row in plan.rows(*domain) {
import_sqlite_billing_row(pool, row, &mut column_cache).await?;
import_sqlite_billing_row(&mut tx, row, &mut column_cache).await?;
imported = imported.saturating_add(1);
}
continue;
}
if *domain == ExportDomain::Wallets {
for row in plan.rows(*domain) {
import_sqlite_wallet_row(pool, row, &mut column_cache).await?;
import_sqlite_wallet_row(&mut tx, row, &mut column_cache).await?;
imported = imported.saturating_add(1);
}
continue;
}
let (table_name, _id_column) = sqlite_domain_table(*domain)?;
let target_columns =
sqlite_import_columns_cached(pool, &mut column_cache, table_name).await?;
sqlite_import_columns_cached(&mut tx, &mut column_cache, table_name).await?;
for row in plan.rows(*domain) {
import_sqlite_row(pool, table_name, *domain, row, &target_columns).await?;
import_sqlite_row(&mut tx, table_name, *domain, row, &target_columns).await?;
imported = imported.saturating_add(1);
}
}
tx.commit().await.map_sql_err()?;
Ok(imported)
}
@@ -107,9 +129,42 @@ fn sqlite_domain_table(
"sqlite billing export uses multiple tables and must be handled as a domain"
.to_string(),
)),
ExportDomain::Auxiliary => Err(DataLayerError::InvalidInput(
"sqlite auxiliary export uses multiple tables and must be handled as a domain"
.to_string(),
)),
}
}
async fn export_sqlite_auxiliary_records(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
records: &mut Vec<DataExportRecord>,
) -> Result<(), DataLayerError> {
for table in AUXILIARY_TABLES {
let table_sql = sqlite_quote_identifier(table.name)?;
let order_sql = table
.primary_key
.iter()
.map(|column| sqlite_quote_identifier(column).map(|column| format!("{column} ASC")))
.collect::<Result<Vec<_>, _>>()?
.join(", ");
let rows = sqlx::query(&format!("SELECT * FROM {table_sql} ORDER BY {order_sql}"))
.fetch_all(&mut **tx)
.await
.map_sql_err()?;
for row in rows {
let payload = sqlite_row_payload(&row)?;
let id = auxiliary_row_id(*table, &payload)?;
records.push(DataExportRecord::row(
ExportDomain::Auxiliary,
id,
payload_with_table(payload, table.name)?,
));
}
}
Ok(())
}
fn sqlite_export_row_id(
domain: ExportDomain,
row: &sqlx::sqlite::SqliteRow,
@@ -140,7 +195,7 @@ fn sqlite_required_export_text(
}
async fn export_sqlite_billing_records(
pool: &crate::driver::sqlite::SqlitePool,
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
records: &mut Vec<DataExportRecord>,
) -> Result<(), DataLayerError> {
for table_name in [
@@ -154,7 +209,7 @@ async fn export_sqlite_billing_records(
"id"
};
let sql = format!("SELECT * FROM {table_name} ORDER BY {id_column} ASC");
let rows = sqlx::query(&sql).fetch_all(pool).await.map_sql_err()?;
let rows = sqlx::query(&sql).fetch_all(&mut **tx).await.map_sql_err()?;
for row in rows {
let id = row
.try_get::<Option<String>, _>(id_column)
@@ -175,12 +230,12 @@ async fn export_sqlite_billing_records(
}
async fn export_sqlite_wallet_records(
pool: &crate::driver::sqlite::SqlitePool,
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
records: &mut Vec<DataExportRecord>,
) -> Result<(), DataLayerError> {
for (table_name, id_column) in sqlite_wallet_tables() {
let sql = format!("SELECT * FROM {table_name} ORDER BY {id_column} ASC");
let rows = sqlx::query(&sql).fetch_all(pool).await.map_sql_err()?;
let rows = sqlx::query(&sql).fetch_all(&mut **tx).await.map_sql_err()?;
for row in rows {
let id = row
.try_get::<Option<String>, _>(id_column)
@@ -201,13 +256,13 @@ async fn export_sqlite_wallet_records(
}
async fn import_sqlite_row(
pool: &crate::driver::sqlite::SqlitePool,
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
table_name: &str,
domain: ExportDomain,
row: &ExportRow,
target_columns: &ImportColumnNames,
target_columns: &SqliteImportColumns,
) -> Result<(), DataLayerError> {
let object = filter_import_payload("sqlite", table_name, domain, row, target_columns)?;
let object = filter_import_payload("sqlite", table_name, domain, row, &target_columns.names)?;
let columns = object.keys().map(String::as_str).collect::<Vec<_>>();
let column_sql = columns
@@ -216,29 +271,55 @@ async fn import_sqlite_row(
.collect::<Result<Vec<_>, _>>()?
.join(", ");
let placeholder_sql = vec!["?"; columns.len()].join(", ");
let sql =
format!("INSERT OR REPLACE INTO {table_name} ({column_sql}) VALUES ({placeholder_sql})");
let conflict_columns = target_columns
.primary_key
.iter()
.map(|column| sqlite_quote_identifier(column))
.collect::<Result<Vec<_>, _>>()?
.join(", ");
let update_sql = columns
.iter()
.filter(|column| !target_columns.primary_key.iter().any(|key| key == *column))
.map(|column| {
let quoted = sqlite_quote_identifier(column)?;
Ok(format!("{quoted} = excluded.{quoted}"))
})
.collect::<Result<Vec<_>, DataLayerError>>()?
.join(", ");
let conflict_sql = if update_sql.is_empty() {
format!("ON CONFLICT ({conflict_columns}) DO NOTHING")
} else {
format!("ON CONFLICT ({conflict_columns}) DO UPDATE SET {update_sql}")
};
let sql = format!(
"INSERT INTO {table_name} ({column_sql}) VALUES ({placeholder_sql}) {conflict_sql}"
);
let mut query = sqlx::query(&sql);
for column in columns {
let value = object
.get(column)
.expect("column name came from payload object keys");
query = bind_sqlite_json_value(query, value)?;
let declared_type = target_columns
.declared_types
.get(column)
.map(String::as_str)
.unwrap_or_default();
query = bind_sqlite_import_value(query, value, table_name, column, declared_type)?;
}
query.execute(pool).await.map_sql_err()?;
query.execute(&mut **tx).await.map_sql_err()?;
Ok(())
}
async fn import_sqlite_billing_row(
pool: &crate::driver::sqlite::SqlitePool,
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
row: &ExportRow,
column_cache: &mut BTreeMap<String, ImportColumnNames>,
column_cache: &mut BTreeMap<String, SqliteImportColumns>,
) -> Result<(), DataLayerError> {
let (table_name, payload) = billing_payload_table(row)?;
let table_name = sqlite_billing_table_name(&table_name)?;
let target_columns = sqlite_import_columns_cached(pool, column_cache, table_name).await?;
let target_columns = sqlite_import_columns_cached(tx, column_cache, table_name).await?;
import_sqlite_row(
pool,
tx,
table_name,
ExportDomain::Billing,
&ExportRow {
@@ -250,6 +331,27 @@ async fn import_sqlite_billing_row(
.await
}
async fn import_sqlite_auxiliary_row(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
row: &ExportRow,
column_cache: &mut BTreeMap<String, SqliteImportColumns>,
) -> Result<(), DataLayerError> {
let (table_name, payload) = domain_payload_table(row, "auxiliary", None)?;
let table = auxiliary_table(&table_name)?;
let target_columns = sqlite_import_columns_cached(tx, column_cache, table.name).await?;
import_sqlite_row(
tx,
table.name,
ExportDomain::Auxiliary,
&ExportRow {
id: row.id.clone(),
payload,
},
&target_columns,
)
.await
}
fn sqlite_billing_table_name(table_name: &str) -> Result<&'static str, DataLayerError> {
match table_name {
"billing_rules" => Ok("billing_rules"),
@@ -262,15 +364,15 @@ fn sqlite_billing_table_name(table_name: &str) -> Result<&'static str, DataLayer
}
async fn import_sqlite_wallet_row(
pool: &crate::driver::sqlite::SqlitePool,
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
row: &ExportRow,
column_cache: &mut BTreeMap<String, ImportColumnNames>,
column_cache: &mut BTreeMap<String, SqliteImportColumns>,
) -> Result<(), DataLayerError> {
let (table_name, payload) = domain_payload_table(row, "wallet", Some("wallets"))?;
let table_name = sqlite_wallet_table_name(&table_name)?;
let target_columns = sqlite_import_columns_cached(pool, column_cache, table_name).await?;
let target_columns = sqlite_import_columns_cached(tx, column_cache, table_name).await?;
import_sqlite_row(
pool,
tx,
table_name,
ExportDomain::Wallets,
&ExportRow {
@@ -308,39 +410,81 @@ fn sqlite_wallet_table_name(table_name: &str) -> Result<&'static str, DataLayerE
}
async fn sqlite_import_columns_cached(
pool: &crate::driver::sqlite::SqlitePool,
cache: &mut BTreeMap<String, ImportColumnNames>,
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
cache: &mut BTreeMap<String, SqliteImportColumns>,
table_name: &str,
) -> Result<ImportColumnNames, DataLayerError> {
) -> Result<SqliteImportColumns, DataLayerError> {
if let Some(columns) = cache.get(table_name) {
return Ok(columns.clone());
}
let columns = load_sqlite_import_columns(pool, table_name).await?;
let columns = load_sqlite_import_columns(tx, table_name).await?;
cache.insert(table_name.to_string(), columns.clone());
Ok(columns)
}
async fn load_sqlite_import_columns(
pool: &crate::driver::sqlite::SqlitePool,
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
table_name: &str,
) -> Result<ImportColumnNames, DataLayerError> {
) -> Result<SqliteImportColumns, DataLayerError> {
let sql = format!("PRAGMA table_info({table_name})");
let rows = sqlx::query(&sql).fetch_all(pool).await.map_sql_err()?;
let mut columns = ImportColumnNames::new();
let rows = sqlx::query(&sql).fetch_all(&mut **tx).await.map_sql_err()?;
let mut columns = SqliteImportColumns::default();
let mut primary_key = BTreeMap::new();
for row in rows {
columns.insert(row.try_get::<String, _>("name").map_sql_err()?);
let name = row.try_get::<String, _>("name").map_sql_err()?;
let declared_type = row
.try_get::<Option<String>, _>("type")
.map_sql_err()?
.unwrap_or_default();
columns.names.insert(name.clone());
columns.declared_types.insert(name.clone(), declared_type);
let primary_key_position = row.try_get::<i64, _>("pk").map_sql_err()?;
if primary_key_position > 0 {
primary_key.insert(primary_key_position, name);
}
}
if columns.is_empty() {
if columns.names.is_empty() {
return Err(DataLayerError::UnexpectedValue(format!(
"sqlite import target table '{table_name}' has no visible columns"
)));
}
if primary_key.is_empty() {
return Err(DataLayerError::UnexpectedValue(format!(
"sqlite import target table '{table_name}' has no primary key"
)));
}
columns.primary_key = primary_key.into_values().collect();
Ok(columns)
}
fn bind_sqlite_import_value<'q>(
query: sqlx::query::Query<'q, sqlx::Sqlite, sqlx::sqlite::SqliteArguments<'q>>,
json_value: &'q Value,
table_name: &str,
column_name: &str,
declared_type: &str,
) -> Result<sqlx::query::Query<'q, sqlx::Sqlite, sqlx::sqlite::SqliteArguments<'q>>, DataLayerError>
{
if declared_type.to_ascii_uppercase().contains("BLOB") {
return match normalize_imported_binary("sqlite", column_name, json_value)? {
Some(bytes) => Ok(query.bind(bytes)),
None => Ok(query.bind(Option::<Vec<u8>>::None)),
};
}
let has_integer_affinity = declared_type.to_ascii_uppercase().contains("INT");
if !has_integer_affinity || !import_column_stores_timestamp(column_name) {
return bind_sqlite_json_value(query, json_value);
}
match normalize_imported_integer_timestamp("sqlite", table_name, column_name, json_value)? {
Some(timestamp) => Ok(query.bind(timestamp)),
None => Ok(query.bind(Option::<i64>::None)),
}
}
fn sqlite_row_payload(row: &sqlx::sqlite::SqliteRow) -> Result<Value, DataLayerError> {
let mut object = serde_json::Map::new();
for (index, column) in row.columns().iter().enumerate() {
crates/aether-data/runtime/src/lifecycle/export/tests.rs
+501 -13
View File
@@ -1,14 +1,16 @@
use std::collections::BTreeMap;
use std::collections::{BTreeMap, BTreeSet};
use serde_json::json;
use super::{
build_import_plan, decode_jsonl, encode_jsonl, export_mysql_core_jsonl, export_mysql_jsonl,
export_postgres_core_jsonl, export_sqlite_core_jsonl, import_mysql_jsonl,
import_postgres_jsonl, import_sqlite_jsonl, mysql_core_export_domains,
normalize_postgres_import_payload, postgres_core_export_domains, sqlite_core_export_domains,
DataExportManifest, DataExportRecord, DataImportPlan, ExportDomain, ExportRow,
PostgresImportColumn,
export_postgres_core_jsonl, export_sqlite_core_jsonl, filter_import_payload,
import_mysql_jsonl, import_postgres_jsonl, import_sqlite_jsonl, mysql_core_export_domains,
normalize_imported_binary, normalize_imported_integer_timestamp,
normalize_postgres_import_payload, postgres_bytea_json_value, postgres_core_export_domains,
sqlite_core_export_domains, sqlite_schema_copy_insert_sql, DataExportManifest,
DataExportRecord, DataImportPlan, ExportDomain, ExportRow, PostgresImportColumn,
SchemaCopyColumn, SchemaCopyTable, SqliteCopyColumn, AUXILIARY_TABLES,
};
use crate::driver::postgres::{PostgresPoolConfig, PostgresPoolFactory};
use crate::lifecycle::migrate::{
@@ -64,6 +66,81 @@ fn jsonl_round_trips_manifest_and_domain_rows() {
fn core_export_domains_match_across_sql_drivers() {
assert_eq!(sqlite_core_export_domains(), mysql_core_export_domains());
assert_eq!(sqlite_core_export_domains(), postgres_core_export_domains());
assert!(sqlite_core_export_domains().contains(&ExportDomain::Auxiliary));
}
#[tokio::test]
async fn sqlite_core_export_covers_every_portable_table() {
let pool = sqlx::sqlite::SqlitePoolOptions::new()
.max_connections(1)
.connect("sqlite::memory:")
.await
.expect("sqlite pool should connect");
run_sqlite_migrations(&pool)
.await
.expect("sqlite migrations should run");
let schema_tables = sqlx::query_scalar::<_, String>(
r#"
SELECT name
FROM sqlite_master
WHERE type = 'table'
AND name NOT LIKE 'sqlite_%'
AND name NOT IN ('_sqlx_migrations', 'schema_backfills')
ORDER BY name
"#,
)
.fetch_all(&pool)
.await
.expect("sqlite schema tables should load")
.into_iter()
.collect::<BTreeSet<_>>();
let mut exported_tables = [
"users",
"api_keys",
"providers",
"provider_api_keys",
"provider_endpoints",
"global_models",
"models",
"auth_modules",
"oauth_providers",
"user_oauth_links",
"user_groups",
"user_group_members",
"proxy_nodes",
"system_configs",
"usage",
"wallets",
"wallet_transactions",
"wallet_daily_usage_ledgers",
"payment_orders",
"payment_callbacks",
"refund_requests",
"redeem_code_batches",
"redeem_codes",
"billing_rules",
"dimension_collectors",
"usage_settlement_snapshots",
]
.into_iter()
.map(str::to_string)
.collect::<BTreeSet<_>>();
exported_tables.extend(AUXILIARY_TABLES.iter().map(|table| table.name.to_string()));
assert_eq!(schema_tables, exported_tables);
}
#[test]
fn version_one_exports_remain_importable_after_full_export_expansion() {
let records = decode_jsonl(
r#"{"record_type":"manifest","manifest":{"format_version":1,"created_at_unix_secs":1,"source_driver":null,"domains":["users"]}}
{"record_type":"row","domain":"users","id":"user-1","payload":{"id":"user-1"}}"#,
)
.expect("version one exports should remain supported");
assert_eq!(records.len(), 2);
}
#[test]
@@ -171,6 +248,70 @@ fn postgres_import_payload_normalizes_sqlite_values_for_target_columns() {
assert!(!normalized.contains_key("legacy_nullable"));
}
#[test]
fn cross_driver_timestamp_normalization_preserves_usage_second_contract() {
assert_eq!(
normalize_imported_integer_timestamp(
"sqlite",
r#""usage""#,
"created_at_unix_ms",
&json!("1970-01-01T00:00:01.234900Z"),
)
.expect("usage timestamp should normalize"),
Some(1),
);
assert_eq!(
normalize_imported_integer_timestamp(
"mysql",
"request_candidates",
"created_at_unix_ms",
&json!("1970-01-01T00:00:01.234900Z"),
)
.expect("millisecond timestamp should normalize"),
Some(1_234),
);
let target_columns = BTreeMap::from([(
"created_at_unix_ms".to_string(),
postgres_column("timestamp with time zone", "timestamptz"),
)]);
let row = ExportRow {
id: "usage-1".to_string(),
payload: json!({ "created_at_unix_ms": 1_700_000_000 }),
};
let normalized = normalize_postgres_import_payload(
"public.usage",
ExportDomain::Usage,
&row,
&target_columns,
)
.expect("postgres usage timestamp should normalize");
assert_eq!(
normalized["created_at_unix_ms"],
json!("2023-11-14T22:13:20+00:00")
);
}
#[test]
fn cross_driver_binary_normalization_preserves_raw_bytes() {
assert_eq!(
normalize_imported_binary("sqlite", "payload_gzip", &json!([0, 1, 127, 255]))
.expect("byte array should normalize"),
Some(vec![0, 1, 127, 255]),
);
assert_eq!(
normalize_imported_binary("mysql", "payload_gzip", &json!("\\x00017fff"))
.expect("postgres hex should normalize"),
Some(vec![0, 1, 127, 255]),
);
assert!(normalize_imported_binary("sqlite", "payload_gzip", &json!([256])).is_err());
assert_eq!(
postgres_bytea_json_value("payload_gzip", &json!([0, 1, 127, 255]))
.expect("postgres bytea should normalize"),
json!("\\x00017fff"),
);
}
#[test]
fn postgres_import_payload_rejects_non_null_unknown_columns() {
let target_columns = BTreeMap::from([(
@@ -197,6 +338,95 @@ fn postgres_import_payload_rejects_non_null_unknown_columns() {
assert!(err.to_string().contains("does not exist"));
}
#[test]
fn mysql_and_sqlite_import_payloads_reject_non_null_unknown_columns() {
let target_columns = BTreeSet::from(["id".to_string()]);
let row = ExportRow {
id: "user-1".to_string(),
payload: json!({
"id": "user-1",
"legacy_nullable": null,
"unexpected_column": "value"
}),
};
for driver_name in ["mysql", "sqlite"] {
let err = filter_import_payload(
driver_name,
"users",
ExportDomain::Users,
&row,
&target_columns,
)
.expect_err("non-null unknown columns should fail");
assert!(err.to_string().contains("unexpected_column"));
assert!(err.to_string().contains("does not exist"));
assert!(err.to_string().contains(driver_name));
}
}
#[test]
fn mysql_and_sqlite_import_payloads_ignore_unknown_null_columns() {
let target_columns = BTreeSet::from(["id".to_string()]);
let row = ExportRow {
id: "user-1".to_string(),
payload: json!({
"id": "user-1",
"legacy_nullable": null
}),
};
let filtered = filter_import_payload(
"sqlite",
"users",
ExportDomain::Users,
&row,
&target_columns,
)
.expect("unknown null columns should remain backward compatible");
assert_eq!(
filtered,
serde_json::Map::from_iter([("id".to_string(), json!("user-1"))])
);
}
#[test]
fn postgres_to_sqlite_copy_uses_primary_key_upsert_instead_of_replace() {
let table = SchemaCopyTable {
table_name: "usage".to_string(),
columns: vec![
SchemaCopyColumn {
sqlite: SqliteCopyColumn {
name: "request_id".to_string(),
declared_type: "TEXT".to_string(),
not_null: true,
has_default: false,
primary_key_position: 1,
},
postgres: postgres_column("character varying", "varchar"),
},
SchemaCopyColumn {
sqlite: SqliteCopyColumn {
name: "status".to_string(),
declared_type: "TEXT".to_string(),
not_null: true,
has_default: false,
primary_key_position: 0,
},
postgres: postgres_column("character varying", "varchar"),
},
],
};
let sql = sqlite_schema_copy_insert_sql(&table).expect("copy SQL should build");
assert!(!sql.contains("OR REPLACE"));
assert!(sql.contains("ON CONFLICT (\"request_id\") DO UPDATE SET"));
assert!(sql.contains("\"status\" = excluded.\"status\""));
}
fn postgres_column(data_type: &str, udt_name: &str) -> PostgresImportColumn {
PostgresImportColumn {
data_type: data_type.to_ascii_lowercase(),
@@ -215,6 +445,183 @@ fn postgres_not_null_default_column(data_type: &str, udt_name: &str) -> Postgres
}
}
#[tokio::test]
async fn sqlite_import_rejects_non_integer_timestamp_values() {
let pool = sqlx::sqlite::SqlitePoolOptions::new()
.max_connections(1)
.connect("sqlite::memory:")
.await
.expect("sqlite pool should connect");
run_sqlite_migrations(&pool)
.await
.expect("sqlite migrations should run");
for invalid_value in [
json!("not-a-timestamp"),
json!(1.5),
json!(true),
json!({"unexpected": "object"}),
] {
let encoded = encode_jsonl(&[
DataExportRecord::manifest(DataExportManifest::new(
1_700_000_000,
Some(DatabaseDriver::Postgres),
vec![ExportDomain::GlobalModels],
)),
DataExportRecord::row(
ExportDomain::GlobalModels,
"invalid-timestamp",
json!({
"id": "invalid-timestamp",
"name": "invalid-timestamp",
"created_at": invalid_value,
"updated_at": 1
}),
),
])
.expect("invalid timestamp fixture should encode");
let err = import_sqlite_jsonl(&pool, &encoded)
.await
.expect_err("non-integer timestamp should be rejected");
assert!(err.to_string().contains(
"timestamp column 'created_at' must contain an integer or supported datetime"
));
}
}
#[tokio::test]
async fn sqlite_import_updates_parent_without_cascading_child_rows() {
let pool = sqlx::sqlite::SqlitePoolOptions::new()
.max_connections(1)
.connect("sqlite::memory:")
.await
.expect("sqlite pool should connect");
run_sqlite_migrations(&pool)
.await
.expect("sqlite migrations should run");
sqlx::query("PRAGMA foreign_keys = ON")
.execute(&pool)
.await
.expect("foreign keys should be enabled");
sqlx::raw_sql(
r#"
INSERT INTO users (id, email, username, created_at, updated_at)
VALUES ('import-user', 'import@example.test', 'import-user', 1, 1);
INSERT INTO user_groups (
id, name, normalized_name, description, priority,
allowed_providers_mode, allowed_api_formats_mode, allowed_models_mode, rate_limit_mode,
created_at, updated_at
)
VALUES (
'import-group', 'Before', 'import-group', 'preserve-me', 0,
'inherit', 'inherit', 'inherit', 'inherit', 1, 1
);
INSERT INTO user_group_members (group_id, user_id, created_at)
VALUES ('import-group', 'import-user', 1);
"#,
)
.execute(&pool)
.await
.expect("parent and child fixtures should insert");
let encoded = encode_jsonl(&[
DataExportRecord::manifest(DataExportManifest::new(
1_700_000_000,
Some(DatabaseDriver::Postgres),
vec![ExportDomain::UserGroups],
)),
DataExportRecord::row(
ExportDomain::UserGroups,
"import-group",
json!({
"id": "import-group",
"name": "After",
"normalized_name": "import-group",
"priority": 10,
"allowed_providers_mode": "inherit",
"allowed_api_formats_mode": "inherit",
"allowed_models_mode": "inherit",
"rate_limit_mode": "inherit",
"created_at": 1,
"updated_at": 2
}),
),
])
.expect("group export should encode");
assert_eq!(
import_sqlite_jsonl(&pool, &encoded)
.await
.expect("group import should update in place"),
1
);
let group = sqlx::query_as::<_, (String, String)>(
"SELECT name, description FROM user_groups WHERE id = 'import-group'",
)
.fetch_one(&pool)
.await
.expect("updated group should load");
assert_eq!(group, ("After".to_string(), "preserve-me".to_string()));
let member_count: i64 = sqlx::query_scalar(
"SELECT COUNT(*) FROM user_group_members WHERE group_id = 'import-group'",
)
.fetch_one(&pool)
.await
.expect("group member count should load");
assert_eq!(member_count, 1);
}
#[tokio::test]
async fn sqlite_import_rolls_back_rows_after_late_failure() {
let pool = sqlx::sqlite::SqlitePoolOptions::new()
.max_connections(1)
.connect("sqlite::memory:")
.await
.expect("sqlite pool should connect");
run_sqlite_migrations(&pool)
.await
.expect("sqlite migrations should run");
let encoded = encode_jsonl(&[
DataExportRecord::manifest(DataExportManifest::new(
1_700_000_000,
Some(DatabaseDriver::Postgres),
vec![ExportDomain::GlobalModels],
)),
DataExportRecord::row(
ExportDomain::GlobalModels,
"rollback-valid",
json!({
"id": "rollback-valid",
"name": "rollback-valid",
"created_at": 1,
"updated_at": 1
}),
),
DataExportRecord::row(
ExportDomain::GlobalModels,
"rollback-invalid",
json!({
"id": "rollback-invalid",
"name": "rollback-invalid",
"created_at": "invalid-timestamp",
"updated_at": 1
}),
),
])
.expect("rollback fixture should encode");
import_sqlite_jsonl(&pool, &encoded)
.await
.expect_err("late invalid row should fail the import");
let count: i64 =
sqlx::query_scalar("SELECT COUNT(*) FROM global_models WHERE id LIKE 'rollback-%'")
.fetch_one(&pool)
.await
.expect("rolled back row count should load");
assert_eq!(count, 0);
}
#[tokio::test]
async fn sqlite_core_export_reads_migrated_database_rows() {
let pool = sqlx::sqlite::SqlitePoolOptions::new()
@@ -243,7 +650,7 @@ VALUES ('provider-key-1', 'provider-1', 'Provider Key', 'ciphertext-provider', '
INSERT INTO provider_endpoints (id, provider_id, name, base_url, created_at, updated_at)
VALUES ('endpoint-1', 'provider-1', 'Primary', 'https://example.test', '1970-01-01T00:00:01Z', '1970-01-01T00:00:02Z');
INSERT INTO global_models (id, name, created_at, updated_at)
VALUES ('global-model-1', 'gpt-test', '1970-01-01T00:00:01Z', '1970-01-01T00:00:02Z');
VALUES ('global-model-1', 'gpt-test', '1970-01-01T00:00:01Z', '1970-01-01 00:00:02.123456');
INSERT INTO models (id, provider_id, global_model_id, provider_model_name, created_at, updated_at)
VALUES ('model-1', 'provider-1', 'global-model-1', 'gpt-test', '1970-01-01T00:00:01Z', '1970-01-01T00:00:02Z');
INSERT INTO billing_rules (id, global_model_id, name, task_type, expression, variables, dimension_mappings, is_enabled, created_at, updated_at)
@@ -255,7 +662,21 @@ VALUES ('config-1', 'billing.enabled', 'true', '1970-01-01T00:00:01Z', '1970-01-
INSERT INTO wallets (id, user_id, created_at, updated_at)
VALUES ('wallet-1', 'user-1', '1970-01-01T00:00:01Z', '1970-01-01T00:00:02Z');
INSERT INTO "usage" (request_id, id, user_id, provider_name, model, status, billing_status, created_at_unix_ms, updated_at_unix_secs)
VALUES ('request-1', 'request-1', 'user-1', 'Provider One', 'gpt-test', 'completed', 'settled', 1, 2);
VALUES ('request-1', 'request-1', 'user-1', 'Provider One', 'gpt-test', 'completed', 'settled', '1970-01-01T00:00:01.234900Z', 2);
INSERT INTO audit_logs (id, event_type, description, request_id, created_at)
VALUES ('audit-1', 'request.completed', 'Exported audit', 'request-1', '1970-01-01T00:00:02Z');
INSERT INTO usage_body_blobs (body_ref, request_id, body_field, payload_gzip, created_at, updated_at)
VALUES ('body-ref-1', 'request-1', 'request', X'00117FFF', '1970-01-01T00:00:01Z', '1970-01-01T00:00:02Z');
INSERT INTO usage_http_audits (request_id, request_body_ref, request_body_state, body_capture_mode, created_at, updated_at)
VALUES ('request-1', 'body-ref-1', 'captured', 'full', '1970-01-01T00:00:01Z', '1970-01-01T00:00:02Z');
INSERT INTO usage_routing_snapshots (
request_id, candidate_id, candidate_index, selected_provider_id,
selected_endpoint_id, selected_provider_api_key_id, created_at, updated_at
)
VALUES (
'request-1', 'candidate-1', 2, 'provider-1',
'endpoint-1', 'provider-key-1', '1970-01-01T00:00:01Z', '1970-01-01T00:00:02Z'
);
"#,
)
.execute(&pool)
@@ -304,6 +725,21 @@ VALUES ('request-1', 'request-1', 'user-1', 'Provider One', 'gpt-test', 'complet
import_plan.rows(ExportDomain::Billing)[0].payload["dimension_mappings"]["input"],
"input_tokens"
);
assert!(import_plan
.rows(ExportDomain::Auxiliary)
.iter()
.any(|row| row.payload["__table"] == "audit_logs" && row.payload["id"] == "audit-1"));
assert!(import_plan
.rows(ExportDomain::Auxiliary)
.iter()
.any(|row| row.payload["__table"] == "usage_body_blobs"
&& row.payload["payload_gzip"] == json!([0, 17, 127, 255])));
assert!(import_plan
.rows(ExportDomain::Auxiliary)
.iter()
.any(|row| row.payload["__table"] == "usage_routing_snapshots"
&& row.payload["candidate_id"] == "candidate-1"
&& row.payload["selected_provider_id"] == "provider-1"));
let target_pool = sqlx::sqlite::SqlitePoolOptions::new()
.max_connections(1)
@@ -316,7 +752,7 @@ VALUES ('request-1', 'request-1', 'user-1', 'Provider One', 'gpt-test', 'complet
let imported = import_sqlite_jsonl(&target_pool, &encoded)
.await
.expect("sqlite import should load exported rows");
assert_eq!(imported, 16);
assert_eq!(imported, 20);
let imported_api_key =
sqlx::query_as::<_, (String,)>("SELECT key_encrypted FROM api_keys WHERE id = 'api-key-1'")
@@ -325,13 +761,31 @@ VALUES ('request-1', 'request-1', 'user-1', 'Provider One', 'gpt-test', 'complet
.expect("imported api key should load");
assert_eq!(imported_api_key.0, "ciphertext-1");
let imported_usage = sqlx::query_as::<_, (String,)>(
"SELECT request_id FROM \"usage\" WHERE request_id = 'request-1'",
let imported_usage = sqlx::query_as::<_, (String, i64, String)>(
"SELECT request_id, created_at_unix_ms, typeof(created_at_unix_ms) FROM \"usage\" WHERE request_id = 'request-1'",
)
.fetch_one(&target_pool)
.await
.expect("imported usage should load");
assert_eq!(imported_usage.0, "request-1");
assert_eq!(
imported_usage,
("request-1".to_string(), 1, "integer".to_string())
);
let imported_global_model_timestamps = sqlx::query_as::<_, (i64, i64, String, String)>(
r#"
SELECT created_at, updated_at, typeof(created_at), typeof(updated_at)
FROM global_models
WHERE id = 'global-model-1'
"#,
)
.fetch_one(&target_pool)
.await
.expect("imported global model timestamps should decode as integers");
assert_eq!(
imported_global_model_timestamps,
(1, 2, "integer".to_string(), "integer".to_string())
);
let imported_group_member = sqlx::query_as::<_, (String, String)>(
"SELECT group_id, user_id FROM user_group_members WHERE group_id = 'group-1' AND user_id = 'user-1'",
@@ -350,6 +804,29 @@ VALUES ('request-1', 'request-1', 'user-1', 'Provider One', 'gpt-test', 'complet
.expect("imported billing rule should load");
assert_eq!(imported_billing_rule.0, "input_tokens * 0.01");
let imported_body: Vec<u8> = sqlx::query_scalar(
"SELECT payload_gzip FROM usage_body_blobs WHERE body_ref = 'body-ref-1'",
)
.fetch_one(&target_pool)
.await
.expect("imported body blob should load");
assert_eq!(imported_body, vec![0, 17, 127, 255]);
let imported_routing = sqlx::query_as::<_, (String, i64, String)>(
r#"
SELECT candidate_id, candidate_index, selected_provider_id
FROM usage_routing_snapshots
WHERE request_id = 'request-1'
"#,
)
.fetch_one(&target_pool)
.await
.expect("imported routing snapshot should load");
assert_eq!(
imported_routing,
("candidate-1".to_string(), 2, "provider-1".to_string())
);
if let Some(database_url) = std::env::var("AETHER_TEST_POSTGRES_URL")
.ok()
.filter(|value| !value.trim().is_empty())
@@ -375,7 +852,7 @@ VALUES ('request-1', 'request-1', 'user-1', 'Provider One', 'gpt-test', 'complet
let imported = import_postgres_jsonl(&postgres_pool, &encoded)
.await
.expect("postgres import should load exported rows");
assert_eq!(imported, 16);
assert_eq!(imported, 20);
let imported_api_key = sqlx::query_as::<_, (String,)>(
"SELECT key_encrypted FROM api_keys WHERE id = 'api-key-1'",
@@ -616,6 +1093,17 @@ async fn postgres_core_export_reads_migrated_database_rows_when_url_is_set() {
.await
.expect("imported sqlite api key should load");
assert_eq!(imported_api_key.0, "ciphertext-1");
let imported_global_model_timestamps = sqlx::query_as::<_, (i64, i64, String, String)>(
"SELECT created_at, updated_at, typeof(created_at), typeof(updated_at) FROM global_models WHERE id = ?",
)
.bind(&global_model_id)
.fetch_one(&target_pool)
.await
.expect("imported sqlite global model timestamps should decode as integers");
assert_eq!(
imported_global_model_timestamps,
(1, 2, "integer".to_string(), "integer".to_string())
);
let imported_group_member = sqlx::query_as::<_, (String, String)>(
"SELECT group_id, user_id FROM user_group_members WHERE group_id = ? AND user_id = ?",
)
crates/aether-data/runtime/src/lifecycle/migrate/tests.rs
+457 -2
View File
@@ -453,15 +453,173 @@ fn create_table_names(sql: &str) -> BTreeSet<String> {
let trimmed = line.trim_start();
let table_part = trimmed
.strip_prefix("CREATE TABLE IF NOT EXISTS public.")
.or_else(|| trimmed.strip_prefix("CREATE TABLE IF NOT EXISTS "))?;
.or_else(|| trimmed.strip_prefix("CREATE TABLE IF NOT EXISTS "))
.or_else(|| trimmed.strip_prefix("CREATE TABLE public."))
.or_else(|| trimmed.strip_prefix("CREATE TABLE "))?;
let table_name = table_part
.split(|ch: char| ch.is_ascii_whitespace() || ch == '(')
.next()?;
Some(table_name.trim_matches('"').to_string())
Some(
table_name
.trim_matches(|ch| ch == '"' || ch == '`')
.to_string(),
)
})
.collect()
}
#[test]
fn portable_driver_migrations_create_the_postgres_table_set() {
let mut postgres_tables = POSTGRES_MIGRATOR
.iter()
.filter(|migration| migration.migration_type.is_up_migration())
.flat_map(|migration| create_table_names(migration.sql.as_ref()))
.collect::<BTreeSet<_>>();
postgres_tables.remove("schema_backfills");
let mysql_tables = super::mysql::MIGRATOR
.iter()
.filter(|migration| migration.migration_type.is_up_migration())
.flat_map(|migration| create_table_names(migration.sql.as_ref()))
.collect::<BTreeSet<_>>();
let sqlite_tables = super::sqlite::MIGRATOR
.iter()
.filter(|migration| migration.migration_type.is_up_migration())
.flat_map(|migration| create_table_names(migration.sql.as_ref()))
.collect::<BTreeSet<_>>();
assert_eq!(mysql_tables, postgres_tables, "MySQL table set drifted");
assert_eq!(sqlite_tables, postgres_tables, "SQLite table set drifted");
}
#[tokio::test]
async fn migrated_sqlite_columns_match_the_generated_logical_schema() {
const GENERATED_SQLITE_SCHEMA: &[&str] = &[
include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/schema/generated/sqlite/baseline/001_identity.sql"
)),
include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/schema/generated/sqlite/baseline/002_provider_catalog.sql"
)),
include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/schema/generated/sqlite/baseline/003_auth_config.sql"
)),
include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/schema/generated/sqlite/baseline/004_proxy_nodes.sql"
)),
include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/schema/generated/sqlite/baseline/005_wallet_billing.sql"
)),
include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/schema/generated/sqlite/baseline/006_usage.sql"
)),
include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/schema/generated/sqlite/baseline/007_stats.sql"
)),
include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/schema/generated/sqlite/baseline/008_background_tasks.sql"
)),
];
let migrated = sqlx::sqlite::SqlitePoolOptions::new()
.max_connections(1)
.connect("sqlite::memory:")
.await
.expect("migrated sqlite pool should connect");
super::run_sqlite_migrations(&migrated)
.await
.expect("sqlite migrations should run");
let generated = sqlx::sqlite::SqlitePoolOptions::new()
.max_connections(1)
.connect("sqlite::memory:")
.await
.expect("generated sqlite pool should connect");
for source in GENERATED_SQLITE_SCHEMA {
sqlx::raw_sql(source)
.execute(&generated)
.await
.expect("generated sqlite schema fragment should run");
}
let migrated_tables = sqlite_portable_table_names(&migrated).await;
let generated_tables = sqlite_portable_table_names(&generated).await;
assert_eq!(migrated_tables, generated_tables);
for table in generated_tables {
let migrated_columns = sqlite_table_column_names(&migrated, &table).await;
let generated_columns = sqlite_table_column_names(&generated, &table).await;
assert_eq!(
migrated_columns, generated_columns,
"SQLite migration columns drifted for table {table}"
);
}
let migrated_indexes = sqlite_named_index_names(&migrated).await;
let generated_indexes = sqlite_named_index_names(&generated).await;
let missing_indexes = generated_indexes
.difference(&migrated_indexes)
.cloned()
.collect::<BTreeSet<_>>();
assert!(
missing_indexes.is_empty(),
"SQLite migrations are missing generated logical indexes: {missing_indexes:?}"
);
}
async fn sqlite_portable_table_names(pool: &SqlitePool) -> BTreeSet<String> {
query_scalar::<_, String>(
r#"
SELECT name
FROM sqlite_master
WHERE type = 'table'
AND name NOT LIKE 'sqlite_%'
AND name NOT IN ('_sqlx_migrations', 'schema_backfills')
ORDER BY name
"#,
)
.fetch_all(pool)
.await
.expect("sqlite table names should load")
.into_iter()
.collect()
}
async fn sqlite_table_column_names(pool: &SqlitePool, table: &str) -> BTreeSet<String> {
query_scalar::<_, String>("SELECT name FROM pragma_table_info(?) ORDER BY cid")
.bind(table)
.fetch_all(pool)
.await
.expect("sqlite table columns should load")
.into_iter()
.collect()
}
async fn sqlite_named_index_names(pool: &SqlitePool) -> BTreeSet<String> {
query_scalar::<_, String>(
r#"
SELECT name
FROM sqlite_master
WHERE type = 'index'
AND sql IS NOT NULL
ORDER BY name
"#,
)
.fetch_all(pool)
.await
.expect("sqlite named indexes should load")
.into_iter()
.collect()
}
#[test]
fn empty_database_snapshot_sql_includes_usage_body_blobs_and_audit_admin_role() {
assert!(EMPTY_DATABASE_SNAPSHOT_SQL.contains("'audit_admin'"));
@@ -862,6 +1020,9 @@ fn mysql_and_sqlite_migrations_include_enabled_incrementals() {
20260527000000,
20260528000000,
20260528020000,
20260725010000,
20260725020000,
20260725030000,
]
);
assert_eq!(
@@ -889,10 +1050,304 @@ fn mysql_and_sqlite_migrations_include_enabled_incrementals() {
20260527000000,
20260528000000,
20260528020000,
20260725000000,
20260725010000,
20260725020000,
20260725030000,
20260725040000,
]
);
}
#[tokio::test]
async fn sqlite_imported_timestamp_migration_normalizes_text_storage() {
let pool = SqlitePool::connect("sqlite::memory:")
.await
.expect("sqlite pool should connect");
super::run_sqlite_migrations(&pool)
.await
.expect("sqlite migrations should run");
query(
r#"
INSERT INTO global_models (id, name, created_at, updated_at)
VALUES
('timestamp-rfc3339', 'timestamp-rfc3339', '1970-01-01T00:00:01Z', '1970-01-01T08:00:02+08:00'),
('timestamp-sqlalchemy', 'timestamp-sqlalchemy', '1970-01-01 00:00:03.123456', '1970-01-01 00:00:04.987654'),
('timestamp-integer', 'timestamp-integer', 5, 6);
"#,
)
.execute(&pool)
.await
.expect("timestamp fixtures should insert");
query(
r#"
INSERT INTO "usage" (request_id, created_at_unix_ms, updated_at_unix_secs)
VALUES ('timestamp-usage', '1970-01-01T00:00:01.234900Z', '1970-01-01T00:00:02Z');
"#,
)
.execute(&pool)
.await
.expect("usage timestamp fixture should insert");
let migration = super::sqlite::MIGRATOR
.iter()
.find(|migration| migration.version == 20260725000000)
.expect("timestamp normalization migration should be embedded");
sqlx::raw_sql(migration.sql.as_ref())
.execute(&pool)
.await
.expect("timestamp normalization migration should apply");
let rows = sqlx::query_as::<_, (String, i64, i64, String, String)>(
r#"
SELECT id, created_at, updated_at, typeof(created_at), typeof(updated_at)
FROM global_models
WHERE id LIKE 'timestamp-%'
ORDER BY id
"#,
)
.fetch_all(&pool)
.await
.expect("normalized timestamps should decode as integers");
assert_eq!(
rows,
vec![
(
"timestamp-integer".to_string(),
5,
6,
"integer".to_string(),
"integer".to_string(),
),
(
"timestamp-rfc3339".to_string(),
1,
2,
"integer".to_string(),
"integer".to_string(),
),
(
"timestamp-sqlalchemy".to_string(),
3,
4,
"integer".to_string(),
"integer".to_string(),
),
]
);
let usage_timestamps = sqlx::query_as::<_, (i64, i64, String, String)>(
r#"
SELECT created_at_unix_ms, updated_at_unix_secs,
typeof(created_at_unix_ms), typeof(updated_at_unix_secs)
FROM "usage"
WHERE request_id = 'timestamp-usage'
"#,
)
.fetch_one(&pool)
.await
.expect("normalized usage timestamps should decode as integers");
assert_eq!(
usage_timestamps,
(1, 2, "integer".to_string(), "integer".to_string())
);
}
#[tokio::test]
async fn sqlite_imported_timestamp_migration_rejects_non_integer_storage() {
let pool = SqlitePool::connect("sqlite::memory:")
.await
.expect("sqlite pool should connect");
super::run_sqlite_migrations(&pool)
.await
.expect("sqlite migrations should run");
query(
r#"
INSERT INTO global_models (id, name, created_at, updated_at)
VALUES ('timestamp-invalid', 'timestamp-invalid', 1.5, 1);
"#,
)
.execute(&pool)
.await
.expect("non-integer timestamp fixture should insert");
let migration = super::sqlite::MIGRATOR
.iter()
.find(|migration| migration.version == 20260725000000)
.expect("timestamp normalization migration should be embedded");
let err = sqlx::raw_sql(migration.sql.as_ref())
.execute(&pool)
.await
.expect_err("non-integer timestamp should fail the migration");
assert!(err
.to_string()
.contains("imported_timestamp_storage_must_be_integer"));
}
#[tokio::test]
async fn sqlite_remaining_timestamp_migration_repairs_other_repository_domains() {
let pool = SqlitePool::connect("sqlite::memory:")
.await
.expect("sqlite pool should connect");
super::run_sqlite_migrations(&pool)
.await
.expect("sqlite migrations should run");
query(
r#"
INSERT INTO users (id, email, username, auth_source, created_at, updated_at)
VALUES ('timestamp-user', 'timestamp@example.com', 'timestamp-user', 'local', 1, 1);
INSERT INTO audit_logs (id, event_type, description, created_at)
VALUES ('timestamp-audit', 'test', 'test', '1970-01-01T00:00:01Z');
INSERT INTO request_candidates (
id, request_id, candidate_index, status, created_at, started_at, finished_at
) VALUES (
'timestamp-candidate', 'timestamp-request', 0, 'success',
'1970-01-01T00:00:02Z', '1970-01-01T00:00:03Z', '1970-01-01T00:00:04Z'
);
INSERT INTO stats_daily (id, date, created_at, updated_at)
VALUES (
'timestamp-stats', '1970-01-02',
'1970-01-01T00:00:05Z', '1970-01-01T00:00:06Z'
);
INSERT INTO user_sessions (
id, user_id, client_device_id, refresh_token_hash,
last_seen_at, expires_at, created_at, updated_at
) VALUES (
'timestamp-session', 'timestamp-user', 'device', 'hash',
'1970-01-01T00:00:07Z', '1970-01-01T00:00:08Z',
'1970-01-01T00:00:09Z', '1970-01-01T00:00:10Z'
);
"#,
)
.execute(&pool)
.await
.expect("remaining timestamp fixtures should insert");
let migration = super::sqlite::MIGRATOR
.iter()
.find(|migration| migration.version == 20260725040000)
.expect("remaining timestamp migration should be embedded");
sqlx::raw_sql(migration.sql.as_ref())
.execute(&pool)
.await
.expect("remaining timestamp migration should apply");
let audit = sqlx::query_as::<_, (i64, String)>(
"SELECT created_at, typeof(created_at) FROM audit_logs WHERE id = 'timestamp-audit'",
)
.fetch_one(&pool)
.await
.expect("normalized audit timestamp should load");
assert_eq!(audit, (1, "integer".to_string()));
let candidate = sqlx::query_as::<_, (i64, i64, i64, String, String, String)>(
r#"
SELECT created_at, started_at, finished_at,
typeof(created_at), typeof(started_at), typeof(finished_at)
FROM request_candidates
WHERE id = 'timestamp-candidate'
"#,
)
.fetch_one(&pool)
.await
.expect("normalized candidate timestamps should load");
assert_eq!(
candidate,
(
2,
3,
4,
"integer".to_string(),
"integer".to_string(),
"integer".to_string(),
)
);
let stats = sqlx::query_as::<_, (i64, i64, i64, String, String, String)>(
r#"
SELECT date, created_at, updated_at,
typeof(date), typeof(created_at), typeof(updated_at)
FROM stats_daily
WHERE id = 'timestamp-stats'
"#,
)
.fetch_one(&pool)
.await
.expect("normalized stats timestamps should load");
assert_eq!(
stats,
(
86_400,
5,
6,
"integer".to_string(),
"integer".to_string(),
"integer".to_string(),
)
);
let session = sqlx::query_as::<_, (i64, i64, i64, i64, String, String, String, String)>(
r#"
SELECT last_seen_at, expires_at, created_at, updated_at,
typeof(last_seen_at), typeof(expires_at), typeof(created_at), typeof(updated_at)
FROM user_sessions
WHERE id = 'timestamp-session'
"#,
)
.fetch_one(&pool)
.await
.expect("normalized session timestamps should load");
assert_eq!(
session,
(
7,
8,
9,
10,
"integer".to_string(),
"integer".to_string(),
"integer".to_string(),
"integer".to_string(),
)
);
}
#[tokio::test]
async fn sqlite_remaining_timestamp_migration_rejects_invalid_storage() {
let pool = SqlitePool::connect("sqlite::memory:")
.await
.expect("sqlite pool should connect");
super::run_sqlite_migrations(&pool)
.await
.expect("sqlite migrations should run");
query(
r#"
INSERT INTO audit_logs (id, event_type, description, created_at)
VALUES ('timestamp-invalid-audit', 'test', 'test', 1.5);
"#,
)
.execute(&pool)
.await
.expect("invalid timestamp fixture should insert");
let migration = super::sqlite::MIGRATOR
.iter()
.find(|migration| migration.version == 20260725040000)
.expect("remaining timestamp migration should be embedded");
let err = sqlx::raw_sql(migration.sql.as_ref())
.execute(&pool)
.await
.expect_err("invalid remaining timestamp should fail the migration");
assert!(err.to_string().contains("invalid_count = 0"));
}
#[tokio::test]
async fn endpoint_api_root_migration_moves_v1_from_stored_default_paths() {
let pool = SqlitePool::connect("sqlite::memory:")
crates/aether-data/runtime/src/repository/usage/mod.rs
-355
View File
@@ -1,358 +1,3 @@
#[cfg(feature = "mysql")]
macro_rules! impl_materialized_usage_read_repository {
($repository:ty) => {
#[async_trait::async_trait]
impl $crate::repository::usage::UsageReadRepository for $repository {
async fn find_by_id(
&self,
id: &str,
) -> Result<
Option<$crate::repository::usage::StoredRequestUsageAudit>,
$crate::DataLayerError,
> {
let repository = self.materialize_read_model().await?;
<$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::find_by_id(&repository, id).await
}
async fn list_by_ids(
&self,
ids: &[String],
) -> Result<
Vec<$crate::repository::usage::StoredRequestUsageAudit>,
$crate::DataLayerError,
> {
let repository = self.materialize_read_model().await?;
<$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::list_by_ids(&repository, ids).await
}
async fn find_by_request_id(
&self,
request_id: &str,
) -> Result<
Option<$crate::repository::usage::StoredRequestUsageAudit>,
$crate::DataLayerError,
> {
let repository = self.materialize_read_model().await?;
<$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::find_by_request_id(&repository, request_id).await
}
async fn resolve_body_ref(
&self,
body_ref: &str,
) -> Result<Option<serde_json::Value>, $crate::DataLayerError> {
let repository = self.materialize_read_model().await?;
<$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::resolve_body_ref(&repository, body_ref).await
}
async fn list_usage_audits(
&self,
query: &$crate::repository::usage::UsageAuditListQuery,
) -> Result<
Vec<$crate::repository::usage::StoredRequestUsageAudit>,
$crate::DataLayerError,
> {
let repository = self.materialize_read_model().await?;
<$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::list_usage_audits(&repository, query).await
}
async fn count_usage_audits(
&self,
query: &$crate::repository::usage::UsageAuditListQuery,
) -> Result<u64, $crate::DataLayerError> {
let repository = self.materialize_read_model().await?;
<$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::count_usage_audits(&repository, query).await
}
async fn list_usage_audits_by_keyword_search(
&self,
query: &$crate::repository::usage::UsageAuditKeywordSearchQuery,
) -> Result<
Vec<$crate::repository::usage::StoredRequestUsageAudit>,
$crate::DataLayerError,
> {
let repository = self.materialize_read_model().await?;
<$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::list_usage_audits_by_keyword_search(&repository, query).await
}
async fn count_usage_audits_by_keyword_search(
&self,
query: &$crate::repository::usage::UsageAuditKeywordSearchQuery,
) -> Result<u64, $crate::DataLayerError> {
let repository = self.materialize_read_model().await?;
<$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::count_usage_audits_by_keyword_search(&repository, query).await
}
async fn aggregate_usage_audits(
&self,
query: &$crate::repository::usage::UsageAuditAggregationQuery,
) -> Result<
Vec<$crate::repository::usage::StoredUsageAuditAggregation>,
$crate::DataLayerError,
> {
let repository = self.materialize_read_model().await?;
<$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::aggregate_usage_audits(&repository, query).await
}
async fn summarize_usage_audits(
&self,
query: &$crate::repository::usage::UsageAuditSummaryQuery,
) -> Result<
$crate::repository::usage::StoredUsageAuditSummary,
$crate::DataLayerError,
> {
let repository = self.materialize_read_model().await?;
<$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_usage_audits(&repository, query).await
}
async fn summarize_usage_totals_by_user_ids(
&self,
user_ids: &[String],
) -> Result<Vec<$crate::repository::usage::StoredUsageUserTotals>, $crate::DataLayerError>
{
<$repository>::summarize_usage_totals_by_user_ids(self, user_ids).await
}
async fn summarize_usage_cache_hit_summary(
&self,
query: &$crate::repository::usage::UsageCacheHitSummaryQuery,
) -> Result<
$crate::repository::usage::StoredUsageCacheHitSummary,
$crate::DataLayerError,
> {
let repository = self.materialize_read_model().await?;
<$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_usage_cache_hit_summary(&repository, query).await
}
async fn summarize_usage_settled_cost(
&self,
query: &$crate::repository::usage::UsageSettledCostSummaryQuery,
) -> Result<
$crate::repository::usage::StoredUsageSettledCostSummary,
$crate::DataLayerError,
> {
let repository = self.materialize_read_model().await?;
<$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_usage_settled_cost(&repository, query).await
}
async fn summarize_usage_cache_affinity_hit_summary(
&self,
query: &$crate::repository::usage::UsageCacheAffinityHitSummaryQuery,
) -> Result<
$crate::repository::usage::StoredUsageCacheAffinityHitSummary,
$crate::DataLayerError,
> {
let repository = self.materialize_read_model().await?;
<$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_usage_cache_affinity_hit_summary(&repository, query).await
}
async fn list_usage_cache_affinity_intervals(
&self,
query: &$crate::repository::usage::UsageCacheAffinityIntervalQuery,
) -> Result<
Vec<$crate::repository::usage::StoredUsageCacheAffinityIntervalRow>,
$crate::DataLayerError,
> {
let repository = self.materialize_read_model().await?;
<$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::list_usage_cache_affinity_intervals(&repository, query).await
}
async fn summarize_dashboard_usage(
&self,
query: &$crate::repository::usage::UsageDashboardSummaryQuery,
) -> Result<
$crate::repository::usage::StoredUsageDashboardSummary,
$crate::DataLayerError,
> {
<$repository>::summarize_dashboard_usage(self, query).await
}
async fn list_dashboard_daily_breakdown(
&self,
query: &$crate::repository::usage::UsageDashboardDailyBreakdownQuery,
) -> Result<
Vec<$crate::repository::usage::StoredUsageDashboardDailyBreakdownRow>,
$crate::DataLayerError,
> {
<$repository>::list_dashboard_daily_breakdown(self, query).await
}
async fn summarize_dashboard_provider_counts(
&self,
query: &$crate::repository::usage::UsageDashboardProviderCountsQuery,
) -> Result<
Vec<$crate::repository::usage::StoredUsageDashboardProviderCount>,
$crate::DataLayerError,
> {
let repository = self.materialize_read_model().await?;
<$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_dashboard_provider_counts(&repository, query).await
}
async fn summarize_usage_breakdown(
&self,
query: &$crate::repository::usage::UsageBreakdownSummaryQuery,
) -> Result<
Vec<$crate::repository::usage::StoredUsageBreakdownSummaryRow>,
$crate::DataLayerError,
> {
let repository = self.materialize_read_model().await?;
<$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_usage_breakdown(&repository, query).await
}
async fn count_monitoring_usage_errors(
&self,
query: &$crate::repository::usage::UsageMonitoringErrorCountQuery,
) -> Result<u64, $crate::DataLayerError> {
let repository = self.materialize_read_model().await?;
<$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::count_monitoring_usage_errors(&repository, query).await
}
async fn list_monitoring_usage_errors(
&self,
query: &$crate::repository::usage::UsageMonitoringErrorListQuery,
) -> Result<
Vec<$crate::repository::usage::StoredRequestUsageAudit>,
$crate::DataLayerError,
> {
let repository = self.materialize_read_model().await?;
<$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::list_monitoring_usage_errors(&repository, query).await
}
async fn summarize_usage_error_distribution(
&self,
query: &$crate::repository::usage::UsageErrorDistributionQuery,
) -> Result<
Vec<$crate::repository::usage::StoredUsageErrorDistributionRow>,
$crate::DataLayerError,
> {
let repository = self.materialize_read_model().await?;
<$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_usage_error_distribution(&repository, query).await
}
async fn summarize_usage_performance_percentiles(
&self,
query: &$crate::repository::usage::UsagePerformancePercentilesQuery,
) -> Result<
Vec<$crate::repository::usage::StoredUsagePerformancePercentilesRow>,
$crate::DataLayerError,
> {
let repository = self.materialize_read_model().await?;
<$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_usage_performance_percentiles(&repository, query).await
}
async fn summarize_usage_provider_performance(
&self,
query: &$crate::repository::usage::UsageProviderPerformanceQuery,
) -> Result<
$crate::repository::usage::StoredUsageProviderPerformance,
$crate::DataLayerError,
> {
let repository = self.materialize_read_model().await?;
<$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_usage_provider_performance(&repository, query).await
}
async fn summarize_usage_cost_savings(
&self,
query: &$crate::repository::usage::UsageCostSavingsSummaryQuery,
) -> Result<
$crate::repository::usage::StoredUsageCostSavingsSummary,
$crate::DataLayerError,
> {
let repository = self.materialize_read_model().await?;
<$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_usage_cost_savings(&repository, query).await
}
async fn summarize_usage_time_series(
&self,
query: &$crate::repository::usage::UsageTimeSeriesQuery,
) -> Result<
Vec<$crate::repository::usage::StoredUsageTimeSeriesBucket>,
$crate::DataLayerError,
> {
let repository = self.materialize_read_model().await?;
<$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_usage_time_series(&repository, query).await
}
async fn summarize_usage_leaderboard(
&self,
query: &$crate::repository::usage::UsageLeaderboardQuery,
) -> Result<
Vec<$crate::repository::usage::StoredUsageLeaderboardSummary>,
$crate::DataLayerError,
> {
let repository = self.materialize_read_model().await?;
<$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_usage_leaderboard(&repository, query).await
}
async fn list_recent_usage_audits(
&self,
user_id: Option<&str>,
limit: usize,
) -> Result<
Vec<$crate::repository::usage::StoredRequestUsageAudit>,
$crate::DataLayerError,
> {
let repository = self.materialize_read_model().await?;
<$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::list_recent_usage_audits(&repository, user_id, limit).await
}
async fn summarize_total_tokens_by_api_key_ids(
&self,
api_key_ids: &[String],
) -> Result<std::collections::BTreeMap<String, u64>, $crate::DataLayerError> {
let repository = self.materialize_read_model().await?;
<$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_total_tokens_by_api_key_ids(&repository, api_key_ids).await
}
async fn summarize_usage_by_provider_api_key_ids(
&self,
provider_api_key_ids: &[String],
) -> Result<
std::collections::BTreeMap<
String,
$crate::repository::usage::StoredProviderApiKeyUsageSummary,
>,
$crate::DataLayerError,
> {
let repository = self.materialize_read_model().await?;
<$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_usage_by_provider_api_key_ids(&repository, provider_api_key_ids).await
}
async fn summarize_usage_by_provider_api_key_windows(
&self,
requests: &[$crate::repository::usage::ProviderApiKeyWindowUsageRequest],
) -> Result<
Vec<$crate::repository::usage::StoredProviderApiKeyWindowUsageSummary>,
$crate::DataLayerError,
> {
let repository = self.materialize_read_model().await?;
<$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_usage_by_provider_api_key_windows(&repository, requests).await
}
async fn summarize_provider_usage_since(
&self,
provider_id: &str,
since_unix_secs: u64,
) -> Result<
$crate::repository::usage::StoredProviderUsageSummary,
$crate::DataLayerError,
> {
let repository = self.materialize_read_model().await?;
<$crate::repository::usage::InMemoryUsageReadRepository as $crate::repository::usage::UsageReadRepository>::summarize_provider_usage_since(&repository, provider_id, since_unix_secs).await
}
async fn summarize_usage_daily_heatmap(
&self,
query: &$crate::repository::usage::UsageDailyHeatmapQuery,
) -> Result<
Vec<$crate::repository::usage::StoredUsageDailySummary>,
$crate::DataLayerError,
> {
<$repository>::summarize_usage_daily_heatmap(self, query).await
}
}
};
}
mod memory;
#[cfg(feature = "mysql")]
mod mysql;
crates/aether-data/runtime/src/repository/usage/mysql.rs
+393 -14
View File
@@ -1,7 +1,23 @@
use std::collections::BTreeMap;
use aether_data_contracts::repository::usage::{
StoredUsageDailySummary, StoredUsageDashboardDailyBreakdownRow, StoredUsageDashboardSummary,
StoredUsageUserTotals, UsageDailyHeatmapQuery, UsageDashboardDailyBreakdownQuery,
UsageDashboardSummaryQuery, UsageReadRepository,
ProviderApiKeyWindowUsageRequest, StoredProviderApiKeyUsageSummary,
StoredProviderApiKeyWindowUsageSummary, StoredProviderUsageSummary, StoredRequestUsageAudit,
StoredUsageAuditAggregation, StoredUsageAuditSummary, StoredUsageBreakdownSummaryRow,
StoredUsageCacheAffinityHitSummary, StoredUsageCacheAffinityIntervalRow,
StoredUsageCacheHitSummary, StoredUsageCostSavingsSummary, StoredUsageDailySummary,
StoredUsageDashboardDailyBreakdownRow, StoredUsageDashboardProviderCount,
StoredUsageDashboardSummary, StoredUsageErrorDistributionRow, StoredUsageLeaderboardSummary,
StoredUsagePerformancePercentilesRow, StoredUsageProviderPerformance,
StoredUsageSettledCostSummary, StoredUsageTimeSeriesBucket, StoredUsageUserTotals,
UsageAuditAggregationQuery, UsageAuditKeywordSearchQuery, UsageAuditListQuery,
UsageAuditSummaryQuery, UsageBreakdownSummaryQuery, UsageCacheAffinityHitSummaryQuery,
UsageCacheAffinityIntervalQuery, UsageCacheHitSummaryQuery, UsageCostSavingsSummaryQuery,
UsageDailyHeatmapQuery, UsageDashboardDailyBreakdownQuery, UsageDashboardProviderCountsQuery,
UsageDashboardSummaryQuery, UsageErrorDistributionQuery, UsageLeaderboardQuery,
UsageMonitoringErrorCountQuery, UsageMonitoringErrorListQuery,
UsagePerformancePercentilesQuery, UsageProviderPerformanceQuery, UsageReadRepository,
UsageSettledCostSummaryQuery, UsageTimeSeriesQuery,
};
use super::InMemoryUsageReadRepository;
@@ -9,30 +25,116 @@ use crate::driver::mysql::MysqlPool;
use crate::DataLayerError;
pub use aether_data_mysql::MysqlUsageWriteRepository;
use aether_data_mysql::{MysqlUsageReadFilter, MysqlUsageStorage};
#[derive(Debug, Clone)]
pub struct MysqlUsageReadRepository {
storage: aether_data_mysql::MysqlUsageStorage,
storage: MysqlUsageStorage,
}
impl MysqlUsageReadRepository {
pub fn new(pool: MysqlPool) -> Self {
Self {
storage: aether_data_mysql::MysqlUsageStorage::new(pool),
storage: MysqlUsageStorage::new(pool),
}
}
async fn materialize_read_model(&self) -> Result<InMemoryUsageReadRepository, DataLayerError> {
async fn materialize_read_model(
&self,
filter: MysqlUsageReadFilter,
) -> Result<InMemoryUsageReadRepository, DataLayerError> {
Ok(InMemoryUsageReadRepository::seed(
self.storage.load_usage_records().await?,
self.storage.load_usage_records_in_range(&filter).await?,
))
}
async fn summarize_usage_daily_heatmap(
fn range(created_from_unix_secs: u64, created_until_unix_secs: u64) -> MysqlUsageReadFilter {
MysqlUsageReadFilter::new(created_from_unix_secs, created_until_unix_secs)
}
}
#[async_trait::async_trait]
impl UsageReadRepository for MysqlUsageReadRepository {
async fn find_by_id(
&self,
query: &UsageDailyHeatmapQuery,
) -> Result<Vec<StoredUsageDailySummary>, DataLayerError> {
self.storage.summarize_usage_daily_heatmap(query).await
id: &str,
) -> Result<Option<StoredRequestUsageAudit>, DataLayerError> {
self.storage.find_by_id(id).await
}
async fn list_by_ids(
&self,
ids: &[String],
) -> Result<Vec<StoredRequestUsageAudit>, DataLayerError> {
self.storage.list_by_ids(ids).await
}
async fn find_by_request_id(
&self,
request_id: &str,
) -> Result<Option<StoredRequestUsageAudit>, DataLayerError> {
self.storage.find_by_request_id(request_id).await
}
async fn resolve_body_ref(
&self,
body_ref: &str,
) -> Result<Option<serde_json::Value>, DataLayerError> {
self.storage.resolve_body_ref(body_ref).await
}
async fn list_usage_audits(
&self,
query: &UsageAuditListQuery,
) -> Result<Vec<StoredRequestUsageAudit>, DataLayerError> {
self.storage.list_usage_audits(query).await
}
async fn count_usage_audits(&self, query: &UsageAuditListQuery) -> Result<u64, DataLayerError> {
self.storage.count_usage_audits(query).await
}
async fn list_usage_audits_by_keyword_search(
&self,
query: &UsageAuditKeywordSearchQuery,
) -> Result<Vec<StoredRequestUsageAudit>, DataLayerError> {
self.storage
.list_usage_audits_by_keyword_search(query)
.await
}
async fn count_usage_audits_by_keyword_search(
&self,
query: &UsageAuditKeywordSearchQuery,
) -> Result<u64, DataLayerError> {
self.storage
.count_usage_audits_by_keyword_search(query)
.await
}
async fn aggregate_usage_audits(
&self,
query: &UsageAuditAggregationQuery,
) -> Result<Vec<StoredUsageAuditAggregation>, DataLayerError> {
let repository = self
.materialize_read_model(Self::range(
query.created_from_unix_secs,
query.created_until_unix_secs,
))
.await?;
repository.aggregate_usage_audits(query).await
}
async fn summarize_usage_audits(
&self,
query: &UsageAuditSummaryQuery,
) -> Result<StoredUsageAuditSummary, DataLayerError> {
let filter = Self::range(query.created_from_unix_secs, query.created_until_unix_secs)
.with_user_id(query.user_id.as_deref())
.with_provider_name(query.provider_name.as_deref())
.with_model(query.model.as_deref());
let repository = self.materialize_read_model(filter).await?;
repository.summarize_usage_audits(query).await
}
async fn summarize_usage_totals_by_user_ids(
@@ -44,10 +146,60 @@ impl MysqlUsageReadRepository {
.await
}
async fn summarize_usage_cache_hit_summary(
&self,
query: &UsageCacheHitSummaryQuery,
) -> Result<StoredUsageCacheHitSummary, DataLayerError> {
let filter = Self::range(query.created_from_unix_secs, query.created_until_unix_secs)
.with_user_id(query.user_id.as_deref());
let repository = self.materialize_read_model(filter).await?;
repository.summarize_usage_cache_hit_summary(query).await
}
async fn summarize_usage_settled_cost(
&self,
query: &UsageSettledCostSummaryQuery,
) -> Result<StoredUsageSettledCostSummary, DataLayerError> {
let filter = Self::range(query.created_from_unix_secs, query.created_until_unix_secs)
.with_user_id(query.user_id.as_deref())
.with_api_key_id(query.api_key_id.as_deref());
let repository = self.materialize_read_model(filter).await?;
repository.summarize_usage_settled_cost(query).await
}
async fn summarize_usage_cache_affinity_hit_summary(
&self,
query: &UsageCacheAffinityHitSummaryQuery,
) -> Result<StoredUsageCacheAffinityHitSummary, DataLayerError> {
let filter = Self::range(query.created_from_unix_secs, query.created_until_unix_secs)
.with_user_id(query.user_id.as_deref())
.with_api_key_id(query.api_key_id.as_deref())
.completed_only();
let repository = self.materialize_read_model(filter).await?;
repository
.summarize_usage_cache_affinity_hit_summary(query)
.await
}
async fn list_usage_cache_affinity_intervals(
&self,
query: &UsageCacheAffinityIntervalQuery,
) -> Result<Vec<StoredUsageCacheAffinityIntervalRow>, DataLayerError> {
let filter = Self::range(query.created_from_unix_secs, query.created_until_unix_secs)
.with_user_id(query.user_id.as_deref())
.with_api_key_id(query.api_key_id.as_deref())
.completed_only();
let repository = self.materialize_read_model(filter).await?;
repository.list_usage_cache_affinity_intervals(query).await
}
async fn summarize_dashboard_usage(
&self,
query: &UsageDashboardSummaryQuery,
) -> Result<StoredUsageDashboardSummary, DataLayerError> {
if query.created_from_unix_secs >= query.created_until_unix_secs {
return Ok(StoredUsageDashboardSummary::default());
}
if let Some(summary) = self
.storage
.summarize_dashboard_usage_from_daily_aggregates(query)
@@ -55,7 +207,10 @@ impl MysqlUsageReadRepository {
{
return Ok(summary);
}
let repository = self.materialize_read_model().await?;
let filter = Self::range(query.created_from_unix_secs, query.created_until_unix_secs)
.with_user_id(query.user_id.as_deref())
.finalized_only();
let repository = self.materialize_read_model(filter).await?;
repository.summarize_dashboard_usage(query).await
}
@@ -63,6 +218,9 @@ impl MysqlUsageReadRepository {
&self,
query: &UsageDashboardDailyBreakdownQuery,
) -> Result<Vec<StoredUsageDashboardDailyBreakdownRow>, DataLayerError> {
if query.created_from_unix_secs >= query.created_until_unix_secs {
return Ok(Vec::new());
}
let rows = self
.storage
.list_dashboard_daily_breakdown_from_daily_aggregates(query)
@@ -70,9 +228,230 @@ impl MysqlUsageReadRepository {
if !rows.is_empty() {
return Ok(rows);
}
let repository = self.materialize_read_model().await?;
let filter = Self::range(query.created_from_unix_secs, query.created_until_unix_secs)
.with_user_id(query.user_id.as_deref())
.finalized_only();
let repository = self.materialize_read_model(filter).await?;
repository.list_dashboard_daily_breakdown(query).await
}
async fn summarize_dashboard_provider_counts(
&self,
query: &UsageDashboardProviderCountsQuery,
) -> Result<Vec<StoredUsageDashboardProviderCount>, DataLayerError> {
let filter = Self::range(query.created_from_unix_secs, query.created_until_unix_secs)
.with_user_id(query.user_id.as_deref())
.finalized_only();
let repository = self.materialize_read_model(filter).await?;
repository.summarize_dashboard_provider_counts(query).await
}
async fn summarize_usage_breakdown(
&self,
query: &UsageBreakdownSummaryQuery,
) -> Result<Vec<StoredUsageBreakdownSummaryRow>, DataLayerError> {
let filter = Self::range(query.created_from_unix_secs, query.created_until_unix_secs)
.with_user_id(query.user_id.as_deref())
.with_provider_name(query.provider_name.as_deref())
.with_model(query.model.as_deref())
.with_api_format(query.api_format.as_deref())
.finalized_only();
let repository = self.materialize_read_model(filter).await?;
repository.summarize_usage_breakdown(query).await
}
async fn count_monitoring_usage_errors(
&self,
query: &UsageMonitoringErrorCountQuery,
) -> Result<u64, DataLayerError> {
self.storage.count_monitoring_usage_errors(query).await
}
async fn list_monitoring_usage_errors(
&self,
query: &UsageMonitoringErrorListQuery,
) -> Result<Vec<StoredRequestUsageAudit>, DataLayerError> {
self.storage.list_monitoring_usage_errors(query).await
}
async fn summarize_usage_error_distribution(
&self,
query: &UsageErrorDistributionQuery,
) -> Result<Vec<StoredUsageErrorDistributionRow>, DataLayerError> {
let repository = self
.materialize_read_model(Self::range(
query.created_from_unix_secs,
query.created_until_unix_secs,
))
.await?;
repository.summarize_usage_error_distribution(query).await
}
async fn summarize_usage_performance_percentiles(
&self,
query: &UsagePerformancePercentilesQuery,
) -> Result<Vec<StoredUsagePerformancePercentilesRow>, DataLayerError> {
let repository = self
.materialize_read_model(
Self::range(query.created_from_unix_secs, query.created_until_unix_secs)
.completed_only(),
)
.await?;
repository
.summarize_usage_performance_percentiles(query)
.await
}
async fn summarize_usage_provider_performance(
&self,
query: &UsageProviderPerformanceQuery,
) -> Result<StoredUsageProviderPerformance, DataLayerError> {
let filter = Self::range(query.created_from_unix_secs, query.created_until_unix_secs)
.with_provider_id(query.provider_id.as_deref())
.with_model(query.model.as_deref())
.with_api_format(query.api_format.as_deref())
.with_endpoint_kind(query.endpoint_kind.as_deref())
.with_is_stream(query.is_stream)
.with_has_format_conversion(query.has_format_conversion)
.finalized_only();
let repository = self.materialize_read_model(filter).await?;
repository.summarize_usage_provider_performance(query).await
}
async fn summarize_usage_cost_savings(
&self,
query: &UsageCostSavingsSummaryQuery,
) -> Result<StoredUsageCostSavingsSummary, DataLayerError> {
let filter = Self::range(query.created_from_unix_secs, query.created_until_unix_secs)
.with_user_id(query.user_id.as_deref())
.with_provider_name(query.provider_name.as_deref())
.with_model(query.model.as_deref());
let repository = self.materialize_read_model(filter).await?;
repository.summarize_usage_cost_savings(query).await
}
async fn summarize_usage_time_series(
&self,
query: &UsageTimeSeriesQuery,
) -> Result<Vec<StoredUsageTimeSeriesBucket>, DataLayerError> {
let filter = Self::range(query.created_from_unix_secs, query.created_until_unix_secs)
.with_user_id(query.user_id.as_deref())
.with_provider_name(query.provider_name.as_deref())
.with_model(query.model.as_deref());
let repository = self.materialize_read_model(filter).await?;
repository.summarize_usage_time_series(query).await
}
async fn summarize_usage_leaderboard(
&self,
query: &UsageLeaderboardQuery,
) -> Result<Vec<StoredUsageLeaderboardSummary>, DataLayerError> {
let filter = Self::range(query.created_from_unix_secs, query.created_until_unix_secs)
.with_user_id(query.user_id.as_deref())
.with_provider_name(query.provider_name.as_deref())
.with_model(query.model.as_deref())
.finalized_only();
let repository = self.materialize_read_model(filter).await?;
repository.summarize_usage_leaderboard(query).await
}
async fn list_recent_usage_audits(
&self,
user_id: Option<&str>,
limit: usize,
) -> Result<Vec<StoredRequestUsageAudit>, DataLayerError> {
self.storage.list_recent_usage_audits(user_id, limit).await
}
async fn summarize_total_tokens_by_api_key_ids(
&self,
api_key_ids: &[String],
) -> Result<BTreeMap<String, u64>, DataLayerError> {
let repository = InMemoryUsageReadRepository::seed(
self.storage
.load_usage_records_by_api_key_ids(api_key_ids)
.await?,
);
repository
.summarize_total_tokens_by_api_key_ids(api_key_ids)
.await
}
async fn summarize_usage_by_provider_api_key_ids(
&self,
provider_api_key_ids: &[String],
) -> Result<BTreeMap<String, StoredProviderApiKeyUsageSummary>, DataLayerError> {
let repository = InMemoryUsageReadRepository::seed(
self.storage
.load_usage_records_by_provider_api_key_ids(provider_api_key_ids)
.await?,
);
repository
.summarize_usage_by_provider_api_key_ids(provider_api_key_ids)
.await
}
async fn summarize_usage_by_provider_api_key_windows(
&self,
requests: &[ProviderApiKeyWindowUsageRequest],
) -> Result<Vec<StoredProviderApiKeyWindowUsageSummary>, DataLayerError> {
let repository = InMemoryUsageReadRepository::seed(
self.storage
.load_usage_records_by_provider_api_key_windows(requests)
.await?,
);
repository
.summarize_usage_by_provider_api_key_windows(requests)
.await
}
async fn summarize_provider_usage_since(
&self,
provider_id: &str,
since_unix_secs: u64,
) -> Result<StoredProviderUsageSummary, DataLayerError> {
let repository = InMemoryUsageReadRepository::seed(
self.storage
.load_usage_records_for_provider_since(provider_id, since_unix_secs)
.await?,
);
repository
.summarize_provider_usage_since(provider_id, since_unix_secs)
.await
}
async fn summarize_usage_daily_heatmap(
&self,
query: &UsageDailyHeatmapQuery,
) -> Result<Vec<StoredUsageDailySummary>, DataLayerError> {
self.storage.summarize_usage_daily_heatmap(query).await
}
async fn read_usage_counter_health(
&self,
) -> Result<aether_data_contracts::repository::usage::UsageCounterHealthSnapshot, DataLayerError>
{
self.storage.read_usage_counter_health().await
}
async fn read_usage_counter_pending_health(
&self,
) -> Result<
aether_data_contracts::repository::usage::UsageCounterPendingHealthSnapshot,
DataLayerError,
> {
self.storage.read_usage_counter_pending_health().await
}
}
impl_materialized_usage_read_repository!(MysqlUsageReadRepository);
#[cfg(test)]
mod tests {
#[test]
fn mysql_usage_reads_do_not_restore_the_unconditional_full_table_loader() {
let source = include_str!("mysql.rs");
let forbidden = ["load_usage_", "records()"].concat();
assert!(!source.contains(&forbidden));
assert!(source.contains("load_usage_records_in_range"));
assert!(source.contains("MysqlUsageReadFilter::new"));
}
}