mirror of
https://github.com/fawney19/Aether.git
synced 2026-10-07 18:07:47 +08:00
feat: revamp analytics dashboards and harden database migrations
Add dashboard and overview analytics, health monitoring, provider expense tracking, and announcement updates across the gateway and frontend. Keep schema migrations free of historical backfills while preserving automatic backfill execution. Bound migration deadlines, run schema preparation before Compose replacement, and anonymize deleted dashboard users. Include the current documentation cleanup and regression coverage.
This commit is contained in:
elky
@@ -18,10 +18,7 @@
|
||||
-- Runs before 20260403000000_baseline.sql so that fresh databases have a
|
||||
-- complete schema by the time baseline (a no-op handoff point) and all
|
||||
-- later ADD COLUMN IF NOT EXISTS migrations execute.
|
||||
SET statement_timeout = 0;
|
||||
|
||||
SET lock_timeout = 0;
|
||||
|
||||
-- Keep the migration runner's statement and lock deadlines in effect.
|
||||
SET idle_in_transaction_session_timeout = 0;
|
||||
|
||||
SET client_encoding = 'UTF8';
|
||||
|
||||
+278
@@ -0,0 +1,278 @@
|
||||
-- Final overview schema and account attribution; no historical usage is backfilled.
|
||||
-- Install transaction-owned dirty events before any later concurrent index build.
|
||||
ALTER TABLE public.usage ADD COLUMN IF NOT EXISTS failure_origin text;
|
||||
ALTER TABLE public.usage ADD COLUMN IF NOT EXISTS failure_stage text;
|
||||
ALTER TABLE public.usage ADD COLUMN IF NOT EXISTS failure_reason text;
|
||||
ALTER TABLE public.usage ADD COLUMN IF NOT EXISTS failure_schema_version integer;
|
||||
ALTER TABLE public.usage_settlement_snapshots
|
||||
ADD COLUMN IF NOT EXISTS quota_covered_amount_usd numeric(20,8),
|
||||
ADD COLUMN IF NOT EXISTS wallet_consumed_amount_usd numeric(20,8),
|
||||
ADD COLUMN IF NOT EXISTS wallet_debit_amount_usd numeric(20,8),
|
||||
ADD COLUMN IF NOT EXISTS wallet_recharge_debit_usd numeric(20,8),
|
||||
ADD COLUMN IF NOT EXISTS wallet_gift_debit_usd numeric(20,8),
|
||||
ADD COLUMN IF NOT EXISTS wallet_overdraft_usd numeric(20,8),
|
||||
ADD COLUMN IF NOT EXISTS allocation_schema_version integer,
|
||||
ADD COLUMN IF NOT EXISTS allocation_status text;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.usage_attribution_snapshots (
|
||||
request_id text PRIMARY KEY,
|
||||
actor_user_id text,
|
||||
credential_owner_id text,
|
||||
attribution_kind text NOT NULL DEFAULT 'unknown',
|
||||
attribution_source text NOT NULL DEFAULT 'unknown',
|
||||
record_kind text NOT NULL DEFAULT 'request',
|
||||
parent_request_id text,
|
||||
schema_version integer NOT NULL DEFAULT 1,
|
||||
attribution_revision bigint NOT NULL DEFAULT 1,
|
||||
recorded_at timestamptz NOT NULL DEFAULT NOW()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS ix_usage_attribution_actor_request
|
||||
ON public.usage_attribution_snapshots(actor_user_id, request_id);
|
||||
-- This attribution table is new and empty; build both lookup indexes here.
|
||||
CREATE INDEX IF NOT EXISTS ix_usage_attribution_owner_request
|
||||
ON public.usage_attribution_snapshots(credential_owner_id, request_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.stats_bucket_state (
|
||||
projection_version text NOT NULL,
|
||||
granularity text NOT NULL,
|
||||
bucket_start timestamptz NOT NULL,
|
||||
source_revision bigint NOT NULL DEFAULT 0,
|
||||
built_revision bigint NOT NULL DEFAULT -1,
|
||||
coverage_status text NOT NULL DEFAULT 'unbuilt',
|
||||
built_at timestamptz,
|
||||
last_error text,
|
||||
last_failed_at timestamptz,
|
||||
PRIMARY KEY (projection_version, granularity, bucket_start)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS ix_stats_bucket_state_dirty
|
||||
ON public.stats_bucket_state(bucket_start)
|
||||
WHERE source_revision > built_revision;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.stats_overview_hourly (
|
||||
projection_version text NOT NULL,
|
||||
bucket_start timestamptz NOT NULL,
|
||||
dimensions jsonb NOT NULL,
|
||||
metrics jsonb NOT NULL,
|
||||
PRIMARY KEY(projection_version, bucket_start, dimensions)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS public.stats_overview_daily (
|
||||
projection_version text NOT NULL,
|
||||
bucket_start timestamptz NOT NULL,
|
||||
dimensions jsonb NOT NULL,
|
||||
metrics jsonb NOT NULL,
|
||||
PRIMARY KEY(projection_version, bucket_start, dimensions)
|
||||
);
|
||||
|
||||
-- These triggers cover old writers, delayed settlement, and maintenance in the fact transaction.
|
||||
-- Only future fact mutations enqueue work; no historical rows are backfilled.
|
||||
-- Each writer owns its transaction's keys, so unrelated requests never contend
|
||||
-- on the current hour/day's stats_bucket_state row.
|
||||
CREATE TABLE IF NOT EXISTS public.stats_overview_dirty_events (
|
||||
transaction_id bigint NOT NULL,
|
||||
projection_version text NOT NULL,
|
||||
granularity text NOT NULL,
|
||||
bucket_start timestamptz NOT NULL,
|
||||
unrecoverable boolean NOT NULL DEFAULT false,
|
||||
PRIMARY KEY (transaction_id, projection_version, granularity, bucket_start)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS ix_stats_overview_dirty_events_bucket
|
||||
ON public.stats_overview_dirty_events(projection_version, granularity, bucket_start);
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.overview_mark_usage_bucket() RETURNS trigger
|
||||
LANGUAGE plpgsql AS $$
|
||||
DECLARE old_time timestamptz; new_time timestamptz; bucket record;
|
||||
BEGIN
|
||||
IF TG_TABLE_NAME = 'usage' THEN
|
||||
IF TG_OP <> 'INSERT' THEN old_time := OLD.created_at; END IF;
|
||||
IF TG_OP <> 'DELETE' THEN new_time := NEW.created_at; END IF;
|
||||
ELSE
|
||||
IF TG_OP <> 'INSERT' THEN
|
||||
SELECT created_at INTO old_time FROM public.usage WHERE request_id = OLD.request_id;
|
||||
END IF;
|
||||
IF TG_OP <> 'DELETE' THEN
|
||||
SELECT created_at INTO new_time FROM public.usage WHERE request_id = NEW.request_id;
|
||||
END IF;
|
||||
END IF;
|
||||
FOR bucket IN
|
||||
SELECT DISTINCT g, date_trunc(g, t AT TIME ZONE 'UTC') AT TIME ZONE 'UTC' AS starts
|
||||
FROM unnest(ARRAY[old_time, new_time]) t CROSS JOIN unnest(ARRAY['day','hour']) g
|
||||
WHERE t IS NOT NULL ORDER BY g, starts
|
||||
LOOP
|
||||
INSERT INTO public.stats_overview_dirty_events
|
||||
(transaction_id, projection_version, granularity, bucket_start, unrecoverable)
|
||||
VALUES (txid_current(), 'overview-v2', bucket.g, bucket.starts,
|
||||
TG_TABLE_NAME = 'usage' AND TG_OP = 'DELETE')
|
||||
ON CONFLICT (transaction_id, projection_version, granularity, bucket_start)
|
||||
DO UPDATE SET unrecoverable = stats_overview_dirty_events.unrecoverable OR EXCLUDED.unrecoverable;
|
||||
END LOOP;
|
||||
RETURN NULL;
|
||||
END $$;
|
||||
|
||||
DROP TRIGGER IF EXISTS overview_usage_dirty ON public.usage;
|
||||
CREATE TRIGGER overview_usage_dirty AFTER INSERT OR UPDATE OR DELETE ON public.usage
|
||||
FOR EACH ROW EXECUTE FUNCTION public.overview_mark_usage_bucket();
|
||||
DROP TRIGGER IF EXISTS overview_settlement_dirty ON public.usage_settlement_snapshots;
|
||||
CREATE TRIGGER overview_settlement_dirty AFTER INSERT OR UPDATE OR DELETE ON public.usage_settlement_snapshots
|
||||
FOR EACH ROW EXECUTE FUNCTION public.overview_mark_usage_bucket();
|
||||
DROP TRIGGER IF EXISTS overview_attribution_dirty ON public.usage_attribution_snapshots;
|
||||
CREATE TRIGGER overview_attribution_dirty AFTER INSERT OR UPDATE OR DELETE ON public.usage_attribution_snapshots
|
||||
FOR EACH ROW EXECUTE FUNCTION public.overview_mark_usage_bucket();
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.overview_capture_usage_identity() RETURNS trigger
|
||||
LANGUAGE plpgsql AS $$
|
||||
DECLARE actor text; owner text; kind text; source text; standalone boolean;
|
||||
BEGIN
|
||||
SELECT id INTO owner FROM public.users WHERE id=NEW.user_id AND NOT is_deleted;
|
||||
SELECT COALESCE(k.is_standalone,
|
||||
CASE WHEN jsonb_typeof(NEW.request_metadata::jsonb #> '{analytics_attribution,is_standalone}')='boolean'
|
||||
THEN (NEW.request_metadata #>> '{analytics_attribution,is_standalone}')::boolean END,
|
||||
CASE WHEN jsonb_typeof(NEW.request_metadata::jsonb->'api_key_is_standalone')='boolean'
|
||||
THEN (NEW.request_metadata->>'api_key_is_standalone')::boolean END,
|
||||
CASE WHEN a.attribution_source='user_account' THEN false
|
||||
WHEN a.attribution_source='standalone_key' THEN true END,
|
||||
CASE WHEN NEW.api_key_id IS NULL THEN false END)
|
||||
INTO standalone FROM (SELECT 1) seed
|
||||
LEFT JOIN public.api_keys k ON k.id=NEW.api_key_id
|
||||
LEFT JOIN public.usage_attribution_snapshots a ON a.request_id=NEW.request_id;
|
||||
kind := CASE WHEN owner IS NULL THEN 'unknown' WHEN standalone THEN 'standalone'
|
||||
WHEN NOT standalone THEN 'employee' ELSE 'unknown' END;
|
||||
actor := CASE WHEN kind='employee' THEN owner END;
|
||||
source := CASE kind WHEN 'employee' THEN 'user_account' WHEN 'standalone' THEN 'standalone_key' ELSE 'unknown' END;
|
||||
INSERT INTO public.usage_attribution_snapshots(request_id, actor_user_id, credential_owner_id,
|
||||
attribution_kind, attribution_source, record_kind, parent_request_id, attribution_revision)
|
||||
VALUES (NEW.request_id, actor, owner, kind, source,
|
||||
COALESCE(NEW.request_metadata #>> '{analytics_attribution,record_kind}', 'request'),
|
||||
NEW.request_metadata #>> '{analytics_attribution,parent_request_id}', 2)
|
||||
ON CONFLICT (request_id) DO UPDATE SET actor_user_id = EXCLUDED.actor_user_id,
|
||||
credential_owner_id = EXCLUDED.credential_owner_id, attribution_kind = EXCLUDED.attribution_kind,
|
||||
attribution_source = EXCLUDED.attribution_source,
|
||||
record_kind = COALESCE(NEW.request_metadata #>> '{analytics_attribution,record_kind}', usage_attribution_snapshots.record_kind),
|
||||
parent_request_id = COALESCE(EXCLUDED.parent_request_id, usage_attribution_snapshots.parent_request_id),
|
||||
attribution_revision = EXCLUDED.attribution_revision, recorded_at = NOW()
|
||||
WHERE usage_attribution_snapshots.attribution_revision <= EXCLUDED.attribution_revision;
|
||||
RETURN NULL;
|
||||
END $$;
|
||||
DROP TRIGGER IF EXISTS overview_usage_identity ON public.usage;
|
||||
CREATE TRIGGER overview_usage_identity AFTER INSERT OR UPDATE OF user_id, api_key_id, request_metadata ON public.usage
|
||||
FOR EACH ROW EXECUTE FUNCTION public.overview_capture_usage_identity();
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.overview_capture_failure() RETURNS trigger
|
||||
LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF NEW.status = 'completed' THEN
|
||||
NEW.failure_origin := NULL; NEW.failure_stage := NULL; NEW.failure_reason := NULL;
|
||||
NEW.failure_schema_version := NULL; RETURN NEW;
|
||||
END IF;
|
||||
IF NEW.request_metadata #>> '{analytics_failure,origin}' IN ('client','gateway','upstream','transport','unknown') THEN
|
||||
NEW.failure_origin := NEW.request_metadata #>> '{analytics_failure,origin}';
|
||||
NEW.failure_stage := NEW.request_metadata #>> '{analytics_failure,stage}';
|
||||
NEW.failure_reason := NEW.request_metadata #>> '{analytics_failure,reason}';
|
||||
NEW.failure_schema_version := 1;
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END $$;
|
||||
DROP TRIGGER IF EXISTS overview_usage_failure ON public.usage;
|
||||
CREATE TRIGGER overview_usage_failure BEFORE INSERT OR UPDATE OF request_metadata, status ON public.usage
|
||||
FOR EACH ROW EXECUTE FUNCTION public.overview_capture_failure();
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.overview_anonymize_user() RETURNS trigger
|
||||
LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF TG_OP = 'DELETE' OR NEW.is_deleted THEN
|
||||
UPDATE public.usage_attribution_snapshots SET actor_user_id = NULL, credential_owner_id = NULL,
|
||||
attribution_kind = 'unknown', attribution_source = 'unknown', attribution_revision = attribution_revision + 10
|
||||
WHERE actor_user_id = OLD.id OR credential_owner_id = OLD.id;
|
||||
UPDATE public.usage SET request_metadata = (request_metadata::jsonb #- '{analytics_attribution,actor_user_id}')::json
|
||||
WHERE request_metadata #>> '{analytics_attribution,actor_user_id}' = OLD.id;
|
||||
DELETE FROM public.stats_overview_hourly WHERE dimensions->>'actor_user_id'=OLD.id OR dimensions->>'credential_owner_id'=OLD.id;
|
||||
DELETE FROM public.stats_overview_daily WHERE dimensions->>'actor_user_id'=OLD.id OR dimensions->>'credential_owner_id'=OLD.id;
|
||||
END IF;
|
||||
RETURN NULL;
|
||||
END $$;
|
||||
DROP TRIGGER IF EXISTS overview_user_anonymize ON public.users;
|
||||
CREATE TRIGGER overview_user_anonymize AFTER DELETE OR UPDATE OF is_deleted ON public.users
|
||||
FOR EACH ROW EXECUTE FUNCTION public.overview_anonymize_user();
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.overview_delete_attribution() RETURNS trigger
|
||||
LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
DELETE FROM public.usage_attribution_snapshots WHERE request_id = OLD.request_id;
|
||||
RETURN OLD;
|
||||
END $$;
|
||||
DROP TRIGGER IF EXISTS overview_usage_delete_attribution ON public.usage;
|
||||
CREATE TRIGGER overview_usage_delete_attribution BEFORE DELETE ON public.usage
|
||||
FOR EACH ROW EXECUTE FUNCTION public.overview_delete_attribution();
|
||||
|
||||
CREATE OR REPLACE VIEW public.usage_analytics_facts_v1 AS
|
||||
SELECT u.request_id, COALESCE(u.id, u.request_id) AS id, u.created_at,
|
||||
CASE WHEN identity.owner_id IS NOT NULL AND identity.is_standalone=false THEN identity.owner_id END AS actor_user_id,
|
||||
identity.owner_id AS credential_owner_id,
|
||||
CASE WHEN identity.owner_id IS NULL THEN 'unknown' WHEN identity.is_standalone THEN 'standalone'
|
||||
WHEN NOT identity.is_standalone THEN 'employee' ELSE 'unknown' END AS attribution_kind,
|
||||
CASE WHEN identity.owner_id IS NULL THEN 'unknown' WHEN identity.is_standalone THEN 'standalone_key'
|
||||
WHEN NOT identity.is_standalone THEN 'user_account' ELSE 'unknown' END AS attribution_source,
|
||||
COALESCE(a.record_kind, 'request') AS record_kind, a.parent_request_id,
|
||||
u.api_key_id, u.model, u.target_model, u.provider_id, u.provider_name,
|
||||
u.api_format, u.endpoint_kind, u.request_type, u.is_stream, u.has_format_conversion,
|
||||
u.status, u.status_code, u.error_category, u.failure_origin, u.failure_stage, u.failure_reason,
|
||||
u.failure_schema_version, u.response_time_ms, u.first_byte_time_ms,
|
||||
COALESCE(s.billing_status, u.billing_status) AS settlement_status,
|
||||
COALESCE(u.request_metadata::jsonb->'usage_available', 'true'::jsonb) <> 'false'::jsonb AS usage_available,
|
||||
COALESCE(u.request_metadata::jsonb->'usage_pricing_available', 'true'::jsonb) <> 'false'::jsonb
|
||||
AND (s.billing_total_cost_usd IS NOT NULL OR COALESCE(s.billing_status, u.billing_status) = 'settled') AS pricing_available,
|
||||
CASE WHEN COALESCE(u.request_metadata::jsonb->'usage_available', 'true'::jsonb) <> 'false'::jsonb
|
||||
THEN b.input_tokens END AS input_tokens,
|
||||
CASE WHEN COALESCE(u.request_metadata::jsonb->'usage_available', 'true'::jsonb) <> 'false'::jsonb
|
||||
THEN b.output_tokens END AS output_tokens,
|
||||
CASE WHEN COALESCE(u.request_metadata::jsonb->'usage_available', 'true'::jsonb) <> 'false'::jsonb
|
||||
THEN b.total_tokens END AS total_tokens,
|
||||
CASE WHEN COALESCE(u.request_metadata::jsonb->'usage_available', 'true'::jsonb) <> 'false'::jsonb
|
||||
THEN b.cache_read_input_tokens END AS cache_read_input_tokens,
|
||||
CASE WHEN COALESCE(u.request_metadata::jsonb->'usage_available', 'true'::jsonb) <> 'false'::jsonb
|
||||
THEN b.cache_creation_input_tokens END AS cache_creation_input_tokens,
|
||||
CASE WHEN COALESCE(u.request_metadata::jsonb->'usage_pricing_available', 'true'::jsonb) <> 'false'::jsonb
|
||||
AND (s.billing_total_cost_usd IS NOT NULL OR COALESCE(s.billing_status, u.billing_status) = 'settled')
|
||||
THEN round(COALESCE(s.billing_total_cost_usd::numeric, u.total_cost_usd::numeric), 8) END AS rated_amount,
|
||||
CASE WHEN COALESCE(u.request_metadata::jsonb->'usage_pricing_available', 'true'::jsonb) <> 'false'::jsonb
|
||||
AND (s.billing_actual_total_cost_usd IS NOT NULL OR COALESCE(s.billing_status, u.billing_status) = 'settled')
|
||||
THEN round(COALESCE(s.billing_actual_total_cost_usd::numeric, u.actual_total_cost_usd::numeric), 8) END AS billable_amount,
|
||||
s.quota_covered_amount_usd AS quota_covered_amount,
|
||||
s.wallet_consumed_amount_usd AS wallet_consumed_amount,
|
||||
s.wallet_debit_amount_usd AS wallet_debit_amount,
|
||||
s.wallet_recharge_debit_usd AS wallet_recharge_debit_amount,
|
||||
s.wallet_gift_debit_usd AS wallet_gift_debit_amount,
|
||||
s.wallet_overdraft_usd AS wallet_overdraft_amount,
|
||||
s.allocation_status, s.finalized_at AS settled_at,
|
||||
CASE WHEN s.billing_total_cost_usd IS NOT NULL THEN 'settlement_snapshot' ELSE 'legacy_float' END AS amount_source,
|
||||
b.upstream_is_stream,
|
||||
CASE WHEN u.request_metadata #>> '{analytics_measurement,source}' IN ('reported','estimated','mixed')
|
||||
THEN u.request_metadata #>> '{analytics_measurement,source}' ELSE 'unknown' END AS token_source,
|
||||
CASE WHEN COALESCE(u.request_metadata::jsonb->'usage_available','true'::jsonb) <> 'false'::jsonb
|
||||
AND COALESCE(u.request_metadata::jsonb->'usage_pricing_available','true'::jsonb) <> 'false'::jsonb
|
||||
AND s.input_price_per_1m IS NOT NULL AND s.billing_cache_read_cost_usd IS NOT NULL
|
||||
THEN round(s.input_price_per_1m::numeric * b.cache_read_input_tokens::numeric / 1000000,8) END AS cache_estimated_full_cost_amount,
|
||||
CASE WHEN COALESCE(u.request_metadata::jsonb->'usage_available','true'::jsonb) <> 'false'::jsonb
|
||||
AND COALESCE(u.request_metadata::jsonb->'usage_pricing_available','true'::jsonb) <> 'false'::jsonb
|
||||
AND s.input_price_per_1m IS NOT NULL AND s.billing_cache_read_cost_usd IS NOT NULL
|
||||
THEN round(s.billing_cache_read_cost_usd::numeric,8) END AS cache_read_cost_amount,
|
||||
CASE WHEN COALESCE(u.request_metadata::jsonb->'usage_available','true'::jsonb) <> 'false'::jsonb
|
||||
AND COALESCE(u.request_metadata::jsonb->'usage_pricing_available','true'::jsonb) <> 'false'::jsonb
|
||||
AND s.input_price_per_1m IS NOT NULL AND s.billing_cache_creation_cost_usd IS NOT NULL
|
||||
THEN round(s.billing_cache_creation_cost_usd::numeric,8) END AS cache_creation_cost_amount
|
||||
FROM public.usage u
|
||||
LEFT JOIN public.usage_settlement_snapshots s USING (request_id)
|
||||
LEFT JOIN public.usage_attribution_snapshots a USING (request_id)
|
||||
JOIN public.usage_billing_facts b USING (request_id)
|
||||
LEFT JOIN public.api_keys k ON k.id=u.api_key_id
|
||||
CROSS JOIN LATERAL (
|
||||
SELECT CASE WHEN a.request_id IS NOT NULL THEN a.credential_owner_id
|
||||
WHEN EXISTS (SELECT 1 FROM public.users WHERE id=u.user_id AND NOT is_deleted) THEN u.user_id END AS owner_id,
|
||||
COALESCE(k.is_standalone,
|
||||
CASE WHEN jsonb_typeof(u.request_metadata::jsonb #> '{analytics_attribution,is_standalone}')='boolean'
|
||||
THEN (u.request_metadata #>> '{analytics_attribution,is_standalone}')::boolean END,
|
||||
CASE WHEN jsonb_typeof(u.request_metadata::jsonb->'api_key_is_standalone')='boolean'
|
||||
THEN (u.request_metadata->>'api_key_is_standalone')::boolean END,
|
||||
CASE WHEN a.attribution_source='user_account' THEN false
|
||||
WHEN a.attribution_source='standalone_key' THEN true END,
|
||||
CASE WHEN u.api_key_id IS NULL THEN false END) AS is_standalone
|
||||
) identity;
|
||||
+146
@@ -0,0 +1,146 @@
|
||||
-- Retain existing migration checksums and historical records; only future writes create v2 buckets.
|
||||
DROP VIEW IF EXISTS public.usage_analytics_facts_v1;
|
||||
ALTER TABLE public.api_keys DROP COLUMN IF EXISTS credential_kind;
|
||||
ALTER TABLE public.usage_attribution_snapshots DROP COLUMN IF EXISTS credential_kind;
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.overview_mark_usage_bucket() RETURNS trigger
|
||||
LANGUAGE plpgsql AS $$
|
||||
DECLARE old_time timestamptz; new_time timestamptz; bucket record;
|
||||
BEGIN
|
||||
IF TG_TABLE_NAME = 'usage' THEN
|
||||
IF TG_OP <> 'INSERT' THEN old_time := OLD.created_at; END IF;
|
||||
IF TG_OP <> 'DELETE' THEN new_time := NEW.created_at; END IF;
|
||||
ELSE
|
||||
IF TG_OP <> 'INSERT' THEN
|
||||
SELECT created_at INTO old_time FROM public.usage WHERE request_id = OLD.request_id;
|
||||
END IF;
|
||||
IF TG_OP <> 'DELETE' THEN
|
||||
SELECT created_at INTO new_time FROM public.usage WHERE request_id = NEW.request_id;
|
||||
END IF;
|
||||
END IF;
|
||||
FOR bucket IN
|
||||
SELECT DISTINCT g, date_trunc(g, t AT TIME ZONE 'UTC') AT TIME ZONE 'UTC' AS starts
|
||||
FROM unnest(ARRAY[old_time, new_time]) t CROSS JOIN unnest(ARRAY['day','hour']) g
|
||||
WHERE t IS NOT NULL ORDER BY g, starts
|
||||
LOOP
|
||||
INSERT INTO public.stats_bucket_state(projection_version, granularity, bucket_start, source_revision)
|
||||
VALUES ('overview-v2', bucket.g, bucket.starts, 1)
|
||||
ON CONFLICT (projection_version, granularity, bucket_start)
|
||||
DO UPDATE SET source_revision = stats_bucket_state.source_revision + 1;
|
||||
IF TG_TABLE_NAME = 'usage' AND TG_OP = 'DELETE' THEN
|
||||
UPDATE public.stats_bucket_state SET coverage_status='unrecoverable',
|
||||
last_error='retained usage facts were deleted'
|
||||
WHERE projection_version='overview-v2' AND granularity=bucket.g AND bucket_start=bucket.starts;
|
||||
END IF;
|
||||
END LOOP;
|
||||
RETURN NULL;
|
||||
END $$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.overview_capture_usage_identity() RETURNS trigger
|
||||
LANGUAGE plpgsql AS $$
|
||||
DECLARE actor text; owner text; kind text; source text; standalone boolean;
|
||||
BEGIN
|
||||
SELECT id INTO owner FROM public.users WHERE id=NEW.user_id AND NOT is_deleted;
|
||||
SELECT COALESCE(k.is_standalone,
|
||||
CASE WHEN jsonb_typeof(NEW.request_metadata::jsonb #> '{analytics_attribution,is_standalone}')='boolean'
|
||||
THEN (NEW.request_metadata #>> '{analytics_attribution,is_standalone}')::boolean END,
|
||||
CASE WHEN jsonb_typeof(NEW.request_metadata::jsonb->'api_key_is_standalone')='boolean'
|
||||
THEN (NEW.request_metadata->>'api_key_is_standalone')::boolean END,
|
||||
CASE WHEN a.attribution_source='user_account' THEN false
|
||||
WHEN a.attribution_source='standalone_key' THEN true END,
|
||||
CASE WHEN NEW.api_key_id IS NULL THEN false END)
|
||||
INTO standalone FROM (SELECT 1) seed
|
||||
LEFT JOIN public.api_keys k ON k.id=NEW.api_key_id
|
||||
LEFT JOIN public.usage_attribution_snapshots a ON a.request_id=NEW.request_id;
|
||||
kind := CASE WHEN owner IS NULL THEN 'unknown' WHEN standalone THEN 'standalone'
|
||||
WHEN NOT standalone THEN 'employee' ELSE 'unknown' END;
|
||||
actor := CASE WHEN kind='employee' THEN owner END;
|
||||
source := CASE kind WHEN 'employee' THEN 'user_account' WHEN 'standalone' THEN 'standalone_key' ELSE 'unknown' END;
|
||||
INSERT INTO public.usage_attribution_snapshots(request_id, actor_user_id, credential_owner_id,
|
||||
attribution_kind, attribution_source, record_kind, parent_request_id, attribution_revision)
|
||||
VALUES (NEW.request_id, actor, owner, kind, source,
|
||||
COALESCE(NEW.request_metadata #>> '{analytics_attribution,record_kind}', 'request'),
|
||||
NEW.request_metadata #>> '{analytics_attribution,parent_request_id}', 2)
|
||||
ON CONFLICT (request_id) DO UPDATE SET actor_user_id = EXCLUDED.actor_user_id,
|
||||
credential_owner_id = EXCLUDED.credential_owner_id, attribution_kind = EXCLUDED.attribution_kind,
|
||||
attribution_source = EXCLUDED.attribution_source,
|
||||
record_kind = COALESCE(NEW.request_metadata #>> '{analytics_attribution,record_kind}', usage_attribution_snapshots.record_kind),
|
||||
parent_request_id = COALESCE(EXCLUDED.parent_request_id, usage_attribution_snapshots.parent_request_id),
|
||||
attribution_revision = EXCLUDED.attribution_revision, recorded_at = NOW()
|
||||
WHERE usage_attribution_snapshots.attribution_revision <= EXCLUDED.attribution_revision;
|
||||
RETURN NULL;
|
||||
END $$;
|
||||
|
||||
CREATE OR REPLACE VIEW public.usage_analytics_facts_v1 AS
|
||||
SELECT u.request_id, COALESCE(u.id, u.request_id) AS id, u.created_at,
|
||||
CASE WHEN identity.owner_id IS NOT NULL AND identity.is_standalone=false THEN identity.owner_id END AS actor_user_id,
|
||||
identity.owner_id AS credential_owner_id,
|
||||
CASE WHEN identity.owner_id IS NULL THEN 'unknown' WHEN identity.is_standalone THEN 'standalone'
|
||||
WHEN NOT identity.is_standalone THEN 'employee' ELSE 'unknown' END AS attribution_kind,
|
||||
CASE WHEN identity.owner_id IS NULL THEN 'unknown' WHEN identity.is_standalone THEN 'standalone_key'
|
||||
WHEN NOT identity.is_standalone THEN 'user_account' ELSE 'unknown' END AS attribution_source,
|
||||
COALESCE(a.record_kind, 'request') AS record_kind, a.parent_request_id,
|
||||
u.api_key_id, u.model, u.target_model, u.provider_id, u.provider_name,
|
||||
u.api_format, u.endpoint_kind, u.request_type, u.is_stream, u.has_format_conversion,
|
||||
u.status, u.status_code, u.error_category, u.failure_origin, u.failure_stage, u.failure_reason,
|
||||
u.failure_schema_version, u.response_time_ms, u.first_byte_time_ms,
|
||||
COALESCE(s.billing_status, u.billing_status) AS settlement_status,
|
||||
COALESCE(u.request_metadata::jsonb->'usage_available', 'true'::jsonb) <> 'false'::jsonb AS usage_available,
|
||||
COALESCE(u.request_metadata::jsonb->'usage_pricing_available', 'true'::jsonb) <> 'false'::jsonb
|
||||
AND (s.billing_total_cost_usd IS NOT NULL OR COALESCE(s.billing_status, u.billing_status) = 'settled') AS pricing_available,
|
||||
CASE WHEN COALESCE(u.request_metadata::jsonb->'usage_available', 'true'::jsonb) <> 'false'::jsonb
|
||||
THEN b.input_tokens END AS input_tokens,
|
||||
CASE WHEN COALESCE(u.request_metadata::jsonb->'usage_available', 'true'::jsonb) <> 'false'::jsonb
|
||||
THEN b.output_tokens END AS output_tokens,
|
||||
CASE WHEN COALESCE(u.request_metadata::jsonb->'usage_available', 'true'::jsonb) <> 'false'::jsonb
|
||||
THEN b.total_tokens END AS total_tokens,
|
||||
CASE WHEN COALESCE(u.request_metadata::jsonb->'usage_available', 'true'::jsonb) <> 'false'::jsonb
|
||||
THEN b.cache_read_input_tokens END AS cache_read_input_tokens,
|
||||
CASE WHEN COALESCE(u.request_metadata::jsonb->'usage_available', 'true'::jsonb) <> 'false'::jsonb
|
||||
THEN b.cache_creation_input_tokens END AS cache_creation_input_tokens,
|
||||
CASE WHEN COALESCE(u.request_metadata::jsonb->'usage_pricing_available', 'true'::jsonb) <> 'false'::jsonb
|
||||
AND (s.billing_total_cost_usd IS NOT NULL OR COALESCE(s.billing_status, u.billing_status) = 'settled')
|
||||
THEN round(COALESCE(s.billing_total_cost_usd::numeric, u.total_cost_usd::numeric), 8) END AS rated_amount,
|
||||
CASE WHEN COALESCE(u.request_metadata::jsonb->'usage_pricing_available', 'true'::jsonb) <> 'false'::jsonb
|
||||
AND (s.billing_actual_total_cost_usd IS NOT NULL OR COALESCE(s.billing_status, u.billing_status) = 'settled')
|
||||
THEN round(COALESCE(s.billing_actual_total_cost_usd::numeric, u.actual_total_cost_usd::numeric), 8) END AS billable_amount,
|
||||
s.quota_covered_amount_usd AS quota_covered_amount,
|
||||
s.wallet_consumed_amount_usd AS wallet_consumed_amount,
|
||||
s.wallet_debit_amount_usd AS wallet_debit_amount,
|
||||
s.wallet_recharge_debit_usd AS wallet_recharge_debit_amount,
|
||||
s.wallet_gift_debit_usd AS wallet_gift_debit_amount,
|
||||
s.wallet_overdraft_usd AS wallet_overdraft_amount,
|
||||
s.allocation_status, s.finalized_at AS settled_at,
|
||||
CASE WHEN s.billing_total_cost_usd IS NOT NULL THEN 'settlement_snapshot' ELSE 'legacy_float' END AS amount_source,
|
||||
b.upstream_is_stream,
|
||||
CASE WHEN u.request_metadata #>> '{analytics_measurement,source}' IN ('reported','estimated','mixed')
|
||||
THEN u.request_metadata #>> '{analytics_measurement,source}' ELSE 'unknown' END AS token_source,
|
||||
CASE WHEN COALESCE(u.request_metadata::jsonb->'usage_available','true'::jsonb) <> 'false'::jsonb
|
||||
AND COALESCE(u.request_metadata::jsonb->'usage_pricing_available','true'::jsonb) <> 'false'::jsonb
|
||||
AND s.input_price_per_1m IS NOT NULL AND s.billing_cache_read_cost_usd IS NOT NULL
|
||||
THEN round(s.input_price_per_1m::numeric * b.cache_read_input_tokens::numeric / 1000000,8) END AS cache_estimated_full_cost_amount,
|
||||
CASE WHEN COALESCE(u.request_metadata::jsonb->'usage_available','true'::jsonb) <> 'false'::jsonb
|
||||
AND COALESCE(u.request_metadata::jsonb->'usage_pricing_available','true'::jsonb) <> 'false'::jsonb
|
||||
AND s.input_price_per_1m IS NOT NULL AND s.billing_cache_read_cost_usd IS NOT NULL
|
||||
THEN round(s.billing_cache_read_cost_usd::numeric,8) END AS cache_read_cost_amount,
|
||||
CASE WHEN COALESCE(u.request_metadata::jsonb->'usage_available','true'::jsonb) <> 'false'::jsonb
|
||||
AND COALESCE(u.request_metadata::jsonb->'usage_pricing_available','true'::jsonb) <> 'false'::jsonb
|
||||
AND s.input_price_per_1m IS NOT NULL AND s.billing_cache_creation_cost_usd IS NOT NULL
|
||||
THEN round(s.billing_cache_creation_cost_usd::numeric,8) END AS cache_creation_cost_amount
|
||||
FROM public.usage u
|
||||
LEFT JOIN public.usage_settlement_snapshots s USING (request_id)
|
||||
LEFT JOIN public.usage_attribution_snapshots a USING (request_id)
|
||||
JOIN public.usage_billing_facts b USING (request_id)
|
||||
LEFT JOIN public.api_keys k ON k.id=u.api_key_id
|
||||
CROSS JOIN LATERAL (
|
||||
SELECT CASE WHEN a.request_id IS NOT NULL THEN a.credential_owner_id
|
||||
WHEN EXISTS (SELECT 1 FROM public.users WHERE id=u.user_id AND NOT is_deleted) THEN u.user_id END AS owner_id,
|
||||
COALESCE(k.is_standalone,
|
||||
CASE WHEN jsonb_typeof(u.request_metadata::jsonb #> '{analytics_attribution,is_standalone}')='boolean'
|
||||
THEN (u.request_metadata #>> '{analytics_attribution,is_standalone}')::boolean END,
|
||||
CASE WHEN jsonb_typeof(u.request_metadata::jsonb->'api_key_is_standalone')='boolean'
|
||||
THEN (u.request_metadata->>'api_key_is_standalone')::boolean END,
|
||||
CASE WHEN a.attribution_source='user_account' THEN false
|
||||
WHEN a.attribution_source='standalone_key' THEN true END,
|
||||
CASE WHEN u.api_key_id IS NULL THEN false END) AS is_standalone
|
||||
) identity;
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
-- Upgrade databases that already applied the original account-attribution migration.
|
||||
-- Keep this before concurrent index builds so their failure cannot leave writers
|
||||
-- contending on shared hourly/daily aggregate rows. Fresh databases are safe too.
|
||||
-- Only future fact mutations enqueue work; no historical rows are backfilled.
|
||||
-- Each writer owns its transaction's keys, so unrelated requests never contend
|
||||
-- on the current hour/day's stats_bucket_state row.
|
||||
CREATE TABLE IF NOT EXISTS public.stats_overview_dirty_events (
|
||||
transaction_id bigint NOT NULL,
|
||||
projection_version text NOT NULL,
|
||||
granularity text NOT NULL,
|
||||
bucket_start timestamptz NOT NULL,
|
||||
unrecoverable boolean NOT NULL DEFAULT false,
|
||||
PRIMARY KEY (transaction_id, projection_version, granularity, bucket_start)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS ix_stats_overview_dirty_events_bucket
|
||||
ON public.stats_overview_dirty_events(projection_version, granularity, bucket_start);
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.overview_mark_usage_bucket() RETURNS trigger
|
||||
LANGUAGE plpgsql AS $$
|
||||
DECLARE old_time timestamptz; new_time timestamptz; bucket record;
|
||||
BEGIN
|
||||
IF TG_TABLE_NAME = 'usage' THEN
|
||||
IF TG_OP <> 'INSERT' THEN old_time := OLD.created_at; END IF;
|
||||
IF TG_OP <> 'DELETE' THEN new_time := NEW.created_at; END IF;
|
||||
ELSE
|
||||
IF TG_OP <> 'INSERT' THEN
|
||||
SELECT created_at INTO old_time FROM public.usage WHERE request_id = OLD.request_id;
|
||||
END IF;
|
||||
IF TG_OP <> 'DELETE' THEN
|
||||
SELECT created_at INTO new_time FROM public.usage WHERE request_id = NEW.request_id;
|
||||
END IF;
|
||||
END IF;
|
||||
FOR bucket IN
|
||||
SELECT DISTINCT g, date_trunc(g, t AT TIME ZONE 'UTC') AT TIME ZONE 'UTC' AS starts
|
||||
FROM unnest(ARRAY[old_time, new_time]) t CROSS JOIN unnest(ARRAY['day','hour']) g
|
||||
WHERE t IS NOT NULL ORDER BY g, starts
|
||||
LOOP
|
||||
INSERT INTO public.stats_overview_dirty_events
|
||||
(transaction_id, projection_version, granularity, bucket_start, unrecoverable)
|
||||
VALUES (txid_current(), 'overview-v2', bucket.g, bucket.starts,
|
||||
TG_TABLE_NAME = 'usage' AND TG_OP = 'DELETE')
|
||||
ON CONFLICT (transaction_id, projection_version, granularity, bucket_start)
|
||||
DO UPDATE SET unrecoverable = stats_overview_dirty_events.unrecoverable OR EXCLUDED.unrecoverable;
|
||||
END LOOP;
|
||||
RETURN NULL;
|
||||
END $$;
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
-- no-transaction
|
||||
-- Keep the existing billing cover while building its replacement. Overview
|
||||
-- totals also read settlement/allocation status; omitting those columns forces
|
||||
-- a scan of the wide snapshot heap even when every monetary field is covered.
|
||||
-- The following migration retires the old index only after this build succeeds.
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_usage_settlement_dashboard_cover_v2
|
||||
ON public.usage_settlement_snapshots (request_id)
|
||||
INCLUDE (
|
||||
billing_input_tokens,
|
||||
billing_effective_input_tokens,
|
||||
billing_output_tokens,
|
||||
billing_cache_creation_tokens,
|
||||
billing_cache_creation_5m_tokens,
|
||||
billing_cache_creation_1h_tokens,
|
||||
billing_cache_read_tokens,
|
||||
billing_total_input_context,
|
||||
billing_cache_creation_cost_usd,
|
||||
billing_cache_read_cost_usd,
|
||||
billing_total_cost_usd,
|
||||
billing_actual_total_cost_usd,
|
||||
input_price_per_1m,
|
||||
billing_status,
|
||||
allocation_status
|
||||
);
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
-- no-transaction
|
||||
-- The preceding migration has published the superset index. Do not keep two
|
||||
-- permanent covering indexes with the same key and almost identical payloads.
|
||||
DROP INDEX CONCURRENTLY IF EXISTS public.idx_usage_settlement_dashboard_cover;
|
||||
+254
@@ -0,0 +1,254 @@
|
||||
-- Future-only dashboard projection. No usage history is read or backfilled.
|
||||
-- Background maintenance bounds detailed minutes to 35 days; lifetime totals
|
||||
-- and narrow activity counters remain available after detail or usage cleanup.
|
||||
CREATE TABLE public.dashboard_stats_state (
|
||||
singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton),
|
||||
stats_since timestamptz NOT NULL,
|
||||
contributions_cleanup_cursor varchar(100)
|
||||
);
|
||||
CREATE TABLE public.dashboard_request_contributions (
|
||||
request_id varchar(100) PRIMARY KEY,
|
||||
created_at timestamptz NOT NULL,
|
||||
actor_user_id varchar(255),
|
||||
metrics jsonb NOT NULL
|
||||
);
|
||||
CREATE TABLE public.dashboard_stats_total (
|
||||
shard smallint PRIMARY KEY CHECK (shard >= 0 AND shard < 16),
|
||||
metrics jsonb NOT NULL DEFAULT '{}'::jsonb
|
||||
);
|
||||
INSERT INTO public.dashboard_stats_total(shard) SELECT generate_series(0,15);
|
||||
CREATE TABLE public.dashboard_stats_minute (
|
||||
bucket_start timestamptz NOT NULL,
|
||||
shard smallint NOT NULL CHECK (shard >= 0 AND shard < 16),
|
||||
metrics jsonb NOT NULL DEFAULT '{}'::jsonb,
|
||||
PRIMARY KEY (bucket_start, shard)
|
||||
);
|
||||
CREATE TABLE public.dashboard_activity_minute (
|
||||
bucket_start timestamptz NOT NULL,
|
||||
shard smallint NOT NULL CHECK (shard >= 0 AND shard < 16),
|
||||
request_count bigint NOT NULL,
|
||||
PRIMARY KEY (bucket_start,shard)
|
||||
);
|
||||
CREATE TABLE public.dashboard_activity_hour (
|
||||
bucket_start timestamptz NOT NULL,
|
||||
shard smallint NOT NULL CHECK (shard >= 0 AND shard < 16),
|
||||
request_count bigint NOT NULL,
|
||||
PRIMARY KEY (bucket_start, shard)
|
||||
);
|
||||
CREATE TABLE public.dashboard_actor_minute (
|
||||
bucket_start timestamptz NOT NULL,
|
||||
shard smallint NOT NULL CHECK (shard >= 0 AND shard < 16),
|
||||
actor_user_id varchar(255) NOT NULL,
|
||||
request_count bigint NOT NULL,
|
||||
PRIMARY KEY (bucket_start, shard, actor_user_id)
|
||||
);
|
||||
-- Rows only exist within a writing transaction. Deferred processing coalesces
|
||||
-- usage, settlement and identity mutations into one final contribution.
|
||||
CREATE TABLE public.dashboard_stats_pending (
|
||||
transaction_id bigint NOT NULL,
|
||||
request_id varchar(100) NOT NULL,
|
||||
deleted_fact jsonb,
|
||||
PRIMARY KEY (transaction_id, request_id)
|
||||
);
|
||||
CREATE TABLE public.dashboard_user_events_minute (
|
||||
bucket_start timestamptz NOT NULL,
|
||||
shard smallint NOT NULL CHECK (shard >= 0 AND shard < 16),
|
||||
created_count bigint NOT NULL DEFAULT 0,
|
||||
deleted_count bigint NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (bucket_start, shard)
|
||||
);
|
||||
|
||||
CREATE FUNCTION public.dashboard_metrics_add(a jsonb, b jsonb, direction integer DEFAULT 1)
|
||||
RETURNS jsonb LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$
|
||||
SELECT COALESCE(jsonb_object_agg(key, value), '{}'::jsonb) FROM (
|
||||
SELECT key, sum(value) AS value FROM (
|
||||
SELECT key, value::numeric FROM jsonb_each_text(COALESCE(a, '{}'::jsonb))
|
||||
UNION ALL
|
||||
SELECT key, value::numeric * direction FROM jsonb_each_text(COALESCE(b, '{}'::jsonb))
|
||||
) entries GROUP BY key
|
||||
) sums
|
||||
$$;
|
||||
|
||||
-- Canonical views are evaluated for ONE indexed request, never for old history.
|
||||
CREATE FUNCTION public.dashboard_request_fact(p_request_id text)
|
||||
RETURNS jsonb LANGUAGE sql STABLE AS $$
|
||||
SELECT jsonb_build_object('created_at', f.created_at,
|
||||
'actor_user_id', f.actor_user_id,
|
||||
'metrics', CASE WHEN f.record_kind = 'session' THEN '{}'::jsonb ELSE
|
||||
jsonb_build_object(
|
||||
'request_count', 1,
|
||||
'input_tokens', COALESCE(f.input_tokens,0),
|
||||
'output_tokens', COALESCE(f.output_tokens,0),
|
||||
'total_tokens', COALESCE(f.total_tokens,0),
|
||||
'usage_available_count', f.usage_available::integer,
|
||||
'pricing_available_count', (f.billable_amount IS NOT NULL)::integer,
|
||||
'billable_amount', COALESCE(f.billable_amount,0),
|
||||
'cache_read_tokens', COALESCE(f.cache_read_input_tokens,0),
|
||||
'cache_creation_tokens', COALESCE(f.cache_creation_input_tokens,0),
|
||||
'cache_input_tokens', CASE WHEN f.usage_available THEN COALESCE(b.total_input_context,0) ELSE 0 END,
|
||||
'first_byte_sum_ms', COALESCE(f.first_byte_time_ms,0),
|
||||
'first_byte_sample_count', (f.first_byte_time_ms IS NOT NULL)::integer,
|
||||
'response_sum_ms', COALESCE(f.response_time_ms,0),
|
||||
'response_sample_count', (f.response_time_ms IS NOT NULL)::integer,
|
||||
'stream_requests', COALESCE(f.upstream_is_stream,false)::integer,
|
||||
'standard_requests', (NOT COALESCE(f.upstream_is_stream,false))::integer
|
||||
) END)
|
||||
FROM public.usage_analytics_facts_v1 f
|
||||
JOIN public.usage_billing_facts b USING (request_id)
|
||||
WHERE f.request_id = p_request_id
|
||||
AND f.created_at >= (SELECT stats_since FROM public.dashboard_stats_state WHERE singleton)
|
||||
$$;
|
||||
|
||||
-- Callers hold the aggregate shard lock. Seed a missing narrow activity counter
|
||||
-- from the detailed minute before applying a correction.
|
||||
CREATE FUNCTION public.dashboard_ensure_activity_minute(p_bucket timestamptz,p_shard smallint)
|
||||
RETURNS void LANGUAGE sql AS $$
|
||||
INSERT INTO public.dashboard_activity_minute(bucket_start,shard,request_count)
|
||||
SELECT bucket_start,shard,COALESCE((metrics->>'request_count')::bigint,0)
|
||||
FROM public.dashboard_stats_minute WHERE bucket_start=p_bucket AND shard=p_shard
|
||||
ON CONFLICT(bucket_start,shard) DO NOTHING
|
||||
$$;
|
||||
|
||||
CREATE FUNCTION public.dashboard_enqueue_request() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
DECLARE rid text;
|
||||
BEGIN
|
||||
IF current_setting('aether.dashboard_restore',true)='on' THEN RETURN COALESCE(NEW,OLD); END IF;
|
||||
rid := CASE WHEN TG_OP = 'DELETE' THEN OLD.request_id ELSE NEW.request_id END;
|
||||
-- Fast path: an update to a pre-activation request must never scan/replay it.
|
||||
IF TG_TABLE_NAME = 'usage' THEN
|
||||
IF (CASE WHEN TG_OP = 'DELETE' THEN OLD.created_at ELSE NEW.created_at END)
|
||||
< (SELECT stats_since FROM public.dashboard_stats_state WHERE singleton) THEN
|
||||
RETURN COALESCE(NEW, OLD);
|
||||
END IF;
|
||||
END IF;
|
||||
INSERT INTO public.dashboard_stats_pending(transaction_id, request_id, deleted_fact)
|
||||
VALUES(txid_current(), rid,
|
||||
CASE WHEN TG_TABLE_NAME = 'usage' AND TG_OP = 'DELETE'
|
||||
THEN public.dashboard_request_fact(rid) END)
|
||||
ON CONFLICT (transaction_id, request_id) DO UPDATE
|
||||
SET deleted_fact = COALESCE(EXCLUDED.deleted_fact, dashboard_stats_pending.deleted_fact);
|
||||
RETURN COALESCE(NEW, OLD);
|
||||
END $$;
|
||||
|
||||
CREATE FUNCTION public.dashboard_apply_pending() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
DECLARE p record; old_fact public.dashboard_request_contributions%ROWTYPE;
|
||||
fact jsonb; next_metrics jsonb; next_at timestamptz; next_actor text;
|
||||
shard_id smallint; old_bucket timestamptz; next_bucket timestamptz; purged boolean;
|
||||
detail_cutoff timestamptz := date_trunc('minute',clock_timestamp()-INTERVAL '35 days');
|
||||
BEGIN
|
||||
IF current_setting('aether.dashboard_restore',true)='on' THEN
|
||||
DELETE FROM public.dashboard_stats_pending WHERE transaction_id=NEW.transaction_id;
|
||||
RETURN NULL;
|
||||
END IF;
|
||||
-- Lock all touched shards in one order before touching any ledger or minute.
|
||||
-- This also serializes concurrent usage/settlement updates for the same request.
|
||||
PERFORM t.shard FROM public.dashboard_stats_total t
|
||||
WHERE t.shard IN (SELECT (hashtextextended(request_id,0) & 15)::smallint
|
||||
FROM public.dashboard_stats_pending WHERE transaction_id=NEW.transaction_id)
|
||||
ORDER BY t.shard FOR UPDATE;
|
||||
FOR p IN SELECT * FROM public.dashboard_stats_pending
|
||||
WHERE transaction_id=NEW.transaction_id ORDER BY request_id LOOP
|
||||
shard_id := (hashtextextended(p.request_id,0) & 15)::smallint;
|
||||
fact := public.dashboard_request_fact(p.request_id);
|
||||
purged := NOT EXISTS(SELECT 1 FROM public.usage WHERE request_id=p.request_id);
|
||||
-- A purge preserves the last contribution, including an update in this tx.
|
||||
fact := COALESCE(fact,p.deleted_fact);
|
||||
IF fact IS NULL THEN
|
||||
IF purged THEN DELETE FROM public.dashboard_request_contributions WHERE request_id=p.request_id; END IF;
|
||||
CONTINUE;
|
||||
END IF;
|
||||
next_metrics := fact->'metrics';
|
||||
next_at := (fact->>'created_at')::timestamptz;
|
||||
next_actor := CASE WHEN COALESCE((next_metrics->>'request_count')::bigint,0)>0
|
||||
THEN fact->>'actor_user_id' END;
|
||||
SELECT * INTO old_fact FROM public.dashboard_request_contributions WHERE request_id=p.request_id;
|
||||
IF FOUND AND old_fact.created_at=next_at
|
||||
AND old_fact.actor_user_id IS NOT DISTINCT FROM next_actor
|
||||
AND old_fact.metrics=next_metrics THEN
|
||||
IF purged THEN DELETE FROM public.dashboard_request_contributions WHERE request_id=p.request_id; END IF;
|
||||
CONTINUE;
|
||||
END IF;
|
||||
next_bucket := date_trunc('minute',next_at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC';
|
||||
IF old_fact.request_id IS NOT NULL THEN
|
||||
old_bucket := date_trunc('minute',old_fact.created_at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC';
|
||||
PERFORM public.dashboard_ensure_activity_minute(old_bucket,shard_id);
|
||||
UPDATE public.dashboard_activity_minute SET request_count=request_count-COALESCE((old_fact.metrics->>'request_count')::bigint,0)
|
||||
WHERE bucket_start=old_bucket AND shard=shard_id;
|
||||
UPDATE public.dashboard_stats_total SET metrics=public.dashboard_metrics_add(metrics,old_fact.metrics,-1) WHERE shard=shard_id;
|
||||
IF old_bucket>=detail_cutoff THEN
|
||||
UPDATE public.dashboard_stats_minute SET metrics=public.dashboard_metrics_add(metrics,old_fact.metrics,-1)
|
||||
WHERE bucket_start=old_bucket AND shard=shard_id;
|
||||
END IF;
|
||||
UPDATE public.dashboard_activity_hour SET request_count=request_count-COALESCE((old_fact.metrics->>'request_count')::bigint,0)
|
||||
WHERE bucket_start=date_trunc('hour',old_fact.created_at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC' AND shard=shard_id;
|
||||
IF old_fact.actor_user_id IS NOT NULL AND old_bucket>=detail_cutoff THEN
|
||||
UPDATE public.dashboard_actor_minute SET request_count=request_count-1
|
||||
WHERE bucket_start=old_bucket AND shard=shard_id AND actor_user_id=old_fact.actor_user_id;
|
||||
END IF;
|
||||
END IF;
|
||||
PERFORM public.dashboard_ensure_activity_minute(next_bucket,shard_id);
|
||||
INSERT INTO public.dashboard_activity_minute(bucket_start,shard,request_count)
|
||||
VALUES(next_bucket,shard_id,COALESCE((next_metrics->>'request_count')::bigint,0))
|
||||
ON CONFLICT(bucket_start,shard) DO UPDATE
|
||||
SET request_count=dashboard_activity_minute.request_count+EXCLUDED.request_count;
|
||||
UPDATE public.dashboard_stats_total SET metrics=public.dashboard_metrics_add(metrics,next_metrics) WHERE shard=shard_id;
|
||||
IF next_bucket>=detail_cutoff THEN
|
||||
INSERT INTO public.dashboard_stats_minute(bucket_start,shard,metrics) VALUES(next_bucket,shard_id,next_metrics)
|
||||
ON CONFLICT(bucket_start,shard) DO UPDATE
|
||||
SET metrics=public.dashboard_metrics_add(dashboard_stats_minute.metrics,EXCLUDED.metrics);
|
||||
END IF;
|
||||
INSERT INTO public.dashboard_activity_hour(bucket_start,shard,request_count)
|
||||
VALUES(date_trunc('hour',next_at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC',shard_id,COALESCE((next_metrics->>'request_count')::bigint,0))
|
||||
ON CONFLICT(bucket_start,shard) DO UPDATE SET request_count=dashboard_activity_hour.request_count+EXCLUDED.request_count;
|
||||
IF next_actor IS NOT NULL AND next_bucket>=detail_cutoff THEN
|
||||
INSERT INTO public.dashboard_actor_minute(bucket_start,shard,actor_user_id,request_count)
|
||||
VALUES(next_bucket,shard_id,next_actor,1)
|
||||
ON CONFLICT(bucket_start,shard,actor_user_id) DO UPDATE SET request_count=dashboard_actor_minute.request_count+1;
|
||||
END IF;
|
||||
IF purged THEN
|
||||
DELETE FROM public.dashboard_request_contributions WHERE request_id=p.request_id;
|
||||
ELSE
|
||||
INSERT INTO public.dashboard_request_contributions(request_id,created_at,actor_user_id,metrics)
|
||||
VALUES(p.request_id,next_at,next_actor,next_metrics)
|
||||
ON CONFLICT(request_id) DO UPDATE SET created_at=EXCLUDED.created_at,
|
||||
actor_user_id=EXCLUDED.actor_user_id,metrics=EXCLUDED.metrics;
|
||||
END IF;
|
||||
END LOOP;
|
||||
DELETE FROM public.dashboard_stats_pending WHERE transaction_id=NEW.transaction_id;
|
||||
RETURN NULL;
|
||||
END $$;
|
||||
CREATE CONSTRAINT TRIGGER dashboard_flush_pending AFTER INSERT ON public.dashboard_stats_pending
|
||||
DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION public.dashboard_apply_pending();
|
||||
|
||||
CREATE FUNCTION public.dashboard_record_user_event() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
DECLARE added bigint := 0; removed bigint := 0; rid text; at timestamptz := clock_timestamp();
|
||||
BEGIN
|
||||
IF current_setting('aether.dashboard_restore',true)='on' THEN RETURN COALESCE(NEW,OLD); END IF;
|
||||
IF TG_OP='INSERT' THEN added:=1; rid:=NEW.id;
|
||||
ELSIF TG_OP='DELETE' THEN removed:=(NOT OLD.is_deleted)::integer; rid:=OLD.id;
|
||||
ELSE removed:=(NOT OLD.is_deleted AND NEW.is_deleted)::integer; rid:=NEW.id;
|
||||
END IF;
|
||||
IF added=0 AND removed=0 THEN RETURN COALESCE(NEW,OLD); END IF;
|
||||
IF NOT EXISTS(SELECT 1 FROM public.dashboard_stats_state WHERE singleton AND stats_since<=at) THEN RETURN COALESCE(NEW,OLD); END IF;
|
||||
INSERT INTO public.dashboard_user_events_minute(bucket_start,shard,created_count,deleted_count)
|
||||
VALUES(date_trunc('minute',at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC',
|
||||
(hashtextextended(rid,0) & 15)::smallint,added,removed)
|
||||
ON CONFLICT(bucket_start,shard) DO UPDATE SET
|
||||
created_count=dashboard_user_events_minute.created_count+EXCLUDED.created_count,
|
||||
deleted_count=dashboard_user_events_minute.deleted_count+EXCLUDED.deleted_count;
|
||||
RETURN COALESCE(NEW,OLD);
|
||||
END $$;
|
||||
|
||||
-- Acquire source DDL locks before recording activation. This excludes transactions
|
||||
-- that could commit writes without observing the new triggers.
|
||||
CREATE TRIGGER dashboard_usage_changed AFTER INSERT OR UPDATE ON public.usage
|
||||
FOR EACH ROW EXECUTE FUNCTION public.dashboard_enqueue_request();
|
||||
CREATE TRIGGER aaa_dashboard_usage_deleted BEFORE DELETE ON public.usage
|
||||
FOR EACH ROW EXECUTE FUNCTION public.dashboard_enqueue_request();
|
||||
CREATE TRIGGER dashboard_settlement_changed AFTER INSERT OR UPDATE OR DELETE ON public.usage_settlement_snapshots
|
||||
FOR EACH ROW EXECUTE FUNCTION public.dashboard_enqueue_request();
|
||||
CREATE TRIGGER dashboard_attribution_changed AFTER INSERT OR UPDATE ON public.usage_attribution_snapshots
|
||||
FOR EACH ROW EXECUTE FUNCTION public.dashboard_enqueue_request();
|
||||
CREATE TRIGGER dashboard_user_changed AFTER INSERT OR DELETE OR UPDATE OF is_deleted ON public.users
|
||||
FOR EACH ROW EXECUTE FUNCTION public.dashboard_record_user_event();
|
||||
INSERT INTO public.dashboard_stats_state(singleton,stats_since) VALUES(true,clock_timestamp());
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
-- no-transaction
|
||||
-- User finance reports use the actual credited period, not order creation time.
|
||||
-- Keep the index build separate from transactional schema changes.
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_payment_orders_status_credited_user
|
||||
ON public.payment_orders USING btree (status, credited_at, user_id);
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
-- Manual purchasing ledger; no foreign-key cascade may erase historical expenditures.
|
||||
CREATE TABLE IF NOT EXISTS public.provider_expenses (
|
||||
id text PRIMARY KEY,
|
||||
client_request_id text NOT NULL UNIQUE,
|
||||
provider_id text NOT NULL,
|
||||
provider_name text NOT NULL,
|
||||
kind text NOT NULL CHECK (kind IN ('recharge', 'subscription', 'other')),
|
||||
amount numeric(20,8) NOT NULL CHECK (amount > 0),
|
||||
currency text NOT NULL CHECK (currency ~ '^[A-Z]{3}$'),
|
||||
paid_at timestamptz NOT NULL,
|
||||
period_start timestamptz,
|
||||
period_end timestamptz,
|
||||
note text,
|
||||
external_reference text,
|
||||
created_by text,
|
||||
created_at timestamptz NOT NULL DEFAULT NOW(),
|
||||
voided_at timestamptz,
|
||||
voided_by text,
|
||||
CONSTRAINT provider_expenses_period_check CHECK (
|
||||
(period_start IS NULL AND period_end IS NULL) OR
|
||||
(period_start IS NOT NULL AND period_end IS NOT NULL AND period_start < period_end)
|
||||
)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS ix_provider_expenses_paid_at ON public.provider_expenses(paid_at, id);
|
||||
CREATE INDEX IF NOT EXISTS ix_provider_expenses_provider_paid_at ON public.provider_expenses(provider_id, paid_at);
|
||||
+147
@@ -0,0 +1,147 @@
|
||||
-- No raw usage history is scanned. Existing dashboard minutes are compacted by
|
||||
-- bounded background batches; lifetime totals and local-calendar activity survive.
|
||||
-- Fresh installations already contain these definitions in the dashboard baseline.
|
||||
ALTER TABLE public.dashboard_stats_state ADD COLUMN IF NOT EXISTS contributions_cleanup_cursor varchar(100);
|
||||
CREATE TABLE IF NOT EXISTS public.dashboard_activity_minute (
|
||||
bucket_start timestamptz NOT NULL,
|
||||
shard smallint NOT NULL CHECK (shard >= 0 AND shard < 16),
|
||||
request_count bigint NOT NULL,
|
||||
PRIMARY KEY (bucket_start,shard)
|
||||
);
|
||||
|
||||
-- Callers hold the aggregate shard lock. Before the first post-upgrade correction
|
||||
-- of a minute, seed its narrow counter from the existing wide aggregate.
|
||||
CREATE OR REPLACE FUNCTION public.dashboard_ensure_activity_minute(p_bucket timestamptz,p_shard smallint)
|
||||
RETURNS void LANGUAGE sql AS $$
|
||||
INSERT INTO public.dashboard_activity_minute(bucket_start,shard,request_count)
|
||||
SELECT bucket_start,shard,COALESCE((metrics->>'request_count')::bigint,0)
|
||||
FROM public.dashboard_stats_minute WHERE bucket_start=p_bucket AND shard=p_shard
|
||||
ON CONFLICT(bucket_start,shard) DO NOTHING
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.dashboard_enqueue_request() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
DECLARE rid text;
|
||||
BEGIN
|
||||
IF current_setting('aether.dashboard_restore',true)='on' THEN RETURN COALESCE(NEW,OLD); END IF;
|
||||
rid := CASE WHEN TG_OP = 'DELETE' THEN OLD.request_id ELSE NEW.request_id END;
|
||||
-- Fast path: an update to a pre-activation request must never scan/replay it.
|
||||
IF TG_TABLE_NAME = 'usage' THEN
|
||||
IF (CASE WHEN TG_OP = 'DELETE' THEN OLD.created_at ELSE NEW.created_at END)
|
||||
< (SELECT stats_since FROM public.dashboard_stats_state WHERE singleton) THEN
|
||||
RETURN COALESCE(NEW, OLD);
|
||||
END IF;
|
||||
END IF;
|
||||
INSERT INTO public.dashboard_stats_pending(transaction_id, request_id, deleted_fact)
|
||||
VALUES(txid_current(), rid,
|
||||
CASE WHEN TG_TABLE_NAME = 'usage' AND TG_OP = 'DELETE'
|
||||
THEN public.dashboard_request_fact(rid) END)
|
||||
ON CONFLICT (transaction_id, request_id) DO UPDATE
|
||||
SET deleted_fact = COALESCE(EXCLUDED.deleted_fact, dashboard_stats_pending.deleted_fact);
|
||||
RETURN COALESCE(NEW, OLD);
|
||||
END $$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.dashboard_apply_pending() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
DECLARE p record; old_fact public.dashboard_request_contributions%ROWTYPE;
|
||||
fact jsonb; next_metrics jsonb; next_at timestamptz; next_actor text;
|
||||
shard_id smallint; old_bucket timestamptz; next_bucket timestamptz; purged boolean;
|
||||
detail_cutoff timestamptz := date_trunc('minute',clock_timestamp()-INTERVAL '35 days');
|
||||
BEGIN
|
||||
IF current_setting('aether.dashboard_restore',true)='on' THEN
|
||||
DELETE FROM public.dashboard_stats_pending WHERE transaction_id=NEW.transaction_id;
|
||||
RETURN NULL;
|
||||
END IF;
|
||||
-- Lock all touched shards in one order before touching any ledger or minute.
|
||||
-- This also serializes concurrent usage/settlement updates for the same request.
|
||||
PERFORM t.shard FROM public.dashboard_stats_total t
|
||||
WHERE t.shard IN (SELECT (hashtextextended(request_id,0) & 15)::smallint
|
||||
FROM public.dashboard_stats_pending WHERE transaction_id=NEW.transaction_id)
|
||||
ORDER BY t.shard FOR UPDATE;
|
||||
FOR p IN SELECT * FROM public.dashboard_stats_pending
|
||||
WHERE transaction_id=NEW.transaction_id ORDER BY request_id LOOP
|
||||
shard_id := (hashtextextended(p.request_id,0) & 15)::smallint;
|
||||
fact := public.dashboard_request_fact(p.request_id);
|
||||
purged := NOT EXISTS(SELECT 1 FROM public.usage WHERE request_id=p.request_id);
|
||||
-- A purge preserves the last contribution, including an update in this tx.
|
||||
fact := COALESCE(fact,p.deleted_fact);
|
||||
IF fact IS NULL THEN
|
||||
IF purged THEN DELETE FROM public.dashboard_request_contributions WHERE request_id=p.request_id; END IF;
|
||||
CONTINUE;
|
||||
END IF;
|
||||
next_metrics := fact->'metrics';
|
||||
next_at := (fact->>'created_at')::timestamptz;
|
||||
next_actor := CASE WHEN COALESCE((next_metrics->>'request_count')::bigint,0)>0
|
||||
THEN fact->>'actor_user_id' END;
|
||||
SELECT * INTO old_fact FROM public.dashboard_request_contributions WHERE request_id=p.request_id;
|
||||
IF FOUND AND old_fact.created_at=next_at
|
||||
AND old_fact.actor_user_id IS NOT DISTINCT FROM next_actor
|
||||
AND old_fact.metrics=next_metrics THEN
|
||||
IF purged THEN DELETE FROM public.dashboard_request_contributions WHERE request_id=p.request_id; END IF;
|
||||
CONTINUE;
|
||||
END IF;
|
||||
next_bucket := date_trunc('minute',next_at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC';
|
||||
IF old_fact.request_id IS NOT NULL THEN
|
||||
old_bucket := date_trunc('minute',old_fact.created_at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC';
|
||||
PERFORM public.dashboard_ensure_activity_minute(old_bucket,shard_id);
|
||||
UPDATE public.dashboard_activity_minute SET request_count=request_count-COALESCE((old_fact.metrics->>'request_count')::bigint,0)
|
||||
WHERE bucket_start=old_bucket AND shard=shard_id;
|
||||
UPDATE public.dashboard_stats_total SET metrics=public.dashboard_metrics_add(metrics,old_fact.metrics,-1) WHERE shard=shard_id;
|
||||
IF old_bucket>=detail_cutoff THEN
|
||||
UPDATE public.dashboard_stats_minute SET metrics=public.dashboard_metrics_add(metrics,old_fact.metrics,-1)
|
||||
WHERE bucket_start=old_bucket AND shard=shard_id;
|
||||
END IF;
|
||||
UPDATE public.dashboard_activity_hour SET request_count=request_count-COALESCE((old_fact.metrics->>'request_count')::bigint,0)
|
||||
WHERE bucket_start=date_trunc('hour',old_fact.created_at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC' AND shard=shard_id;
|
||||
IF old_fact.actor_user_id IS NOT NULL AND old_bucket>=detail_cutoff THEN
|
||||
UPDATE public.dashboard_actor_minute SET request_count=request_count-1
|
||||
WHERE bucket_start=old_bucket AND shard=shard_id AND actor_user_id=old_fact.actor_user_id;
|
||||
END IF;
|
||||
END IF;
|
||||
PERFORM public.dashboard_ensure_activity_minute(next_bucket,shard_id);
|
||||
INSERT INTO public.dashboard_activity_minute(bucket_start,shard,request_count)
|
||||
VALUES(next_bucket,shard_id,COALESCE((next_metrics->>'request_count')::bigint,0))
|
||||
ON CONFLICT(bucket_start,shard) DO UPDATE
|
||||
SET request_count=dashboard_activity_minute.request_count+EXCLUDED.request_count;
|
||||
UPDATE public.dashboard_stats_total SET metrics=public.dashboard_metrics_add(metrics,next_metrics) WHERE shard=shard_id;
|
||||
IF next_bucket>=detail_cutoff THEN
|
||||
INSERT INTO public.dashboard_stats_minute(bucket_start,shard,metrics) VALUES(next_bucket,shard_id,next_metrics)
|
||||
ON CONFLICT(bucket_start,shard) DO UPDATE
|
||||
SET metrics=public.dashboard_metrics_add(dashboard_stats_minute.metrics,EXCLUDED.metrics);
|
||||
END IF;
|
||||
INSERT INTO public.dashboard_activity_hour(bucket_start,shard,request_count)
|
||||
VALUES(date_trunc('hour',next_at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC',shard_id,COALESCE((next_metrics->>'request_count')::bigint,0))
|
||||
ON CONFLICT(bucket_start,shard) DO UPDATE SET request_count=dashboard_activity_hour.request_count+EXCLUDED.request_count;
|
||||
IF next_actor IS NOT NULL AND next_bucket>=detail_cutoff THEN
|
||||
INSERT INTO public.dashboard_actor_minute(bucket_start,shard,actor_user_id,request_count)
|
||||
VALUES(next_bucket,shard_id,next_actor,1)
|
||||
ON CONFLICT(bucket_start,shard,actor_user_id) DO UPDATE SET request_count=dashboard_actor_minute.request_count+1;
|
||||
END IF;
|
||||
IF purged THEN
|
||||
DELETE FROM public.dashboard_request_contributions WHERE request_id=p.request_id;
|
||||
ELSE
|
||||
INSERT INTO public.dashboard_request_contributions(request_id,created_at,actor_user_id,metrics)
|
||||
VALUES(p.request_id,next_at,next_actor,next_metrics)
|
||||
ON CONFLICT(request_id) DO UPDATE SET created_at=EXCLUDED.created_at,
|
||||
actor_user_id=EXCLUDED.actor_user_id,metrics=EXCLUDED.metrics;
|
||||
END IF;
|
||||
END LOOP;
|
||||
DELETE FROM public.dashboard_stats_pending WHERE transaction_id=NEW.transaction_id;
|
||||
RETURN NULL;
|
||||
END $$;
|
||||
CREATE OR REPLACE FUNCTION public.dashboard_record_user_event() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
DECLARE added bigint := 0; removed bigint := 0; rid text; at timestamptz := clock_timestamp();
|
||||
BEGIN
|
||||
IF current_setting('aether.dashboard_restore',true)='on' THEN RETURN COALESCE(NEW,OLD); END IF;
|
||||
IF TG_OP='INSERT' THEN added:=1; rid:=NEW.id;
|
||||
ELSIF TG_OP='DELETE' THEN removed:=(NOT OLD.is_deleted)::integer; rid:=OLD.id;
|
||||
ELSE removed:=(NOT OLD.is_deleted AND NEW.is_deleted)::integer; rid:=NEW.id;
|
||||
END IF;
|
||||
IF added=0 AND removed=0 THEN RETURN COALESCE(NEW,OLD); END IF;
|
||||
IF NOT EXISTS(SELECT 1 FROM public.dashboard_stats_state WHERE singleton AND stats_since<=at) THEN RETURN COALESCE(NEW,OLD); END IF;
|
||||
INSERT INTO public.dashboard_user_events_minute(bucket_start,shard,created_count,deleted_count)
|
||||
VALUES(date_trunc('minute',at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC',
|
||||
(hashtextextended(rid,0) & 15)::smallint,added,removed)
|
||||
ON CONFLICT(bucket_start,shard) DO UPDATE SET
|
||||
created_count=dashboard_user_events_minute.created_count+EXCLUDED.created_count,
|
||||
deleted_count=dashboard_user_events_minute.deleted_count+EXCLUDED.deleted_count;
|
||||
RETURN COALESCE(NEW,OLD);
|
||||
END $$;
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
-- no-transaction
|
||||
-- Anonymization looks up either side of the account attribution. Keep both
|
||||
-- branches indexed so deleting an unrelated user never scans all snapshots.
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_usage_attribution_owner_request
|
||||
ON public.usage_attribution_snapshots (credential_owner_id, request_id);
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
-- no-transaction
|
||||
-- Historical trusted-identity metadata must also be anonymized. Index that
|
||||
-- exact predicate without storing the much larger metadata document.
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_usage_analytics_actor_metadata
|
||||
ON public.usage ((request_metadata #>> '{analytics_attribution,actor_user_id}'))
|
||||
WHERE (request_metadata #>> '{analytics_attribution,actor_user_id}') IS NOT NULL;
|
||||
+139
@@ -0,0 +1,139 @@
|
||||
-- Install future user anonymization without scanning or rewriting historical rows.
|
||||
-- Legacy orphan actors are excluded by the dashboard read path. User deletion
|
||||
-- cleans the retained actor window and contribution identities at transaction end.
|
||||
CREATE TABLE public.dashboard_user_anonymization_pending (
|
||||
transaction_id bigint NOT NULL,
|
||||
user_id varchar(255) NOT NULL,
|
||||
PRIMARY KEY (transaction_id,user_id)
|
||||
);
|
||||
|
||||
CREATE FUNCTION public.dashboard_enqueue_user_anonymization() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF current_setting('aether.dashboard_restore',true)='on' THEN RETURN NULL; END IF;
|
||||
IF TG_OP='DELETE' OR NEW.is_deleted THEN
|
||||
INSERT INTO public.dashboard_user_anonymization_pending(transaction_id,user_id)
|
||||
VALUES(txid_current(),OLD.id) ON CONFLICT DO NOTHING;
|
||||
END IF;
|
||||
RETURN NULL;
|
||||
END $$;
|
||||
CREATE TRIGGER dashboard_user_anonymize AFTER DELETE OR UPDATE OF is_deleted ON public.users
|
||||
FOR EACH ROW EXECUTE FUNCTION public.dashboard_enqueue_user_anonymization();
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.dashboard_apply_pending() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
DECLARE p record; anonymous_request record; old_fact public.dashboard_request_contributions%ROWTYPE;
|
||||
fact jsonb; next_metrics jsonb; next_at timestamptz; next_actor text;
|
||||
shard_id smallint; old_bucket timestamptz; next_bucket timestamptz; purged boolean; anonymize boolean;
|
||||
detail_cutoff timestamptz := date_trunc('minute',clock_timestamp()-INTERVAL '35 days');
|
||||
BEGIN
|
||||
IF current_setting('aether.dashboard_restore',true)='on' THEN
|
||||
DELETE FROM public.dashboard_stats_pending WHERE transaction_id=NEW.transaction_id;
|
||||
DELETE FROM public.dashboard_user_anonymization_pending WHERE transaction_id=NEW.transaction_id;
|
||||
RETURN NULL;
|
||||
END IF;
|
||||
SELECT EXISTS (SELECT 1 FROM public.dashboard_user_anonymization_pending
|
||||
WHERE transaction_id=NEW.transaction_id) INTO anonymize;
|
||||
-- Lock all touched shards in one order before touching any ledger or minute.
|
||||
-- This also serializes concurrent usage/settlement updates for the same request.
|
||||
PERFORM t.shard FROM public.dashboard_stats_total t
|
||||
WHERE t.shard IN (SELECT (hashtextextended(request_id,0) & 15)::smallint
|
||||
FROM public.dashboard_stats_pending WHERE transaction_id=NEW.transaction_id)
|
||||
OR anonymize
|
||||
ORDER BY t.shard FOR UPDATE;
|
||||
FOR p IN SELECT * FROM public.dashboard_stats_pending
|
||||
WHERE transaction_id=NEW.transaction_id ORDER BY request_id LOOP
|
||||
shard_id := (hashtextextended(p.request_id,0) & 15)::smallint;
|
||||
fact := public.dashboard_request_fact(p.request_id);
|
||||
purged := NOT EXISTS(SELECT 1 FROM public.usage WHERE request_id=p.request_id);
|
||||
-- A purge preserves the last contribution, including an update in this tx.
|
||||
fact := COALESCE(fact,p.deleted_fact);
|
||||
IF fact IS NULL THEN
|
||||
IF purged THEN DELETE FROM public.dashboard_request_contributions WHERE request_id=p.request_id; END IF;
|
||||
CONTINUE;
|
||||
END IF;
|
||||
next_metrics := fact->'metrics';
|
||||
next_at := (fact->>'created_at')::timestamptz;
|
||||
next_actor := CASE WHEN COALESCE((next_metrics->>'request_count')::bigint,0)>0
|
||||
THEN (SELECT id FROM public.users WHERE id=fact->>'actor_user_id' AND NOT is_deleted) END;
|
||||
SELECT * INTO old_fact FROM public.dashboard_request_contributions WHERE request_id=p.request_id;
|
||||
IF FOUND AND old_fact.created_at=next_at
|
||||
AND old_fact.actor_user_id IS NOT DISTINCT FROM next_actor
|
||||
AND old_fact.metrics=next_metrics THEN
|
||||
IF purged THEN DELETE FROM public.dashboard_request_contributions WHERE request_id=p.request_id; END IF;
|
||||
CONTINUE;
|
||||
END IF;
|
||||
next_bucket := date_trunc('minute',next_at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC';
|
||||
IF old_fact.request_id IS NOT NULL THEN
|
||||
old_bucket := date_trunc('minute',old_fact.created_at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC';
|
||||
PERFORM public.dashboard_ensure_activity_minute(old_bucket,shard_id);
|
||||
UPDATE public.dashboard_activity_minute SET request_count=request_count-COALESCE((old_fact.metrics->>'request_count')::bigint,0)
|
||||
WHERE bucket_start=old_bucket AND shard=shard_id;
|
||||
UPDATE public.dashboard_stats_total SET metrics=public.dashboard_metrics_add(metrics,old_fact.metrics,-1) WHERE shard=shard_id;
|
||||
IF old_bucket>=detail_cutoff THEN
|
||||
UPDATE public.dashboard_stats_minute SET metrics=public.dashboard_metrics_add(metrics,old_fact.metrics,-1)
|
||||
WHERE bucket_start=old_bucket AND shard=shard_id;
|
||||
END IF;
|
||||
UPDATE public.dashboard_activity_hour SET request_count=request_count-COALESCE((old_fact.metrics->>'request_count')::bigint,0)
|
||||
WHERE bucket_start=date_trunc('hour',old_fact.created_at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC' AND shard=shard_id;
|
||||
IF old_fact.actor_user_id IS NOT NULL AND old_bucket>=detail_cutoff THEN
|
||||
UPDATE public.dashboard_actor_minute SET request_count=request_count-1
|
||||
WHERE bucket_start=old_bucket AND shard=shard_id AND actor_user_id=old_fact.actor_user_id;
|
||||
END IF;
|
||||
END IF;
|
||||
PERFORM public.dashboard_ensure_activity_minute(next_bucket,shard_id);
|
||||
INSERT INTO public.dashboard_activity_minute(bucket_start,shard,request_count)
|
||||
VALUES(next_bucket,shard_id,COALESCE((next_metrics->>'request_count')::bigint,0))
|
||||
ON CONFLICT(bucket_start,shard) DO UPDATE
|
||||
SET request_count=dashboard_activity_minute.request_count+EXCLUDED.request_count;
|
||||
UPDATE public.dashboard_stats_total SET metrics=public.dashboard_metrics_add(metrics,next_metrics) WHERE shard=shard_id;
|
||||
IF next_bucket>=detail_cutoff THEN
|
||||
INSERT INTO public.dashboard_stats_minute(bucket_start,shard,metrics) VALUES(next_bucket,shard_id,next_metrics)
|
||||
ON CONFLICT(bucket_start,shard) DO UPDATE
|
||||
SET metrics=public.dashboard_metrics_add(dashboard_stats_minute.metrics,EXCLUDED.metrics);
|
||||
END IF;
|
||||
INSERT INTO public.dashboard_activity_hour(bucket_start,shard,request_count)
|
||||
VALUES(date_trunc('hour',next_at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC',shard_id,COALESCE((next_metrics->>'request_count')::bigint,0))
|
||||
ON CONFLICT(bucket_start,shard) DO UPDATE SET request_count=dashboard_activity_hour.request_count+EXCLUDED.request_count;
|
||||
IF next_actor IS NOT NULL AND next_bucket>=detail_cutoff THEN
|
||||
INSERT INTO public.dashboard_actor_minute(bucket_start,shard,actor_user_id,request_count)
|
||||
VALUES(next_bucket,shard_id,next_actor,1)
|
||||
ON CONFLICT(bucket_start,shard,actor_user_id) DO UPDATE SET request_count=dashboard_actor_minute.request_count+1;
|
||||
END IF;
|
||||
IF purged THEN
|
||||
DELETE FROM public.dashboard_request_contributions WHERE request_id=p.request_id;
|
||||
ELSE
|
||||
INSERT INTO public.dashboard_request_contributions(request_id,created_at,actor_user_id,metrics)
|
||||
VALUES(p.request_id,next_at,next_actor,next_metrics)
|
||||
ON CONFLICT(request_id) DO UPDATE SET created_at=EXCLUDED.created_at,
|
||||
actor_user_id=EXCLUDED.actor_user_id,metrics=EXCLUDED.metrics;
|
||||
END IF;
|
||||
END LOOP;
|
||||
-- Flush identity removal only after request corrections, under the same shard
|
||||
-- locks. This also covers requests whose source usage was already purged.
|
||||
-- Surviving contribution identities were corrected above by request_id; do
|
||||
-- not scan the lifetime contribution table while holding the shard locks.
|
||||
IF anonymize THEN
|
||||
-- A writer can commit between the user's deletion statement and this flush.
|
||||
-- Its new attribution was invisible to the user's earlier trigger. Reuse
|
||||
-- the actor index to find only those requests, then clear each ledger by PK.
|
||||
-- Do not lock attribution rows here: writers lock them before their shard.
|
||||
FOR anonymous_request IN
|
||||
SELECT a.request_id, pending_user.user_id
|
||||
FROM public.dashboard_user_anonymization_pending pending_user
|
||||
JOIN public.usage_attribution_snapshots a ON a.actor_user_id=pending_user.user_id
|
||||
WHERE pending_user.transaction_id=NEW.transaction_id
|
||||
LOOP
|
||||
UPDATE public.dashboard_request_contributions SET actor_user_id=NULL
|
||||
WHERE request_id=anonymous_request.request_id AND actor_user_id=anonymous_request.user_id;
|
||||
END LOOP;
|
||||
DELETE FROM public.dashboard_actor_minute a USING public.dashboard_user_anonymization_pending pending_user
|
||||
WHERE pending_user.transaction_id=NEW.transaction_id AND a.actor_user_id=pending_user.user_id;
|
||||
END IF;
|
||||
DELETE FROM public.dashboard_stats_pending WHERE transaction_id=NEW.transaction_id;
|
||||
DELETE FROM public.dashboard_user_anonymization_pending WHERE transaction_id=NEW.transaction_id;
|
||||
RETURN NULL;
|
||||
END $$;
|
||||
-- Both request and user events share one deferred flush. It locks every touched
|
||||
-- shard in order before corrections or deletion, regardless of trigger order.
|
||||
CREATE CONSTRAINT TRIGGER dashboard_flush_user_anonymization
|
||||
AFTER INSERT ON public.dashboard_user_anonymization_pending
|
||||
DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION public.dashboard_apply_pending();
|
||||
Reference in New Issue
Block a user