mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 01:10:23 +08:00
failover/stream_telemetry/recording/stream 中的同步 DB 操作(commit/execute/query) 会阻塞 asyncio 事件循环,导致 Hub PING 心跳无法发送、worker idle timeout 断连。 将这些操作包装到 asyncio.to_thread() 中执行。 同时提取 Alembic 迁移脚本中重复的幂等性辅助函数到 alembic/helpers.py, 用批量查询缓存替代逐条 information_schema 查询,backfill SQL 合并为 LEFT JOIN。 新增 failover 中客户端断连的快速终止路径,避免继续无意义的重试。
172 lines
5.8 KiB
Python
172 lines
5.8 KiB
Python
"""Shared idempotent helpers for Alembic migrations.
|
|
|
|
All metadata lookups are batched: one query loads an entire table's column info
|
|
or all FK delete rules, then results are cached for the migration's lifetime.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections import defaultdict
|
|
from collections.abc import Callable
|
|
|
|
import sqlalchemy as sa
|
|
|
|
from alembic import op
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Batch metadata cache
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class _SchemaCache:
|
|
"""Lazy, per-migration cache for information_schema lookups.
|
|
|
|
Call ``load_columns(tables)`` / ``load_fk_rules(tables)`` once at the top
|
|
of ``upgrade()`` or ``downgrade()`` to prime the cache. Subsequent
|
|
``column_exists`` / ``column_type`` / ``fk_ondelete`` calls are pure
|
|
dict lookups -- zero extra DB round-trips.
|
|
"""
|
|
|
|
def __init__(self) -> None:
|
|
# {table_name: {column_name: data_type}}
|
|
self._columns: dict[str, dict[str, str]] = {}
|
|
# {(table_name, constraint_name): delete_rule}
|
|
self._fk_rules: dict[tuple[str, str], str] = {}
|
|
self._fk_loaded_tables: set[str] = set()
|
|
|
|
# -- loaders (one query per call) --------------------------------------
|
|
|
|
def load_columns(self, tables: list[str]) -> None:
|
|
"""Fetch column names + data types for *tables* in one query."""
|
|
need = [t for t in tables if t not in self._columns]
|
|
if not need:
|
|
return
|
|
bind = op.get_bind()
|
|
rows = bind.execute(
|
|
sa.text(
|
|
"SELECT table_name, column_name, data_type "
|
|
"FROM information_schema.columns "
|
|
"WHERE table_name = ANY(:tables)"
|
|
),
|
|
{"tables": need},
|
|
).fetchall()
|
|
# Initialise even empty tables so we don't re-query
|
|
for t in need:
|
|
self._columns.setdefault(t, {})
|
|
for table, col, dtype in rows:
|
|
self._columns[table][col] = dtype
|
|
|
|
def load_fk_rules(self, tables: list[str]) -> None:
|
|
"""Fetch FK delete rules for *tables* in one query."""
|
|
need = [t for t in tables if t not in self._fk_loaded_tables]
|
|
if not need:
|
|
return
|
|
bind = op.get_bind()
|
|
rows = bind.execute(
|
|
sa.text(
|
|
"SELECT tc.table_name, tc.constraint_name, rc.delete_rule "
|
|
"FROM information_schema.referential_constraints rc "
|
|
"JOIN information_schema.table_constraints tc "
|
|
" ON rc.constraint_name = tc.constraint_name "
|
|
"WHERE tc.table_name = ANY(:tables)"
|
|
),
|
|
{"tables": need},
|
|
).fetchall()
|
|
for table, name, rule in rows:
|
|
self._fk_rules[(table, name)] = rule
|
|
self._fk_loaded_tables.update(need)
|
|
|
|
# -- lookups (pure dict, zero DB) --------------------------------------
|
|
|
|
def column_exists(self, table: str, column: str) -> bool:
|
|
return column in self._columns.get(table, {})
|
|
|
|
def column_type(self, table: str, column: str) -> str | None:
|
|
return self._columns.get(table, {}).get(column)
|
|
|
|
def is_numeric(self, table: str, column: str) -> bool:
|
|
return self.column_type(table, column) == "numeric"
|
|
|
|
def fk_ondelete(self, table: str, constraint: str) -> str | None:
|
|
return self._fk_rules.get((table, constraint))
|
|
|
|
def invalidate_columns(self, table: str) -> None:
|
|
"""Force re-load on next load_columns() for *table* (after ADD/DROP COLUMN)."""
|
|
self._columns.pop(table, None)
|
|
|
|
|
|
def new_cache() -> _SchemaCache:
|
|
"""Create a fresh schema cache for a single migration run."""
|
|
return _SchemaCache()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Idempotent DDL helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def replace_fk_if_needed(
|
|
cache: _SchemaCache,
|
|
constraint_name: str,
|
|
table_name: str,
|
|
ref_table: str,
|
|
local_cols: list[str],
|
|
remote_cols: list[str],
|
|
desired_ondelete: str,
|
|
) -> None:
|
|
"""Drop and recreate a FK only if the current ON DELETE rule differs."""
|
|
current = cache.fk_ondelete(table_name, constraint_name)
|
|
if current and current.upper() == desired_ondelete.upper():
|
|
return
|
|
if current:
|
|
op.drop_constraint(constraint_name, table_name, type_="foreignkey")
|
|
op.create_foreign_key(
|
|
constraint_name,
|
|
table_name,
|
|
ref_table,
|
|
local_cols,
|
|
remote_cols,
|
|
ondelete=desired_ondelete,
|
|
)
|
|
|
|
|
|
def index_exists(index_name: str) -> bool:
|
|
bind = op.get_bind()
|
|
result = bind.execute(
|
|
sa.text("SELECT 1 FROM pg_indexes WHERE indexname = :name"),
|
|
{"name": index_name},
|
|
)
|
|
return result.scalar() is not None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Batch ALTER TYPE helper
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def batch_alter_type(
|
|
cache: _SchemaCache,
|
|
columns: list[tuple[str, str, bool, str | None]],
|
|
cast_suffix: str,
|
|
type_fn: Callable[[str], str],
|
|
) -> None:
|
|
"""Group columns by table and issue ONE ``ALTER TABLE`` per table.
|
|
|
|
Skips columns that don't exist. Each tuple is
|
|
``(table_name, column_name, nullable, server_default)``.
|
|
"""
|
|
by_table: dict[str, list[tuple[str, str]]] = defaultdict(list)
|
|
for table, col, _nullable, _default in columns:
|
|
if not cache.column_exists(table, col):
|
|
continue
|
|
by_table[table].append((col, type_fn(col)))
|
|
|
|
bind = op.get_bind()
|
|
for table, col_types in by_table.items():
|
|
parts = [
|
|
f"ALTER COLUMN {col} TYPE {target} USING {col}::{cast_suffix}"
|
|
for col, target in col_types
|
|
]
|
|
if parts:
|
|
bind.execute(sa.text(f"ALTER TABLE {table} " + ", ".join(parts)))
|