mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 09:20:22 +08:00
refactor(maintenance): body 压缩改为逐条独立事务，降低内存占用与锁粒度
将 _cleanup_body_fields 从批量加载完整记录改为先查询 ID 列表， 再逐条独立会话处理压缩，避免大批量事务导致的内存和锁问题。 批次大小上限从 100 降至 25，新增排序保证处理顺序确定性。
This commit is contained in:
fawney19
@@ -1025,117 +1025,29 @@ class MaintenanceScheduler:
|
||||
|
||||
total_compressed = 0
|
||||
no_progress_count = 0
|
||||
memory_safe_batch_size = max(1, min(batch_size, 100))
|
||||
memory_safe_batch_size = max(1, min(batch_size, 25))
|
||||
|
||||
while True:
|
||||
batch_db = create_session()
|
||||
try:
|
||||
records = (
|
||||
batch_db.query(
|
||||
Usage.id,
|
||||
Usage.request_body,
|
||||
Usage.response_body,
|
||||
Usage.provider_request_body,
|
||||
Usage.client_response_body,
|
||||
record_ids = [
|
||||
row.id
|
||||
for row in (
|
||||
batch_db.query(Usage.id)
|
||||
.filter(Usage.created_at < cutoff_time)
|
||||
.filter(
|
||||
(Usage.request_body.isnot(None))
|
||||
| (Usage.response_body.isnot(None))
|
||||
| (Usage.provider_request_body.isnot(None))
|
||||
| (Usage.client_response_body.isnot(None))
|
||||
)
|
||||
.order_by(Usage.created_at.asc(), Usage.id.asc())
|
||||
.limit(memory_safe_batch_size)
|
||||
.all()
|
||||
)
|
||||
.filter(Usage.created_at < cutoff_time)
|
||||
.filter(
|
||||
(Usage.request_body.isnot(None))
|
||||
| (Usage.response_body.isnot(None))
|
||||
| (Usage.provider_request_body.isnot(None))
|
||||
| (Usage.client_response_body.isnot(None))
|
||||
)
|
||||
.limit(memory_safe_batch_size)
|
||||
.all()
|
||||
)
|
||||
|
||||
if not records:
|
||||
break
|
||||
|
||||
valid_records = [
|
||||
r
|
||||
for r in records
|
||||
if r.request_body is not None
|
||||
or r.response_body is not None
|
||||
or r.provider_request_body is not None
|
||||
or r.client_response_body is not None
|
||||
]
|
||||
|
||||
if not valid_records:
|
||||
logger.warning(
|
||||
f"检测到 {len(records)} 条记录的 body 字段为 JSON null，进行清理"
|
||||
)
|
||||
for r in records:
|
||||
batch_db.execute(
|
||||
update(Usage)
|
||||
.where(Usage.id == r.id)
|
||||
.values(
|
||||
request_body=null(),
|
||||
response_body=null(),
|
||||
provider_request_body=null(),
|
||||
client_response_body=null(),
|
||||
)
|
||||
)
|
||||
batch_db.commit()
|
||||
continue
|
||||
|
||||
batch_success = 0
|
||||
|
||||
for r in valid_records:
|
||||
try:
|
||||
result = batch_db.execute(
|
||||
update(Usage)
|
||||
.where(Usage.id == r.id)
|
||||
.values(
|
||||
request_body=null(),
|
||||
response_body=null(),
|
||||
provider_request_body=null(),
|
||||
client_response_body=null(),
|
||||
request_body_compressed=(
|
||||
compress_json(r.request_body) if r.request_body else None
|
||||
),
|
||||
response_body_compressed=(
|
||||
compress_json(r.response_body) if r.response_body else None
|
||||
),
|
||||
provider_request_body_compressed=(
|
||||
compress_json(r.provider_request_body)
|
||||
if r.provider_request_body
|
||||
else None
|
||||
),
|
||||
client_response_body_compressed=(
|
||||
compress_json(r.client_response_body)
|
||||
if r.client_response_body
|
||||
else None
|
||||
),
|
||||
)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
if result.rowcount > 0:
|
||||
batch_success += 1
|
||||
except Exception as e:
|
||||
logger.warning(f"压缩记录 {r.id} 失败: {e}")
|
||||
continue
|
||||
|
||||
batch_db.commit()
|
||||
|
||||
if batch_success == 0:
|
||||
no_progress_count += 1
|
||||
if no_progress_count >= 3:
|
||||
logger.error(
|
||||
f"压缩 body 字段连续 {no_progress_count} 批无进展，"
|
||||
"终止循环以避免死循环"
|
||||
)
|
||||
break
|
||||
else:
|
||||
no_progress_count = 0
|
||||
|
||||
total_compressed += batch_success
|
||||
logger.debug(
|
||||
f"已压缩 {batch_success} 条记录的 body 字段，累计 {total_compressed} 条"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"压缩 body 字段失败: {e}")
|
||||
logger.exception("加载待压缩 body 记录 ID 失败: {}", e)
|
||||
try:
|
||||
batch_db.rollback()
|
||||
except Exception:
|
||||
@@ -1144,6 +1056,114 @@ class MaintenanceScheduler:
|
||||
finally:
|
||||
batch_db.close()
|
||||
|
||||
if not record_ids:
|
||||
break
|
||||
|
||||
batch_success = 0
|
||||
batch_progress = False
|
||||
|
||||
for record_id in record_ids:
|
||||
record_db = create_session()
|
||||
try:
|
||||
record = (
|
||||
record_db.query(
|
||||
Usage.id,
|
||||
Usage.request_body,
|
||||
Usage.response_body,
|
||||
Usage.provider_request_body,
|
||||
Usage.client_response_body,
|
||||
)
|
||||
.filter(Usage.id == record_id)
|
||||
.first()
|
||||
)
|
||||
|
||||
if record is None:
|
||||
continue
|
||||
|
||||
has_body_payload = (
|
||||
record.request_body is not None
|
||||
or record.response_body is not None
|
||||
or record.provider_request_body is not None
|
||||
or record.client_response_body is not None
|
||||
)
|
||||
|
||||
if not has_body_payload:
|
||||
result = record_db.execute(
|
||||
update(Usage)
|
||||
.where(Usage.id == record.id)
|
||||
.values(
|
||||
request_body=null(),
|
||||
response_body=null(),
|
||||
provider_request_body=null(),
|
||||
client_response_body=null(),
|
||||
)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
record_db.commit()
|
||||
if result.rowcount > 0:
|
||||
batch_progress = True
|
||||
continue
|
||||
|
||||
result = record_db.execute(
|
||||
update(Usage)
|
||||
.where(Usage.id == record.id)
|
||||
.values(
|
||||
request_body=null(),
|
||||
response_body=null(),
|
||||
provider_request_body=null(),
|
||||
client_response_body=null(),
|
||||
request_body_compressed=(
|
||||
compress_json(record.request_body) if record.request_body else None
|
||||
),
|
||||
response_body_compressed=(
|
||||
compress_json(record.response_body)
|
||||
if record.response_body
|
||||
else None
|
||||
),
|
||||
provider_request_body_compressed=(
|
||||
compress_json(record.provider_request_body)
|
||||
if record.provider_request_body
|
||||
else None
|
||||
),
|
||||
client_response_body_compressed=(
|
||||
compress_json(record.client_response_body)
|
||||
if record.client_response_body
|
||||
else None
|
||||
),
|
||||
)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
record_db.commit()
|
||||
|
||||
if result.rowcount > 0:
|
||||
batch_success += 1
|
||||
batch_progress = True
|
||||
except Exception as e:
|
||||
logger.warning("压缩记录 {} 失败: {}", record_id, e)
|
||||
try:
|
||||
record_db.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
continue
|
||||
finally:
|
||||
record_db.close()
|
||||
|
||||
if not batch_progress:
|
||||
no_progress_count += 1
|
||||
if no_progress_count >= 3:
|
||||
logger.error(
|
||||
f"压缩 body 字段连续 {no_progress_count} 批无进展，" "终止循环以避免死循环"
|
||||
)
|
||||
break
|
||||
else:
|
||||
no_progress_count = 0
|
||||
|
||||
total_compressed += batch_success
|
||||
if batch_success > 0:
|
||||
logger.debug(
|
||||
f"已压缩 {batch_success} 条记录的 body 字段，累计 {total_compressed} 条"
|
||||
)
|
||||
|
||||
return total_compressed
|
||||
|
||||
def _cleanup_compressed_fields(self, cutoff_time: datetime, batch_size: int) -> int:
|
||||
|
||||
Reference in New Issue
Block a user