perf: 优化流式响应内存占用和正则预编译

- 添加 record_parsed_chunks 标志,仅在 FULL 日志级别时累积 parsed_chunks
- 预编译 CJK 和敏感信息正则表达式,避免重复编译
- 优化 header 脱敏循环,预计算 lowercase 集合
- 移除 consumer_streams.py 中未使用的 message_fields 变量
- 将 SystemConfigService import 移至文件顶部
This commit is contained in:
fawney19
2026-02-03 21:32:53 +08:00
parent d6597121f5
commit 6f363a7703
9 changed files with 40 additions and 23 deletions

View File

@@ -500,10 +500,11 @@ class SystemConfigService:
return headers
sensitive_headers = cls.get_sensitive_headers(db)
sensitive_lower = {h.lower() for h in sensitive_headers if isinstance(h, str) and h}
masked_headers = {}
for key, value in headers.items():
if key.lower() in [h.lower() for h in sensitive_headers]:
if key.lower() in sensitive_lower:
# 保留前后各4个字符中间用星号替换
if len(str(value)) > 8:
masked_value = str(value)[:4] + "****" + str(value)[-4:]

View File

@@ -1,5 +1,6 @@
from __future__ import annotations
import re
from collections.abc import Callable
from typing import Any
@@ -18,6 +19,11 @@ from src.services.task.exceptions import TaskNotFoundError
from src.services.task.protocol import AttemptKind, AttemptResult
from src.services.task.schema import ExecutionResult, TaskStatusResult
_SENSITIVE_PATTERN = re.compile(
r"(api[_-]?key|token|bearer|authorization)[=:\s]+\S+",
re.IGNORECASE,
)
class TaskService:
"""
@@ -883,7 +889,6 @@ class TaskService:
- stop on "client error" (raise UpstreamClientRequestError)
- if all failed, raise AllCandidatesFailedError
"""
import re
from datetime import datetime, timezone
import httpx
@@ -901,11 +906,6 @@ class TaskService:
from src.services.orchestration.error_classifier import ErrorClassifier
from src.services.system.config import SystemConfigService
_SENSITIVE_PATTERN = re.compile(
r"(api[_-]?key|token|bearer|authorization)[=:\s]+\S+",
re.IGNORECASE,
)
def _sanitize(message: str, max_length: int = 200) -> str:
if not message:
return "request_failed"

View File

@@ -283,12 +283,10 @@ class UsageQueueConsumer:
# 准备批量记录数据
records: list[dict[str, Any]] = []
message_ids: list[str] = []
message_fields: list[dict[str, Any]] = []
for message_id, fields, event in messages:
records.append(_event_to_record(event))
message_ids.append(message_id)
message_fields.append(fields)
# 批量写入
await UsageService.record_usage_batch(db, records)