mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-02 09:20:22 +08:00
refactor: 移除 Python 后端源码，全面迁移至 Rust gateway 架构
- 删除全部 Python 源码 (src/) 及 Alembic 迁移脚本，归档至 _deprecated_py_src/ - 重构 Rust gateway ai_pipeline: 拆分 planner/finalize 模块，新增 contracts/adaptation 层 - 重组 handlers 模块为 admin/public/proxy/internal/shared 子模块结构 - 新增 executor 模块，引入 Rust 原生数据库迁移 (aether-data/migrations) - 简化 CI/Docker 构建流程，移除 base image 二级构建，统一为单一 app image - 移除 Python 相关基础设施文件 (entrypoint.sh, gunicorn_conf.py, Dockerfile.base)
This commit is contained in:
fawney19
@@ -0,0 +1,6 @@
|
||||
"""AWS Event Stream parser for Kiro."""
|
||||
|
||||
from .decoder import EventStreamDecoder
|
||||
from .frame import Frame
|
||||
|
||||
__all__ = ["EventStreamDecoder", "Frame"]
|
||||
@@ -0,0 +1,13 @@
|
||||
"""CRC helpers for AWS Event Stream frames."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import binascii
|
||||
|
||||
|
||||
def crc32(data: bytes) -> int:
|
||||
"""Compute unsigned CRC32 (IEEE)."""
|
||||
return binascii.crc32(data) & 0xFFFFFFFF
|
||||
|
||||
|
||||
__all__ = ["crc32"]
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Incremental AWS Event Stream decoder."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from .error import BufferOverflowError, EventStreamParseError
|
||||
from .frame import MAX_MESSAGE_SIZE, Frame, parse_frame
|
||||
|
||||
DEFAULT_MAX_BUFFER_SIZE = MAX_MESSAGE_SIZE
|
||||
DEFAULT_MAX_ERRORS = 5
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class DecoderStats:
|
||||
frames_decoded: int = 0
|
||||
bytes_skipped: int = 0
|
||||
error_count: int = 0
|
||||
|
||||
|
||||
class EventStreamDecoder:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
max_buffer_size: int = DEFAULT_MAX_BUFFER_SIZE,
|
||||
max_errors: int = DEFAULT_MAX_ERRORS,
|
||||
) -> None:
|
||||
self._buffer = bytearray()
|
||||
self._max_buffer_size = int(max_buffer_size)
|
||||
self._max_errors = int(max_errors)
|
||||
self._stopped = False
|
||||
self.stats = DecoderStats()
|
||||
|
||||
@property
|
||||
def stopped(self) -> bool:
|
||||
return self._stopped
|
||||
|
||||
def feed(self, data: bytes) -> None:
|
||||
if self._stopped:
|
||||
return
|
||||
if not data:
|
||||
return
|
||||
new_size = len(self._buffer) + len(data)
|
||||
if new_size > self._max_buffer_size:
|
||||
self._stopped = True
|
||||
raise BufferOverflowError(size=new_size, max_size=self._max_buffer_size)
|
||||
self._buffer.extend(data)
|
||||
|
||||
def decode_available(self) -> list[Frame]:
|
||||
"""Decode all complete frames currently in buffer."""
|
||||
out: list[Frame] = []
|
||||
if self._stopped:
|
||||
return out
|
||||
|
||||
while True:
|
||||
try:
|
||||
# Use memoryview to avoid full buffer copy on each iteration
|
||||
parsed = parse_frame(memoryview(self._buffer))
|
||||
except EventStreamParseError:
|
||||
self.stats.error_count += 1
|
||||
if self.stats.error_count >= self._max_errors:
|
||||
self._stopped = True
|
||||
raise
|
||||
|
||||
# Recovery: skip a byte and keep scanning.
|
||||
if self._buffer:
|
||||
del self._buffer[0]
|
||||
self.stats.bytes_skipped += 1
|
||||
else:
|
||||
break
|
||||
continue
|
||||
|
||||
if parsed is None:
|
||||
break
|
||||
|
||||
frame, consumed = parsed
|
||||
if consumed <= 0:
|
||||
break
|
||||
|
||||
out.append(frame)
|
||||
del self._buffer[:consumed]
|
||||
self.stats.frames_decoded += 1
|
||||
self.stats.error_count = 0
|
||||
|
||||
return out
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DecoderStats",
|
||||
"EventStreamDecoder",
|
||||
]
|
||||
@@ -0,0 +1,72 @@
|
||||
"""AWS Event Stream parsing errors."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
class EventStreamParseError(Exception):
|
||||
"""Base error for AWS Event Stream decoding."""
|
||||
|
||||
|
||||
class IncompleteFrameError(EventStreamParseError):
|
||||
def __init__(self, *, needed: int, available: int) -> None:
|
||||
super().__init__(f"incomplete frame: needed={needed} available={available}")
|
||||
self.needed = needed
|
||||
self.available = available
|
||||
|
||||
|
||||
class MessageTooSmallError(EventStreamParseError):
|
||||
def __init__(self, *, length: int, min_length: int) -> None:
|
||||
super().__init__(f"message too small: length={length} min={min_length}")
|
||||
self.length = length
|
||||
self.min_length = min_length
|
||||
|
||||
|
||||
class MessageTooLargeError(EventStreamParseError):
|
||||
def __init__(self, *, length: int, max_length: int) -> None:
|
||||
super().__init__(f"message too large: length={length} max={max_length}")
|
||||
self.length = length
|
||||
self.max_length = max_length
|
||||
|
||||
|
||||
class PreludeCrcMismatchError(EventStreamParseError):
|
||||
def __init__(self, *, expected: int, actual: int) -> None:
|
||||
super().__init__(f"prelude crc mismatch: expected={expected} actual={actual}")
|
||||
self.expected = expected
|
||||
self.actual = actual
|
||||
|
||||
|
||||
class MessageCrcMismatchError(EventStreamParseError):
|
||||
def __init__(self, *, expected: int, actual: int) -> None:
|
||||
super().__init__(f"message crc mismatch: expected={expected} actual={actual}")
|
||||
self.expected = expected
|
||||
self.actual = actual
|
||||
|
||||
|
||||
class InvalidHeaderTypeError(EventStreamParseError):
|
||||
def __init__(self, type_id: int) -> None:
|
||||
super().__init__(f"invalid header type: {type_id}")
|
||||
self.type_id = type_id
|
||||
|
||||
|
||||
class HeaderParseError(EventStreamParseError):
|
||||
pass
|
||||
|
||||
|
||||
class BufferOverflowError(EventStreamParseError):
|
||||
def __init__(self, *, size: int, max_size: int) -> None:
|
||||
super().__init__(f"buffer overflow: size={size} max={max_size}")
|
||||
self.size = size
|
||||
self.max_size = max_size
|
||||
|
||||
|
||||
__all__ = [
|
||||
"BufferOverflowError",
|
||||
"EventStreamParseError",
|
||||
"HeaderParseError",
|
||||
"IncompleteFrameError",
|
||||
"InvalidHeaderTypeError",
|
||||
"MessageCrcMismatchError",
|
||||
"MessageTooLargeError",
|
||||
"MessageTooSmallError",
|
||||
"PreludeCrcMismatchError",
|
||||
]
|
||||
@@ -0,0 +1,95 @@
|
||||
"""AWS Event Stream message frame parsing."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from .crc import crc32
|
||||
from .error import (
|
||||
HeaderParseError,
|
||||
IncompleteFrameError,
|
||||
MessageCrcMismatchError,
|
||||
MessageTooLargeError,
|
||||
MessageTooSmallError,
|
||||
PreludeCrcMismatchError,
|
||||
)
|
||||
from .header import Headers, parse_headers
|
||||
|
||||
PRELUDE_SIZE = 12
|
||||
MIN_MESSAGE_SIZE = PRELUDE_SIZE + 4
|
||||
MAX_MESSAGE_SIZE = 16 * 1024 * 1024
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Frame:
|
||||
headers: Headers
|
||||
payload: bytes
|
||||
|
||||
def message_type(self) -> str | None:
|
||||
return self.headers.message_type()
|
||||
|
||||
def event_type(self) -> str | None:
|
||||
return self.headers.event_type()
|
||||
|
||||
def payload_as_text(self) -> str:
|
||||
return self.payload.decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def parse_frame(buffer: bytes | memoryview) -> tuple[Frame, int] | None:
|
||||
"""Parse a single frame from the front of buffer.
|
||||
|
||||
Returns:
|
||||
(frame, consumed_bytes) if a full frame is available, otherwise None.
|
||||
|
||||
Raises:
|
||||
EventStreamParseError subclasses on validation errors.
|
||||
"""
|
||||
if len(buffer) < PRELUDE_SIZE:
|
||||
return None
|
||||
|
||||
total_length = int.from_bytes(buffer[0:4], "big", signed=False)
|
||||
header_length = int.from_bytes(buffer[4:8], "big", signed=False)
|
||||
prelude_crc = int.from_bytes(buffer[8:12], "big", signed=False)
|
||||
|
||||
if total_length < MIN_MESSAGE_SIZE:
|
||||
raise MessageTooSmallError(length=total_length, min_length=MIN_MESSAGE_SIZE)
|
||||
if total_length > MAX_MESSAGE_SIZE:
|
||||
raise MessageTooLargeError(length=total_length, max_length=MAX_MESSAGE_SIZE)
|
||||
|
||||
if len(buffer) < total_length:
|
||||
return None
|
||||
|
||||
actual_prelude_crc = crc32(buffer[0:8])
|
||||
if actual_prelude_crc != prelude_crc:
|
||||
raise PreludeCrcMismatchError(expected=prelude_crc, actual=actual_prelude_crc)
|
||||
|
||||
message_crc = int.from_bytes(buffer[total_length - 4 : total_length], "big", signed=False)
|
||||
actual_message_crc = crc32(buffer[0 : total_length - 4])
|
||||
if actual_message_crc != message_crc:
|
||||
raise MessageCrcMismatchError(expected=message_crc, actual=actual_message_crc)
|
||||
|
||||
headers_start = PRELUDE_SIZE
|
||||
headers_end = headers_start + header_length
|
||||
|
||||
if headers_end > total_length - 4:
|
||||
raise HeaderParseError("header length exceeds frame boundary")
|
||||
|
||||
headers = parse_headers(bytes(buffer[headers_start:headers_end]), header_length)
|
||||
|
||||
payload_start = headers_end
|
||||
payload_end = total_length - 4
|
||||
if payload_end < payload_start:
|
||||
raise IncompleteFrameError(needed=payload_start, available=payload_end)
|
||||
|
||||
payload = bytes(buffer[payload_start:payload_end])
|
||||
|
||||
return Frame(headers=headers, payload=payload), total_length
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Frame",
|
||||
"MAX_MESSAGE_SIZE",
|
||||
"MIN_MESSAGE_SIZE",
|
||||
"PRELUDE_SIZE",
|
||||
"parse_frame",
|
||||
]
|
||||
@@ -0,0 +1,144 @@
|
||||
"""AWS Event Stream header parsing."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import IntEnum
|
||||
|
||||
from .error import HeaderParseError, IncompleteFrameError, InvalidHeaderTypeError
|
||||
|
||||
|
||||
class HeaderValueType(IntEnum):
|
||||
BOOL_TRUE = 0
|
||||
BOOL_FALSE = 1
|
||||
BYTE = 2
|
||||
SHORT = 3
|
||||
INTEGER = 4
|
||||
LONG = 5
|
||||
BYTE_ARRAY = 6
|
||||
STRING = 7
|
||||
TIMESTAMP = 8
|
||||
UUID = 9
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Headers:
|
||||
values: dict[str, object]
|
||||
|
||||
def get(self, name: str) -> object | None:
|
||||
return self.values.get(name)
|
||||
|
||||
def get_string(self, name: str) -> str | None:
|
||||
v = self.values.get(name)
|
||||
return v if isinstance(v, str) else None
|
||||
|
||||
def message_type(self) -> str | None:
|
||||
return self.get_string(":message-type")
|
||||
|
||||
def event_type(self) -> str | None:
|
||||
return self.get_string(":event-type")
|
||||
|
||||
def exception_type(self) -> str | None:
|
||||
return self.get_string(":exception-type")
|
||||
|
||||
def error_code(self) -> str | None:
|
||||
return self.get_string(":error-code")
|
||||
|
||||
|
||||
def _ensure_bytes(data: bytes, offset: int, needed: int) -> None:
|
||||
available = len(data) - offset
|
||||
if available < needed:
|
||||
raise IncompleteFrameError(needed=needed, available=available)
|
||||
|
||||
|
||||
def parse_headers(data: bytes, header_length: int) -> Headers:
|
||||
if len(data) < header_length:
|
||||
raise IncompleteFrameError(needed=header_length, available=len(data))
|
||||
|
||||
values: dict[str, object] = {}
|
||||
offset = 0
|
||||
|
||||
while offset < header_length:
|
||||
_ensure_bytes(data, offset, 1)
|
||||
name_len = data[offset]
|
||||
offset += 1
|
||||
if name_len == 0:
|
||||
raise HeaderParseError("header name length cannot be 0")
|
||||
|
||||
_ensure_bytes(data, offset, name_len)
|
||||
name = data[offset : offset + name_len].decode("utf-8", errors="replace")
|
||||
offset += name_len
|
||||
|
||||
_ensure_bytes(data, offset, 1)
|
||||
type_id = data[offset]
|
||||
offset += 1
|
||||
try:
|
||||
value_type = HeaderValueType(type_id)
|
||||
except ValueError as e:
|
||||
raise InvalidHeaderTypeError(type_id) from e
|
||||
|
||||
if value_type == HeaderValueType.BOOL_TRUE:
|
||||
values[name] = True
|
||||
continue
|
||||
if value_type == HeaderValueType.BOOL_FALSE:
|
||||
values[name] = False
|
||||
continue
|
||||
|
||||
if value_type == HeaderValueType.BYTE:
|
||||
_ensure_bytes(data, offset, 1)
|
||||
values[name] = int.from_bytes(data[offset : offset + 1], "big", signed=True)
|
||||
offset += 1
|
||||
continue
|
||||
|
||||
if value_type == HeaderValueType.SHORT:
|
||||
_ensure_bytes(data, offset, 2)
|
||||
values[name] = int.from_bytes(data[offset : offset + 2], "big", signed=True)
|
||||
offset += 2
|
||||
continue
|
||||
|
||||
if value_type == HeaderValueType.INTEGER:
|
||||
_ensure_bytes(data, offset, 4)
|
||||
values[name] = int.from_bytes(data[offset : offset + 4], "big", signed=True)
|
||||
offset += 4
|
||||
continue
|
||||
|
||||
if value_type in (HeaderValueType.LONG, HeaderValueType.TIMESTAMP):
|
||||
_ensure_bytes(data, offset, 8)
|
||||
values[name] = int.from_bytes(data[offset : offset + 8], "big", signed=True)
|
||||
offset += 8
|
||||
continue
|
||||
|
||||
if value_type == HeaderValueType.BYTE_ARRAY:
|
||||
_ensure_bytes(data, offset, 2)
|
||||
length = int.from_bytes(data[offset : offset + 2], "big", signed=False)
|
||||
offset += 2
|
||||
_ensure_bytes(data, offset, length)
|
||||
values[name] = data[offset : offset + length]
|
||||
offset += length
|
||||
continue
|
||||
|
||||
if value_type == HeaderValueType.STRING:
|
||||
_ensure_bytes(data, offset, 2)
|
||||
length = int.from_bytes(data[offset : offset + 2], "big", signed=False)
|
||||
offset += 2
|
||||
_ensure_bytes(data, offset, length)
|
||||
values[name] = data[offset : offset + length].decode("utf-8", errors="replace")
|
||||
offset += length
|
||||
continue
|
||||
|
||||
if value_type == HeaderValueType.UUID:
|
||||
_ensure_bytes(data, offset, 16)
|
||||
values[name] = bytes(data[offset : offset + 16])
|
||||
offset += 16
|
||||
continue
|
||||
|
||||
raise HeaderParseError(f"unhandled header type: {value_type}")
|
||||
|
||||
return Headers(values=values)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"HeaderValueType",
|
||||
"Headers",
|
||||
"parse_headers",
|
||||
]
|
||||
Reference in New Issue
Block a user