mirror of
https://github.com/fawney19/Aether.git
synced 2026-09-14 23:20:19 +08:00
feat(security): harden gateway boundaries and usage policies
Consolidate subscription usage policy enforcement, privacy-safe persistence, and gateway security hardening into one reviewable change. Includes bounded HTTP and execution envelopes, header and protocol guards, DNS and relay validation, authentication and secret projection hardening, secure backup/install paths, and regression coverage.
This commit is contained in:
+39
-7
@@ -15,6 +15,11 @@ APP_PORT=8084
|
||||
# APP_IMAGE=ghcr.io/fawney19/aether:beta
|
||||
# APP_IMAGE=ghcr.io/fawney19/aether:0.7.0-rc.1
|
||||
|
||||
# Compose 应用容器的非 root 数字身份。Single Node 的 ./data 必须归该身份所有;
|
||||
# install.sh 会自动写入安装用户的 UID/GID 并迁移旧数据目录。
|
||||
AETHER_CONTAINER_UID=65532
|
||||
AETHER_CONTAINER_GID=65532
|
||||
|
||||
# API Key 前缀(默认 sk)
|
||||
API_KEY_PREFIX=sk
|
||||
|
||||
@@ -35,12 +40,16 @@ DB_HOST=localhost
|
||||
DB_PORT=5432
|
||||
DB_USER=postgres
|
||||
DB_NAME=aether
|
||||
DB_PASSWORD=aether
|
||||
DB_PASSWORD=
|
||||
|
||||
# Redis 配置
|
||||
REDIS_HOST=localhost
|
||||
REDIS_PORT=6379
|
||||
REDIS_PASSWORD=aether
|
||||
REDIS_PASSWORD=
|
||||
|
||||
# 可选 MySQL profile 的应用用户与 root 密码
|
||||
MYSQL_PASSWORD=
|
||||
MYSQL_ROOT_PASSWORD=
|
||||
|
||||
# JWT密钥(使用 ./generate_keys.sh 生成)
|
||||
# 用于用户登录 token 签名,更换后所有用户需重新登录
|
||||
@@ -50,6 +59,10 @@ JWT_SECRET_KEY=change-this-to-a-secure-random-string
|
||||
# 注意:更换此密钥后需要在管理面板重新配置所有 Provider API Key
|
||||
ENCRYPTION_KEY=change-this-to-another-secure-random-string
|
||||
|
||||
# S3 备份的独立加密密钥(推荐)。未配置时为兼容旧部署,会回退到 ENCRYPTION_KEY。
|
||||
# 密钥轮换前必须保留旧值,离线恢复工具需要它解密历史备份。
|
||||
# AETHER_BACKUP_ENCRYPTION_KEY=change-this-to-a-dedicated-secure-random-string
|
||||
|
||||
# 启动自举管理员(仅在当前库里还没有活动管理员时生效)
|
||||
# 首次启动前必须设置 ADMIN_PASSWORD;install.sh 首次生成配置时会提示输入。
|
||||
ADMIN_EMAIL=admin@example.com
|
||||
@@ -77,13 +90,32 @@ ADMIN_USERNAME=admin123456
|
||||
# AETHER_GATEWAY_DATA_POSTGRES_MAX_CONNECTIONS=80
|
||||
# AETHER_GATEWAY_MAX_IN_FLIGHT_REQUESTS=2048
|
||||
# AETHER_GATEWAY_REQUEST_BODY_BUFFER_BUDGET_MB=256
|
||||
# AETHER_GATEWAY_REQUEST_BODY_READ_TIMEOUT_MS=120000
|
||||
# 可选的 Payload 上限(MiB);默认及 0 均表示不限制。
|
||||
# AETHER_MAX_REQUEST_BODY_MB=0
|
||||
# 请求体完整读取总超时默认关闭;确需限制时配置 1000-600000 毫秒的非零值。
|
||||
# AETHER_GATEWAY_REQUEST_BODY_READ_TIMEOUT_MS=0
|
||||
# 单请求解压后 Payload 上限(MiB),默认 256;显式设为 0 才表示不限制。
|
||||
# AETHER_MAX_REQUEST_BODY_MB=256
|
||||
# AETHER_GATEWAY_SECURITY_CACHE_TTL_MS=1000
|
||||
# AETHER_MAX_REDACTED_SYNC_RESPONSE_BODY_MB=0
|
||||
# AETHER_MAX_INTERNAL_BUFFERED_BODY_MB=0
|
||||
# AETHER_MAX_REDACTED_SYNC_RESPONSE_BODY_MB=64
|
||||
# AETHER_MAX_INTERNAL_BUFFERED_BODY_MB=64
|
||||
# AETHER_TUNNEL_NODE_STATUS_QUEUE_CAPACITY=1024
|
||||
# Tunnel relay 使用的独立 HMAC 密钥。启用 HTTP tunnel relay 或多网关 owner 转发时必须配置,
|
||||
# 所有网关实例必须使用同一个至少 32 字节的随机值;不要复用 JWT 或数据加密密钥。
|
||||
# AETHER_TUNNEL_RELAY_AUTH_SECRET=
|
||||
# 旧版 /api/internal/gateway/* 控制面默认关闭。确需独立服务调用时,配置至少 32 字节的
|
||||
# 独立 HMAC 密钥;不要复用 JWT、数据加密或 tunnel relay 密钥。多节点必须使用相同值和共享 Redis。
|
||||
# AETHER_INTERNAL_GATEWAY_AUTH_SECRET=
|
||||
# 远程 relay 地址必须使用 HTTPS;HTTP 仅允许 localhost 或回环 IP。
|
||||
# AETHER_TUNNEL_RELAY_BASE_URL=https://gateway-a.example.com
|
||||
# Bark 自建服务默认仅允许公网 HTTPS。确需明文 HTTP 或内网目标时分别显式开启:
|
||||
# AETHER_BARK_ALLOW_HTTP=false
|
||||
# AETHER_BARK_ALLOW_PRIVATE_TARGETS=false
|
||||
|
||||
# 可选 Provider OAuth 客户端。使用 Gemini CLI / Antigravity 浏览器授权时必须配置
|
||||
# 对应的 client secret;client ID 未配置时使用内置的公开 native-app client ID。
|
||||
# AETHER_GEMINI_CLI_OAUTH_CLIENT_ID=
|
||||
# AETHER_GEMINI_CLI_OAUTH_CLIENT_SECRET=
|
||||
# AETHER_ANTIGRAVITY_OAUTH_CLIENT_ID=
|
||||
# AETHER_ANTIGRAVITY_OAUTH_CLIENT_SECRET=
|
||||
|
||||
# PostgreSQL 容器调优:docker-compose.yml 已内置通用默认值,通常不用配置。
|
||||
# 只有在 Postgres 独占大内存、或压测显示 DB 缓存/排序/维护任务成为瓶颈时再覆盖。
|
||||
|
||||
@@ -6,7 +6,8 @@ on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: build-tunnel-${{ github.ref }}
|
||||
@@ -17,7 +18,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Ensure tunnel tag matches Cargo version
|
||||
shell: bash
|
||||
@@ -78,10 +79,10 @@ jobs:
|
||||
use_cross: false
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
@@ -89,14 +90,14 @@ jobs:
|
||||
run: rustup target add ${{ matrix.target }}
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
workspaces: apps/aether-tunnel -> target
|
||||
key: ${{ matrix.target }}
|
||||
|
||||
- name: Install cross
|
||||
if: matrix.use_cross
|
||||
uses: taiki-e/install-action@cross
|
||||
uses: taiki-e/install-action@1ae7257be536a92d9218a6b343dc6e6ba650f7e1 # cross
|
||||
|
||||
- name: Build
|
||||
working-directory: apps/aether-tunnel
|
||||
@@ -122,9 +123,10 @@ jobs:
|
||||
run: |
|
||||
cd target/${{ matrix.target }}/release
|
||||
7z a ../../../aether-tunnel-${{ matrix.name }}.zip aether-tunnel.exe
|
||||
tar czf ../../../aether-tunnel-${{ matrix.name }}.tar.gz aether-tunnel.exe
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: aether-tunnel-${{ matrix.name }}
|
||||
path: |
|
||||
@@ -137,9 +139,14 @@ jobs:
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
permissions:
|
||||
actions: read
|
||||
attestations: write
|
||||
contents: write
|
||||
id-token: write
|
||||
steps:
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
merge-multiple: true
|
||||
path: artifacts
|
||||
@@ -148,6 +155,20 @@ jobs:
|
||||
working-directory: artifacts
|
||||
run: sha256sum aether-tunnel-* > SHA256SUMS.txt
|
||||
|
||||
- name: Attest tunnel release provenance
|
||||
id: attest-release
|
||||
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||
with:
|
||||
subject-path: |
|
||||
artifacts/aether-tunnel-*.tar.gz
|
||||
artifacts/aether-tunnel-*.zip
|
||||
artifacts/SHA256SUMS.txt
|
||||
|
||||
- name: Bundle tunnel release provenance
|
||||
env:
|
||||
ATTESTATION_BUNDLE: ${{ steps.attest-release.outputs.bundle-path }}
|
||||
run: install -m 0644 "${ATTESTATION_BUNDLE}" artifacts/AETHER_TUNNEL_RELEASE_PROVENANCE.sigstore.json
|
||||
|
||||
- name: Delete stale draft releases for tag
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
@@ -170,12 +191,13 @@ jobs:
|
||||
done <<< "${draft_ids}"
|
||||
|
||||
- name: Create GitHub Release
|
||||
uses: softprops/action-gh-release@v2
|
||||
uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2
|
||||
with:
|
||||
name: "${{ github.ref_name }}"
|
||||
generate_release_notes: true
|
||||
files: |
|
||||
artifacts/aether-tunnel-*
|
||||
artifacts/AETHER_TUNNEL_RELEASE_PROVENANCE.sigstore.json
|
||||
artifacts/SHA256SUMS.txt
|
||||
fail_on_unmatched_files: true
|
||||
|
||||
@@ -183,8 +205,10 @@ jobs:
|
||||
needs: release
|
||||
runs-on: ubuntu-latest
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
ref: main
|
||||
|
||||
|
||||
@@ -7,8 +7,6 @@ on:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pages: write
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: pages
|
||||
@@ -46,10 +44,10 @@ jobs:
|
||||
if: needs.preflight.outputs.deploy_pages == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v5
|
||||
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: 'npm'
|
||||
@@ -77,10 +75,10 @@ jobs:
|
||||
run: cp frontend/dist/index.html frontend/dist/404.html
|
||||
|
||||
- name: Setup Pages
|
||||
uses: actions/configure-pages@v5
|
||||
uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-pages-artifact@v3
|
||||
uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3
|
||||
with:
|
||||
path: frontend/dist
|
||||
|
||||
@@ -90,7 +88,10 @@ jobs:
|
||||
url: ${{ steps.deployment.outputs.page_url }}
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
permissions:
|
||||
id-token: write
|
||||
pages: write
|
||||
steps:
|
||||
- name: Deploy to GitHub Pages
|
||||
id: deployment
|
||||
uses: actions/deploy-pages@v4
|
||||
uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4
|
||||
|
||||
@@ -66,12 +66,12 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 90
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
ref: ${{ needs.source.outputs.sha }}
|
||||
|
||||
- name: Install pinned Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
|
||||
@@ -79,13 +79,13 @@ jobs:
|
||||
run: rustc -Vv
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: nightly-rust-1.95-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/sccache-action@v0.0.9
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Check all workspace targets
|
||||
env:
|
||||
@@ -112,12 +112,12 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
ref: ${{ needs.source.outputs.sha }}
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v5
|
||||
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: npm
|
||||
@@ -157,7 +157,7 @@ jobs:
|
||||
run: npm run build
|
||||
|
||||
- name: Upload frontend artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: nightly-frontend-dist
|
||||
path: frontend/dist/
|
||||
@@ -171,12 +171,12 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
ref: ${{ needs.source.outputs.sha }}
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v5
|
||||
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5
|
||||
with:
|
||||
node-version: '22'
|
||||
|
||||
@@ -260,25 +260,25 @@ jobs:
|
||||
os: macos-15
|
||||
use_cross: false
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
ref: ${{ needs.source.outputs.sha }}
|
||||
|
||||
- name: Install pinned Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: nightly-release-${{ matrix.target }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Install cross
|
||||
if: matrix.use_cross
|
||||
uses: taiki-e/install-action@cross
|
||||
uses: taiki-e/install-action@1ae7257be536a92d9218a6b343dc6e6ba650f7e1 # cross
|
||||
|
||||
- name: Build release binary
|
||||
env:
|
||||
@@ -295,7 +295,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Upload binary artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: nightly-gateway-${{ matrix.platform }}-${{ matrix.arch }}
|
||||
path: target/${{ matrix.target }}/release/aether-gateway
|
||||
@@ -313,12 +313,12 @@ jobs:
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
ref: ${{ needs.source.outputs.sha }}
|
||||
|
||||
- name: Download Linux binaries and frontend
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
pattern: nightly-*
|
||||
path: artifacts
|
||||
@@ -335,20 +335,20 @@ jobs:
|
||||
cp -R artifacts/nightly-frontend-dist/. dist/frontend/
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6 # v3
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build and push nightly image
|
||||
uses: docker/build-push-action@v6
|
||||
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile.app
|
||||
@@ -372,12 +372,12 @@ jobs:
|
||||
actions: read
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
ref: ${{ needs.source.outputs.sha }}
|
||||
|
||||
- name: Download nightly artifacts
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
pattern: nightly-*
|
||||
path: artifacts
|
||||
@@ -434,7 +434,7 @@ jobs:
|
||||
done
|
||||
|
||||
- name: Upload nightly package artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: nightly-release-assets
|
||||
path: release-assets/*
|
||||
@@ -452,7 +452,7 @@ jobs:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Download nightly package artifact
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
name: nightly-release-assets
|
||||
path: release-assets
|
||||
|
||||
@@ -6,8 +6,8 @@ on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
packages: write
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: release-aether-${{ github.ref }}
|
||||
@@ -70,10 +70,10 @@ jobs:
|
||||
needs: preflight
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: npm
|
||||
@@ -94,7 +94,7 @@ jobs:
|
||||
npm run build
|
||||
|
||||
- name: Upload frontend artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: frontend-dist
|
||||
path: frontend/dist/
|
||||
@@ -106,10 +106,10 @@ jobs:
|
||||
needs: preflight
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: npm
|
||||
@@ -159,7 +159,7 @@ jobs:
|
||||
unzip -l "aether-vscodex-${version}.vsix" | grep 'extension/node_modules/ws/index.js' >/dev/null
|
||||
|
||||
- name: Upload VSIX artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: aether-vscodex-vsix
|
||||
path: aether-vscodex/vscode-extension/aether-vscodex-*.vsix
|
||||
@@ -199,22 +199,22 @@ jobs:
|
||||
os: macos-15
|
||||
use_cross: false
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: release-${{ matrix.target }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Install cross
|
||||
if: matrix.use_cross
|
||||
uses: taiki-e/install-action@cross
|
||||
uses: taiki-e/install-action@1ae7257be536a92d9218a6b343dc6e6ba650f7e1 # cross
|
||||
|
||||
- name: Build
|
||||
env:
|
||||
@@ -230,7 +230,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Upload binary artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: aether-gateway-${{ matrix.platform }}-${{ matrix.arch }}
|
||||
path: target/${{ matrix.target }}/release/aether-gateway
|
||||
@@ -242,11 +242,17 @@ jobs:
|
||||
needs: [preflight, frontend, build]
|
||||
if: needs.preflight.outputs.publish == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: read
|
||||
attestations: write
|
||||
contents: read
|
||||
id-token: write
|
||||
packages: write
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
path: artifacts
|
||||
|
||||
@@ -259,27 +265,27 @@ jobs:
|
||||
cp -r artifacts/frontend-dist dist/frontend
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5
|
||||
with:
|
||||
images: |
|
||||
${{ env.REGISTRY }}/${{ env.GHCR_IMAGE }}
|
||||
@@ -295,7 +301,8 @@ jobs:
|
||||
latest=false
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
id: push
|
||||
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile.app
|
||||
@@ -304,15 +311,36 @@ jobs:
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
platforms: linux/amd64,linux/arm64
|
||||
|
||||
- name: Attest GHCR image provenance
|
||||
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||
with:
|
||||
subject-name: ${{ env.REGISTRY }}/${{ env.GHCR_IMAGE }}
|
||||
subject-digest: ${{ steps.push.outputs.digest }}
|
||||
push-to-registry: true
|
||||
create-storage-record: false
|
||||
|
||||
- name: Attest Docker Hub image provenance
|
||||
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||
with:
|
||||
subject-name: docker.io/${{ env.DOCKERHUB_IMAGE }}
|
||||
subject-digest: ${{ steps.push.outputs.digest }}
|
||||
push-to-registry: true
|
||||
create-storage-record: false
|
||||
|
||||
package:
|
||||
name: Release tarballs
|
||||
needs: [preflight, frontend, build]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: read
|
||||
attestations: write
|
||||
contents: read
|
||||
id-token: write
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
path: artifacts
|
||||
|
||||
@@ -362,8 +390,24 @@ jobs:
|
||||
chmod +x release-assets/install.sh
|
||||
(cd release-assets && sha256sum *.tar.gz > SHA256SUMS)
|
||||
|
||||
- name: Attest release package provenance
|
||||
id: attest-release
|
||||
if: needs.preflight.outputs.publish == 'true'
|
||||
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||
with:
|
||||
subject-path: |
|
||||
release-assets/*.tar.gz
|
||||
release-assets/install.sh
|
||||
release-assets/SHA256SUMS
|
||||
|
||||
- name: Bundle release package provenance
|
||||
if: needs.preflight.outputs.publish == 'true'
|
||||
env:
|
||||
ATTESTATION_BUNDLE: ${{ steps.attest-release.outputs.bundle-path }}
|
||||
run: install -m 0644 "${ATTESTATION_BUNDLE}" release-assets/AETHER_RELEASE_PROVENANCE.sigstore.json
|
||||
|
||||
- name: Upload release package artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: release-assets
|
||||
path: release-assets/*
|
||||
@@ -375,15 +419,18 @@ jobs:
|
||||
needs: [preflight, docker, package, vscodex]
|
||||
if: needs.preflight.outputs.publish == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: read
|
||||
contents: write
|
||||
steps:
|
||||
- name: Download release package artifact
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
name: release-assets
|
||||
path: release-assets
|
||||
|
||||
- name: Download VSIX artifact
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
name: aether-vscodex-vsix
|
||||
path: release-assets
|
||||
@@ -410,13 +457,14 @@ jobs:
|
||||
done <<< "${draft_ids}"
|
||||
|
||||
- name: Publish GitHub Release assets
|
||||
uses: softprops/action-gh-release@v2
|
||||
uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2
|
||||
with:
|
||||
generate_release_notes: true
|
||||
prerelease: ${{ needs.preflight.outputs.prerelease }}
|
||||
make_latest: ${{ needs.preflight.outputs.make_latest }}
|
||||
files: |
|
||||
release-assets/*.tar.gz
|
||||
release-assets/AETHER_RELEASE_PROVENANCE.sigstore.json
|
||||
release-assets/SHA256SUMS
|
||||
release-assets/install.sh
|
||||
release-assets/*.vsix
|
||||
|
||||
+112
-56
@@ -11,6 +11,23 @@ on:
|
||||
- "Cargo.lock"
|
||||
- "crates/**"
|
||||
- "apps/**"
|
||||
- "install.sh"
|
||||
- "deploy.sh"
|
||||
- "update.sh"
|
||||
- "generate_keys.sh"
|
||||
- ".env.example"
|
||||
- "README.md"
|
||||
- "Dockerfile.app"
|
||||
- "docker-compose.yml"
|
||||
- "docker-compose.single-node.yml"
|
||||
- "tests/install_*_test.sh"
|
||||
- "tests/deploy_*_test.sh"
|
||||
- "tests/update_*_test.sh"
|
||||
- "tests/release_supply_chain_test.sh"
|
||||
- "tests/tunnel_installer_config_security_test.sh"
|
||||
- ".github/workflows/build-tunnel.yml"
|
||||
- ".github/workflows/deploy-pages.yml"
|
||||
- ".github/workflows/release.yml"
|
||||
- ".github/workflows/rust-ci.yml"
|
||||
- ".github/workflows/nightly.yml"
|
||||
pull_request:
|
||||
@@ -19,6 +36,23 @@ on:
|
||||
- "Cargo.lock"
|
||||
- "crates/**"
|
||||
- "apps/**"
|
||||
- "install.sh"
|
||||
- "deploy.sh"
|
||||
- "update.sh"
|
||||
- "generate_keys.sh"
|
||||
- ".env.example"
|
||||
- "README.md"
|
||||
- "Dockerfile.app"
|
||||
- "docker-compose.yml"
|
||||
- "docker-compose.single-node.yml"
|
||||
- "tests/install_*_test.sh"
|
||||
- "tests/deploy_*_test.sh"
|
||||
- "tests/update_*_test.sh"
|
||||
- "tests/release_supply_chain_test.sh"
|
||||
- "tests/tunnel_installer_config_security_test.sh"
|
||||
- ".github/workflows/build-tunnel.yml"
|
||||
- ".github/workflows/deploy-pages.yml"
|
||||
- ".github/workflows/release.yml"
|
||||
- ".github/workflows/rust-ci.yml"
|
||||
- ".github/workflows/nightly.yml"
|
||||
|
||||
@@ -36,14 +70,34 @@ env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
jobs:
|
||||
shell_security:
|
||||
name: Shell security fixtures
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Run installer and supply-chain fixtures
|
||||
shell: bash
|
||||
run: |
|
||||
bash tests/deploy_state_safety_test.sh
|
||||
bash tests/install_archive_safety_test.sh
|
||||
bash tests/install_container_runtime_security_test.sh
|
||||
bash tests/install_current_release_link_test.sh
|
||||
bash tests/install_local_bundle_safety_test.sh
|
||||
bash tests/install_privileged_write_safety_test.sh
|
||||
bash tests/install_source_trust_test.sh
|
||||
bash tests/release_supply_chain_test.sh
|
||||
bash tests/update_compose_safety_test.sh
|
||||
bash tests/tunnel_installer_config_security_test.sh
|
||||
|
||||
fmt:
|
||||
name: Format
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
components: rustfmt
|
||||
@@ -55,22 +109,22 @@ jobs:
|
||||
name: Clippy (Gateway)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
components: clippy
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/sccache-action@v0.0.9
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Clippy
|
||||
env:
|
||||
@@ -89,22 +143,22 @@ jobs:
|
||||
name: Clippy (Data)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
components: clippy
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/sccache-action@v0.0.9
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Clippy
|
||||
env:
|
||||
@@ -123,22 +177,22 @@ jobs:
|
||||
name: Clippy (Workspace Rest)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
components: clippy
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/sccache-action@v0.0.9
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Clippy
|
||||
env:
|
||||
@@ -175,28 +229,28 @@ jobs:
|
||||
name: Test (Gateway)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
|
||||
- name: Show Rust toolchain
|
||||
run: rustup show active-toolchain
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/sccache-action@v0.0.9
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Setup mold
|
||||
uses: rui314/setup-mold@v1
|
||||
uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1
|
||||
|
||||
- name: Install nextest
|
||||
uses: taiki-e/install-action@nextest
|
||||
uses: taiki-e/install-action@d5f9268ff7620505a81ada10ddf18cdd72240185 # nextest
|
||||
|
||||
- name: Test lib
|
||||
env:
|
||||
@@ -225,25 +279,25 @@ jobs:
|
||||
name: Test (Data)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
|
||||
- name: Show Rust toolchain
|
||||
run: rustup show active-toolchain
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/sccache-action@v0.0.9
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Install nextest
|
||||
uses: taiki-e/install-action@nextest
|
||||
uses: taiki-e/install-action@d5f9268ff7620505a81ada10ddf18cdd72240185 # nextest
|
||||
|
||||
- name: Test
|
||||
env:
|
||||
@@ -270,19 +324,19 @@ jobs:
|
||||
- sqlite
|
||||
- all-drivers
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/sccache-action@v0.0.9
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Check selected data driver
|
||||
env:
|
||||
@@ -301,25 +355,25 @@ jobs:
|
||||
name: Test (Workspace Rest)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
|
||||
- name: Show Rust toolchain
|
||||
run: rustup show active-toolchain
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/sccache-action@v0.0.9
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Install nextest
|
||||
uses: taiki-e/install-action@nextest
|
||||
uses: taiki-e/install-action@d5f9268ff7620505a81ada10ddf18cdd72240185 # nextest
|
||||
|
||||
- name: Test
|
||||
env:
|
||||
@@ -345,22 +399,22 @@ jobs:
|
||||
- aether-data-mysql
|
||||
- aether-data-sqlite
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/sccache-action@v0.0.9
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Install nextest
|
||||
uses: taiki-e/install-action@nextest
|
||||
uses: taiki-e/install-action@d5f9268ff7620505a81ada10ddf18cdd72240185 # nextest
|
||||
|
||||
- name: Test adapter
|
||||
env:
|
||||
@@ -379,19 +433,19 @@ jobs:
|
||||
name: Test (Integration Scenarios)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/sccache-action@v0.0.9
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Test scenario binaries and end-to-end suites
|
||||
env:
|
||||
@@ -434,22 +488,22 @@ jobs:
|
||||
name: Data DB Smoke (SQLite)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
|
||||
- name: Show Rust toolchain
|
||||
run: rustup show active-toolchain
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/sccache-action@v0.0.9
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Run SQLite data smoke tests
|
||||
env:
|
||||
@@ -482,22 +536,22 @@ jobs:
|
||||
--health-timeout=5s
|
||||
--health-retries=20
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
|
||||
- name: Show Rust toolchain
|
||||
run: rustup show active-toolchain
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/sccache-action@v0.0.9
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Add PostgreSQL server binaries to PATH
|
||||
run: echo "$(pg_config --bindir)" >> "$GITHUB_PATH"
|
||||
@@ -568,22 +622,22 @@ jobs:
|
||||
--health-timeout=5s
|
||||
--health-retries=20
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
|
||||
- name: Show Rust toolchain
|
||||
run: rustup show active-toolchain
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/sccache-action@v0.0.9
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Run MySQL migration smoke test
|
||||
env:
|
||||
@@ -674,6 +728,7 @@ jobs:
|
||||
- clippy
|
||||
- test
|
||||
- data_db_smoke
|
||||
- shell_security
|
||||
if: ${{ always() }}
|
||||
steps:
|
||||
- name: Verify required jobs
|
||||
@@ -681,7 +736,8 @@ jobs:
|
||||
if [ "${{ needs.fmt.result }}" != "success" ] || \
|
||||
[ "${{ needs.clippy.result }}" != "success" ] || \
|
||||
[ "${{ needs.test.result }}" != "success" ] || \
|
||||
[ "${{ needs.data_db_smoke.result }}" != "success" ]; then
|
||||
[ "${{ needs.data_db_smoke.result }}" != "success" ] || \
|
||||
[ "${{ needs.shell_security.result }}" != "success" ]; then
|
||||
echo "Rust CI failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
Generated
+594
-345
File diff suppressed because it is too large
Load Diff
+3
-1
@@ -101,6 +101,7 @@ aether-runtime = { path = "crates/aether-runtime/base" }
|
||||
aether-testkit = { path = "crates/aether-testing/testkit" }
|
||||
aes = "0.8"
|
||||
aes-gcm = "0.10"
|
||||
aws-lc-rs = { version = "1.16.2", default-features = false, features = ["alloc", "aws-lc-sys"] }
|
||||
async-stream = "0.3"
|
||||
async-trait = "0.1"
|
||||
axum = "0.8"
|
||||
@@ -117,8 +118,9 @@ flate2 = "1"
|
||||
futures-util = "0.3"
|
||||
hmac = "0.12"
|
||||
http = "1"
|
||||
object_store = { version = "0.12", default-features = false, features = ["aws"] }
|
||||
object_store = { version = "0.14.1", default-features = false, features = ["aws"] }
|
||||
pbkdf2 = { version = "0.12", default-features = false, features = ["hmac"] }
|
||||
percent-encoding = "2"
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "rustls-tls", "http2", "socks"] }
|
||||
redis = { version = "0.28", default-features = false, features = ["tokio-comp", "script", "streams", "connection-manager"] }
|
||||
regex = "1"
|
||||
|
||||
+9
-4
@@ -10,20 +10,24 @@
|
||||
|
||||
# --- layout stage: create /opt/aether directory structure with symlink ---
|
||||
# distroless has no shell, so we use busybox to set up the symlink.
|
||||
FROM busybox:1.37-musl AS layout
|
||||
FROM busybox:1.37.0-musl@sha256:fc6dddc4c44b1bfe37f41cae8e67d1693828e8f42a91862816d7953e2c9d3f23 AS layout
|
||||
|
||||
ARG TARGETARCH
|
||||
|
||||
RUN mkdir -p /opt/aether/releases/image/bin /opt/aether/releases/image/frontend /opt/aether/logs
|
||||
|
||||
COPY dist/aether-gateway-${TARGETARCH} /opt/aether/releases/image/bin/aether-gateway
|
||||
RUN chmod 0755 /opt/aether/releases/image/bin/aether-gateway
|
||||
COPY dist/frontend/ /opt/aether/releases/image/frontend/
|
||||
|
||||
# Keep the immutable release root-owned while guaranteeing that the runtime
|
||||
# identity can traverse and read every packaged asset.
|
||||
RUN chmod -R u=rwX,go=rX /opt/aether/releases/image \
|
||||
&& chmod 0755 /opt/aether/releases/image/bin/aether-gateway
|
||||
|
||||
RUN ln -s /opt/aether/releases/image /opt/aether/current
|
||||
|
||||
# --- final stage: distroless runtime ---
|
||||
FROM gcr.io/distroless/static-debian12
|
||||
FROM gcr.io/distroless/static-debian12@sha256:6447365a6337c3732f412d1b74357b30a633831955b2bc45552b0086be907687
|
||||
|
||||
COPY --from=layout /opt/aether /opt/aether
|
||||
|
||||
@@ -31,6 +35,7 @@ WORKDIR /opt/aether
|
||||
|
||||
ENV RUST_LOG=aether_gateway=info \
|
||||
APP_PORT=8084 \
|
||||
HOME=/tmp/aether-home \
|
||||
AETHER_UPDATE_STRATEGY=docker \
|
||||
AETHER_GATEWAY_STATIC_DIR=/opt/aether/current/frontend
|
||||
|
||||
@@ -39,5 +44,5 @@ EXPOSE 8084
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
||||
CMD ["/opt/aether/current/bin/aether-gateway", "--healthcheck"]
|
||||
|
||||
USER root
|
||||
USER 65532:65532
|
||||
ENTRYPOINT ["/opt/aether/current/bin/aether-gateway"]
|
||||
|
||||
@@ -44,17 +44,30 @@ cd Aether
|
||||
|
||||
# 2. 配置环境变量
|
||||
cp .env.example .env
|
||||
# 生成 JWT_SECRET_KEY / ENCRYPTION_KEY, 并填入 .env
|
||||
# .env 包含数据库、JWT 和数据加密密钥,先限制为仅当前用户可读写
|
||||
chmod 600 .env
|
||||
# 生成 JWT / 加密 / Postgres / Redis / MySQL 独立随机密钥,并填入 .env
|
||||
./generate_keys.sh
|
||||
# 编辑 .env 设置 ADMIN_PASSWORD
|
||||
|
||||
# 3. 首次部署 / 更新 (从以下部署形态任选其一)
|
||||
# Postgres + Redis (适用于企业或多人使用)
|
||||
# Postgres + Redis (推荐)
|
||||
docker compose pull && docker compose up -d
|
||||
# Single Node (适用于个人用户或朋友分享)
|
||||
# Single Node:默认容器身份为 65532:65532,先停止旧容器并检查/迁移 SQLite bind 目录
|
||||
docker compose -f docker-compose.single-node.yml stop app
|
||||
mkdir -p data
|
||||
test -z "$(find data ! -type d ! -type f -print -quit)" || { echo "data 中存在 symlink/FIFO/socket/device,拒绝迁移" >&2; exit 1; }
|
||||
test -z "$(find data -type f -links +1 -print -quit)" || { echo "data 中存在硬链接,拒绝迁移" >&2; exit 1; }
|
||||
sudo chown -R -P 65532:65532 ./data
|
||||
sudo find data -type d -exec chmod 0700 {} +
|
||||
sudo find data -type f -exec chmod 0600 {} +
|
||||
docker compose -f docker-compose.single-node.yml pull && docker compose -f docker-compose.single-node.yml up -d
|
||||
```
|
||||
|
||||
应用镜像默认以固定非 root 身份 `65532:65532` 运行;Compose 同时移除全部 Linux capabilities、禁止提权、启用只读根文件系统,并仅提供带 `nosuid,nodev,noexec` 的 `/tmp` 临时文件系统。若宿主机不适合使用固定 UID/GID,可在 `.env` 中把 `AETHER_CONTAINER_UID` / `AETHER_CONTAINER_GID` 改成其他非零数字身份,并让 Single Node 的 `./data` 归该身份所有。`install.sh --mode compose-single-node` 会按安装用户自动生成这两个值;使用 `sudo` 运行时会采用原调用用户身份,并安全迁移已有 SQLite 数据。
|
||||
|
||||
从旧版 root 容器升级 Single Node 时,必须先停止旧 `app` 容器,再在第一次启动新版 Compose 前完成一次数据目录迁移;安装器检测到容器仍在运行会拒绝迁移,避免并发改写造成检查竞态。停止容器后用 `sudo` 重新执行一键安装器会自动处理;非 root 安装器发现旧数据所有权不匹配时会拒绝启动并提示迁移,不会放宽目录权限。手工部署且仍使用默认身份时执行上面的检查、`chown` 和 `find ... chmod` 命令即可。迁移只改变 `./data` 的所有权和权限,不会删除数据库、WAL 或备份文件。
|
||||
|
||||
### 一键更新
|
||||
|
||||
Docker Compose 部署后,可在部署目录直接执行:
|
||||
@@ -69,10 +82,24 @@ Docker Compose 部署后,可在部署目录直接执行:
|
||||
./update.sh --mode single-node
|
||||
```
|
||||
|
||||
仓库自带的 Docker Compose 默认把应用日志输出到容器 `stdout/stderr`,直接用 `docker compose logs -f app` 查看,并由 Docker 轮转日志,避免正式发布镜像切换到非 root 用户后再被宿主机挂载日志目录的权限问题拖垮启动。如果你确实需要文件日志,需要在 compose 里把 `AETHER_LOG_DESTINATION` 改成 `file|both`,并额外挂载一个容器用户可写的目录到 `/opt/aether/logs`。
|
||||
仓库自带的 Docker Compose 默认把应用日志输出到容器 `stdout/stderr`,直接用 `docker compose logs -f app` 查看,并由 Docker 轮转日志,避免非 root 用户被宿主机日志目录权限拖垮启动。如果你确实需要文件日志,需要在 compose 里把 `AETHER_LOG_DESTINATION` 改成 `file|both`,额外挂载目录到 `/opt/aether/logs`,并让它归 `.env` 中配置的容器 UID/GID 所有;只读根文件系统不会阻止显式可写挂载。
|
||||
|
||||
管理后台右上角“版本信息”会检测新版本。Docker Compose 部署只提示版本,实际更新继续执行 `./update.sh`;systemd / launchd / 二进制部署才使用后台自更新,流程是下载对应平台的 GitHub Release 包、强制校验 `SHA256SUMS`、解压到 `/opt/aether/releases/<version>`,再切换 `/opt/aether/current` 并退出进程,交给 systemd / launchd 拉起新版本。
|
||||
|
||||
正式 Release 还会发布由 GitHub Actions OIDC / Sigstore 签发的 SLSA build provenance。需要验证发布者身份时,下载目标 tarball 和 `AETHER_RELEASE_PROVENANCE.sigstore.json`,并把 `TAG` 设置为对应 Release tag:
|
||||
|
||||
```bash
|
||||
gh attestation verify "aether-${TAG}-linux-amd64.tar.gz" \
|
||||
--repo fawney19/Aether \
|
||||
--signer-workflow fawney19/Aether/.github/workflows/release.yml \
|
||||
--source-ref "refs/tags/${TAG}" \
|
||||
--bundle AETHER_RELEASE_PROVENANCE.sigstore.json
|
||||
```
|
||||
|
||||
`docker-compose.yml` 中的官方 PostgreSQL、Redis 和 MySQL 镜像均固定到多架构 OCI index digest。升级这些依赖时应在发布变更中显式更新 digest,避免同名 tag 在无人审查的情况下改变部署内容。
|
||||
|
||||
正式发布到 GHCR 和 Docker Hub 的多架构 Aether 镜像也带有同一 GitHub Actions OIDC / Sigstore provenance;生产 `Dockerfile.app` 的 BusyBox 与 Distroless 基础镜像同样固定到多架构 OCI index digest。
|
||||
|
||||
源码或本地构建版本不会启用后台在线更新,请继续使用源码更新流程。Docker Compose 用户如果希望“容器重建后也保持镜像层面的新版本”,仍建议定期运行 `./update.sh` 拉取并重建 app 镜像。服务器访问 GitHub 需要代理时,可设置 `AETHER_UPDATE_PROXY_URL`,也兼容 `UPDATE_PROXY_URL`、`HTTPS_PROXY`、`ALL_PROXY`、`HTTP_PROXY` 以及 `NO_PROXY`。共享出口触发 GitHub API 限流时,可设置只读 `AETHER_UPDATE_GITHUB_TOKEN`,也兼容 `GITHUB_TOKEN` / `GH_TOKEN`。下载总超时默认 600 秒,连续无响应/无数据默认 30 秒,可通过 `AETHER_UPDATE_DOWNLOAD_TIMEOUT_SECS` 和 `AETHER_UPDATE_DOWNLOAD_IDLE_TIMEOUT_SECS` 调整。
|
||||
|
||||
标准 Docker Compose 使用 Docker named volumes 存放 Postgres/Redis/MySQL 数据;Single Node 使用部署目录下的 `./data` 存放 SQLite 数据。
|
||||
@@ -172,22 +199,40 @@ Aether Tunnel 是配套的正向代理节点,部署在海外 VPS 上,为墙
|
||||
- `AETHER_GATEWAY_DATA_POSTGRES_MIN_CONNECTIONS` / `AETHER_GATEWAY_DATA_POSTGRES_MAX_CONNECTIONS`:数据库连接池手动覆盖值;未配置时 SQLite 固定 `1/1`,Postgres/MySQL 按每核 `4` 条自动推导,总池范围为 `32-100`。该预算按进程计算,多实例部署应按数据库连接上限显式分配
|
||||
- `AETHER_GATEWAY_MAX_IN_FLIGHT_REQUESTS`:单实例请求并发上限;未配置时按 CPU 自动推导(基础范围 `512-65536`),低文件描述符预算时会进一步下调
|
||||
- `AETHER_GATEWAY_REQUEST_BODY_BUFFER_BUDGET_MB`:单实例同时读取和解压请求体的加权内存预算,默认 `256MB`
|
||||
- `AETHER_GATEWAY_REQUEST_BODY_READ_TIMEOUT_MS`:请求体完整读取超时,默认 `120000ms`
|
||||
- `AETHER_MAX_REQUEST_BODY_MB`:可选的单请求解压后请求体上限;未配置或设为 `0` 时不限制
|
||||
- `AETHER_MAX_INTERNAL_BUFFERED_BODY_MB`:可选的 heartbeat、管理探测等内部整包响应体上限;未配置或设为 `0` 时不限制
|
||||
- `AETHER_GATEWAY_REQUEST_BODY_READ_TIMEOUT_MS`:可选的请求体完整读取超时;默认或显式设为 `0` 时关闭,非零值限制在 `1000-600000ms`
|
||||
- `AETHER_MAX_REQUEST_BODY_MB`:单请求解压后请求体上限,默认 `256MB`;显式设为 `0` 表示不再收紧默认值,但仍受 `256MB` 安全硬上限约束
|
||||
- `AETHER_MAX_INTERNAL_BUFFERED_BODY_MB`:heartbeat、管理探测等内部整包响应体上限,默认 `64MB`;显式设为 `0` 表示不再收紧默认值,但仍受 `256MB` 安全硬上限约束
|
||||
- `AETHER_TUNNEL_NODE_STATUS_QUEUE_CAPACITY`:隧道节点状态上报队列容量,默认 `1024`;满载时拒绝新事件,避免控制面故障导致无界内存增长
|
||||
- `AETHER_INTERNAL_GATEWAY_AUTH_SECRET`:旧版 `/api/internal/gateway/*` 高权限控制面的独立 HMAC 密钥,至少 `32` 字节;未配置时该控制面返回 `404`。不要复用 JWT、数据加密或 tunnel relay 密钥,多节点必须使用同一值及共享 Redis 防重放
|
||||
- `AETHER_GATEWAY_SECURITY_CACHE_TTL_MS`:IP 黑白名单本地缓存时间,默认 `1000ms`,写操作会主动失效相关缓存
|
||||
- `AETHER_MAX_REDACTED_SYNC_RESPONSE_BODY_MB`:可选的 PII 恢复同步响应缓冲上限;未配置或设为 `0` 时不限制
|
||||
- `AETHER_MAX_REDACTED_SYNC_RESPONSE_BODY_MB`:PII 恢复同步响应缓冲上限,默认 `64MB`;显式设为 `0` 表示不再收紧默认值,但仍受 `256MB` 安全硬上限约束
|
||||
- `REDIS_URL`:Redis 连接串;仅 Postgres + Redis 的 Docker Compose 部署需要配置
|
||||
- `AETHER_RUNTIME_BACKEND=memory|redis`:运行时缓存/协调后端。SQLite 默认用 `memory`,不会连接 Redis;多节点部署和需要跨 gateway 重启恢复 OpenAI Responses continuation history 的部署必须使用共享 Redis
|
||||
- `AETHER_GATEWAY_DATABASE_MODE=auto|verify-only`:数据库启动策略,默认 `auto`,自动完成挂起的 schema migration 和 backfill;`verify-only` 仅检查并在数据库落后时拒绝启动
|
||||
- `AETHER_GATEWAY_AUTO_PREPARE_DATABASE`:旧版兼容开关;新配置请使用 `AETHER_GATEWAY_DATABASE_MODE`
|
||||
- `JWT_SECRET_KEY` / `ENCRYPTION_KEY`:认证和敏感数据加密所需密钥
|
||||
- `AETHER_BACKUP_ENCRYPTION_KEY`:推荐的 S3 备份独立加密密钥;缺省回退到 `ENCRYPTION_KEY`。新备份使用带 key ID 的 AES-256-GCM v2 envelope,轮换前必须保留旧密钥
|
||||
- `API_KEY_PREFIX`:用户和管理员新建 API Key 时使用的前缀,默认 `sk`
|
||||
- `ADMIN_USERNAME` / `ADMIN_PASSWORD` / `ADMIN_EMAIL`:首次启动时自举首个本地管理员;`install.sh` 会提示输入管理员密码
|
||||
- `CORS_ORIGINS` / `CORS_ALLOW_CREDENTIALS`:前端跨域来源控制;如果要跨域带登录 Cookie,`CORS_ORIGINS` 不能写 `*`
|
||||
- `RUST_LOG`:Rust 日志过滤,例如 `aether_gateway=info`、`aether_gateway=debug,sqlx=warn`
|
||||
- Docker Compose 的 `DB_PASSWORD` / `REDIS_PASSWORD` 默认使用 `aether`
|
||||
- `DB_PASSWORD` / `REDIS_PASSWORD` / `MYSQL_PASSWORD` / `MYSQL_ROOT_PASSWORD`:Docker Compose 后端密码,首次安装时分别随机生成;手工部署必须替换示例占位值,不要互相复用
|
||||
|
||||
### S3 备份离线恢复
|
||||
|
||||
先从 S3 下载完整的 `.json.zst.aes256gcm` 对象,再使用原始的完整 S3 object key 做认证解密。恢复工具只验证并输出本地 JSON,不会直接写数据库;数据库导入仍应在维护窗口通过管理端完成。
|
||||
|
||||
```bash
|
||||
AETHER_BACKUP_ENCRYPTION_KEY='原备份密钥' \
|
||||
cargo run -p aether-gateway --bin aether-backup-restore -- \
|
||||
--input ./backup.json.zst.aes256gcm \
|
||||
--object-key 'aether/backups/aether-data-backup-20260822-010000.json.zst.aes256gcm' \
|
||||
--output ./restored-backup.json
|
||||
```
|
||||
|
||||
工具默认拒绝覆盖,输出采用原子写并在 Unix 上设置为 `0600`;Unix 可用 `--overwrite` 原子替换,Windows 为避免非原子删除窗口会要求选择新输出路径。密钥不能作为命令行参数。可使用 `AETHER_BACKUP_ENCRYPTION_KEY`、兼容用 `AETHER_GATEWAY_DATA_ENCRYPTION_KEY` / `ENCRYPTION_KEY`、受保护的 `--key-file`,或 `AETHER_BACKUP_KEYRING_FILE`。Keyring JSON 格式为 `{"version":1,"keys":["当前或历史 v2 secret"],"legacy_v1":["旧 v1 secret"]}`;条目也可写成 `{"secret":"..."}`(兼容字段名 `key`)。也可由 `AETHER_BACKUP_HISTORICAL_KEYS_JSON` 提供同一结构。密钥文件必须是非符号链接的普通文件,Unix 下权限需为 `0600` 或更严格。
|
||||
|
||||
默认限制密文为 `512MiB`、解压后 JSON 为 `1GiB`,可通过受限的 `--max-encrypted-mib` / `--max-json-mib` 调整。网关最多扫描同一备份前缀下 10,000 个对象,并且不会自动删除 S3 对象:`backup_s3_retention_count` 只用于报告超出保留数量的清理候选。旧明文备份在创建并验证加密副本后仍会保留,必须通过 bucket lifecycle 或支持版本条件的外部清理工具移除;启用 Versioning 时还需清理 noncurrent versions,Object Lock/retention 可能阻止物理删除。
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -65,14 +65,14 @@ http.workspace = true
|
||||
http-body-util = "0.1"
|
||||
hyper = { version = "1", features = ["client", "server", "http1", "http2"] }
|
||||
hyper-util = { version = "0.1", features = ["client-legacy", "client-pool", "server-auto", "service", "tokio"] }
|
||||
ldap3 = { version = "0.11", default-features = false, features = ["sync", "tls-rustls"] }
|
||||
ldap3 = { version = "0.12.1", default-features = false, features = ["sync", "tls-rustls-ring"] }
|
||||
libc = "0.2"
|
||||
md-5 = "0.10"
|
||||
object_store.workspace = true
|
||||
parking_lot = "0.12"
|
||||
percent-encoding.workspace = true
|
||||
regex.workspace = true
|
||||
reqwest.workspace = true
|
||||
rsa = "0.9.10"
|
||||
rustls.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
@@ -103,4 +103,5 @@ tikv-jemalloc-sys = { version = "0.6", optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
aether-test-support.workspace = true
|
||||
aws-lc-rs.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
|
||||
@@ -30,7 +30,7 @@ struct Args {
|
||||
#[arg(
|
||||
long,
|
||||
env = "AETHER_EXECUTION_RUNTIME_UNIX_SOCKET",
|
||||
default_value = "/tmp/aether-execution-runtime.sock"
|
||||
default_value = "/tmp/aether-execution-runtime/aether-execution-runtime.sock"
|
||||
)]
|
||||
unix_socket: PathBuf,
|
||||
|
||||
|
||||
@@ -1,17 +1,18 @@
|
||||
pub(crate) use crate::handlers::admin::{
|
||||
admin_provider_ops_local_action_response, admin_provider_pool_config,
|
||||
build_internal_control_error_response, create_provider_oauth_catalog_key,
|
||||
find_duplicate_provider_oauth_key, maybe_build_local_admin_pool_response,
|
||||
maybe_build_local_admin_response, persist_provider_quota_refresh_state,
|
||||
provider_oauth_maintenance_endpoint_for_provider, provider_oauth_runtime_endpoint_for_provider,
|
||||
provider_quota_refresh_endpoint_for_provider, provider_type_supports_quota_refresh,
|
||||
reconcile_admin_fixed_provider_template_endpoints,
|
||||
execute_admin_system_import_exclusively, find_duplicate_provider_oauth_key,
|
||||
maybe_build_local_admin_pool_response, maybe_build_local_admin_response,
|
||||
persist_provider_quota_refresh_state, provider_oauth_maintenance_endpoint_for_provider,
|
||||
provider_oauth_runtime_endpoint_for_provider, provider_quota_refresh_endpoint_for_provider,
|
||||
provider_type_supports_quota_refresh, reconcile_admin_fixed_provider_template_endpoints,
|
||||
refresh_provider_oauth_account_state_after_update, refresh_provider_pool_quota_locally,
|
||||
store_admin_provider_ops_balance_cache, update_existing_provider_oauth_catalog_key,
|
||||
release_admin_system_import_lease, store_admin_provider_ops_balance_cache,
|
||||
try_acquire_admin_system_import_lease, update_existing_provider_oauth_catalog_key,
|
||||
AdminAppState, AdminGatewayProviderTransportSnapshot, AdminLocalOAuthRefreshError,
|
||||
AdminRequestContext, AdminRouteRequest, AdminRouteResponse, AdminRouteResult,
|
||||
AdminStatsTimeRange, AdminStatsUsageFilter, OAUTH_ACCOUNT_BLOCK_PREFIX,
|
||||
OAUTH_REQUEST_FAILED_PREFIX,
|
||||
AdminStatsTimeRange, AdminStatsUsageFilter, AdminSystemImportLockError, SystemExportMode,
|
||||
OAUTH_ACCOUNT_BLOCK_PREFIX, OAUTH_REQUEST_FAILED_PREFIX,
|
||||
};
|
||||
|
||||
use crate::handlers::admin::{
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use aether_usage_runtime::decode_internal_report_body_base64;
|
||||
use base64::Engine as _;
|
||||
use serde_json::Value;
|
||||
|
||||
@@ -43,9 +44,8 @@ pub(crate) fn maybe_normalize_provider_private_sync_report_payload(
|
||||
}
|
||||
|
||||
if let Some(body_base64) = payload.body_base64.as_deref() {
|
||||
let body_bytes = base64::engine::general_purpose::STANDARD
|
||||
.decode(body_base64)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
let body_bytes =
|
||||
decode_internal_report_body_base64(body_base64).map_err(GatewayError::Internal)?;
|
||||
let Some(normalized_bytes) =
|
||||
normalize_provider_private_stream_bytes(report_context, &body_bytes)?
|
||||
else {
|
||||
|
||||
@@ -2325,16 +2325,11 @@ mod tests {
|
||||
assert_eq!(stored.len(), 1);
|
||||
assert_eq!(stored[0].key_id.as_deref(), Some("normal-key"));
|
||||
assert_eq!(stored[0].candidate_index, 2);
|
||||
assert_eq!(
|
||||
stored[0]
|
||||
.extra_data
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("dispatch_ref"))
|
||||
.and_then(|value| value.get("SingleKey"))
|
||||
.and_then(|value| value.get("key"))
|
||||
.and_then(|value| value.get("key_id")),
|
||||
Some(&json!("normal-key"))
|
||||
);
|
||||
assert!(stored[0]
|
||||
.extra_data
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("dispatch_ref"))
|
||||
.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2624,16 +2619,11 @@ mod tests {
|
||||
);
|
||||
assert_eq!(stored[1].key_id.as_deref(), Some("normal-key"));
|
||||
assert_eq!(stored[1].candidate_index, 1);
|
||||
assert_eq!(
|
||||
stored[1]
|
||||
.extra_data
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("dispatch_ref"))
|
||||
.and_then(|value| value.get("SingleKey"))
|
||||
.and_then(|value| value.get("key"))
|
||||
.and_then(|value| value.get("key_id")),
|
||||
Some(&json!("normal-key"))
|
||||
);
|
||||
assert!(stored[1]
|
||||
.extra_data
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("dispatch_ref"))
|
||||
.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2703,7 +2693,7 @@ mod tests {
|
||||
.as_ref()
|
||||
.and_then(serde_json::Value::as_object)
|
||||
.expect("ranking metadata should persist as object extra data");
|
||||
assert_eq!(extra_data.get("existing"), Some(&json!("value")));
|
||||
assert!(extra_data.get("existing").is_none());
|
||||
assert_eq!(
|
||||
extra_data.get("ranking_mode"),
|
||||
Some(&json!("CacheAffinity"))
|
||||
@@ -2716,14 +2706,7 @@ mod tests {
|
||||
Some(&json!("cached_affinity"))
|
||||
);
|
||||
assert_eq!(extra_data.get("demoted_by"), Some(&json!("cross_format")));
|
||||
assert_eq!(
|
||||
extra_data
|
||||
.get("dispatch_ref")
|
||||
.and_then(|value| value.get("SingleKey"))
|
||||
.and_then(|value| value.get("key"))
|
||||
.and_then(|value| value.get("key_id")),
|
||||
Some(&json!("ranked-key"))
|
||||
);
|
||||
assert!(extra_data.get("dispatch_ref").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -3061,7 +3044,7 @@ mod tests {
|
||||
.as_ref()
|
||||
.and_then(serde_json::Value::as_object)
|
||||
.expect("skipped ranking metadata should persist");
|
||||
assert_eq!(extra_data.get("existing"), Some(&json!("value")));
|
||||
assert!(extra_data.get("existing").is_none());
|
||||
assert_eq!(
|
||||
extra_data.get("ranking_mode"),
|
||||
Some(&json!("CacheAffinity"))
|
||||
|
||||
@@ -278,13 +278,21 @@ mod tests {
|
||||
|
||||
assert_eq!(metadata["transport_diagnostics"]["provider_type"], "codex");
|
||||
assert_eq!(
|
||||
metadata["transport_diagnostics"]["fingerprint"]["transport_profile"]["profile_id"],
|
||||
"chrome_136"
|
||||
metadata["transport_diagnostics"]["key_fingerprint_configured"],
|
||||
Value::Bool(true)
|
||||
);
|
||||
assert_eq!(
|
||||
metadata["transport_diagnostics"]["key_transport_profile_configured"],
|
||||
Value::Bool(true)
|
||||
);
|
||||
assert_eq!(
|
||||
metadata["transport_diagnostics"]["resolved_transport_profile_id"],
|
||||
"chrome_136"
|
||||
);
|
||||
assert_eq!(
|
||||
metadata["transport_diagnostics"]["resolved_transport_profile"]["profile_id"],
|
||||
"chrome_136"
|
||||
);
|
||||
assert_eq!(
|
||||
metadata["transport_diagnostics"]["request_pair"]["conversion_enabled"],
|
||||
Value::Bool(true)
|
||||
|
||||
@@ -251,6 +251,7 @@ mod tests {
|
||||
use aether_ai_serving::{
|
||||
ai_ranking_context, build_ai_rankable_candidate, AiRankableCandidateParts,
|
||||
};
|
||||
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||
use aether_data::repository::{
|
||||
provider_catalog::InMemoryProviderCatalogReadRepository,
|
||||
routing_profiles::InMemoryRoutingGroupRepository,
|
||||
@@ -566,6 +567,15 @@ mod tests {
|
||||
api_formats: Option<serde_json::Value>,
|
||||
allowed_models: Option<serde_json::Value>,
|
||||
) -> StoredProviderCatalogKey {
|
||||
let credential_state = AppState::new()
|
||||
.expect("credential state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::disabled()
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY),
|
||||
);
|
||||
let encrypted_api_key = credential_state
|
||||
.seal_provider_catalog_key_api_key(provider_id, id, "plain-upstream-key")
|
||||
.expect("api key should encrypt");
|
||||
StoredProviderCatalogKey::new(
|
||||
id.to_string(),
|
||||
provider_id.to_string(),
|
||||
@@ -577,7 +587,7 @@ mod tests {
|
||||
.expect("key should build")
|
||||
.with_transport_fields(
|
||||
api_formats,
|
||||
"plain-upstream-key".to_string(),
|
||||
encrypted_api_key,
|
||||
None,
|
||||
None,
|
||||
Some(json!({"openai:chat": 1})),
|
||||
@@ -691,7 +701,7 @@ mod tests {
|
||||
let observed_at_unix_secs = current_unix_secs();
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
)
|
||||
.with_system_config_values_for_tests(vec![
|
||||
("provider_priority_mode".to_string(), json!("provider")),
|
||||
@@ -700,6 +710,7 @@ mod tests {
|
||||
serde_json::to_value(TunnelAttachmentRecord {
|
||||
gateway_instance_id: "gateway-b".to_string(),
|
||||
relay_base_url: "http://gateway-b:8080".to_string(),
|
||||
tunnel_generation: "test-generation-remote".to_string(),
|
||||
conn_count: 1,
|
||||
observed_at_unix_secs,
|
||||
})
|
||||
@@ -710,6 +721,7 @@ mod tests {
|
||||
serde_json::to_value(TunnelAttachmentRecord {
|
||||
gateway_instance_id: "gateway-a".to_string(),
|
||||
relay_base_url: "http://gateway-a:8080".to_string(),
|
||||
tunnel_generation: "test-generation-local".to_string(),
|
||||
conn_count: 1,
|
||||
observed_at_unix_secs,
|
||||
})
|
||||
@@ -768,7 +780,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -821,7 +833,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
)
|
||||
.with_system_config_values_for_tests(vec![(
|
||||
"scheduling_mode".to_string(),
|
||||
@@ -878,7 +890,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -953,7 +965,7 @@ mod tests {
|
||||
.expect("routing strategy should be created");
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
)
|
||||
.with_routing_group_repository_for_tests(routing_repository);
|
||||
let state = AppState::new()
|
||||
@@ -1015,7 +1027,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
)
|
||||
.with_system_config_values_for_tests(vec![(
|
||||
"provider_priority_mode".to_string(),
|
||||
@@ -1081,7 +1093,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1134,7 +1146,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1208,7 +1220,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1288,7 +1300,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1364,7 +1376,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1431,7 +1443,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1514,7 +1526,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1579,7 +1591,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1668,7 +1680,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1754,7 +1766,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1851,7 +1863,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1956,7 +1968,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -2050,7 +2062,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
|
||||
@@ -1473,6 +1473,7 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::data::GatewayDataState;
|
||||
use crate::AppState;
|
||||
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||
use aether_data::repository::candidate_selection::InMemoryMinimalCandidateSelectionReadRepository;
|
||||
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
||||
use aether_data::DataLayerError;
|
||||
@@ -2174,6 +2175,19 @@ mod tests {
|
||||
None,
|
||||
)
|
||||
.expect("endpoint transport should build");
|
||||
let credential_state = AppState::new()
|
||||
.expect("credential state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::disabled()
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY),
|
||||
);
|
||||
let encrypted_api_key = credential_state
|
||||
.seal_provider_catalog_key_api_key(
|
||||
row.provider_id.as_str(),
|
||||
row.key_id.as_str(),
|
||||
"plain-upstream-key",
|
||||
)
|
||||
.expect("api key should encrypt");
|
||||
let key = StoredProviderCatalogKey::new(
|
||||
row.key_id.clone(),
|
||||
row.provider_id.clone(),
|
||||
@@ -2185,7 +2199,7 @@ mod tests {
|
||||
.expect("key should build")
|
||||
.with_transport_fields(
|
||||
Some(serde_json::json!([row.endpoint_api_format.clone()])),
|
||||
"plain-upstream-key".to_string(),
|
||||
encrypted_api_key,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
@@ -2540,7 +2554,7 @@ mod tests {
|
||||
provider_repository,
|
||||
candidate_repository,
|
||||
)
|
||||
.with_encryption_key_for_tests("development-key");
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY);
|
||||
let app = AppState::new()
|
||||
.expect("gateway state should build")
|
||||
.with_data_state_for_tests(data_state);
|
||||
@@ -2660,7 +2674,7 @@ mod tests {
|
||||
provider_repository,
|
||||
candidate_repository,
|
||||
)
|
||||
.with_encryption_key_for_tests("development-key")
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY)
|
||||
// Legacy keys deliberately disagree with the routing policy: the
|
||||
// resolved policy must be the only source of scheduler ordering.
|
||||
.with_system_config_values_for_tests([
|
||||
|
||||
@@ -11,6 +11,7 @@ use crate::ai_serving::planner::route::{
|
||||
is_matching_stream_request, resolve_execution_runtime_stream_plan_kind,
|
||||
};
|
||||
use crate::ai_serving::{resolve_decision_execution_runtime_auth_context, GatewayControlDecision};
|
||||
use crate::state::VideoTaskRouteAccess;
|
||||
use crate::{AiExecutionDecision, AppState, GatewayError};
|
||||
|
||||
pub(crate) async fn maybe_build_stream_decision_payload(
|
||||
@@ -155,16 +156,37 @@ async fn maybe_build_local_video_task_content_stream_decision_payload(
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let _ = state
|
||||
.hydrate_video_task_for_route(decision.route_family.as_deref(), parts.uri.path())
|
||||
.await?;
|
||||
let Some(user_id) = decision
|
||||
.auth_context
|
||||
.as_ref()
|
||||
.filter(|auth_context| auth_context.access_allowed)
|
||||
.map(|auth_context| auth_context.user_id.trim())
|
||||
.filter(|value| !value.is_empty())
|
||||
else {
|
||||
return Err(crate::video_tasks::not_found_error());
|
||||
};
|
||||
if state
|
||||
.hydrate_video_task_for_route_for_user(
|
||||
decision.route_family.as_deref(),
|
||||
parts.uri.path(),
|
||||
user_id,
|
||||
)
|
||||
.await?
|
||||
!= VideoTaskRouteAccess::Allowed
|
||||
{
|
||||
return Err(crate::video_tasks::not_found_error());
|
||||
}
|
||||
|
||||
let Some(action) = state.video_tasks.prepare_openai_content_stream_action(
|
||||
parts.uri.path(),
|
||||
parts.uri.query(),
|
||||
trace_id,
|
||||
) else {
|
||||
return Ok(None);
|
||||
let Some(action) = state
|
||||
.video_tasks
|
||||
.prepare_openai_content_stream_action_for_user(
|
||||
parts.uri.path(),
|
||||
parts.uri.query(),
|
||||
trace_id,
|
||||
user_id,
|
||||
)
|
||||
else {
|
||||
return Err(crate::video_tasks::not_found_error());
|
||||
};
|
||||
|
||||
let crate::video_tasks::LocalVideoTaskContentAction::StreamPlan(plan) = action else {
|
||||
|
||||
@@ -16,6 +16,7 @@ use crate::ai_serving::{
|
||||
build_execution_runtime_auth_context, resolve_execution_runtime_auth_context,
|
||||
GatewayControlDecision,
|
||||
};
|
||||
use crate::state::VideoTaskRouteAccess;
|
||||
use crate::{AiExecutionDecision, AppState, GatewayError};
|
||||
|
||||
pub(crate) async fn maybe_build_sync_decision_payload(
|
||||
@@ -191,10 +192,6 @@ async fn maybe_build_local_video_task_follow_up_sync_decision_payload(
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let _ = state
|
||||
.hydrate_video_task_for_route(decision.route_family.as_deref(), parts.uri.path())
|
||||
.await?;
|
||||
|
||||
let auth_context = resolve_execution_runtime_auth_context(
|
||||
state,
|
||||
decision,
|
||||
@@ -204,16 +201,30 @@ async fn maybe_build_local_video_task_follow_up_sync_decision_payload(
|
||||
)
|
||||
.await?;
|
||||
let Some(auth_context) = auth_context else {
|
||||
return Ok(None);
|
||||
return Err(crate::video_tasks::not_found_error());
|
||||
};
|
||||
let Some(follow_up) = state.video_tasks.prepare_follow_up_sync_plan(
|
||||
if !auth_context.access_allowed || auth_context.user_id.trim().is_empty() {
|
||||
return Err(crate::video_tasks::not_found_error());
|
||||
}
|
||||
if state
|
||||
.hydrate_video_task_for_route_for_user(
|
||||
decision.route_family.as_deref(),
|
||||
parts.uri.path(),
|
||||
&auth_context.user_id,
|
||||
)
|
||||
.await?
|
||||
!= VideoTaskRouteAccess::Allowed
|
||||
{
|
||||
return Err(crate::video_tasks::not_found_error());
|
||||
}
|
||||
let Some(follow_up) = state.video_tasks.prepare_follow_up_sync_plan_for_user(
|
||||
plan_kind,
|
||||
parts.uri.path(),
|
||||
Some(body_json),
|
||||
Some(&auth_context),
|
||||
trace_id,
|
||||
) else {
|
||||
return Ok(None);
|
||||
return Err(crate::video_tasks::not_found_error());
|
||||
};
|
||||
|
||||
let aether_video_tasks_core::LocalVideoTaskFollowUpPlan {
|
||||
@@ -236,8 +247,7 @@ async fn maybe_build_local_video_task_follow_up_sync_decision_payload(
|
||||
downstream_path = %parts.uri.path(),
|
||||
provider_api_format = %plan.provider_api_format,
|
||||
client_api_format = %plan.client_api_format,
|
||||
upstream_base_url = ?upstream_base_url,
|
||||
upstream_url = %plan.url,
|
||||
upstream_origin = %crate::handlers::shared::security_log_url_origin(&plan.url),
|
||||
"gateway built local video follow-up sync decision payload"
|
||||
);
|
||||
|
||||
|
||||
@@ -37,6 +37,10 @@ const ROUTING_GROUP_SELECTION_CACHE_TTL: Duration = Duration::from_secs(30);
|
||||
const ROUTING_GROUP_SELECTION_CACHE_STALE_TTL: Duration = Duration::from_secs(120);
|
||||
const CODEX_ACCOUNT_ID_HEADER: &str = "chatgpt-account-id";
|
||||
const CODEX_FEDRAMP_HEADER: &str = "x-openai-fedramp";
|
||||
const INVALID_ROUTING_PROVIDER_CONTRACT_MESSAGE: &str =
|
||||
"routing provider request violates provider contract";
|
||||
const INVALID_ROUTING_PROVIDER_HEADERS_MESSAGE: &str =
|
||||
"invalid provider request headers in routing mutation";
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct ResolvedLocalDecisionAuthInput {
|
||||
@@ -312,10 +316,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
||||
)
|
||||
}
|
||||
}
|
||||
.map_err(|violation| GatewayError::Client {
|
||||
status: StatusCode::BAD_REQUEST,
|
||||
message: format!("routing provider_request violates provider contract: {violation:?}"),
|
||||
})?;
|
||||
.map_err(|_| invalid_routing_provider_contract())?;
|
||||
}
|
||||
let provider_model = provider_request_body
|
||||
.get("model")
|
||||
@@ -863,10 +864,32 @@ fn routing_selection_error(error: GatewayRoutingSelectionError) -> GatewayError
|
||||
status: StatusCode::SERVICE_UNAVAILABLE,
|
||||
message: "no enabled routing strategy is configured for this request".to_string(),
|
||||
},
|
||||
error => GatewayError::Client {
|
||||
GatewayRoutingSelectionError::NotFound(_) => GatewayError::Client {
|
||||
status: StatusCode::FORBIDDEN,
|
||||
message: error.to_string(),
|
||||
message: "requested routing group was not found".to_string(),
|
||||
},
|
||||
GatewayRoutingSelectionError::Disabled(_) => GatewayError::Client {
|
||||
status: StatusCode::FORBIDDEN,
|
||||
message: "requested routing group is not enabled".to_string(),
|
||||
},
|
||||
GatewayRoutingSelectionError::Forbidden(_) => GatewayError::Client {
|
||||
status: StatusCode::FORBIDDEN,
|
||||
message: "requested routing group is not allowed for this principal".to_string(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn invalid_routing_provider_contract() -> GatewayError {
|
||||
GatewayError::Client {
|
||||
status: StatusCode::BAD_REQUEST,
|
||||
message: INVALID_ROUTING_PROVIDER_CONTRACT_MESSAGE.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn invalid_routing_provider_headers() -> GatewayError {
|
||||
GatewayError::Client {
|
||||
status: StatusCode::BAD_REQUEST,
|
||||
message: INVALID_ROUTING_PROVIDER_HEADERS_MESSAGE.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -921,14 +944,9 @@ fn btree_headers_to_header_map(
|
||||
) -> Result<HeaderMap, GatewayError> {
|
||||
let mut output = HeaderMap::new();
|
||||
for (name, value) in headers {
|
||||
let name = HeaderName::from_bytes(name.as_bytes()).map_err(|err| GatewayError::Client {
|
||||
status: StatusCode::BAD_REQUEST,
|
||||
message: format!("invalid provider request header name in routing mutation: {err}"),
|
||||
})?;
|
||||
let value = HeaderValue::from_str(value).map_err(|err| GatewayError::Client {
|
||||
status: StatusCode::BAD_REQUEST,
|
||||
message: format!("invalid provider request header value in routing mutation: {err}"),
|
||||
})?;
|
||||
let name = HeaderName::from_bytes(name.as_bytes())
|
||||
.map_err(|_| invalid_routing_provider_headers())?;
|
||||
let value = HeaderValue::from_str(value).map_err(|_| invalid_routing_provider_headers())?;
|
||||
output.insert(name, value);
|
||||
}
|
||||
Ok(output)
|
||||
@@ -1151,6 +1169,50 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn routing_selection_errors_do_not_echo_explicit_group() {
|
||||
let secret = "private-group?token=Bearer-secret";
|
||||
|
||||
for error in [
|
||||
GatewayRoutingSelectionError::NotFound(secret.to_string()),
|
||||
GatewayRoutingSelectionError::Disabled(secret.to_string()),
|
||||
GatewayRoutingSelectionError::Forbidden(secret.to_string()),
|
||||
] {
|
||||
let error = routing_selection_error(error);
|
||||
assert!(matches!(
|
||||
error,
|
||||
GatewayError::Client {
|
||||
status: StatusCode::FORBIDDEN,
|
||||
ref message,
|
||||
} if !message.contains(secret)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn routing_provider_errors_do_not_echo_dynamic_details() {
|
||||
let secret = "https://internal.example/?token=Bearer-secret";
|
||||
let contract_error = invalid_routing_provider_contract();
|
||||
let header_error = btree_headers_to_header_map(&BTreeMap::from([(
|
||||
format!("Authorization: {secret}"),
|
||||
secret.to_string(),
|
||||
)]))
|
||||
.expect_err("invalid header should fail");
|
||||
|
||||
for (error, expected_message) in [
|
||||
(contract_error, INVALID_ROUTING_PROVIDER_CONTRACT_MESSAGE),
|
||||
(header_error, INVALID_ROUTING_PROVIDER_HEADERS_MESSAGE),
|
||||
] {
|
||||
assert!(matches!(
|
||||
error,
|
||||
GatewayError::Client {
|
||||
status: StatusCode::BAD_REQUEST,
|
||||
ref message,
|
||||
} if message == expected_message && !message.contains(secret)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn explicit_auth_snapshot_override_does_not_fall_back_to_the_planner_cache() {
|
||||
// AppState::new has no auth snapshot repository. Without the explicit
|
||||
|
||||
@@ -76,10 +76,12 @@ pub(crate) fn build_local_execution_report_context(
|
||||
let RequestOrigin {
|
||||
client_ip,
|
||||
user_agent,
|
||||
forwarded_headers_trusted,
|
||||
} = parts
|
||||
.request_origin
|
||||
.unwrap_or_else(|| request_origin_from_headers(parts.original_headers));
|
||||
let original_headers = crate::ai_serving::collect_control_headers(parts.original_headers);
|
||||
let original_headers =
|
||||
collect_report_context_original_headers(parts.original_headers, forwarded_headers_trusted);
|
||||
let original_request_body = crate::ai_serving::build_report_context_original_request_echo(
|
||||
parts.original_request_body_json,
|
||||
parts.original_request_body_base64,
|
||||
@@ -106,10 +108,12 @@ pub(crate) fn build_local_execution_report_context(
|
||||
value,
|
||||
);
|
||||
}
|
||||
if let Some(incoming_tls) =
|
||||
crate::ai_serving::tls_fingerprint_from_headers(parts.original_headers)
|
||||
{
|
||||
merge_incoming_tls_fingerprint(&mut extra_fields, incoming_tls);
|
||||
if forwarded_headers_trusted {
|
||||
if let Some(incoming_tls) =
|
||||
crate::ai_serving::tls_fingerprint_from_headers(parts.original_headers)
|
||||
{
|
||||
merge_incoming_tls_fingerprint(&mut extra_fields, incoming_tls);
|
||||
}
|
||||
}
|
||||
insert_pool_key_lease_report_context_fields(&mut extra_fields, parts.pool_key_lease);
|
||||
insert_scheduler_affinity_policy_report_context_field(&mut extra_fields, parts.routing_policy);
|
||||
@@ -189,6 +193,17 @@ pub(crate) fn build_local_execution_report_context(
|
||||
})
|
||||
}
|
||||
|
||||
fn collect_report_context_original_headers(
|
||||
headers: &http::HeaderMap,
|
||||
forwarded_headers_trusted: bool,
|
||||
) -> BTreeMap<String, String> {
|
||||
let mut collected = crate::ai_serving::collect_control_headers(headers);
|
||||
if !forwarded_headers_trusted {
|
||||
collected.retain(|name, _| !name.starts_with("x-aether-tls-"));
|
||||
}
|
||||
collected
|
||||
}
|
||||
|
||||
fn insert_request_path_fields(
|
||||
extra_fields: &mut Map<String, Value>,
|
||||
request_path: Option<&str>,
|
||||
@@ -258,8 +273,8 @@ mod tests {
|
||||
use serde_json::{json, Map, Value};
|
||||
|
||||
use super::{
|
||||
build_local_execution_report_context, provider_stream_event_api_format_for_provider_type,
|
||||
LocalExecutionReportContextParts,
|
||||
build_local_execution_report_context, collect_report_context_original_headers,
|
||||
provider_stream_event_api_format_for_provider_type, LocalExecutionReportContextParts,
|
||||
};
|
||||
use crate::ai_serving::ExecutionRuntimeAuthContext;
|
||||
use crate::ai_serving::RequestOrigin;
|
||||
@@ -289,6 +304,26 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn untrusted_tls_forwarding_headers_are_excluded_from_report_context() {
|
||||
let mut headers = http::HeaderMap::new();
|
||||
headers.insert("x-aether-tls-ja3", "spoofed-ja3".parse().unwrap());
|
||||
headers.insert(http::header::USER_AGENT, "test-client".parse().unwrap());
|
||||
|
||||
let untrusted = collect_report_context_original_headers(&headers, false);
|
||||
assert!(!untrusted.contains_key("x-aether-tls-ja3"));
|
||||
assert_eq!(
|
||||
untrusted.get("user-agent").map(String::as_str),
|
||||
Some("test-client")
|
||||
);
|
||||
|
||||
let trusted = collect_report_context_original_headers(&headers, true);
|
||||
assert_eq!(
|
||||
trusted.get("x-aether-tls-ja3").map(String::as_str),
|
||||
Some("spoofed-ja3")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_execution_report_context_records_request_origin_and_session_affinity() {
|
||||
let auth_context = ExecutionRuntimeAuthContext {
|
||||
@@ -339,6 +374,7 @@ mod tests {
|
||||
request_origin: Some(RequestOrigin {
|
||||
client_ip: Some("203.0.113.8".to_string()),
|
||||
user_agent: Some("Claude-Code/1.0".to_string()),
|
||||
forwarded_headers_trusted: false,
|
||||
}),
|
||||
original_request_body_json: Some(&json!({"model": "gpt-5"})),
|
||||
original_request_body_base64: None,
|
||||
@@ -491,7 +527,11 @@ mod tests {
|
||||
original_headers: &original_headers,
|
||||
request_path: None,
|
||||
request_query_string: None,
|
||||
request_origin: None,
|
||||
request_origin: Some(RequestOrigin {
|
||||
client_ip: None,
|
||||
user_agent: None,
|
||||
forwarded_headers_trusted: true,
|
||||
}),
|
||||
original_request_body_json: Some(&json!({"model": "gpt-5"})),
|
||||
original_request_body_base64: None,
|
||||
client_session_affinity: None,
|
||||
|
||||
@@ -8,7 +8,10 @@ use crate::ai_serving::transport::{
|
||||
GeminiFilesRequestBodyError,
|
||||
};
|
||||
use crate::ai_serving::GEMINI_FILES_UPLOAD_PLAN_KIND;
|
||||
use crate::ai_serving::{CandidateFailureDiagnostic, GatewayProviderTransportSnapshot};
|
||||
use crate::ai_serving::{
|
||||
CandidateFailureDiagnostic, GatewayProviderTransportSnapshot, GEMINI_FILES_DELETE_PLAN_KIND,
|
||||
GEMINI_FILES_DOWNLOAD_PLAN_KIND, GEMINI_FILES_GET_PLAN_KIND,
|
||||
};
|
||||
use crate::AppState;
|
||||
|
||||
use super::support::{
|
||||
@@ -47,6 +50,26 @@ pub(super) async fn resolve_local_gemini_files_candidate_payload_parts(
|
||||
let transport = &attempt.eligible.transport;
|
||||
let effective_headers = input.effective_headers(&parts.headers);
|
||||
|
||||
if matches!(
|
||||
spec_metadata.decision_kind,
|
||||
GEMINI_FILES_GET_PLAN_KIND
|
||||
| GEMINI_FILES_DELETE_PLAN_KIND
|
||||
| GEMINI_FILES_DOWNLOAD_PLAN_KIND
|
||||
) && !candidate_matches_owned_gemini_file_mapping(state, parts, input, attempt).await
|
||||
{
|
||||
mark_skipped_local_gemini_files_candidate(
|
||||
state,
|
||||
input,
|
||||
trace_id,
|
||||
candidate,
|
||||
attempt.candidate_index,
|
||||
&attempt.candidate_id,
|
||||
"gemini_file_mapping_mismatch",
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
}
|
||||
|
||||
if let Some(skip_reason) =
|
||||
gemini_files_transport_unsupported_reason(transport, GEMINI_FILES_CANDIDATE_API_FORMAT)
|
||||
{
|
||||
@@ -191,3 +214,64 @@ pub(super) async fn resolve_local_gemini_files_candidate_payload_parts(
|
||||
file_name,
|
||||
})
|
||||
}
|
||||
|
||||
async fn candidate_matches_owned_gemini_file_mapping(
|
||||
state: &AppState,
|
||||
parts: &http::request::Parts,
|
||||
input: &LocalGeminiFilesDecisionInput,
|
||||
attempt: &LocalGeminiFilesCandidateAttempt,
|
||||
) -> bool {
|
||||
let Some(file_name) = normalize_gemini_file_name_from_path(parts.uri.path()) else {
|
||||
return false;
|
||||
};
|
||||
let user_id = input.auth_context.user_id.trim();
|
||||
if user_id.is_empty() || !state.has_gemini_file_mapping_data_reader() {
|
||||
return false;
|
||||
}
|
||||
let Ok(Some(mapping)) = state
|
||||
.find_active_gemini_file_mapping_for_owner(
|
||||
file_name.as_str(),
|
||||
&attempt.eligible.transport.key.id,
|
||||
user_id,
|
||||
crate::clock::current_unix_secs(),
|
||||
)
|
||||
.await
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
|
||||
mapping.user_id.as_deref().map(str::trim) == Some(user_id)
|
||||
&& mapping.key_id == attempt.eligible.transport.key.id
|
||||
}
|
||||
|
||||
pub(crate) fn normalize_gemini_file_name_from_path(path: &str) -> Option<String> {
|
||||
let suffix = path.strip_prefix("/v1beta/files/")?.trim_matches('/');
|
||||
let suffix = suffix.strip_suffix(":download").unwrap_or(suffix).trim();
|
||||
let suffix = suffix.strip_prefix("files/").unwrap_or(suffix).trim();
|
||||
if suffix.is_empty() || suffix.contains('/') {
|
||||
return None;
|
||||
}
|
||||
Some(format!("files/{suffix}"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::normalize_gemini_file_name_from_path;
|
||||
|
||||
#[test]
|
||||
fn normalizes_supported_gemini_file_object_paths() {
|
||||
assert_eq!(
|
||||
normalize_gemini_file_name_from_path("/v1beta/files/file-123"),
|
||||
Some("files/file-123".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
normalize_gemini_file_name_from_path("/v1beta/files/file-123:download"),
|
||||
Some("files/file-123".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
normalize_gemini_file_name_from_path("/v1beta/files/files/abc-123"),
|
||||
Some("files/abc-123".to_string())
|
||||
);
|
||||
assert_eq!(normalize_gemini_file_name_from_path("/v1beta/files"), None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -587,7 +587,7 @@ pub(crate) async fn resolve_local_standard_candidate_payload_parts(
|
||||
}
|
||||
};
|
||||
crate::ai_serving::hydrate_openai_response_history(
|
||||
state.runtime_state(),
|
||||
state,
|
||||
body_json,
|
||||
spec_metadata.api_format,
|
||||
provider_api_format,
|
||||
|
||||
+12
-27
@@ -1,7 +1,6 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use aether_contracts::RequestBody;
|
||||
use tracing::debug;
|
||||
|
||||
use super::super::{
|
||||
augment_sync_report_context, build_ai_execution_plan_from_decision,
|
||||
@@ -10,7 +9,6 @@ use super::super::{
|
||||
AiStreamAttempt,
|
||||
};
|
||||
use crate::ai_serving::planner::common::enforce_provider_body_stream_policy;
|
||||
use crate::ai_serving::planner::redaction::sanitize_upstream_url_for_log;
|
||||
use crate::ai_serving::provider_adaptation_requires_eventstream_accept;
|
||||
use crate::ai_serving::transport::{
|
||||
build_standard_plan_fallback_headers, build_standard_plan_fallback_openai_chat_url,
|
||||
@@ -157,21 +155,16 @@ pub(crate) fn build_openai_responses_stream_plan_from_decision(
|
||||
let Some(auth_pair) = take_ai_upstream_auth_pair(&mut payload) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let (url, url_source) = if let Some(upstream_url) =
|
||||
take_non_empty_string(&mut payload.upstream_url)
|
||||
{
|
||||
(upstream_url, "upstream_url")
|
||||
let url = if let Some(upstream_url) = take_non_empty_string(&mut payload.upstream_url) {
|
||||
upstream_url
|
||||
} else {
|
||||
let Some(upstream_base_url) = take_non_empty_string(&mut payload.upstream_base_url) else {
|
||||
return Ok(None);
|
||||
};
|
||||
(
|
||||
build_standard_plan_fallback_openai_responses_url(
|
||||
&upstream_base_url,
|
||||
parts.uri.query(),
|
||||
compact,
|
||||
),
|
||||
"upstream_base_url",
|
||||
build_standard_plan_fallback_openai_responses_url(
|
||||
&upstream_base_url,
|
||||
parts.uri.query(),
|
||||
compact,
|
||||
)
|
||||
};
|
||||
let Some(provider_request_body_value) = payload.provider_request_body.take() else {
|
||||
@@ -238,16 +231,7 @@ pub(crate) fn build_openai_responses_stream_plan_from_decision(
|
||||
.uri
|
||||
.query()
|
||||
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
||||
let log_decision_upstream_base_url = payload
|
||||
.upstream_base_url
|
||||
.as_deref()
|
||||
.map(sanitize_upstream_url_for_log);
|
||||
let log_decision_upstream_url = payload
|
||||
.upstream_url
|
||||
.as_deref()
|
||||
.map(sanitize_upstream_url_for_log);
|
||||
let log_plan_url = sanitize_upstream_url_for_log(plan.url.as_str());
|
||||
debug!(
|
||||
tracing::debug!(
|
||||
event_name = "local_openai_responses_stream_plan_built",
|
||||
log_type = "debug",
|
||||
request_id = %plan.request_id,
|
||||
@@ -255,12 +239,13 @@ pub(crate) fn build_openai_responses_stream_plan_from_decision(
|
||||
provider_id = %plan.provider_id,
|
||||
endpoint_id = %plan.endpoint_id,
|
||||
key_id = %plan.key_id,
|
||||
downstream_path_and_query = %crate::ai_serving::pure::sanitize_request_path_and_query(
|
||||
parts.uri.path(),
|
||||
parts.uri.query(),
|
||||
).unwrap_or_else(|| "/".to_string()),
|
||||
upstream_origin = %crate::handlers::shared::security_log_url_origin(&plan.url),
|
||||
downstream_path = %parts.uri.path(),
|
||||
downstream_query = ?log_downstream_query,
|
||||
url_source,
|
||||
decision_upstream_base_url = ?log_decision_upstream_base_url,
|
||||
decision_upstream_url = ?log_decision_upstream_url,
|
||||
plan_url = %log_plan_url,
|
||||
client_api_format = %plan.client_api_format,
|
||||
provider_api_format = %plan.provider_api_format,
|
||||
upstream_is_stream = effective_upstream_is_stream,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use aether_contracts::RequestBody;
|
||||
use tracing::debug;
|
||||
|
||||
use super::super::{
|
||||
augment_sync_report_context, build_ai_execution_plan_from_decision,
|
||||
@@ -10,7 +9,6 @@ use super::super::{
|
||||
AiSyncAttempt,
|
||||
};
|
||||
use crate::ai_serving::planner::common::enforce_provider_body_stream_policy;
|
||||
use crate::ai_serving::planner::redaction::sanitize_upstream_url_for_log;
|
||||
use crate::ai_serving::transport::{
|
||||
build_standard_plan_fallback_headers, build_standard_plan_fallback_openai_chat_url,
|
||||
build_standard_plan_fallback_openai_responses_url, StandardPlanFallbackAcceptPolicy,
|
||||
@@ -142,21 +140,16 @@ pub(crate) fn build_openai_responses_sync_plan_from_decision(
|
||||
let Some(auth_pair) = take_ai_upstream_auth_pair(&mut payload) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let (url, url_source) = if let Some(upstream_url) =
|
||||
take_non_empty_string(&mut payload.upstream_url)
|
||||
{
|
||||
(upstream_url, "upstream_url")
|
||||
let url = if let Some(upstream_url) = take_non_empty_string(&mut payload.upstream_url) {
|
||||
upstream_url
|
||||
} else {
|
||||
let Some(upstream_base_url) = take_non_empty_string(&mut payload.upstream_base_url) else {
|
||||
return Ok(None);
|
||||
};
|
||||
(
|
||||
build_standard_plan_fallback_openai_responses_url(
|
||||
&upstream_base_url,
|
||||
parts.uri.query(),
|
||||
compact,
|
||||
),
|
||||
"upstream_base_url",
|
||||
build_standard_plan_fallback_openai_responses_url(
|
||||
&upstream_base_url,
|
||||
parts.uri.query(),
|
||||
compact,
|
||||
)
|
||||
};
|
||||
let Some(provider_request_body_value) = payload.provider_request_body.take() else {
|
||||
@@ -205,16 +198,7 @@ pub(crate) fn build_openai_responses_sync_plan_from_decision(
|
||||
.uri
|
||||
.query()
|
||||
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
||||
let log_decision_upstream_base_url = payload
|
||||
.upstream_base_url
|
||||
.as_deref()
|
||||
.map(sanitize_upstream_url_for_log);
|
||||
let log_decision_upstream_url = payload
|
||||
.upstream_url
|
||||
.as_deref()
|
||||
.map(sanitize_upstream_url_for_log);
|
||||
let log_plan_url = sanitize_upstream_url_for_log(plan.url.as_str());
|
||||
debug!(
|
||||
tracing::debug!(
|
||||
event_name = "local_openai_responses_sync_plan_built",
|
||||
log_type = "debug",
|
||||
request_id = %plan.request_id,
|
||||
@@ -222,12 +206,13 @@ pub(crate) fn build_openai_responses_sync_plan_from_decision(
|
||||
provider_id = %plan.provider_id,
|
||||
endpoint_id = %plan.endpoint_id,
|
||||
key_id = %plan.key_id,
|
||||
downstream_path_and_query = %crate::ai_serving::pure::sanitize_request_path_and_query(
|
||||
parts.uri.path(),
|
||||
parts.uri.query(),
|
||||
).unwrap_or_else(|| "/".to_string()),
|
||||
upstream_origin = %crate::handlers::shared::security_log_url_origin(&plan.url),
|
||||
downstream_path = %parts.uri.path(),
|
||||
downstream_query = ?log_downstream_query,
|
||||
url_source,
|
||||
decision_upstream_base_url = ?log_decision_upstream_base_url,
|
||||
decision_upstream_url = ?log_decision_upstream_url,
|
||||
plan_url = %log_plan_url,
|
||||
client_api_format = %plan.client_api_format,
|
||||
provider_api_format = %plan.provider_api_format,
|
||||
upstream_is_stream = payload.upstream_is_stream,
|
||||
|
||||
+5
-5
@@ -3,7 +3,6 @@ use tracing::debug;
|
||||
|
||||
use crate::ai_serving::build_request_trace_proxy_value;
|
||||
use crate::ai_serving::planner::decision_input::apply_provider_request_routing_policy_to_decision_with_websocket_mode;
|
||||
use crate::ai_serving::planner::redaction::sanitize_upstream_url_for_log;
|
||||
use crate::ai_serving::planner::report_context::{
|
||||
build_local_execution_report_context, insert_native_client_envelope_name,
|
||||
insert_provider_stream_event_api_format, LocalExecutionReportContextParts,
|
||||
@@ -205,12 +204,10 @@ pub(crate) async fn maybe_build_local_openai_responses_decision_payload_for_cand
|
||||
&resolved.transport,
|
||||
);
|
||||
|
||||
let log_base_url = sanitize_upstream_url_for_log(resolved.transport.endpoint.base_url.as_str());
|
||||
let log_request_query = parts
|
||||
.uri
|
||||
.query()
|
||||
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
||||
let log_upstream_url = sanitize_upstream_url_for_log(resolved.upstream_url.as_str());
|
||||
debug!(
|
||||
event_name = "local_openai_responses_decision_payload_built",
|
||||
log_type = "debug",
|
||||
@@ -227,9 +224,12 @@ pub(crate) async fn maybe_build_local_openai_responses_decision_payload_for_cand
|
||||
client_api_format = spec_metadata.api_format,
|
||||
provider_api_format = %resolved.provider_api_format,
|
||||
request_path = %parts.uri.path(),
|
||||
request_path_and_query = %crate::ai_serving::pure::sanitize_request_path_and_query(
|
||||
parts.uri.path(),
|
||||
parts.uri.query(),
|
||||
).unwrap_or_else(|| "/".to_string()),
|
||||
upstream_origin = %crate::handlers::shared::security_log_url_origin(&resolved.upstream_url),
|
||||
request_query = ?log_request_query,
|
||||
upstream_base_url = %log_base_url,
|
||||
upstream_url = %log_upstream_url,
|
||||
upstream_is_stream = resolved.upstream_is_stream,
|
||||
has_envelope = resolved.envelope_name.is_some(),
|
||||
"gateway built local openai responses decision payload"
|
||||
|
||||
+7
-15
@@ -24,7 +24,6 @@ use crate::ai_serving::planner::gemini_cli::{
|
||||
};
|
||||
use crate::ai_serving::planner::redaction::{
|
||||
request_identity_response_encoding_when_redacted, resolve_provider_chat_pii_redaction,
|
||||
sanitize_upstream_url_for_log,
|
||||
};
|
||||
use crate::ai_serving::planner::spec_metadata::local_openai_responses_spec_metadata;
|
||||
use crate::ai_serving::planner::standard::{
|
||||
@@ -428,7 +427,7 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts_with_
|
||||
}
|
||||
};
|
||||
crate::ai_serving::hydrate_openai_response_history(
|
||||
state.runtime_state(),
|
||||
state,
|
||||
body_json,
|
||||
spec_metadata.api_format,
|
||||
provider_api_format,
|
||||
@@ -867,17 +866,10 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts_with_
|
||||
|
||||
let (execution_strategy, conversion_mode) =
|
||||
ai_local_execution_contract_for_formats(spec_metadata.api_format, provider_api_format);
|
||||
let log_base_url = sanitize_upstream_url_for_log(transport.endpoint.base_url.as_str());
|
||||
let log_custom_path = transport
|
||||
.endpoint
|
||||
.custom_path
|
||||
.as_deref()
|
||||
.map(sanitize_upstream_url_for_log);
|
||||
let log_request_query = parts
|
||||
.uri
|
||||
.query()
|
||||
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
||||
let log_upstream_url = sanitize_upstream_url_for_log(upstream_url.as_str());
|
||||
|
||||
debug!(
|
||||
event_name = "local_openai_responses_upstream_url_resolved",
|
||||
@@ -893,12 +885,14 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts_with_
|
||||
provider_api_format = %provider_api_format,
|
||||
execution_strategy = execution_strategy.as_str(),
|
||||
conversion_mode = conversion_mode.as_str(),
|
||||
base_url = %log_base_url,
|
||||
custom_path = ?log_custom_path,
|
||||
request_path_and_query = %crate::ai_serving::pure::sanitize_request_path_and_query(
|
||||
parts.uri.path(),
|
||||
parts.uri.query(),
|
||||
).unwrap_or_else(|| "/".to_string()),
|
||||
upstream_origin = %crate::handlers::shared::security_log_url_origin(&upstream_url),
|
||||
request_path = %parts.uri.path(),
|
||||
request_query = ?log_request_query,
|
||||
mapped_model = %mapped_model,
|
||||
upstream_url = %log_upstream_url,
|
||||
upstream_is_stream,
|
||||
"gateway resolved local openai responses upstream url"
|
||||
);
|
||||
@@ -2011,8 +2005,6 @@ async fn build_kiro_openai_responses_payload_parts(
|
||||
};
|
||||
let (execution_strategy, conversion_mode) =
|
||||
ai_local_execution_contract_for_formats(client_api_format, provider_api_format);
|
||||
let log_upstream_url = sanitize_upstream_url_for_log(upstream_url.as_str());
|
||||
|
||||
debug!(
|
||||
event_name = "local_openai_responses_kiro_upstream_url_resolved",
|
||||
log_type = "debug",
|
||||
@@ -2027,7 +2019,7 @@ async fn build_kiro_openai_responses_payload_parts(
|
||||
provider_api_format = %provider_api_format,
|
||||
execution_strategy = execution_strategy.as_str(),
|
||||
conversion_mode = conversion_mode.as_str(),
|
||||
upstream_url = %log_upstream_url,
|
||||
upstream_origin = %crate::handlers::shared::security_log_url_origin(&upstream_url),
|
||||
upstream_is_stream,
|
||||
"gateway resolved local openai responses kiro upstream url"
|
||||
);
|
||||
|
||||
@@ -2,14 +2,15 @@ use crate::ai_serving::{
|
||||
hydrate_response_history, normalize_api_format_alias, record_converted_response_history,
|
||||
response_history_is_loaded, response_history_storage_key, ResponseHistoryRecord,
|
||||
};
|
||||
use aether_runtime_state::RuntimeState;
|
||||
use serde_json::Value;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::GatewayError;
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
const RESPONSE_HISTORY_SECRET_PURPOSE: &str = "openai-response-history";
|
||||
|
||||
pub(crate) async fn hydrate_openai_response_history(
|
||||
runtime_state: &RuntimeState,
|
||||
state: &AppState,
|
||||
request: &Value,
|
||||
client_api_format: &str,
|
||||
provider_api_format: &str,
|
||||
@@ -33,6 +34,7 @@ pub(crate) async fn hydrate_openai_response_history(
|
||||
}
|
||||
|
||||
let storage_key = response_history_storage_key(previous_response_id, Some(history_scope));
|
||||
let runtime_state = state.runtime_state();
|
||||
let payload = runtime_state.kv_get(&storage_key).await.map_err(|error| {
|
||||
warn!(
|
||||
event_name = "openai_response_history_read_failed",
|
||||
@@ -46,8 +48,24 @@ pub(crate) async fn hydrate_openai_response_history(
|
||||
let Some(payload) = payload else {
|
||||
return Ok(());
|
||||
};
|
||||
let Some(payload) = crate::handlers::shared::open_runtime_secret_payload(
|
||||
state,
|
||||
RESPONSE_HISTORY_SECRET_PURPOSE,
|
||||
&payload,
|
||||
) else {
|
||||
let _ = runtime_state.kv_delete(&storage_key).await;
|
||||
warn!(
|
||||
event_name = "openai_response_history_decryption_failed",
|
||||
log_type = "ops",
|
||||
backend = runtime_state.backend_kind().as_str(),
|
||||
"gateway rejected undecryptable shared OpenAI response history"
|
||||
);
|
||||
return Err(GatewayError::Internal(
|
||||
"OpenAI response history decryption failed".to_string(),
|
||||
));
|
||||
};
|
||||
if let Err(error) =
|
||||
hydrate_response_history(previous_response_id, Some(history_scope), &payload)
|
||||
hydrate_response_history(previous_response_id, Some(history_scope), payload.as_str())
|
||||
{
|
||||
let _ = runtime_state.kv_delete(&storage_key).await;
|
||||
warn!(
|
||||
@@ -65,11 +83,25 @@ pub(crate) async fn hydrate_openai_response_history(
|
||||
}
|
||||
|
||||
pub(crate) async fn persist_response_history_record(
|
||||
runtime_state: &RuntimeState,
|
||||
state: &AppState,
|
||||
record: ResponseHistoryRecord,
|
||||
) {
|
||||
let runtime_state = state.runtime_state();
|
||||
let Some(sealed_payload) = crate::handlers::shared::seal_runtime_secret_payload(
|
||||
state,
|
||||
RESPONSE_HISTORY_SECRET_PURPOSE,
|
||||
&record.payload,
|
||||
) else {
|
||||
warn!(
|
||||
event_name = "openai_response_history_encryption_unavailable",
|
||||
log_type = "ops",
|
||||
backend = runtime_state.backend_kind().as_str(),
|
||||
"gateway refused to persist unencrypted OpenAI response history"
|
||||
);
|
||||
return;
|
||||
};
|
||||
if let Err(error) = runtime_state
|
||||
.kv_set(&record.storage_key, record.payload, Some(record.ttl))
|
||||
.kv_set(&record.storage_key, sealed_payload, Some(record.ttl))
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
@@ -83,7 +115,7 @@ pub(crate) async fn persist_response_history_record(
|
||||
}
|
||||
|
||||
pub(crate) async fn persist_converted_response_history(
|
||||
runtime_state: &RuntimeState,
|
||||
state: &AppState,
|
||||
report_context: &Value,
|
||||
response: Option<&Value>,
|
||||
) {
|
||||
@@ -91,6 +123,126 @@ pub(crate) async fn persist_converted_response_history(
|
||||
return;
|
||||
};
|
||||
if let Some(record) = record_converted_response_history(report_context, response) {
|
||||
persist_response_history_record(runtime_state, record).await;
|
||||
persist_response_history_record(state, record).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||
use aether_runtime_state::{MemoryRuntimeStateConfig, RuntimeState};
|
||||
use serde_json::json;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::{
|
||||
hydrate_openai_response_history, persist_response_history_record, ResponseHistoryRecord,
|
||||
};
|
||||
use crate::{ai_serving::response_history_storage_key, data::GatewayDataState, AppState};
|
||||
|
||||
fn response_history_test_state() -> AppState {
|
||||
AppState::new()
|
||||
.expect("test state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::disabled()
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY),
|
||||
)
|
||||
.with_runtime_state(Arc::new(RuntimeState::memory(
|
||||
MemoryRuntimeStateConfig::default(),
|
||||
)))
|
||||
}
|
||||
|
||||
fn response_history_payload(response_id: &str, scope: &str, marker: &str) -> String {
|
||||
let expires_at_unix_secs = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
.saturating_add(3600);
|
||||
json!({
|
||||
"version": 1,
|
||||
"response_id": response_id,
|
||||
"scope_fingerprint": format!("{:x}", Sha256::digest(scope.trim().as_bytes())),
|
||||
"expires_at_unix_secs": expires_at_unix_secs,
|
||||
"transcript": [{"type": "message", "content": marker}],
|
||||
})
|
||||
.to_string()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn response_history_is_encrypted_at_rest_and_hydrates() {
|
||||
let state = response_history_test_state();
|
||||
let response_id = "resp_gateway_encrypted_history_v1";
|
||||
let scope = "response-history-encrypted-scope";
|
||||
let marker = "private-response-history-marker";
|
||||
let storage_key = response_history_storage_key(response_id, Some(scope));
|
||||
let payload = response_history_payload(response_id, scope, marker);
|
||||
|
||||
persist_response_history_record(
|
||||
&state,
|
||||
ResponseHistoryRecord {
|
||||
storage_key: storage_key.clone(),
|
||||
payload,
|
||||
ttl: Duration::from_secs(6 * 60 * 60),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
let stored = state
|
||||
.runtime_kv_get(&storage_key)
|
||||
.await
|
||||
.expect("history lookup should succeed")
|
||||
.expect("history should be persisted");
|
||||
assert!(crate::handlers::shared::runtime_secret_payload_is_sealed(
|
||||
&stored
|
||||
));
|
||||
assert!(!stored.contains(marker));
|
||||
|
||||
hydrate_openai_response_history(
|
||||
&state,
|
||||
&json!({"previous_response_id": response_id}),
|
||||
"openai:responses",
|
||||
"openai:chat",
|
||||
scope,
|
||||
)
|
||||
.await
|
||||
.expect("encrypted history should hydrate");
|
||||
assert!(crate::ai_serving::response_history_is_loaded(
|
||||
response_id,
|
||||
Some(scope)
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn response_history_reader_rejects_and_deletes_legacy_plaintext() {
|
||||
let state = response_history_test_state();
|
||||
let response_id = "resp_gateway_legacy_history_v1";
|
||||
let scope = "response-history-legacy-scope";
|
||||
let storage_key = response_history_storage_key(response_id, Some(scope));
|
||||
let payload = response_history_payload(response_id, scope, "legacy-private-history");
|
||||
state
|
||||
.runtime_kv_setex(&storage_key, &payload, 6 * 60 * 60)
|
||||
.await
|
||||
.expect("legacy history should store");
|
||||
|
||||
let result = hydrate_openai_response_history(
|
||||
&state,
|
||||
&json!({"previous_response_id": response_id}),
|
||||
"openai:responses",
|
||||
"openai:chat",
|
||||
scope,
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_err());
|
||||
assert!(!crate::ai_serving::response_history_is_loaded(
|
||||
response_id,
|
||||
Some(scope)
|
||||
));
|
||||
assert!(state
|
||||
.runtime_kv_get(&storage_key)
|
||||
.await
|
||||
.expect("history lookup should succeed")
|
||||
.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,7 +133,7 @@ pub(crate) async fn frontdoor_manifest(State(state): State<AppState>) -> impl In
|
||||
"internal_gateway": {
|
||||
"route_groups": INTERNAL_GATEWAY_ROUTE_GROUPS,
|
||||
"path_prefixes": INTERNAL_GATEWAY_PATH_PREFIXES,
|
||||
"status": "rust_native_control_plane",
|
||||
"status": state.internal_gateway_auth_status(),
|
||||
},
|
||||
},
|
||||
"features": {
|
||||
|
||||
@@ -1,5 +1,14 @@
|
||||
use std::net::SocketAddr;
|
||||
|
||||
use axum::body::Body;
|
||||
use axum::extract::{ConnectInfo, Request, State};
|
||||
use axum::http::{self, HeaderValue, StatusCode};
|
||||
use axum::middleware::{self, Next};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use axum::routing::{get, post};
|
||||
use axum::Router;
|
||||
use axum::{Json, Router};
|
||||
use serde_json::json;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::async_task::{
|
||||
cancel_video_task, get_video_task_detail, get_video_task_stats, get_video_task_video,
|
||||
@@ -10,8 +19,18 @@ use crate::hooks::{get_request_audit_bundle, get_request_usage_audit};
|
||||
use crate::router::metrics;
|
||||
use crate::state::AppState;
|
||||
|
||||
pub(crate) fn mount_operational_routes(router: Router<AppState>) -> Router<AppState> {
|
||||
router
|
||||
#[derive(Clone, Copy)]
|
||||
struct OperationalPermission {
|
||||
required_permissions: &'static [&'static str],
|
||||
write: bool,
|
||||
requires_full_admin_role: bool,
|
||||
}
|
||||
|
||||
pub(crate) fn mount_operational_routes(
|
||||
router: Router<AppState>,
|
||||
state: AppState,
|
||||
) -> Router<AppState> {
|
||||
let operational = Router::<AppState>::new()
|
||||
.route("/_gateway/metrics", get(metrics))
|
||||
.route("/_gateway/async-tasks/video-tasks", get(list_video_tasks))
|
||||
.route(
|
||||
@@ -50,4 +69,236 @@ pub(crate) fn mount_operational_routes(router: Router<AppState>) -> Router<AppSt
|
||||
"/_gateway/audit/request-usage/{request_id}",
|
||||
get(get_request_usage_audit),
|
||||
)
|
||||
.route_layer(middleware::from_fn_with_state(
|
||||
state,
|
||||
authorize_operational_request,
|
||||
));
|
||||
router.merge(operational)
|
||||
}
|
||||
|
||||
async fn authorize_operational_request(
|
||||
State(state): State<AppState>,
|
||||
request: Request,
|
||||
next: Next,
|
||||
) -> Response<Body> {
|
||||
let Some(permission) = operational_permission(request.method(), request.uri().path()) else {
|
||||
return operational_error_response(
|
||||
StatusCode::FORBIDDEN,
|
||||
"operational route permission is not configured",
|
||||
None,
|
||||
);
|
||||
};
|
||||
let Some(remote_addr) = request
|
||||
.extensions()
|
||||
.get::<ConnectInfo<SocketAddr>>()
|
||||
.map(|value| value.0)
|
||||
else {
|
||||
return operational_error_response(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"operational authentication unavailable",
|
||||
None,
|
||||
);
|
||||
};
|
||||
let headers = request.headers().clone();
|
||||
let uri = request.uri().clone();
|
||||
if headers.get_all(http::header::AUTHORIZATION).iter().count() > 1 {
|
||||
return operational_auth_required_response();
|
||||
}
|
||||
|
||||
match crate::control::resolve_local_admin_session_principal(&state, &headers, &uri).await {
|
||||
Ok(Some(principal)) => {
|
||||
if permission.requires_full_admin_role
|
||||
&& !crate::roles::is_full_admin_role(&principal.user_role)
|
||||
{
|
||||
return operational_permission_denied_response(permission.required_permissions[0]);
|
||||
}
|
||||
if permission.write && !crate::roles::can_write_admin_console(&principal.user_role) {
|
||||
return operational_permission_denied_response(permission.required_permissions[0]);
|
||||
}
|
||||
}
|
||||
Ok(None) => {
|
||||
let client_ip = crate::headers::effective_client_ip(&headers, &remote_addr);
|
||||
let authenticated = match crate::management_token_auth::authenticate_management_token(
|
||||
&state, &headers, client_ip,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(authenticated) => authenticated,
|
||||
Err(
|
||||
crate::management_token_auth::ManagementTokenAuthError::Missing
|
||||
| crate::management_token_auth::ManagementTokenAuthError::Invalid,
|
||||
) => return operational_auth_required_response(),
|
||||
Err(crate::management_token_auth::ManagementTokenAuthError::Unavailable) => {
|
||||
return operational_error_response(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"operational authentication unavailable",
|
||||
None,
|
||||
)
|
||||
}
|
||||
};
|
||||
|
||||
if permission.requires_full_admin_role
|
||||
&& !crate::roles::is_full_admin_role(&authenticated.user.role)
|
||||
{
|
||||
return operational_permission_denied_response(permission.required_permissions[0]);
|
||||
}
|
||||
if permission.write && !crate::roles::can_write_admin_console(&authenticated.user.role)
|
||||
{
|
||||
return operational_permission_denied_response(permission.required_permissions[0]);
|
||||
}
|
||||
let missing_permission =
|
||||
permission
|
||||
.required_permissions
|
||||
.iter()
|
||||
.copied()
|
||||
.find(|required| {
|
||||
!management_token_has_operational_permission(
|
||||
&authenticated.permissions,
|
||||
required,
|
||||
)
|
||||
});
|
||||
if let Some(required_permission) = missing_permission {
|
||||
return operational_permission_denied_response(required_permission);
|
||||
}
|
||||
|
||||
let client_ip = client_ip.to_string();
|
||||
if let Err(err) = state
|
||||
.record_management_token_usage(&authenticated.token.id, Some(client_ip.as_str()))
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
token_id = %authenticated.token.id,
|
||||
error = ?err,
|
||||
"gateway failed to record operational management token usage"
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(error = ?err, "operational admin session authentication failed");
|
||||
return operational_error_response(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"operational authentication unavailable",
|
||||
None,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let mut response = next.run(request).await;
|
||||
response.headers_mut().insert(
|
||||
http::header::CACHE_CONTROL,
|
||||
HeaderValue::from_static("no-store"),
|
||||
);
|
||||
response
|
||||
}
|
||||
|
||||
fn operational_permission(method: &http::Method, path: &str) -> Option<OperationalPermission> {
|
||||
if path == "/_gateway/metrics" {
|
||||
return Some(OperationalPermission {
|
||||
required_permissions: &["admin:monitoring:read"],
|
||||
write: false,
|
||||
requires_full_admin_role: false,
|
||||
});
|
||||
}
|
||||
if path.starts_with("/_gateway/async-tasks/video-tasks") {
|
||||
let write = *method == http::Method::POST && path.ends_with("/cancel");
|
||||
return Some(OperationalPermission {
|
||||
required_permissions: if write {
|
||||
&["admin:video_tasks:write"]
|
||||
} else {
|
||||
&["admin:video_tasks:read"]
|
||||
},
|
||||
write,
|
||||
requires_full_admin_role: false,
|
||||
});
|
||||
}
|
||||
if path.starts_with("/_gateway/audit/auth/users/") {
|
||||
return Some(OperationalPermission {
|
||||
required_permissions: &["admin:api_keys:read"],
|
||||
write: false,
|
||||
requires_full_admin_role: false,
|
||||
});
|
||||
}
|
||||
if path.starts_with("/_gateway/audit/request-audit/") {
|
||||
return Some(OperationalPermission {
|
||||
required_permissions: &[
|
||||
"admin:monitoring:admin",
|
||||
"admin:usage:read",
|
||||
"admin:api_keys:read",
|
||||
],
|
||||
write: false,
|
||||
requires_full_admin_role: true,
|
||||
});
|
||||
}
|
||||
if path.starts_with("/_gateway/audit/request-candidates/")
|
||||
|| path.starts_with("/_gateway/audit/decision-trace/")
|
||||
{
|
||||
return Some(OperationalPermission {
|
||||
required_permissions: &["admin:monitoring:admin"],
|
||||
write: false,
|
||||
requires_full_admin_role: true,
|
||||
});
|
||||
}
|
||||
if path.starts_with("/_gateway/audit/") {
|
||||
return Some(OperationalPermission {
|
||||
required_permissions: &["admin:usage:read"],
|
||||
write: false,
|
||||
requires_full_admin_role: false,
|
||||
});
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn management_token_has_operational_permission(
|
||||
permissions: &[String],
|
||||
required_permission: &str,
|
||||
) -> bool {
|
||||
let scope = required_permission
|
||||
.rsplit_once(':')
|
||||
.map(|(scope, _)| scope)
|
||||
.unwrap_or(required_permission);
|
||||
let admin_permission = format!("{scope}:admin");
|
||||
permissions
|
||||
.iter()
|
||||
.any(|permission| permission == required_permission || permission == &admin_permission)
|
||||
}
|
||||
|
||||
fn operational_auth_required_response() -> Response<Body> {
|
||||
let mut response = operational_error_response(
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"admin authentication required",
|
||||
None,
|
||||
);
|
||||
response.headers_mut().insert(
|
||||
http::header::WWW_AUTHENTICATE,
|
||||
HeaderValue::from_static("Bearer"),
|
||||
);
|
||||
response
|
||||
}
|
||||
|
||||
fn operational_permission_denied_response(required_permission: &'static str) -> Response<Body> {
|
||||
operational_error_response(
|
||||
StatusCode::FORBIDDEN,
|
||||
"operational permission denied",
|
||||
Some(required_permission),
|
||||
)
|
||||
}
|
||||
|
||||
fn operational_error_response(
|
||||
status: StatusCode,
|
||||
detail: &'static str,
|
||||
required_permission: Option<&'static str>,
|
||||
) -> Response<Body> {
|
||||
let mut response = (
|
||||
status,
|
||||
Json(json!({
|
||||
"detail": detail,
|
||||
"required_permission": required_permission,
|
||||
})),
|
||||
)
|
||||
.into_response();
|
||||
response.headers_mut().insert(
|
||||
http::header::CACHE_CONTROL,
|
||||
HeaderValue::from_static("no-store"),
|
||||
);
|
||||
response
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ use crate::constants::*;
|
||||
use crate::control::GatewayControlDecision;
|
||||
use crate::control::GatewayLocalAuthRejection;
|
||||
use crate::headers::should_skip_response_header;
|
||||
use crate::plan_usage_policy::PlanUsagePolicyRejection;
|
||||
use crate::rate_limit::FrontdoorUserRpmRejection;
|
||||
use crate::{insert_header_if_missing, GatewayError};
|
||||
|
||||
@@ -52,22 +53,34 @@ pub(crate) fn apply_streaming_response_headers(headers: &mut http::HeaderMap) {
|
||||
);
|
||||
}
|
||||
|
||||
fn apply_gateway_browser_security_headers(headers: &mut http::HeaderMap) {
|
||||
// Provider responses are API data, even when an untrusted provider labels
|
||||
// them as HTML or SVG. Keep a direct navigation to a gateway API route
|
||||
// from becoming same-origin active content, and prevent referrer leakage
|
||||
// if a user follows a link rendered from such a response.
|
||||
headers.insert(
|
||||
http::header::X_CONTENT_TYPE_OPTIONS,
|
||||
HeaderValue::from_static("nosniff"),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("content-security-policy"),
|
||||
HeaderValue::from_static(
|
||||
"default-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'; sandbox",
|
||||
),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("referrer-policy"),
|
||||
HeaderValue::from_static("no-referrer"),
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn build_client_response(
|
||||
upstream_response: reqwest::Response,
|
||||
trace_id: &str,
|
||||
control_decision: Option<&GatewayControlDecision>,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let status = upstream_response.status();
|
||||
let upstream_headers = upstream_response
|
||||
.headers()
|
||||
.iter()
|
||||
.map(|(name, value)| {
|
||||
(
|
||||
name.as_str().to_string(),
|
||||
value.to_str().unwrap_or_default().to_string(),
|
||||
)
|
||||
})
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let upstream_headers = collect_safe_response_headers(upstream_response.headers());
|
||||
let upstream_stream = upstream_response.bytes_stream();
|
||||
build_client_response_from_parts(
|
||||
status.as_u16(),
|
||||
@@ -78,6 +91,40 @@ pub(crate) fn build_client_response(
|
||||
)
|
||||
}
|
||||
|
||||
fn collect_safe_response_headers(headers: &http::HeaderMap) -> BTreeMap<String, String> {
|
||||
let connection_declared = aether_http::connection_declared_header_names(
|
||||
headers
|
||||
.get_all(http::header::CONNECTION)
|
||||
.iter()
|
||||
.filter_map(|value| value.to_str().ok()),
|
||||
);
|
||||
headers
|
||||
.iter()
|
||||
.filter_map(|(name, value)| {
|
||||
let normalized = name.as_str().to_ascii_lowercase();
|
||||
if should_skip_client_response_header(&normalized)
|
||||
|| connection_declared.contains(&normalized)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
value
|
||||
.to_str()
|
||||
.ok()
|
||||
.map(|value| (normalized, value.to_string()))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn should_skip_client_response_header(name: &str) -> bool {
|
||||
should_skip_response_header(name)
|
||||
// A provider Location is relative to the provider, not to the gateway.
|
||||
// Forwarding it lets redirect-following clients bypass the gateway and
|
||||
// can disclose their gateway Authorization header to another origin.
|
||||
// Keep Location available inside execution reports, but never expose
|
||||
// it on the client-facing response boundary.
|
||||
|| name.eq_ignore_ascii_case(http::header::LOCATION.as_str())
|
||||
}
|
||||
|
||||
pub(crate) fn build_client_response_from_parts(
|
||||
status_code: u16,
|
||||
upstream_headers: &BTreeMap<String, String>,
|
||||
@@ -111,8 +158,17 @@ where
|
||||
.body(body)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
|
||||
let connection_declared = aether_http::connection_declared_header_names(
|
||||
upstream_headers
|
||||
.iter()
|
||||
.filter(|(name, _)| name.eq_ignore_ascii_case(http::header::CONNECTION.as_str()))
|
||||
.map(|(_, value)| value.as_str()),
|
||||
);
|
||||
|
||||
for (name, value) in upstream_headers {
|
||||
if should_skip_response_header(name.as_str()) {
|
||||
if should_skip_client_response_header(name.as_str())
|
||||
|| connection_declared.contains(&name.to_ascii_lowercase())
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let header_name = HeaderName::from_bytes(name.as_bytes())
|
||||
@@ -123,6 +179,7 @@ where
|
||||
}
|
||||
mutate_headers(response.headers_mut())?;
|
||||
apply_streaming_response_headers(response.headers_mut());
|
||||
apply_gateway_browser_security_headers(response.headers_mut());
|
||||
insert_header_if_missing(response.headers_mut(), TRACE_ID_HEADER, trace_id)?;
|
||||
insert_header_if_missing(response.headers_mut(), GATEWAY_HEADER, "rust-phase3b")?;
|
||||
if let Some(decision) = control_decision {
|
||||
@@ -258,6 +315,57 @@ pub(crate) fn build_local_user_rpm_limited_response(
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn build_local_plan_usage_limited_response(
|
||||
trace_id: &str,
|
||||
control_decision: Option<&GatewayControlDecision>,
|
||||
rejection: &PlanUsagePolicyRejection,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let message = "套餐使用限制已达到上限,请稍后重试";
|
||||
let fallback_payload = json!({
|
||||
"error": {
|
||||
"type": "plan_usage_limit_exceeded",
|
||||
"message": message,
|
||||
"details": {
|
||||
"metric": rejection.metric,
|
||||
"window": rejection.window,
|
||||
"limit": rejection.limit,
|
||||
"retry_after": rejection.retry_after,
|
||||
}
|
||||
}
|
||||
});
|
||||
let payload = build_local_error_payload(
|
||||
control_decision,
|
||||
None,
|
||||
message,
|
||||
LocalCoreSyncErrorKind::RateLimit,
|
||||
fallback_payload,
|
||||
);
|
||||
let body =
|
||||
serde_json::to_vec(&payload).map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
let headers = BTreeMap::from([
|
||||
("content-type".to_string(), "application/json".to_string()),
|
||||
("Retry-After".to_string(), rejection.retry_after.to_string()),
|
||||
("X-RateLimit-Limit".to_string(), rejection.limit.to_string()),
|
||||
("X-RateLimit-Remaining".to_string(), "0".to_string()),
|
||||
("X-RateLimit-Scope".to_string(), "plan".to_string()),
|
||||
(
|
||||
"X-RateLimit-Metric".to_string(),
|
||||
rejection.metric.to_string(),
|
||||
),
|
||||
(
|
||||
"X-RateLimit-Window".to_string(),
|
||||
rejection.window.to_string(),
|
||||
),
|
||||
]);
|
||||
build_client_response_from_parts(
|
||||
StatusCode::TOO_MANY_REQUESTS.as_u16(),
|
||||
&headers,
|
||||
Body::from(body),
|
||||
trace_id,
|
||||
control_decision,
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn build_local_http_error_response(
|
||||
trace_id: &str,
|
||||
control_decision: Option<&GatewayControlDecision>,
|
||||
@@ -454,11 +562,13 @@ fn local_error_kind_for_status(status: StatusCode) -> LocalCoreSyncErrorKind {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
build_client_response_from_parts, build_local_auth_rejection_response,
|
||||
build_client_response, build_client_response_from_parts,
|
||||
build_client_response_from_parts_with_mutator, build_local_auth_rejection_response,
|
||||
build_local_http_error_response_with_request_path, build_local_overloaded_response,
|
||||
build_local_user_rpm_limited_response,
|
||||
build_local_plan_usage_limited_response, build_local_user_rpm_limited_response,
|
||||
};
|
||||
use crate::control::{GatewayControlDecision, GatewayLocalAuthRejection};
|
||||
use crate::plan_usage_policy::PlanUsagePolicyRejection;
|
||||
use crate::rate_limit::FrontdoorUserRpmRejection;
|
||||
use axum::body::{to_bytes, Body};
|
||||
use std::collections::BTreeMap;
|
||||
@@ -490,6 +600,163 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upstream_security_headers_are_stripped_before_gateway_headers_are_added() {
|
||||
let response = build_client_response_from_parts_with_mutator(
|
||||
200,
|
||||
&BTreeMap::from([
|
||||
("set-cookie".to_string(), "session=attacker".to_string()),
|
||||
(
|
||||
"x-aether-gateway".to_string(),
|
||||
"attacker-gateway".to_string(),
|
||||
),
|
||||
(
|
||||
"x-aether-control-action".to_string(),
|
||||
"attacker-action".to_string(),
|
||||
),
|
||||
(
|
||||
"x-aether-future-control".to_string(),
|
||||
"attacker-future".to_string(),
|
||||
),
|
||||
(
|
||||
"x-accel-redirect".to_string(),
|
||||
"/internal/private-file".to_string(),
|
||||
),
|
||||
("x-sendfile".to_string(), "/etc/passwd".to_string()),
|
||||
(
|
||||
"x-reproxy-url".to_string(),
|
||||
"http://127.0.0.1:9000/private".to_string(),
|
||||
),
|
||||
(
|
||||
"access-control-allow-origin".to_string(),
|
||||
"https://attacker.example".to_string(),
|
||||
),
|
||||
(
|
||||
"access-control-allow-credentials".to_string(),
|
||||
"true".to_string(),
|
||||
),
|
||||
("content-length".to_string(), "999999".to_string()),
|
||||
(
|
||||
"content-security-policy".to_string(),
|
||||
"default-src * 'unsafe-inline' 'unsafe-eval'".to_string(),
|
||||
),
|
||||
(
|
||||
"content-security-policy-report-only".to_string(),
|
||||
"default-src 'none'; report-uri https://attacker.example/csp".to_string(),
|
||||
),
|
||||
(
|
||||
"reporting-endpoints".to_string(),
|
||||
"attacker=\"https://attacker.example/reports\"".to_string(),
|
||||
),
|
||||
("report-to".to_string(), "attacker".to_string()),
|
||||
(
|
||||
"nel".to_string(),
|
||||
"{\"report_to\":\"attacker\"}".to_string(),
|
||||
),
|
||||
(
|
||||
"refresh".to_string(),
|
||||
"0; url=https://attacker.example".to_string(),
|
||||
),
|
||||
("referrer-policy".to_string(), "unsafe-url".to_string()),
|
||||
("x-content-type-options".to_string(), "invalid".to_string()),
|
||||
(
|
||||
"location".to_string(),
|
||||
"https://provider.example/direct".to_string(),
|
||||
),
|
||||
("x-upstream-visible".to_string(), "ok".to_string()),
|
||||
]),
|
||||
Body::empty(),
|
||||
"trace-upstream-header-filter",
|
||||
None,
|
||||
|headers| {
|
||||
headers.insert(
|
||||
http::HeaderName::from_static("x-aether-control-action"),
|
||||
http::HeaderValue::from_static("gateway-action"),
|
||||
);
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.expect("response should build");
|
||||
|
||||
assert!(response.headers().get(http::header::SET_COOKIE).is_none());
|
||||
assert!(response.headers().get("x-aether-future-control").is_none());
|
||||
assert!(response.headers().get("x-accel-redirect").is_none());
|
||||
assert!(response.headers().get("x-sendfile").is_none());
|
||||
assert!(response.headers().get("x-reproxy-url").is_none());
|
||||
assert!(response
|
||||
.headers()
|
||||
.get("access-control-allow-origin")
|
||||
.is_none());
|
||||
assert!(response
|
||||
.headers()
|
||||
.get("access-control-allow-credentials")
|
||||
.is_none());
|
||||
assert!(response
|
||||
.headers()
|
||||
.get(http::header::CONTENT_LENGTH)
|
||||
.is_none());
|
||||
assert!(response
|
||||
.headers()
|
||||
.get("content-security-policy-report-only")
|
||||
.is_none());
|
||||
assert!(response.headers().get("reporting-endpoints").is_none());
|
||||
assert!(response.headers().get("report-to").is_none());
|
||||
assert!(response.headers().get("nel").is_none());
|
||||
assert!(response.headers().get("refresh").is_none());
|
||||
assert!(response.headers().get(http::header::LOCATION).is_none());
|
||||
assert_eq!(
|
||||
response.headers()["content-security-policy"],
|
||||
"default-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'; sandbox"
|
||||
);
|
||||
assert_eq!(response.headers()["referrer-policy"], "no-referrer");
|
||||
assert_eq!(
|
||||
response.headers()[http::header::X_CONTENT_TYPE_OPTIONS],
|
||||
"nosniff"
|
||||
);
|
||||
assert_eq!(response.headers()["x-aether-gateway"], "rust-phase3b");
|
||||
assert_eq!(
|
||||
response.headers()["x-aether-control-action"],
|
||||
"gateway-action"
|
||||
);
|
||||
assert_eq!(response.headers()["x-upstream-visible"], "ok");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raw_response_collector_honors_all_connection_header_lines() {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("listener");
|
||||
let addr = listener.local_addr().expect("address");
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.expect("connection");
|
||||
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
|
||||
let mut request = [0_u8; 1024];
|
||||
let _ = stream.read(&mut request).await.expect("request read");
|
||||
stream
|
||||
.write_all(
|
||||
b"HTTP/1.1 200 OK\r\nConnection: x-first-hop\r\nConnection: x-second-hop\r\nX-First-Hop: first-secret\r\nX-Second-Hop: second-secret\r\nContent-Length: 2\r\n\r\nok",
|
||||
)
|
||||
.await
|
||||
.expect("response write");
|
||||
});
|
||||
let upstream = reqwest::Client::builder()
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.build()
|
||||
.expect("client")
|
||||
.get(format!("http://{addr}/"))
|
||||
.send()
|
||||
.await
|
||||
.expect("upstream response");
|
||||
|
||||
let response = build_client_response(upstream, "trace-connection-lines", None)
|
||||
.expect("client response");
|
||||
server.await.expect("server");
|
||||
|
||||
assert!(response.headers().get("connection").is_none());
|
||||
assert!(response.headers().get("x-first-hop").is_none());
|
||||
assert!(response.headers().get("x-second-hop").is_none());
|
||||
}
|
||||
|
||||
fn claude_decision() -> GatewayControlDecision {
|
||||
GatewayControlDecision::synthetic(
|
||||
"/v1/messages",
|
||||
@@ -581,4 +848,25 @@ mod tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn plan_usage_rejection_exposes_machine_readable_limit_headers() {
|
||||
let response = build_local_plan_usage_limited_response(
|
||||
"trace-plan-limit",
|
||||
None,
|
||||
&PlanUsagePolicyRejection {
|
||||
metric: "request_count",
|
||||
limit: 100.0,
|
||||
retry_after: 42,
|
||||
window: "calendar_week",
|
||||
},
|
||||
)
|
||||
.expect("response");
|
||||
assert_eq!(response.status(), http::StatusCode::TOO_MANY_REQUESTS);
|
||||
assert_eq!(response.headers()["retry-after"], "42");
|
||||
assert_eq!(response.headers()["x-ratelimit-scope"], "plan");
|
||||
assert_eq!(response.headers()["x-ratelimit-window"], "calendar_week");
|
||||
let payload = response_json(response).await;
|
||||
assert_eq!(payload["error"]["type"], "plan_usage_limit_exceeded");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use aether_contracts::ExecutionResult;
|
||||
@@ -21,7 +22,9 @@ use super::{
|
||||
};
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
pub(crate) use self::cancel::{cancel_video_task_record, CancelVideoTaskError};
|
||||
pub(crate) use self::cancel::{
|
||||
cancel_video_task_record, cancel_video_task_record_for_user, CancelVideoTaskError,
|
||||
};
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(crate) struct ListVideoTasksQuery {
|
||||
@@ -146,18 +149,21 @@ pub(crate) async fn get_video_task_video(
|
||||
}
|
||||
|
||||
pub(crate) async fn build_video_task_video_response(
|
||||
state: &AppState,
|
||||
_state: &AppState,
|
||||
task_id: &str,
|
||||
source: VideoTaskVideoSource,
|
||||
) -> Result<axum::response::Response, GatewayError> {
|
||||
match source {
|
||||
VideoTaskVideoSource::Redirect { url } => Ok(Redirect::temporary(&url).into_response()),
|
||||
VideoTaskVideoSource::Redirect { url } => {
|
||||
resolve_public_video_target(&url).await?;
|
||||
Ok(Redirect::temporary(url.as_str()).into_response())
|
||||
}
|
||||
VideoTaskVideoSource::Proxy {
|
||||
url,
|
||||
header_name,
|
||||
header_value,
|
||||
filename,
|
||||
} => proxy_video_stream(state, task_id, &url, &header_name, &header_value, &filename).await,
|
||||
} => proxy_video_stream(task_id, &url, &header_name, &header_value, &filename).await,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -209,25 +215,34 @@ fn video_task_status_name(status: VideoTaskStatus) -> &'static str {
|
||||
}
|
||||
|
||||
async fn proxy_video_stream(
|
||||
state: &AppState,
|
||||
task_id: &str,
|
||||
url: &str,
|
||||
url: &url::Url,
|
||||
header_name: &str,
|
||||
header_value: &str,
|
||||
filename: &str,
|
||||
) -> Result<axum::response::Response, GatewayError> {
|
||||
let response = state
|
||||
.client
|
||||
.get(url)
|
||||
let target = resolve_public_video_target(url).await?;
|
||||
let client = build_pinned_video_client(&target)?;
|
||||
let response = client
|
||||
.get(url.clone())
|
||||
.header(header_name, header_value)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| GatewayError::UpstreamUnavailable {
|
||||
trace_id: task_id.to_string(),
|
||||
message: err.to_string(),
|
||||
message: video_request_failure_message(&err).to_string(),
|
||||
})?;
|
||||
|
||||
if response.status().is_client_error() || response.status().is_server_error() {
|
||||
if response.status().is_redirection() {
|
||||
return Err(GatewayError::UpstreamUnavailable {
|
||||
trace_id: task_id.to_string(),
|
||||
message: format!(
|
||||
"video upstream redirect was rejected with HTTP {}",
|
||||
response.status()
|
||||
),
|
||||
});
|
||||
}
|
||||
if !response.status().is_success() {
|
||||
return Err(GatewayError::UpstreamUnavailable {
|
||||
trace_id: task_id.to_string(),
|
||||
message: format!("video upstream returned HTTP {}", response.status()),
|
||||
@@ -235,47 +250,321 @@ async fn proxy_video_stream(
|
||||
}
|
||||
|
||||
let status = response.status();
|
||||
let content_type = response
|
||||
.headers()
|
||||
.get(axum::http::header::CONTENT_TYPE)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| axum::http::HeaderValue::from_static("video/mp4"));
|
||||
let content_length = response
|
||||
.headers()
|
||||
.get(axum::http::header::CONTENT_LENGTH)
|
||||
.cloned();
|
||||
let cache_control = response
|
||||
.headers()
|
||||
.get(axum::http::header::CACHE_CONTROL)
|
||||
.cloned();
|
||||
// Do not copy the provider's Content-Length onto a newly wrapped stream.
|
||||
// Reqwest may decode transfer/content encodings and the provider controls
|
||||
// the declaration; forwarding a stale value would make the client-facing
|
||||
// HTTP framing disagree with the bytes produced by this Body. Axum/Hyper
|
||||
// will select safe framing for the actual stream.
|
||||
let upstream_headers = response.headers().clone();
|
||||
let body = Body::from_stream(response.bytes_stream());
|
||||
|
||||
let mut outbound = axum::http::Response::builder()
|
||||
.status(status)
|
||||
.body(body)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
outbound
|
||||
.headers_mut()
|
||||
.insert(axum::http::header::CONTENT_TYPE, content_type);
|
||||
outbound.headers_mut().insert(
|
||||
axum::http::header::CONTENT_DISPOSITION,
|
||||
axum::http::HeaderValue::from_str(&format!("inline; filename=\"{filename}\""))
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?,
|
||||
);
|
||||
if let Some(content_length) = content_length {
|
||||
outbound
|
||||
.headers_mut()
|
||||
.insert(axum::http::header::CONTENT_LENGTH, content_length);
|
||||
}
|
||||
if let Some(cache_control) = cache_control {
|
||||
outbound
|
||||
.headers_mut()
|
||||
.insert(axum::http::header::CACHE_CONTROL, cache_control);
|
||||
} else {
|
||||
outbound.headers_mut().insert(
|
||||
axum::http::header::CACHE_CONTROL,
|
||||
axum::http::HeaderValue::from_static("private, max-age=3600"),
|
||||
);
|
||||
}
|
||||
apply_safe_video_response_metadata(outbound.headers_mut(), &upstream_headers, filename)?;
|
||||
Ok(outbound)
|
||||
}
|
||||
|
||||
fn apply_safe_video_response_metadata(
|
||||
outbound: &mut axum::http::HeaderMap,
|
||||
upstream: &axum::http::HeaderMap,
|
||||
filename: &str,
|
||||
) -> Result<(), GatewayError> {
|
||||
let content_type = upstream
|
||||
.get(axum::http::header::CONTENT_TYPE)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(safe_video_content_type)
|
||||
.unwrap_or_else(|| axum::http::HeaderValue::from_static("application/octet-stream"));
|
||||
outbound.insert(axum::http::header::CONTENT_TYPE, content_type);
|
||||
outbound.insert(
|
||||
axum::http::header::CONTENT_DISPOSITION,
|
||||
axum::http::HeaderValue::from_str(&format!(
|
||||
"inline; filename=\"{}\"",
|
||||
safe_video_filename(filename)
|
||||
))
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?,
|
||||
);
|
||||
outbound.remove(axum::http::header::CONTENT_LENGTH);
|
||||
outbound.insert(
|
||||
axum::http::header::CACHE_CONTROL,
|
||||
axum::http::HeaderValue::from_static("private, no-store"),
|
||||
);
|
||||
outbound.insert(
|
||||
axum::http::header::X_CONTENT_TYPE_OPTIONS,
|
||||
axum::http::HeaderValue::from_static("nosniff"),
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn safe_video_content_type(raw_value: &str) -> Option<axum::http::HeaderValue> {
|
||||
let media_type = raw_value.split(';').next()?.trim().to_ascii_lowercase();
|
||||
let subtype = media_type.strip_prefix("video/")?;
|
||||
if subtype.is_empty()
|
||||
|| !subtype.bytes().all(|byte| {
|
||||
byte.is_ascii_alphanumeric()
|
||||
|| matches!(
|
||||
byte,
|
||||
b'!' | b'#' | b'$' | b'&' | b'-' | b'^' | b'_' | b'.' | b'+'
|
||||
)
|
||||
})
|
||||
{
|
||||
return None;
|
||||
}
|
||||
axum::http::HeaderValue::from_str(raw_value).ok()
|
||||
}
|
||||
|
||||
fn safe_video_filename(filename: &str) -> String {
|
||||
let filename = filename
|
||||
.chars()
|
||||
.take(255)
|
||||
.map(|character| {
|
||||
if character.is_ascii_alphanumeric() || matches!(character, '.' | '-' | '_') {
|
||||
character
|
||||
} else {
|
||||
'_'
|
||||
}
|
||||
})
|
||||
.collect::<String>();
|
||||
if filename.is_empty() {
|
||||
"video.mp4".to_string()
|
||||
} else {
|
||||
filename
|
||||
}
|
||||
}
|
||||
|
||||
struct ResolvedVideoTarget {
|
||||
host: String,
|
||||
addrs: Vec<SocketAddr>,
|
||||
}
|
||||
|
||||
async fn resolve_public_video_target(url: &url::Url) -> Result<ResolvedVideoTarget, GatewayError> {
|
||||
if !matches!(url.scheme(), "http" | "https")
|
||||
|| !url.username().is_empty()
|
||||
|| url.password().is_some()
|
||||
{
|
||||
return Err(video_target_rejected(
|
||||
"video URL must be an absolute HTTP(S) URL without credentials",
|
||||
));
|
||||
}
|
||||
let port = url
|
||||
.port_or_known_default()
|
||||
.ok_or_else(|| video_target_rejected("video URL is missing a port"))?;
|
||||
let (host, addrs) = match url.host() {
|
||||
Some(url::Host::Ipv4(ip)) => (ip.to_string(), vec![SocketAddr::new(IpAddr::V4(ip), port)]),
|
||||
Some(url::Host::Ipv6(ip)) => (ip.to_string(), vec![SocketAddr::new(IpAddr::V6(ip), port)]),
|
||||
Some(url::Host::Domain(host)) if !host.is_empty() => {
|
||||
let addrs = aether_http::lookup_host_with_limits(
|
||||
host,
|
||||
port,
|
||||
aether_http::DEFAULT_DNS_LOOKUP_TIMEOUT,
|
||||
)
|
||||
.await
|
||||
.map_err(|_| video_target_rejected("video URL DNS resolution failed"))?;
|
||||
(host.to_string(), addrs)
|
||||
}
|
||||
_ => return Err(video_target_rejected("video URL is missing a host")),
|
||||
};
|
||||
if addrs.is_empty()
|
||||
|| addrs
|
||||
.iter()
|
||||
.any(|addr| aether_http::is_private_or_reserved_ip(addr.ip()))
|
||||
{
|
||||
return Err(video_target_rejected(
|
||||
"video URL resolves to a private or reserved address",
|
||||
));
|
||||
}
|
||||
Ok(ResolvedVideoTarget { host, addrs })
|
||||
}
|
||||
|
||||
fn build_pinned_video_client(
|
||||
target: &ResolvedVideoTarget,
|
||||
) -> Result<reqwest::Client, GatewayError> {
|
||||
let mut builder = aether_http::apply_http_client_config(
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.redirect(reqwest::redirect::Policy::none()),
|
||||
&aether_http::HttpClientConfig {
|
||||
connect_timeout_ms: Some(10_000),
|
||||
request_timeout_ms: Some(300_000),
|
||||
http2_adaptive_window: true,
|
||||
..aether_http::HttpClientConfig::default()
|
||||
},
|
||||
);
|
||||
if target.host.parse::<IpAddr>().is_err() {
|
||||
builder = builder.resolve_to_addrs(&target.host, &target.addrs);
|
||||
}
|
||||
builder
|
||||
.build()
|
||||
.map_err(|_| GatewayError::Internal("video HTTP client initialization failed".to_string()))
|
||||
}
|
||||
|
||||
fn video_target_rejected(message: &str) -> GatewayError {
|
||||
GatewayError::Client {
|
||||
status: axum::http::StatusCode::BAD_GATEWAY,
|
||||
message: message.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn video_request_failure_message(error: &reqwest::Error) -> &'static str {
|
||||
if error.is_timeout() {
|
||||
"video upstream request timed out"
|
||||
} else if error.is_connect() {
|
||||
"video upstream connection failed"
|
||||
} else if error.is_body() || error.is_decode() {
|
||||
"video upstream response failed"
|
||||
} else {
|
||||
"video upstream request failed"
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use axum::response::IntoResponse;
|
||||
|
||||
use super::{
|
||||
apply_safe_video_response_metadata, build_video_task_video_response,
|
||||
resolve_public_video_target, safe_video_content_type, safe_video_filename,
|
||||
VideoTaskVideoSource,
|
||||
};
|
||||
use crate::AppState;
|
||||
|
||||
#[tokio::test]
|
||||
async fn video_redirect_response_accepts_public_target() {
|
||||
let state = AppState::new().expect("gateway state should build");
|
||||
let target = "https://8.8.8.8/video.mp4";
|
||||
|
||||
let response = build_video_task_video_response(
|
||||
&state,
|
||||
"task-public-redirect",
|
||||
VideoTaskVideoSource::Redirect {
|
||||
url: url::Url::parse(target).expect("public target should parse"),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("public redirect should build");
|
||||
|
||||
assert_eq!(
|
||||
response.status(),
|
||||
axum::http::StatusCode::TEMPORARY_REDIRECT
|
||||
);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get(axum::http::header::LOCATION)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some(target)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn video_redirect_response_rejects_private_and_reserved_targets() {
|
||||
let state = AppState::new().expect("gateway state should build");
|
||||
|
||||
for raw_url in [
|
||||
"http://127.0.0.1/video.mp4",
|
||||
"http://169.254.169.254/latest/meta-data",
|
||||
"http://10.0.0.1/video.mp4",
|
||||
"http://[::1]/video.mp4",
|
||||
] {
|
||||
let error = build_video_task_video_response(
|
||||
&state,
|
||||
"task-rejected-redirect",
|
||||
VideoTaskVideoSource::Redirect {
|
||||
url: url::Url::parse(raw_url).expect("target should parse"),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("private or reserved redirect target should be rejected");
|
||||
|
||||
assert_eq!(
|
||||
error.into_response().status(),
|
||||
axum::http::StatusCode::BAD_GATEWAY,
|
||||
"unexpected status for {raw_url}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn video_target_resolution_rejects_private_and_reserved_ip_literals() {
|
||||
for raw_url in [
|
||||
"http://127.0.0.1/video.mp4",
|
||||
"http://169.254.169.254/latest/meta-data",
|
||||
"http://10.0.0.1/video.mp4",
|
||||
"http://[::1]/video.mp4",
|
||||
"http://[::ffff:127.0.0.1]/video.mp4",
|
||||
] {
|
||||
let url = url::Url::parse(raw_url).unwrap();
|
||||
assert!(
|
||||
resolve_public_video_target(&url).await.is_err(),
|
||||
"target should be rejected: {raw_url}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn video_target_resolution_accepts_public_ip_literals() {
|
||||
for raw_url in [
|
||||
"https://8.8.8.8/video.mp4",
|
||||
"https://[2606:4700:4700::1111]/video.mp4",
|
||||
] {
|
||||
let url = url::Url::parse(raw_url).unwrap();
|
||||
assert!(
|
||||
resolve_public_video_target(&url).await.is_ok(),
|
||||
"target should be accepted: {raw_url}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_response_metadata_rejects_active_content_and_sanitizes_filename() {
|
||||
assert!(safe_video_content_type("video/mp4").is_some());
|
||||
assert!(safe_video_content_type("video/webm; charset=binary").is_some());
|
||||
assert!(safe_video_content_type("video/").is_none());
|
||||
assert!(safe_video_content_type("video/; charset=binary").is_none());
|
||||
assert!(safe_video_content_type("text/html").is_none());
|
||||
assert!(safe_video_content_type("video/mp4\r\nx-test: injected").is_none());
|
||||
assert_eq!(
|
||||
safe_video_filename("video_123.mp4\"; filename=\"attack.html"),
|
||||
"video_123.mp4___filename__attack.html"
|
||||
);
|
||||
assert_eq!(safe_video_filename(&"x".repeat(1024)).len(), 255);
|
||||
|
||||
let mut upstream = axum::http::HeaderMap::new();
|
||||
upstream.insert(
|
||||
axum::http::header::CONTENT_TYPE,
|
||||
axum::http::HeaderValue::from_static("text/html"),
|
||||
);
|
||||
upstream.insert(
|
||||
axum::http::header::CONTENT_LENGTH,
|
||||
axum::http::HeaderValue::from_static("999999"),
|
||||
);
|
||||
let mut outbound = upstream.clone();
|
||||
apply_safe_video_response_metadata(
|
||||
&mut outbound,
|
||||
&upstream,
|
||||
"video.mp4\"; filename=\"attack.html",
|
||||
)
|
||||
.expect("video metadata should build");
|
||||
|
||||
assert_eq!(
|
||||
outbound
|
||||
.get(axum::http::header::CONTENT_TYPE)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("application/octet-stream")
|
||||
);
|
||||
assert!(outbound.get(axum::http::header::CONTENT_LENGTH).is_none());
|
||||
assert_eq!(
|
||||
outbound
|
||||
.get(axum::http::header::X_CONTENT_TYPE_OPTIONS)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("nosniff")
|
||||
);
|
||||
assert_eq!(
|
||||
outbound
|
||||
.get(axum::http::header::CONTENT_DISPOSITION)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("inline; filename=\"video.mp4___filename__attack.html\"")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,12 +3,13 @@ use aether_data_contracts::repository::video_tasks::{
|
||||
};
|
||||
use axum::response::IntoResponse;
|
||||
use axum::Json;
|
||||
use serde_json::{json, Map, Value};
|
||||
use serde_json::json;
|
||||
|
||||
use crate::state::VideoTaskRouteAccess;
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
use super::super::finalize_video_task_if_terminal;
|
||||
use super::super::read_video_task_detail;
|
||||
use super::super::{read_video_task_detail, read_video_task_detail_for_user};
|
||||
use super::current_unix_secs;
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -29,7 +30,31 @@ pub(crate) async fn cancel_video_task_record(
|
||||
state: &AppState,
|
||||
task_id: &str,
|
||||
) -> Result<StoredVideoTask, CancelVideoTaskError> {
|
||||
let Some(task) = read_video_task_detail(state, task_id).await? else {
|
||||
cancel_video_task_record_inner(state, task_id, None).await
|
||||
}
|
||||
|
||||
pub(crate) async fn cancel_video_task_record_for_user(
|
||||
state: &AppState,
|
||||
task_id: &str,
|
||||
user_id: &str,
|
||||
) -> Result<StoredVideoTask, CancelVideoTaskError> {
|
||||
let user_id = user_id.trim();
|
||||
if user_id.is_empty() {
|
||||
return Err(CancelVideoTaskError::NotFound);
|
||||
}
|
||||
cancel_video_task_record_inner(state, task_id, Some(user_id)).await
|
||||
}
|
||||
|
||||
async fn cancel_video_task_record_inner(
|
||||
state: &AppState,
|
||||
task_id: &str,
|
||||
expected_user_id: Option<&str>,
|
||||
) -> Result<StoredVideoTask, CancelVideoTaskError> {
|
||||
let task = match expected_user_id {
|
||||
Some(user_id) => read_video_task_detail_for_user(state, task_id, user_id).await?,
|
||||
None => read_video_task_detail(state, task_id).await?,
|
||||
};
|
||||
let Some(task) = task else {
|
||||
return Err(CancelVideoTaskError::NotFound);
|
||||
};
|
||||
|
||||
@@ -45,39 +70,84 @@ pub(crate) async fn cancel_video_task_record(
|
||||
}
|
||||
|
||||
let trace_id = format!("async-task-admin-cancel-{task_id}");
|
||||
let mut finalize_mutation = None;
|
||||
if let Some(cancel_plan) = build_video_task_cancel_plan(&task) {
|
||||
state
|
||||
.hydrate_video_task_for_route(Some(cancel_plan.route_family), &cancel_plan.request_path)
|
||||
.await?;
|
||||
|
||||
let body_json = json!({});
|
||||
let follow_up = state.video_tasks.prepare_follow_up_sync_plan(
|
||||
cancel_plan.plan_kind,
|
||||
&cancel_plan.request_path,
|
||||
Some(&body_json),
|
||||
None,
|
||||
&trace_id,
|
||||
);
|
||||
let follow_up = if let Some(user_id) = expected_user_id {
|
||||
if state
|
||||
.hydrate_video_task_for_route_for_user(
|
||||
Some(cancel_plan.route_family),
|
||||
&cancel_plan.request_path,
|
||||
user_id,
|
||||
)
|
||||
.await?
|
||||
!= VideoTaskRouteAccess::Allowed
|
||||
{
|
||||
return Err(CancelVideoTaskError::NotFound);
|
||||
}
|
||||
state.video_tasks.prepare_follow_up_sync_plan_for_user_id(
|
||||
cancel_plan.plan_kind,
|
||||
&cancel_plan.request_path,
|
||||
Some(&body_json),
|
||||
user_id,
|
||||
task.api_key_id.as_deref(),
|
||||
&trace_id,
|
||||
)
|
||||
} else {
|
||||
state
|
||||
.hydrate_video_task_for_route(
|
||||
Some(cancel_plan.route_family),
|
||||
&cancel_plan.request_path,
|
||||
)
|
||||
.await?;
|
||||
state.video_tasks.prepare_follow_up_sync_plan(
|
||||
cancel_plan.plan_kind,
|
||||
&cancel_plan.request_path,
|
||||
Some(&body_json),
|
||||
None,
|
||||
&trace_id,
|
||||
)
|
||||
};
|
||||
|
||||
if let Some(follow_up) = follow_up {
|
||||
execute_video_task_cancel_plan(state, &trace_id, follow_up.plan)
|
||||
.await
|
||||
.map_err(CancelVideoTaskError::Response)?;
|
||||
finalize_mutation = Some((
|
||||
cancel_plan.request_path,
|
||||
cancel_plan.report_kind.to_string(),
|
||||
));
|
||||
} else if expected_user_id.is_none() {
|
||||
finalize_mutation = Some((
|
||||
cancel_plan.request_path,
|
||||
cancel_plan.report_kind.to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
state
|
||||
.video_tasks
|
||||
.apply_finalize_mutation(&cancel_plan.request_path, cancel_plan.report_kind);
|
||||
}
|
||||
|
||||
let request_metadata = build_cancelled_request_metadata(state, &task).await?;
|
||||
let stored = persist_cancelled_video_task(state, &task, request_metadata)
|
||||
.await?
|
||||
.ok_or_else(|| {
|
||||
CancelVideoTaskError::Gateway(GatewayError::Internal(
|
||||
let stored = match persist_cancelled_video_task(state, &task).await? {
|
||||
Some(stored) => stored,
|
||||
None => {
|
||||
let current = match expected_user_id {
|
||||
Some(user_id) => read_video_task_detail_for_user(state, task_id, user_id).await?,
|
||||
None => read_video_task_detail(state, task_id).await?,
|
||||
};
|
||||
let Some(current) = current else {
|
||||
return Err(CancelVideoTaskError::NotFound);
|
||||
};
|
||||
if !current.status.is_active() {
|
||||
return Err(CancelVideoTaskError::InvalidStatus(current.status));
|
||||
}
|
||||
return Err(CancelVideoTaskError::Gateway(GatewayError::Internal(
|
||||
"video task repository is unavailable".to_string(),
|
||||
))
|
||||
})?;
|
||||
)));
|
||||
}
|
||||
};
|
||||
if let Some((request_path, report_kind)) = finalize_mutation {
|
||||
state
|
||||
.video_tasks
|
||||
.apply_finalize_mutation(&request_path, &report_kind);
|
||||
}
|
||||
finalize_video_task_if_terminal(state, &stored).await;
|
||||
Ok(stored)
|
||||
}
|
||||
@@ -91,12 +161,7 @@ struct VideoTaskCancelPlan<'a> {
|
||||
}
|
||||
|
||||
fn build_video_task_cancel_plan(task: &StoredVideoTask) -> Option<VideoTaskCancelPlan<'_>> {
|
||||
let provider_api_format = task
|
||||
.provider_api_format
|
||||
.as_deref()
|
||||
.or(task.client_api_format.as_deref())
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())?;
|
||||
let provider_api_format = task.effective_api_format()?;
|
||||
|
||||
match provider_api_format {
|
||||
"openai:video" => Some(VideoTaskCancelPlan {
|
||||
@@ -131,99 +196,52 @@ async fn execute_video_task_cancel_plan(
|
||||
let result =
|
||||
crate::execution_runtime::execute_execution_runtime_sync_plan(state, Some(trace_id), &plan)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
.map_err(|_| {
|
||||
GatewayError::UpstreamUnavailable {
|
||||
trace_id: trace_id.to_string(),
|
||||
message: format!("{err:?}"),
|
||||
message: "video cancellation request failed".to_string(),
|
||||
}
|
||||
.into_response()
|
||||
})?;
|
||||
|
||||
if result.status_code >= 400 {
|
||||
let status = axum::http::StatusCode::from_u16(result.status_code)
|
||||
.unwrap_or(axum::http::StatusCode::BAD_GATEWAY);
|
||||
let body_json = result
|
||||
.body
|
||||
.and_then(|body| body.json_body)
|
||||
.unwrap_or_else(|| {
|
||||
json!({
|
||||
"error": {
|
||||
"message": result
|
||||
.error
|
||||
.as_ref()
|
||||
.map(|error| error.message.clone())
|
||||
.unwrap_or_else(|| {
|
||||
format!("execution runtime returned {}", result.status_code)
|
||||
}),
|
||||
}
|
||||
})
|
||||
});
|
||||
return Err((status, Json(body_json)).into_response());
|
||||
return Err(build_video_task_cancel_upstream_error_response(&result));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn build_cancelled_request_metadata(
|
||||
state: &AppState,
|
||||
task: &StoredVideoTask,
|
||||
) -> Result<Option<Value>, GatewayError> {
|
||||
let mut metadata = match task.request_metadata.clone() {
|
||||
Some(Value::Object(object)) => object,
|
||||
_ => Map::new(),
|
||||
};
|
||||
let mut snapshot_value = metadata.get("rust_local_snapshot").cloned();
|
||||
if snapshot_value.is_none() {
|
||||
snapshot_value = state
|
||||
.reconstruct_video_task_snapshot(task)
|
||||
.await?
|
||||
.map(|snapshot| {
|
||||
serde_json::to_value(snapshot)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))
|
||||
})
|
||||
.transpose()?;
|
||||
}
|
||||
if let Some(snapshot_value_ref) = snapshot_value.as_mut() {
|
||||
mark_snapshot_value_cancelled(snapshot_value_ref);
|
||||
metadata.insert(
|
||||
"rust_owner".to_string(),
|
||||
Value::String("async_task".to_string()),
|
||||
);
|
||||
metadata.insert(
|
||||
"rust_local_snapshot".to_string(),
|
||||
snapshot_value_ref.clone(),
|
||||
);
|
||||
return Ok(Some(Value::Object(metadata)));
|
||||
}
|
||||
|
||||
Ok(task.request_metadata.clone())
|
||||
}
|
||||
|
||||
fn mark_snapshot_value_cancelled(snapshot_value: &mut Value) {
|
||||
if let Some(object) = snapshot_value
|
||||
.get_mut("OpenAi")
|
||||
.and_then(Value::as_object_mut)
|
||||
{
|
||||
object.insert("status".to_string(), Value::String("Cancelled".to_string()));
|
||||
return;
|
||||
}
|
||||
if let Some(object) = snapshot_value
|
||||
.get_mut("Gemini")
|
||||
.and_then(Value::as_object_mut)
|
||||
{
|
||||
object.insert("status".to_string(), Value::String("Cancelled".to_string()));
|
||||
}
|
||||
fn build_video_task_cancel_upstream_error_response(
|
||||
result: &aether_contracts::ExecutionResult,
|
||||
) -> axum::response::Response {
|
||||
let status = axum::http::StatusCode::from_u16(result.status_code)
|
||||
.unwrap_or(axum::http::StatusCode::BAD_GATEWAY);
|
||||
tracing::warn!(
|
||||
event_name = "video_task_cancel_upstream_error",
|
||||
upstream_status = result.status_code,
|
||||
"video cancellation upstream response body discarded"
|
||||
);
|
||||
(
|
||||
status,
|
||||
Json(json!({
|
||||
"error": {
|
||||
"message": format!(
|
||||
"video cancellation upstream returned HTTP {}",
|
||||
result.status_code
|
||||
),
|
||||
}
|
||||
})),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
async fn persist_cancelled_video_task(
|
||||
state: &AppState,
|
||||
task: &StoredVideoTask,
|
||||
request_metadata: Option<Value>,
|
||||
) -> Result<Option<StoredVideoTask>, GatewayError> {
|
||||
let now_unix_secs = current_unix_secs();
|
||||
state
|
||||
.data
|
||||
.upsert_video_task(UpsertVideoTask {
|
||||
.update_active_video_task(UpsertVideoTask {
|
||||
id: task.id.clone(),
|
||||
short_id: task.short_id.clone(),
|
||||
request_id: task.request_id.clone(),
|
||||
@@ -240,14 +258,14 @@ async fn persist_cancelled_video_task(
|
||||
format_converted: task.format_converted,
|
||||
model: task.model.clone(),
|
||||
prompt: task.prompt.clone(),
|
||||
original_request_body: task.original_request_body.clone(),
|
||||
original_request_body: None,
|
||||
duration_seconds: task.duration_seconds,
|
||||
resolution: task.resolution.clone(),
|
||||
aspect_ratio: task.aspect_ratio.clone(),
|
||||
size: task.size.clone(),
|
||||
status: VideoTaskStatus::Cancelled,
|
||||
progress_percent: task.progress_percent,
|
||||
progress_message: task.progress_message.clone(),
|
||||
progress_message: None,
|
||||
retry_count: task.retry_count,
|
||||
poll_interval_seconds: task.poll_interval_seconds,
|
||||
next_poll_at_unix_secs: None,
|
||||
@@ -258,10 +276,73 @@ async fn persist_cancelled_video_task(
|
||||
completed_at_unix_secs: Some(now_unix_secs),
|
||||
updated_at_unix_secs: now_unix_secs,
|
||||
error_code: task.error_code.clone(),
|
||||
error_message: task.error_message.clone(),
|
||||
error_message: None,
|
||||
video_url: task.video_url.clone(),
|
||||
request_metadata,
|
||||
request_metadata: None,
|
||||
})
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use aether_contracts::{
|
||||
ExecutionError, ExecutionErrorKind, ExecutionPhase, ExecutionResult, ResponseBody,
|
||||
};
|
||||
use axum::body::to_bytes;
|
||||
use serde_json::json;
|
||||
|
||||
use super::build_video_task_cancel_upstream_error_response;
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancellation_upstream_errors_do_not_expose_runtime_payloads() {
|
||||
let result = ExecutionResult {
|
||||
request_id: "cancel-secret-request-id".to_string(),
|
||||
candidate_id: Some("cancel-secret-candidate-id".to_string()),
|
||||
status_code: 502,
|
||||
headers: BTreeMap::from([(
|
||||
"x-internal-secret".to_string(),
|
||||
"cancel-secret-header".to_string(),
|
||||
)]),
|
||||
response_observation: None,
|
||||
body: Some(ResponseBody {
|
||||
json_body: Some(json!({
|
||||
"error": {
|
||||
"message": "cancel-secret-upstream-body",
|
||||
}
|
||||
})),
|
||||
body_bytes_b64: None,
|
||||
}),
|
||||
telemetry: None,
|
||||
error: Some(ExecutionError {
|
||||
kind: ExecutionErrorKind::Upstream5xx,
|
||||
phase: ExecutionPhase::FirstByte,
|
||||
message: "cancel-secret-runtime-error".to_string(),
|
||||
upstream_status: Some(502),
|
||||
retryable: true,
|
||||
failover_recommended: false,
|
||||
}),
|
||||
};
|
||||
|
||||
let response = build_video_task_cancel_upstream_error_response(&result);
|
||||
assert_eq!(response.status(), axum::http::StatusCode::BAD_GATEWAY);
|
||||
assert!(response.headers().get("x-internal-secret").is_none());
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("response body should read");
|
||||
let payload: serde_json::Value =
|
||||
serde_json::from_slice(&body).expect("response body should parse");
|
||||
|
||||
assert_eq!(
|
||||
payload,
|
||||
json!({
|
||||
"error": {
|
||||
"message": "video cancellation upstream returned HTTP 502",
|
||||
}
|
||||
})
|
||||
);
|
||||
let body = String::from_utf8(body.to_vec()).expect("response body should be utf-8");
|
||||
assert!(!body.contains("cancel-secret"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,13 +6,14 @@ pub(crate) use crate::video_tasks::VideoTaskService;
|
||||
pub use crate::video_tasks::VideoTaskTruthSourceMode;
|
||||
pub(crate) use http::{
|
||||
build_video_task_video_response, cancel_video_task, cancel_video_task_record,
|
||||
get_video_task_detail, get_video_task_stats, get_video_task_video, list_video_tasks,
|
||||
CancelVideoTaskError,
|
||||
cancel_video_task_record_for_user, get_video_task_detail, get_video_task_stats,
|
||||
get_video_task_video, list_video_tasks, CancelVideoTaskError,
|
||||
};
|
||||
pub(crate) use query::{
|
||||
read_video_task_detail, read_video_task_page, read_video_task_page_summary,
|
||||
read_video_task_stats, read_video_task_video_source, VideoTaskPageResponse,
|
||||
VideoTaskStatsResponse, VideoTaskVideoSource,
|
||||
read_video_task_detail, read_video_task_detail_for_user, read_video_task_page,
|
||||
read_video_task_page_summary, read_video_task_stats, read_video_task_video_source,
|
||||
video_task_video_source_from_task, VideoTaskPageResponse, VideoTaskStatsResponse,
|
||||
VideoTaskVideoSource,
|
||||
};
|
||||
pub(crate) use runtime::{
|
||||
execute_video_task_refresh_plan, finalize_video_task_if_terminal, spawn_video_task_poller,
|
||||
|
||||
@@ -26,13 +26,12 @@ pub(crate) struct VideoTaskStatsResponse {
|
||||
pub(crate) processing_count: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) enum VideoTaskVideoSource {
|
||||
Redirect {
|
||||
url: String,
|
||||
url: url::Url,
|
||||
},
|
||||
Proxy {
|
||||
url: String,
|
||||
url: url::Url,
|
||||
header_name: String,
|
||||
header_value: String,
|
||||
filename: String,
|
||||
@@ -102,6 +101,14 @@ pub(crate) async fn read_video_task_detail(
|
||||
state.find_video_task_by_id(task_id).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_video_task_detail_for_user(
|
||||
state: &AppState,
|
||||
task_id: &str,
|
||||
user_id: &str,
|
||||
) -> Result<Option<StoredVideoTask>, GatewayError> {
|
||||
state.find_video_task_by_id_for_user(task_id, user_id).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_video_task_video_source(
|
||||
state: &AppState,
|
||||
task_id: &str,
|
||||
@@ -109,6 +116,13 @@ pub(crate) async fn read_video_task_video_source(
|
||||
let Some(task) = read_video_task_detail(state, task_id).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
video_task_video_source_from_task(state, &task).await
|
||||
}
|
||||
|
||||
pub(crate) async fn video_task_video_source_from_task(
|
||||
state: &AppState,
|
||||
task: &StoredVideoTask,
|
||||
) -> Result<Option<VideoTaskVideoSource>, GatewayError> {
|
||||
let Some(video_url) = task
|
||||
.video_url
|
||||
.as_deref()
|
||||
@@ -119,7 +133,9 @@ pub(crate) async fn read_video_task_video_source(
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
if !video_url.contains("generativelanguage.googleapis.com") {
|
||||
let video_url = parse_video_url(&video_url)?;
|
||||
|
||||
if task.effective_api_format() != Some("gemini:video") {
|
||||
return Ok(Some(VideoTaskVideoSource::Redirect { url: video_url }));
|
||||
}
|
||||
|
||||
@@ -148,6 +164,15 @@ pub(crate) async fn read_video_task_video_source(
|
||||
));
|
||||
};
|
||||
|
||||
let endpoint_url = parse_video_url(transport.endpoint.base_url.trim()).map_err(|_| {
|
||||
GatewayError::Internal("provider endpoint URL is invalid for proxied video".to_string())
|
||||
})?;
|
||||
if !video_urls_share_origin(&endpoint_url, &video_url) {
|
||||
return Err(GatewayError::Client {
|
||||
status: axum::http::StatusCode::BAD_GATEWAY,
|
||||
message: "video URL origin does not match its provider endpoint".to_string(),
|
||||
});
|
||||
}
|
||||
let api_key = transport.key.decrypted_api_key.trim();
|
||||
if api_key.is_empty() {
|
||||
return Err(GatewayError::Internal(
|
||||
@@ -159,10 +184,34 @@ pub(crate) async fn read_video_task_video_source(
|
||||
url: video_url,
|
||||
header_name: "x-goog-api-key".to_string(),
|
||||
header_value: api_key.to_string(),
|
||||
filename: format!("video_{task_id}.mp4"),
|
||||
filename: format!("video_{}.mp4", task.id),
|
||||
}))
|
||||
}
|
||||
|
||||
fn parse_video_url(raw_url: &str) -> Result<url::Url, GatewayError> {
|
||||
let url = url::Url::parse(raw_url.trim()).map_err(|_| GatewayError::Client {
|
||||
status: axum::http::StatusCode::BAD_GATEWAY,
|
||||
message: "video URL is invalid".to_string(),
|
||||
})?;
|
||||
if !matches!(url.scheme(), "http" | "https")
|
||||
|| url.host_str().is_none()
|
||||
|| !url.username().is_empty()
|
||||
|| url.password().is_some()
|
||||
{
|
||||
return Err(GatewayError::Client {
|
||||
status: axum::http::StatusCode::BAD_GATEWAY,
|
||||
message: "video URL must be an absolute HTTP(S) URL without credentials".to_string(),
|
||||
});
|
||||
}
|
||||
Ok(url)
|
||||
}
|
||||
|
||||
fn video_urls_share_origin(left: &url::Url, right: &url::Url) -> bool {
|
||||
left.scheme() == right.scheme()
|
||||
&& left.host() == right.host()
|
||||
&& left.port_or_known_default() == right.port_or_known_default()
|
||||
}
|
||||
|
||||
pub(crate) async fn read_video_task_stats(
|
||||
state: &AppState,
|
||||
filter: &VideoTaskQueryFilter,
|
||||
@@ -226,3 +275,212 @@ fn status_key(status: VideoTaskStatus) -> String {
|
||||
fn start_of_utc_day(now_unix_secs: u64) -> u64 {
|
||||
now_unix_secs - (now_unix_secs % 86_400)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
||||
use aether_data::repository::video_tasks::InMemoryVideoTaskRepository;
|
||||
use aether_data_contracts::repository::provider_catalog::{
|
||||
ProviderCatalogReadRepository, StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
|
||||
StoredProviderCatalogProvider,
|
||||
};
|
||||
use aether_data_contracts::repository::video_tasks::{UpsertVideoTask, VideoTaskStatus};
|
||||
use serde_json::json;
|
||||
|
||||
use super::{
|
||||
parse_video_url, video_task_video_source_from_task, video_urls_share_origin,
|
||||
VideoTaskVideoSource,
|
||||
};
|
||||
use crate::{data::GatewayDataState, AppState};
|
||||
|
||||
fn legacy_gemini_video_task() -> aether_data_contracts::repository::video_tasks::StoredVideoTask
|
||||
{
|
||||
UpsertVideoTask {
|
||||
id: "legacy-gemini-task".to_string(),
|
||||
short_id: Some("legacy-short".to_string()),
|
||||
request_id: "legacy-request".to_string(),
|
||||
user_id: Some("user-1".to_string()),
|
||||
api_key_id: Some("client-key-1".to_string()),
|
||||
username: None,
|
||||
api_key_name: None,
|
||||
external_task_id: Some("operations/upstream-1".to_string()),
|
||||
provider_id: Some("provider-1".to_string()),
|
||||
endpoint_id: Some("endpoint-1".to_string()),
|
||||
key_id: Some("provider-key-1".to_string()),
|
||||
client_api_format: Some("gemini:video".to_string()),
|
||||
provider_api_format: None,
|
||||
format_converted: false,
|
||||
model: Some("veo-3".to_string()),
|
||||
prompt: None,
|
||||
original_request_body: None,
|
||||
duration_seconds: Some(8),
|
||||
resolution: Some("720p".to_string()),
|
||||
aspect_ratio: Some("16:9".to_string()),
|
||||
size: Some("1280x720".to_string()),
|
||||
status: VideoTaskStatus::Completed,
|
||||
progress_percent: 100,
|
||||
progress_message: None,
|
||||
retry_count: 0,
|
||||
poll_interval_seconds: 10,
|
||||
next_poll_at_unix_secs: None,
|
||||
poll_count: 1,
|
||||
max_poll_count: 360,
|
||||
created_at_unix_ms: 1,
|
||||
submitted_at_unix_secs: Some(1),
|
||||
completed_at_unix_secs: Some(2),
|
||||
updated_at_unix_secs: 2,
|
||||
error_code: None,
|
||||
error_message: None,
|
||||
video_url: Some(
|
||||
"https://generativelanguage.googleapis.com/v1beta/files/video-1:download?alt=media"
|
||||
.to_string(),
|
||||
),
|
||||
request_metadata: None,
|
||||
}
|
||||
.into_stored()
|
||||
}
|
||||
|
||||
fn state_with_gemini_transport() -> AppState {
|
||||
let state = AppState::new().expect("gateway state should build");
|
||||
let provider = StoredProviderCatalogProvider::new(
|
||||
"provider-1".to_string(),
|
||||
"Gemini".to_string(),
|
||||
Some("https://ai.google.dev".to_string()),
|
||||
"gemini".to_string(),
|
||||
)
|
||||
.expect("provider should build");
|
||||
let endpoint = StoredProviderCatalogEndpoint::new(
|
||||
"endpoint-1".to_string(),
|
||||
"provider-1".to_string(),
|
||||
"gemini:video".to_string(),
|
||||
None,
|
||||
None,
|
||||
true,
|
||||
)
|
||||
.expect("endpoint should build")
|
||||
.with_transport_fields(
|
||||
"https://generativelanguage.googleapis.com".to_string(),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.expect("endpoint transport should build");
|
||||
let encrypted_api_key = state
|
||||
.seal_provider_catalog_key_api_key(
|
||||
"provider-1",
|
||||
"provider-key-1",
|
||||
"gemini-provider-secret",
|
||||
)
|
||||
.expect("provider key should encrypt");
|
||||
let key = StoredProviderCatalogKey::new(
|
||||
"provider-key-1".to_string(),
|
||||
"provider-1".to_string(),
|
||||
"default".to_string(),
|
||||
"api_key".to_string(),
|
||||
None,
|
||||
true,
|
||||
)
|
||||
.expect("provider key should build")
|
||||
.with_transport_fields(
|
||||
Some(json!(["gemini:video"])),
|
||||
encrypted_api_key,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.expect("provider key transport should build");
|
||||
let provider_catalog: Arc<dyn ProviderCatalogReadRepository> = Arc::new(
|
||||
InMemoryProviderCatalogReadRepository::seed(vec![provider], vec![endpoint], vec![key]),
|
||||
);
|
||||
let video_tasks = Arc::new(InMemoryVideoTaskRepository::default());
|
||||
let data = GatewayDataState::with_video_task_repository_and_provider_transport_for_tests(
|
||||
video_tasks,
|
||||
provider_catalog,
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
state.with_data_state_for_tests(data)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_url_parser_rejects_non_http_and_embedded_credentials() {
|
||||
for raw_url in [
|
||||
"file:///etc/passwd",
|
||||
"data:video/mp4;base64,AAAA",
|
||||
"https://user@example.com/video.mp4",
|
||||
"https://user:password@example.com/video.mp4",
|
||||
"/relative/video.mp4",
|
||||
] {
|
||||
assert!(
|
||||
parse_video_url(raw_url).is_err(),
|
||||
"URL should be rejected: {raw_url}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_origin_comparison_uses_scheme_host_and_effective_port() {
|
||||
let base = parse_video_url("https://generativelanguage.googleapis.com/v1beta").unwrap();
|
||||
for same_origin in [
|
||||
"https://generativelanguage.googleapis.com/file",
|
||||
"https://generativelanguage.googleapis.com:443/file",
|
||||
] {
|
||||
assert!(video_urls_share_origin(
|
||||
&base,
|
||||
&parse_video_url(same_origin).unwrap()
|
||||
));
|
||||
}
|
||||
for different_origin in [
|
||||
"http://generativelanguage.googleapis.com/file",
|
||||
"https://generativelanguage.googleapis.com:444/file",
|
||||
"https://generativelanguage.googleapis.com.evil.test/file",
|
||||
"https://evil.test/generativelanguage.googleapis.com/file",
|
||||
] {
|
||||
assert!(!video_urls_share_origin(
|
||||
&base,
|
||||
&parse_video_url(different_origin).unwrap()
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_gemini_client_format_uses_authenticated_proxy_source() {
|
||||
let source = video_task_video_source_from_task(
|
||||
&state_with_gemini_transport(),
|
||||
&legacy_gemini_video_task(),
|
||||
)
|
||||
.await
|
||||
.expect("video source should resolve")
|
||||
.expect("video source should exist");
|
||||
|
||||
match source {
|
||||
VideoTaskVideoSource::Proxy {
|
||||
url,
|
||||
header_name,
|
||||
header_value,
|
||||
filename,
|
||||
} => {
|
||||
assert_eq!(
|
||||
url.as_str(),
|
||||
"https://generativelanguage.googleapis.com/v1beta/files/video-1:download?alt=media"
|
||||
);
|
||||
assert_eq!(header_name, "x-goog-api-key");
|
||||
assert_eq!(header_value, "gemini-provider-secret");
|
||||
assert_eq!(filename, "video_legacy-gemini-task.mp4");
|
||||
}
|
||||
VideoTaskVideoSource::Redirect { .. } => {
|
||||
panic!("legacy Gemini video must not bypass the authenticated proxy")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@ const VIDEO_TASK_POLL_CLAIM_SECONDS: u64 = 30;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct VideoTaskRefreshError {
|
||||
message: String,
|
||||
category: &'static str,
|
||||
permanent: bool,
|
||||
}
|
||||
|
||||
@@ -55,7 +55,7 @@ pub(crate) async fn execute_video_task_refresh_plan(
|
||||
warn!(
|
||||
event_name = "video_task_refresh_failed",
|
||||
log_type = "event",
|
||||
error = %err.message,
|
||||
error_category = err.category,
|
||||
permanent = err.permanent,
|
||||
"gateway video task refresh failed"
|
||||
);
|
||||
@@ -79,23 +79,32 @@ async fn poll_video_tasks_once(state: &AppState, batch_size: usize) -> Result<us
|
||||
let mut refreshed = 0usize;
|
||||
for (index, task) in tasks.into_iter().enumerate() {
|
||||
let trace_id = format!("video-task-poller-{index}");
|
||||
let Some(snapshot) = state.reconstruct_video_task_snapshot(&task).await? else {
|
||||
continue;
|
||||
};
|
||||
let Some(refresh_plan) = state
|
||||
.video_tasks
|
||||
.prepare_poll_refresh_plan_for_stored_task(&task, &trace_id)
|
||||
.prepare_poll_refresh_plan_for_snapshot(snapshot.clone(), &trace_id)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
|
||||
match fetch_video_task_refresh_attempt(state, &refresh_plan).await? {
|
||||
VideoTaskRefreshAttempt::Success { provider_body } => {
|
||||
let Some(updated) =
|
||||
build_successful_poll_update(&task, &provider_body, now_unix_secs)?
|
||||
let Some(updated) = build_successful_poll_update(
|
||||
&task,
|
||||
snapshot.clone(),
|
||||
&provider_body,
|
||||
now_unix_secs,
|
||||
)?
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
match state.update_active_video_task(updated).await? {
|
||||
Some(stored) => {
|
||||
if let Some(snapshot) = LocalVideoTaskSnapshot::from_stored_task(&stored) {
|
||||
if let Some(snapshot) =
|
||||
state.reconstruct_video_task_snapshot(&stored).await?
|
||||
{
|
||||
state.video_tasks.record_snapshot(snapshot);
|
||||
}
|
||||
info!(
|
||||
@@ -116,7 +125,9 @@ async fn poll_video_tasks_once(state: &AppState, batch_size: usize) -> Result<us
|
||||
let updated = build_failed_poll_update(&task, &err, now_unix_secs);
|
||||
match state.update_active_video_task(updated).await? {
|
||||
Some(stored) => {
|
||||
if let Some(snapshot) = LocalVideoTaskSnapshot::from_stored_task(&stored) {
|
||||
if let Some(snapshot) =
|
||||
state.reconstruct_video_task_snapshot(&stored).await?
|
||||
{
|
||||
state.video_tasks.record_snapshot(snapshot);
|
||||
}
|
||||
info!(
|
||||
@@ -190,9 +201,9 @@ async fn fetch_video_task_refresh_attempt(
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(err) => {
|
||||
Err(_) => {
|
||||
return Ok(VideoTaskRefreshAttempt::Error(VideoTaskRefreshError {
|
||||
message: format!("{err:?}"),
|
||||
category: "transport_error",
|
||||
permanent: false,
|
||||
}));
|
||||
}
|
||||
@@ -209,7 +220,7 @@ async fn fetch_video_task_refresh_attempt(
|
||||
.and_then(|body| body.as_object().cloned())
|
||||
else {
|
||||
return Ok(VideoTaskRefreshAttempt::Error(VideoTaskRefreshError {
|
||||
message: "video task refresh missing json provider body".to_string(),
|
||||
category: "invalid_provider_response",
|
||||
permanent: false,
|
||||
}));
|
||||
};
|
||||
@@ -223,20 +234,19 @@ fn classify_refresh_result_error(result: &ExecutionResult) -> VideoTaskRefreshEr
|
||||
.as_ref()
|
||||
.and_then(|error| error.upstream_status)
|
||||
.unwrap_or(result.status_code);
|
||||
let message = result
|
||||
.error
|
||||
.as_ref()
|
||||
.map(|error| error.message.clone())
|
||||
.or_else(|| {
|
||||
result
|
||||
.body
|
||||
.as_ref()
|
||||
.and_then(|body| body.json_body.as_ref())
|
||||
.and_then(|value| value.get("error"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_string)
|
||||
})
|
||||
.unwrap_or_else(|| format!("upstream returned {status_code}"));
|
||||
let category = if status_code == 401 {
|
||||
"authentication_error"
|
||||
} else if status_code == 403 {
|
||||
"permission_denied"
|
||||
} else if status_code == 404 {
|
||||
"not_found"
|
||||
} else if status_code == 429 {
|
||||
"rate_limit"
|
||||
} else if status_code >= 500 {
|
||||
"server_error"
|
||||
} else {
|
||||
"provider_error"
|
||||
};
|
||||
let permanent = result.error.as_ref().map_or(
|
||||
matches!(status_code, 400 | 401 | 403 | 404 | 422),
|
||||
|error| match error.kind {
|
||||
@@ -253,17 +263,18 @@ fn classify_refresh_result_error(result: &ExecutionResult) -> VideoTaskRefreshEr
|
||||
},
|
||||
);
|
||||
|
||||
VideoTaskRefreshError { message, permanent }
|
||||
VideoTaskRefreshError {
|
||||
category,
|
||||
permanent,
|
||||
}
|
||||
}
|
||||
|
||||
fn build_successful_poll_update(
|
||||
task: &StoredVideoTask,
|
||||
mut snapshot: LocalVideoTaskSnapshot,
|
||||
provider_body: &Map<String, Value>,
|
||||
now_unix_secs: u64,
|
||||
) -> Result<Option<UpsertVideoTask>, GatewayError> {
|
||||
let Some(mut snapshot) = LocalVideoTaskSnapshot::from_stored_task(task) else {
|
||||
return Ok(None);
|
||||
};
|
||||
snapshot.apply_provider_body(provider_body);
|
||||
|
||||
let mut record = snapshot.to_upsert_record();
|
||||
@@ -283,10 +294,7 @@ fn build_successful_poll_update(
|
||||
record.format_converted = task.format_converted;
|
||||
record.model = task.model.clone().or(record.model);
|
||||
record.prompt = task.prompt.clone().or(record.prompt);
|
||||
record.original_request_body = task
|
||||
.original_request_body
|
||||
.clone()
|
||||
.or(record.original_request_body);
|
||||
record.original_request_body = None;
|
||||
record.duration_seconds = task.duration_seconds.or(record.duration_seconds);
|
||||
record.resolution = task.resolution.clone().or(record.resolution);
|
||||
record.aspect_ratio = task.aspect_ratio.clone().or(record.aspect_ratio);
|
||||
@@ -309,17 +317,11 @@ fn build_successful_poll_update(
|
||||
if record.status.is_active() && record.poll_count >= record.max_poll_count {
|
||||
record.status = VideoTaskStatus::Failed;
|
||||
record.error_code = Some("poll_timeout".to_string());
|
||||
record.error_message = Some(format!("Task timed out after {} polls", record.poll_count));
|
||||
record.error_message = None;
|
||||
record.completed_at_unix_secs = Some(now_unix_secs);
|
||||
record.next_poll_at_unix_secs = None;
|
||||
}
|
||||
record.request_metadata = merge_video_task_request_metadata(
|
||||
task.request_metadata.clone(),
|
||||
&snapshot,
|
||||
Some(provider_body),
|
||||
None,
|
||||
)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
record.request_metadata = None;
|
||||
|
||||
Ok(Some(record))
|
||||
}
|
||||
@@ -332,11 +334,11 @@ fn build_failed_poll_update(
|
||||
let mut record = stored_task_to_upsert(task);
|
||||
record.updated_at_unix_secs = now_unix_secs;
|
||||
record.poll_count = task.poll_count.saturating_add(1);
|
||||
record.progress_message = Some(format!("Poll error: {}", err.message));
|
||||
record.progress_message = None;
|
||||
if err.permanent {
|
||||
record.status = VideoTaskStatus::Failed;
|
||||
record.error_code = Some("poll_permanent_error".to_string());
|
||||
record.error_message = Some(err.message.clone());
|
||||
record.error_message = None;
|
||||
record.completed_at_unix_secs = Some(now_unix_secs);
|
||||
record.next_poll_at_unix_secs = None;
|
||||
} else {
|
||||
@@ -348,28 +350,15 @@ fn build_failed_poll_update(
|
||||
if record.status.is_active() && record.poll_count >= record.max_poll_count {
|
||||
record.status = VideoTaskStatus::Failed;
|
||||
record.error_code = Some("poll_timeout".to_string());
|
||||
record.error_message = Some(format!("Task timed out after {} polls", record.poll_count));
|
||||
record.error_message = None;
|
||||
record.completed_at_unix_secs = Some(now_unix_secs);
|
||||
record.next_poll_at_unix_secs = None;
|
||||
}
|
||||
record.request_metadata = LocalVideoTaskSnapshot::from_stored_task(task)
|
||||
.and_then(|snapshot| {
|
||||
merge_video_task_request_metadata(
|
||||
task.request_metadata.clone(),
|
||||
&snapshot,
|
||||
None,
|
||||
Some(err),
|
||||
)
|
||||
.ok()
|
||||
.flatten()
|
||||
})
|
||||
.or(task.request_metadata.clone());
|
||||
record.request_metadata = None;
|
||||
record
|
||||
}
|
||||
|
||||
fn stored_task_to_upsert(task: &StoredVideoTask) -> UpsertVideoTask {
|
||||
let snapshot_record =
|
||||
LocalVideoTaskSnapshot::from_stored_task(task).map(|snapshot| snapshot.to_upsert_record());
|
||||
UpsertVideoTask {
|
||||
id: task.id.clone(),
|
||||
short_id: task.short_id.clone(),
|
||||
@@ -386,39 +375,15 @@ fn stored_task_to_upsert(task: &StoredVideoTask) -> UpsertVideoTask {
|
||||
provider_api_format: task.provider_api_format.clone(),
|
||||
format_converted: task.format_converted,
|
||||
model: task.model.clone(),
|
||||
prompt: task.prompt.clone().or_else(|| {
|
||||
snapshot_record
|
||||
.as_ref()
|
||||
.and_then(|record| record.prompt.clone())
|
||||
}),
|
||||
original_request_body: task.original_request_body.clone().or_else(|| {
|
||||
snapshot_record
|
||||
.as_ref()
|
||||
.and_then(|record| record.original_request_body.clone())
|
||||
}),
|
||||
duration_seconds: task.duration_seconds.or_else(|| {
|
||||
snapshot_record
|
||||
.as_ref()
|
||||
.and_then(|record| record.duration_seconds)
|
||||
}),
|
||||
resolution: task.resolution.clone().or_else(|| {
|
||||
snapshot_record
|
||||
.as_ref()
|
||||
.and_then(|record| record.resolution.clone())
|
||||
}),
|
||||
aspect_ratio: task.aspect_ratio.clone().or_else(|| {
|
||||
snapshot_record
|
||||
.as_ref()
|
||||
.and_then(|record| record.aspect_ratio.clone())
|
||||
}),
|
||||
size: task.size.clone().or_else(|| {
|
||||
snapshot_record
|
||||
.as_ref()
|
||||
.and_then(|record| record.size.clone())
|
||||
}),
|
||||
prompt: task.prompt.clone(),
|
||||
original_request_body: None,
|
||||
duration_seconds: task.duration_seconds,
|
||||
resolution: task.resolution.clone(),
|
||||
aspect_ratio: task.aspect_ratio.clone(),
|
||||
size: task.size.clone(),
|
||||
status: task.status,
|
||||
progress_percent: task.progress_percent,
|
||||
progress_message: task.progress_message.clone(),
|
||||
progress_message: None,
|
||||
retry_count: task.retry_count,
|
||||
poll_interval_seconds: task.poll_interval_seconds.max(1),
|
||||
next_poll_at_unix_secs: task.next_poll_at_unix_secs,
|
||||
@@ -429,9 +394,9 @@ fn stored_task_to_upsert(task: &StoredVideoTask) -> UpsertVideoTask {
|
||||
completed_at_unix_secs: task.completed_at_unix_secs,
|
||||
updated_at_unix_secs: task.updated_at_unix_secs,
|
||||
error_code: task.error_code.clone(),
|
||||
error_message: task.error_message.clone(),
|
||||
error_message: None,
|
||||
video_url: task.video_url.clone(),
|
||||
request_metadata: task.request_metadata.clone(),
|
||||
request_metadata: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -443,44 +408,6 @@ fn compute_poll_backoff_seconds(poll_interval_seconds: u32, retry_count: u32) ->
|
||||
.min(MAX_VIDEO_TASK_POLL_BACKOFF_SECONDS)
|
||||
}
|
||||
|
||||
fn merge_video_task_request_metadata(
|
||||
existing: Option<Value>,
|
||||
snapshot: &LocalVideoTaskSnapshot,
|
||||
provider_body: Option<&Map<String, Value>>,
|
||||
poll_error: Option<&VideoTaskRefreshError>,
|
||||
) -> Result<Option<Value>, serde_json::Error> {
|
||||
let mut metadata = match existing {
|
||||
Some(Value::Object(object)) => object,
|
||||
_ => Map::new(),
|
||||
};
|
||||
metadata.insert(
|
||||
"rust_owner".to_string(),
|
||||
Value::String("async_task".to_string()),
|
||||
);
|
||||
metadata.insert(
|
||||
"rust_local_snapshot".to_string(),
|
||||
serde_json::to_value(snapshot)?,
|
||||
);
|
||||
if let Some(provider_body) = provider_body {
|
||||
metadata.insert(
|
||||
"poll_raw_response".to_string(),
|
||||
Value::Object(provider_body.clone()),
|
||||
);
|
||||
metadata.remove("poll_error");
|
||||
}
|
||||
if let Some(poll_error) = poll_error {
|
||||
metadata.insert(
|
||||
"poll_error".to_string(),
|
||||
serde_json::json!({
|
||||
"message": poll_error.message,
|
||||
"permanent": poll_error.permanent,
|
||||
"observed_at_unix_secs": now_unix_secs(),
|
||||
}),
|
||||
);
|
||||
}
|
||||
Ok(Some(Value::Object(metadata)))
|
||||
}
|
||||
|
||||
pub(crate) async fn finalize_video_task_if_terminal(state: &AppState, task: &StoredVideoTask) {
|
||||
let Some(event) = build_video_task_terminal_usage_event(task) else {
|
||||
return;
|
||||
@@ -543,9 +470,9 @@ fn build_video_task_terminal_usage_event(task: &StoredVideoTask) -> Option<Usage
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let provider_name = LocalVideoTaskSnapshot::from_stored_task(task)
|
||||
.and_then(|snapshot| snapshot.provider_name().map(str::to_string))
|
||||
.or_else(|| task.provider_id.clone())
|
||||
let provider_name = task
|
||||
.provider_id
|
||||
.clone()
|
||||
.unwrap_or_else(|| "unknown".to_string());
|
||||
let response_time_ms = task
|
||||
.submitted_at_unix_secs
|
||||
@@ -580,10 +507,10 @@ fn build_video_task_terminal_usage_event(task: &StoredVideoTask) -> Option<Usage
|
||||
has_format_conversion: Some(task.format_converted),
|
||||
is_stream: Some(false),
|
||||
status_code,
|
||||
error_message: task.error_message.clone().or(task.error_code.clone()),
|
||||
error_message: task.error_code.clone(),
|
||||
response_time_ms,
|
||||
request_body: task.original_request_body.clone(),
|
||||
request_metadata: task.request_metadata.clone(),
|
||||
request_body: None,
|
||||
request_metadata: None,
|
||||
..UsageEventData::default()
|
||||
},
|
||||
))
|
||||
@@ -701,48 +628,36 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stored_task_to_upsert_restores_sparse_fields_from_snapshot() {
|
||||
fn stored_task_to_upsert_does_not_restore_sensitive_legacy_snapshot_fields() {
|
||||
let record = stored_task_to_upsert(&sample_sparse_stored_task());
|
||||
|
||||
assert_eq!(record.prompt.as_deref(), Some("hello"));
|
||||
assert_eq!(
|
||||
record.original_request_body,
|
||||
Some(json!({
|
||||
"prompt": "hello",
|
||||
"seconds": "4",
|
||||
"resolution": "720p",
|
||||
"aspect_ratio": "16:9",
|
||||
"size": "1280x720"
|
||||
}))
|
||||
);
|
||||
assert_eq!(record.duration_seconds, Some(4));
|
||||
assert_eq!(record.resolution.as_deref(), Some("720p"));
|
||||
assert_eq!(record.aspect_ratio.as_deref(), Some("16:9"));
|
||||
assert_eq!(record.size.as_deref(), Some("1280x720"));
|
||||
assert!(record.prompt.is_none());
|
||||
assert!(record.original_request_body.is_none());
|
||||
assert!(record.duration_seconds.is_none());
|
||||
assert!(record.resolution.is_none());
|
||||
assert!(record.aspect_ratio.is_none());
|
||||
assert!(record.size.is_none());
|
||||
assert!(record.progress_message.is_none());
|
||||
assert!(record.error_message.is_none());
|
||||
assert!(record.request_metadata.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failed_poll_update_keeps_snapshot_backed_request_body() {
|
||||
fn failed_poll_update_drops_snapshot_backed_sensitive_fields() {
|
||||
let record = build_failed_poll_update(
|
||||
&sample_sparse_stored_task(),
|
||||
&VideoTaskRefreshError {
|
||||
message: "temporary failure".to_string(),
|
||||
category: "transport_error",
|
||||
permanent: false,
|
||||
},
|
||||
100,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
record.original_request_body,
|
||||
Some(json!({
|
||||
"prompt": "hello",
|
||||
"seconds": "4",
|
||||
"resolution": "720p",
|
||||
"aspect_ratio": "16:9",
|
||||
"size": "1280x720"
|
||||
}))
|
||||
);
|
||||
assert_eq!(record.prompt.as_deref(), Some("hello"));
|
||||
assert_eq!(record.resolution.as_deref(), Some("720p"));
|
||||
assert!(record.original_request_body.is_none());
|
||||
assert!(record.prompt.is_none());
|
||||
assert!(record.resolution.is_none());
|
||||
assert!(record.progress_message.is_none());
|
||||
assert!(record.error_message.is_none());
|
||||
assert!(record.request_metadata.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,6 +34,7 @@ pub(crate) fn emit_admin_audit(
|
||||
path_and_query: &str,
|
||||
control_decision: Option<&GatewayControlDecision>,
|
||||
) {
|
||||
let sanitized_path_and_query = sanitize_admin_audit_path(path_and_query);
|
||||
let Some(decision) = control_decision else {
|
||||
return;
|
||||
};
|
||||
@@ -64,9 +65,10 @@ pub(crate) fn emit_admin_audit(
|
||||
},
|
||||
route_kind,
|
||||
default_target_type(route_family),
|
||||
path_and_query.to_string(),
|
||||
sanitized_path_and_query.clone(),
|
||||
)
|
||||
};
|
||||
let target_id = sanitize_admin_audit_target_id(target_id);
|
||||
|
||||
let (audit_status, log_level) = classify_admin_audit_response(method, response.status());
|
||||
if log_level == AdminAuditLogLevel::Info {
|
||||
@@ -83,7 +85,7 @@ pub(crate) fn emit_admin_audit(
|
||||
route_family,
|
||||
route_kind,
|
||||
method = %method,
|
||||
path = %path_and_query,
|
||||
path = %sanitized_path_and_query,
|
||||
action,
|
||||
target_type,
|
||||
target_id = %target_id,
|
||||
@@ -103,7 +105,7 @@ pub(crate) fn emit_admin_audit(
|
||||
route_family,
|
||||
route_kind,
|
||||
method = %method,
|
||||
path = %path_and_query,
|
||||
path = %sanitized_path_and_query,
|
||||
action,
|
||||
target_type,
|
||||
target_id = %target_id,
|
||||
@@ -112,6 +114,17 @@ pub(crate) fn emit_admin_audit(
|
||||
}
|
||||
}
|
||||
|
||||
fn sanitize_admin_audit_path(path_and_query: &str) -> String {
|
||||
crate::middleware::sanitize_access_log_path(path_and_query)
|
||||
}
|
||||
|
||||
fn sanitize_admin_audit_target_id(target_id: String) -> String {
|
||||
if target_id.trim_start().starts_with('/') {
|
||||
return sanitize_admin_audit_path(&target_id);
|
||||
}
|
||||
target_id
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum AdminAuditLogLevel {
|
||||
Info,
|
||||
@@ -151,7 +164,10 @@ fn is_admin_read_method(method: &http::Method) -> bool {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{classify_admin_audit_response, AdminAuditLogLevel};
|
||||
use super::{
|
||||
classify_admin_audit_response, sanitize_admin_audit_path, sanitize_admin_audit_target_id,
|
||||
AdminAuditLogLevel,
|
||||
};
|
||||
use axum::http::{Method, StatusCode};
|
||||
|
||||
#[test]
|
||||
@@ -169,4 +185,32 @@ mod tests {
|
||||
("failed", AdminAuditLogLevel::Warn)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audit_paths_drop_sensitive_query_values() {
|
||||
assert_eq!(
|
||||
sanitize_admin_audit_path(
|
||||
"/api/admin/providers?token=secret&api_key=live-key&limit=25"
|
||||
),
|
||||
"/api/admin/providers?limit=25"
|
||||
);
|
||||
assert_eq!(
|
||||
sanitize_admin_audit_path("/install/one-time-secret?view=raw"),
|
||||
"/install/[redacted]?view=raw"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn path_shaped_audit_targets_drop_sensitive_query_values() {
|
||||
assert_eq!(
|
||||
sanitize_admin_audit_target_id(
|
||||
"/api/admin/monitoring/trace/request-1?token=secret&limit=25".to_string(),
|
||||
),
|
||||
"/api/admin/monitoring/trace/request-1?limit=25"
|
||||
);
|
||||
assert_eq!(
|
||||
sanitize_admin_audit_target_id("resource-id?literal".to_string()),
|
||||
"resource-id?literal"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,7 +26,10 @@ pub(crate) async fn get_request_candidate_trace(
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()).into_response())?;
|
||||
|
||||
match trace {
|
||||
Some(trace) => Ok(Json(trace)),
|
||||
Some(mut trace) => {
|
||||
trace.sanitize_sensitive_diagnostics();
|
||||
Ok(Json(trace))
|
||||
}
|
||||
None => Err((
|
||||
axum::http::StatusCode::NOT_FOUND,
|
||||
Json(json!({
|
||||
@@ -52,7 +55,10 @@ pub(crate) async fn get_decision_trace(
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()).into_response())?;
|
||||
|
||||
match trace {
|
||||
Some(trace) => Ok(Json(trace)),
|
||||
Some(mut trace) => {
|
||||
trace.sanitize_sensitive_diagnostics();
|
||||
Ok(Json(trace))
|
||||
}
|
||||
None => Err((
|
||||
axum::http::StatusCode::NOT_FOUND,
|
||||
Json(json!({
|
||||
|
||||
@@ -5,7 +5,7 @@ use serde_json::{Map, Value};
|
||||
use super::schedule::{BackupSchedule, BackupScheduleUnit};
|
||||
use super::scopes::BackupScope;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub(crate) struct S3BackupConfig {
|
||||
pub(crate) enabled: bool,
|
||||
pub(crate) scope: BackupScope,
|
||||
@@ -22,6 +22,28 @@ pub(crate) struct S3BackupConfig {
|
||||
pub(crate) retention_count: u32,
|
||||
}
|
||||
|
||||
impl fmt::Debug for S3BackupConfig {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
let endpoint_origin = sanitized_endpoint_origin(&self.endpoint);
|
||||
formatter
|
||||
.debug_struct("S3BackupConfig")
|
||||
.field("enabled", &self.enabled)
|
||||
.field("scope", &self.scope)
|
||||
.field("endpoint_origin", &endpoint_origin)
|
||||
.field("region", &self.region)
|
||||
.field("user_agent", &self.user_agent)
|
||||
.field("bucket", &self.bucket)
|
||||
.field("prefix", &self.prefix)
|
||||
.field("has_access_key_id", &!self.access_key_id.is_empty())
|
||||
.field("has_secret_access_key", &!self.secret_access_key.is_empty())
|
||||
.field("path_style", &self.path_style)
|
||||
.field("compression", &self.compression)
|
||||
.field("schedule", &self.schedule)
|
||||
.field("retention_count", &self.retention_count)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct BackupConfigError {
|
||||
message: String,
|
||||
@@ -84,6 +106,9 @@ impl S3BackupConfig {
|
||||
"Endpoint(S3 地址)",
|
||||
enabled,
|
||||
)?;
|
||||
if enabled {
|
||||
validate_s3_endpoint(&endpoint)?;
|
||||
}
|
||||
let bucket =
|
||||
required_or_disabled_string(entries, "backup_s3_bucket", "Bucket(存储桶)", enabled)?;
|
||||
let access_key_id = required_or_disabled_string(
|
||||
@@ -99,6 +124,11 @@ impl S3BackupConfig {
|
||||
enabled,
|
||||
)?;
|
||||
|
||||
let prefix = normalize_s3_prefix(
|
||||
&optional_string(entries, "backup_s3_prefix")?
|
||||
.unwrap_or_else(|| "aether/backups/".to_string()),
|
||||
)?;
|
||||
|
||||
Ok(Self {
|
||||
enabled,
|
||||
scope,
|
||||
@@ -108,8 +138,7 @@ impl S3BackupConfig {
|
||||
user_agent: optional_string(entries, "backup_s3_user_agent")?
|
||||
.unwrap_or_else(|| "rclone/v1.68.0".to_string()),
|
||||
bucket,
|
||||
prefix: optional_string(entries, "backup_s3_prefix")?
|
||||
.unwrap_or_else(|| "aether/backups/".to_string()),
|
||||
prefix,
|
||||
access_key_id,
|
||||
secret_access_key,
|
||||
path_style: optional_bool(entries, "backup_s3_path_style")?.unwrap_or(true),
|
||||
@@ -121,6 +150,48 @@ impl S3BackupConfig {
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_s3_prefix(prefix: &str) -> Result<String, BackupConfigError> {
|
||||
let prefix = prefix.trim().trim_matches('/');
|
||||
if prefix.is_empty() {
|
||||
return Ok(String::new());
|
||||
}
|
||||
if prefix
|
||||
.split('/')
|
||||
.any(|segment| segment.is_empty() || segment == "." || segment == "..")
|
||||
|| prefix.contains('\\')
|
||||
{
|
||||
return Err(BackupConfigError::new(
|
||||
"Prefix(备份前缀)不能包含空路径段、相对路径段或反斜杠",
|
||||
));
|
||||
}
|
||||
|
||||
Ok(format!("{prefix}/"))
|
||||
}
|
||||
|
||||
fn validate_s3_endpoint(endpoint: &str) -> Result<(), BackupConfigError> {
|
||||
let parsed = url::Url::parse(endpoint)
|
||||
.map_err(|_| BackupConfigError::new("Endpoint(S3 地址)必须是有效的 HTTPS URL"))?;
|
||||
if parsed.scheme() != "https"
|
||||
|| parsed.host_str().is_none()
|
||||
|| !parsed.username().is_empty()
|
||||
|| parsed.password().is_some()
|
||||
|| parsed.query().is_some()
|
||||
|| parsed.fragment().is_some()
|
||||
{
|
||||
return Err(BackupConfigError::new(
|
||||
"Endpoint(S3 地址)必须使用 HTTPS,且不能包含用户凭据、查询参数或片段",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn sanitized_endpoint_origin(endpoint: &str) -> String {
|
||||
url::Url::parse(endpoint)
|
||||
.ok()
|
||||
.map(|parsed| parsed.origin().ascii_serialization())
|
||||
.unwrap_or_else(|| "<invalid>".to_string())
|
||||
}
|
||||
|
||||
fn validate_range(label: &str, value: u32, min: u32, max: u32) -> Result<(), BackupConfigError> {
|
||||
if (min..=max).contains(&value) {
|
||||
Ok(())
|
||||
@@ -374,6 +445,69 @@ mod tests {
|
||||
assert!(err.to_string().contains("Endpoint"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_insecure_or_credential_bearing_endpoints() {
|
||||
for endpoint in [
|
||||
"http://s3.example.com",
|
||||
"https://user:password@s3.example.com",
|
||||
"https://s3.example.com?token=secret",
|
||||
"https://s3.example.com/#fragment",
|
||||
] {
|
||||
let entries = serde_json::json!({
|
||||
"backup_s3_enabled": true,
|
||||
"backup_s3_endpoint": endpoint,
|
||||
"backup_s3_bucket": "aether-backups",
|
||||
"backup_s3_access_key_id": "access",
|
||||
"backup_s3_secret_access_key": "secret"
|
||||
});
|
||||
|
||||
let error = S3BackupConfig::from_json_map(entries.as_object().unwrap())
|
||||
.expect_err("unsafe endpoint should fail closed");
|
||||
assert!(error.to_string().contains("Endpoint"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn debug_output_does_not_expose_s3_credentials() {
|
||||
let entries = serde_json::json!({
|
||||
"backup_s3_enabled": true,
|
||||
"backup_s3_endpoint": "https://s3.example.com/path",
|
||||
"backup_s3_bucket": "aether-backups",
|
||||
"backup_s3_access_key_id": "access-key-value",
|
||||
"backup_s3_secret_access_key": "secret-key-value"
|
||||
});
|
||||
let config = S3BackupConfig::from_json_map(entries.as_object().unwrap())
|
||||
.expect("config should parse");
|
||||
|
||||
let debug = format!("{config:?}");
|
||||
assert!(debug.contains("https://s3.example.com"));
|
||||
assert!(!debug.contains("/path"));
|
||||
assert!(!debug.contains("access-key-value"));
|
||||
assert!(!debug.contains("secret-key-value"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonicalizes_s3_backup_prefix_once() {
|
||||
let entries = serde_json::json!({
|
||||
"backup_s3_enabled": true,
|
||||
"backup_s3_endpoint": "https://s3.example.com",
|
||||
"backup_s3_bucket": "aether-backups",
|
||||
"backup_s3_prefix": "/prod/backups//",
|
||||
"backup_s3_access_key_id": "access",
|
||||
"backup_s3_secret_access_key": "secret"
|
||||
});
|
||||
let config = S3BackupConfig::from_json_map(entries.as_object().unwrap())
|
||||
.expect("prefix should be canonicalized");
|
||||
|
||||
assert_eq!(config.prefix, "prod/backups/");
|
||||
|
||||
for invalid_prefix in ["prod//backups", "prod/../backups", "prod\\backups"] {
|
||||
let mut entries = entries.clone();
|
||||
entries["backup_s3_prefix"] = serde_json::json!(invalid_prefix);
|
||||
assert!(S3BackupConfig::from_json_map(entries.as_object().unwrap()).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn applies_default_values_from_system_config_contract() {
|
||||
let entries = serde_json::json!({
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -6,5 +6,167 @@ pub(crate) mod store;
|
||||
pub(crate) mod task;
|
||||
pub(crate) mod worker;
|
||||
|
||||
pub use executor::{
|
||||
restore_backup_json, BackupDecryptionKey, BackupRestoreError, BackupRestoreLimits,
|
||||
RestoredBackupJson, DEFAULT_BACKUP_MAX_ENCRYPTED_BYTES, DEFAULT_BACKUP_MAX_JSON_BYTES,
|
||||
};
|
||||
|
||||
use axum::body::Bytes;
|
||||
use serde_json::Value;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum BackupRestoreScope {
|
||||
Config,
|
||||
Users,
|
||||
Data,
|
||||
}
|
||||
|
||||
impl BackupRestoreScope {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Config => "config",
|
||||
Self::Users => "users",
|
||||
Self::Data => "data",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("backup database apply failed: {0}")]
|
||||
pub struct BackupApplyError(String);
|
||||
|
||||
pub async fn apply_restored_backup(
|
||||
app: &crate::AppState,
|
||||
restored: RestoredBackupJson,
|
||||
scope: BackupRestoreScope,
|
||||
operator_id: Option<&str>,
|
||||
) -> Result<Result<Value, (http::StatusCode, Value)>, BackupApplyError> {
|
||||
let (json_bytes, authority) = restored.into_authenticated_parts();
|
||||
if authority.scope() != scope {
|
||||
return Err(BackupApplyError(format!(
|
||||
"authenticated {} backup cannot be applied to {} scope",
|
||||
authority.scope().as_str(),
|
||||
scope.as_str(),
|
||||
)));
|
||||
}
|
||||
let request_body = Bytes::from(json_bytes);
|
||||
let state = crate::admin_api::AdminAppState::new(app);
|
||||
let result = crate::admin_api::execute_admin_system_import_exclusively(app, async {
|
||||
match scope {
|
||||
BackupRestoreScope::Config => {
|
||||
state
|
||||
.restore_admin_system_config_backup(&request_body, authority)
|
||||
.await
|
||||
}
|
||||
BackupRestoreScope::Users => {
|
||||
state
|
||||
.restore_admin_system_users_backup(&request_body, operator_id, authority)
|
||||
.await
|
||||
}
|
||||
BackupRestoreScope::Data => {
|
||||
state
|
||||
.restore_admin_system_data_backup(&request_body, operator_id, authority)
|
||||
.await
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|error| {
|
||||
let message = match error {
|
||||
crate::admin_api::AdminSystemImportLockError::Conflict => {
|
||||
"another system import or restore is already running"
|
||||
}
|
||||
crate::admin_api::AdminSystemImportLockError::Unavailable => {
|
||||
"system import coordination is unavailable"
|
||||
}
|
||||
crate::admin_api::AdminSystemImportLockError::Lost => {
|
||||
"system import coordination lease was lost; restore was cancelled and may have partially applied changes"
|
||||
}
|
||||
};
|
||||
BackupApplyError(message.to_string())
|
||||
})?;
|
||||
result.map_err(|error| BackupApplyError(error.into_message()))
|
||||
}
|
||||
|
||||
pub(crate) const S3_BACKUP_ENABLED_KEY: &str = "backup_s3_enabled";
|
||||
pub(crate) const S3_BACKUP_LAST_SLOT_KEY: &str = "backup_s3_last_slot";
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
apply_restored_backup, BackupDecryptionKey, BackupRestoreLimits, BackupRestoreScope,
|
||||
RestoredBackupJson,
|
||||
};
|
||||
use crate::backup::executor::encrypt_backup_bytes;
|
||||
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||
use serde_json::json;
|
||||
|
||||
fn authenticated_users_backup() -> RestoredBackupJson {
|
||||
let object_key = "prod/aether-users-backup-20260830-120000.json.zst.aes256gcm";
|
||||
let compressed = zstd::stream::encode_all(
|
||||
serde_json::to_vec(&json!({
|
||||
"version": "1.5",
|
||||
"exported_at": "2026-08-30T12:00:00Z",
|
||||
"users": [],
|
||||
"standalone_keys": [],
|
||||
}))
|
||||
.expect("test backup should serialize")
|
||||
.as_slice(),
|
||||
0,
|
||||
)
|
||||
.expect("test backup should compress");
|
||||
let (envelope, _) =
|
||||
encrypt_backup_bytes(DEVELOPMENT_ENCRYPTION_KEY, object_key, &compressed)
|
||||
.expect("test backup should encrypt");
|
||||
super::restore_backup_json(
|
||||
object_key,
|
||||
&envelope,
|
||||
&[BackupDecryptionKey::current(DEVELOPMENT_ENCRYPTION_KEY)
|
||||
.expect("test restore key should build")],
|
||||
BackupRestoreLimits::default(),
|
||||
)
|
||||
.expect("test backup should authenticate")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authenticated_backup_cannot_be_applied_to_a_different_scope() {
|
||||
let restored = authenticated_users_backup();
|
||||
|
||||
let error = apply_restored_backup(
|
||||
&crate::AppState::new().expect("test state should build"),
|
||||
restored,
|
||||
BackupRestoreScope::Config,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect_err("scope mismatch must fail before database access");
|
||||
|
||||
assert_eq!(
|
||||
error.to_string(),
|
||||
"backup database apply failed: authenticated users backup cannot be applied to config scope"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authenticated_backup_apply_uses_the_shared_system_import_lock() {
|
||||
let app = crate::AppState::new().expect("test state should build");
|
||||
let lock = crate::admin_api::try_acquire_admin_system_import_lease(&app)
|
||||
.await
|
||||
.expect("test should acquire the shared import lease");
|
||||
|
||||
let error = apply_restored_backup(
|
||||
&app,
|
||||
authenticated_users_backup(),
|
||||
BackupRestoreScope::Users,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect_err("restore must not interleave with another system import");
|
||||
|
||||
assert_eq!(
|
||||
error.to_string(),
|
||||
"backup database apply failed: another system import or restore is already running"
|
||||
);
|
||||
crate::admin_api::release_admin_system_import_lease(&app, &lock).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
use std::fmt;
|
||||
|
||||
const ENCRYPTED_BACKUP_FILE_SUFFIX: &str = ".json.zst.aes256gcm";
|
||||
const LEGACY_PLAINTEXT_BACKUP_FILE_SUFFIX: &str = ".json.zst";
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum BackupScope {
|
||||
Config,
|
||||
@@ -52,10 +55,81 @@ impl BackupScope {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn from_encrypted_object_key(object_key: &str) -> Option<Self> {
|
||||
if object_key.is_empty()
|
||||
|| object_key.starts_with('/')
|
||||
|| object_key.contains('\0')
|
||||
|| object_key.contains('\\')
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let mut segments = object_key.split('/').peekable();
|
||||
let mut file_name = None;
|
||||
while let Some(segment) = segments.next() {
|
||||
if segment.is_empty()
|
||||
|| segment == "."
|
||||
|| segment == ".."
|
||||
|| segment.chars().any(char::is_control)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
if segments.peek().is_none() {
|
||||
file_name = Some(segment);
|
||||
}
|
||||
}
|
||||
let file_name = file_name?;
|
||||
|
||||
[Self::Config, Self::Users, Self::Data]
|
||||
.into_iter()
|
||||
.find(|scope| {
|
||||
file_name
|
||||
.strip_prefix(&format!("{}-", scope.file_stem()))
|
||||
.and_then(|rest| rest.strip_suffix(ENCRYPTED_BACKUP_FILE_SUFFIX))
|
||||
.is_some_and(is_aether_backup_object_id)
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn matching_backup_keys(
|
||||
self,
|
||||
prefix: &str,
|
||||
keys: impl IntoIterator<Item = String>,
|
||||
) -> Vec<String> {
|
||||
self.matching_backup_keys_with_suffixes(
|
||||
prefix,
|
||||
keys,
|
||||
&[
|
||||
ENCRYPTED_BACKUP_FILE_SUFFIX,
|
||||
LEGACY_PLAINTEXT_BACKUP_FILE_SUFFIX,
|
||||
],
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn matching_encrypted_backup_keys(
|
||||
self,
|
||||
prefix: &str,
|
||||
keys: impl IntoIterator<Item = String>,
|
||||
) -> Vec<String> {
|
||||
self.matching_backup_keys_with_suffixes(prefix, keys, &[ENCRYPTED_BACKUP_FILE_SUFFIX])
|
||||
}
|
||||
|
||||
pub(crate) fn matching_legacy_plaintext_backup_keys(
|
||||
self,
|
||||
prefix: &str,
|
||||
keys: impl IntoIterator<Item = String>,
|
||||
) -> Vec<String> {
|
||||
self.matching_backup_keys_with_suffixes(
|
||||
prefix,
|
||||
keys,
|
||||
&[LEGACY_PLAINTEXT_BACKUP_FILE_SUFFIX],
|
||||
)
|
||||
}
|
||||
|
||||
fn matching_backup_keys_with_suffixes(
|
||||
self,
|
||||
prefix: &str,
|
||||
keys: impl IntoIterator<Item = String>,
|
||||
file_suffixes: &[&str],
|
||||
) -> Vec<String> {
|
||||
let normalized_prefix = normalized_prefix(prefix);
|
||||
let expected_prefix = if normalized_prefix.is_empty() {
|
||||
@@ -64,7 +138,6 @@ impl BackupScope {
|
||||
format!("{normalized_prefix}/")
|
||||
};
|
||||
let file_prefix = format!("{}-", self.file_stem());
|
||||
let file_suffix = ".json.zst";
|
||||
|
||||
keys.into_iter()
|
||||
.filter(|key| {
|
||||
@@ -74,20 +147,24 @@ impl BackupScope {
|
||||
if file_name.contains('/') {
|
||||
return false;
|
||||
}
|
||||
let Some(timestamp) = file_name
|
||||
.strip_prefix(&file_prefix)
|
||||
.and_then(|rest| rest.strip_suffix(file_suffix))
|
||||
else {
|
||||
let Some(timestamp) = file_name.strip_prefix(&file_prefix).and_then(|rest| {
|
||||
file_suffixes
|
||||
.iter()
|
||||
.find_map(|suffix| rest.strip_suffix(suffix))
|
||||
}) else {
|
||||
return false;
|
||||
};
|
||||
|
||||
is_aether_backup_timestamp(timestamp)
|
||||
is_aether_backup_object_id(timestamp)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn file_name(self, timestamp: &str) -> String {
|
||||
format!("{}-{timestamp}.json.zst", self.file_stem())
|
||||
format!(
|
||||
"{}-{timestamp}{ENCRYPTED_BACKUP_FILE_SUFFIX}",
|
||||
self.file_stem()
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -110,6 +187,25 @@ fn is_aether_backup_timestamp(timestamp: &str) -> bool {
|
||||
&& bytes[9..].iter().all(|byte| byte.is_ascii_digit())
|
||||
}
|
||||
|
||||
fn is_aether_backup_object_id(value: &str) -> bool {
|
||||
if is_aether_backup_timestamp(value) {
|
||||
return true;
|
||||
}
|
||||
|
||||
let Some((timestamp, collision_digest)) = value.split_once('-').and_then(|(date, rest)| {
|
||||
let (time, digest) = rest.split_once('-')?;
|
||||
Some((format!("{date}-{time}"), digest))
|
||||
}) else {
|
||||
return false;
|
||||
};
|
||||
|
||||
is_aether_backup_timestamp(×tamp)
|
||||
&& collision_digest.len() == 64
|
||||
&& collision_digest
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::BackupScope;
|
||||
@@ -130,15 +226,15 @@ mod tests {
|
||||
|
||||
assert_eq!(
|
||||
BackupScope::Config.object_key("prod/", "20260524-031500"),
|
||||
"prod/aether-config-backup-20260524-031500.json.zst"
|
||||
"prod/aether-config-backup-20260524-031500.json.zst.aes256gcm"
|
||||
);
|
||||
assert_eq!(
|
||||
BackupScope::Users.object_key("prod/", "20260524-031500"),
|
||||
"prod/aether-users-backup-20260524-031500.json.zst"
|
||||
"prod/aether-users-backup-20260524-031500.json.zst.aes256gcm"
|
||||
);
|
||||
assert_eq!(
|
||||
BackupScope::Data.object_key("prod/", "20260524-031500"),
|
||||
"prod/aether-data-backup-20260524-031500.json.zst"
|
||||
"prod/aether-data-backup-20260524-031500.json.zst.aes256gcm"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -146,7 +242,7 @@ mod tests {
|
||||
fn retention_filter_only_matches_same_scope() {
|
||||
let keys = vec![
|
||||
"prod/aether-config-backup-20260524-010000.json.zst".to_string(),
|
||||
"prod/aether-users-backup-20260524-010000.json.zst".to_string(),
|
||||
"prod/aether-users-backup-20260524-010000.json.zst.aes256gcm".to_string(),
|
||||
"prod/aether-data-backup-20260524-010000.json.zst".to_string(),
|
||||
"prod/random.json.zst".to_string(),
|
||||
];
|
||||
@@ -155,14 +251,18 @@ mod tests {
|
||||
|
||||
assert_eq!(
|
||||
matched,
|
||||
vec!["prod/aether-users-backup-20260524-010000.json.zst"]
|
||||
vec!["prod/aether-users-backup-20260524-010000.json.zst.aes256gcm"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retention_filter_requires_aether_timestamp_format() {
|
||||
let collision_digest = "a".repeat(64);
|
||||
let keys = vec![
|
||||
"prod/aether-users-backup-20260524-010000.json.zst".to_string(),
|
||||
format!(
|
||||
"prod/aether-users-backup-20260524-010000-{collision_digest}.json.zst.aes256gcm"
|
||||
),
|
||||
"prod/aether-users-backup-foo.json.zst".to_string(),
|
||||
"prod/aether-users-backup-2026052-010000.json.zst".to_string(),
|
||||
"prod/aether-users-backup-202605240-010000.json.zst".to_string(),
|
||||
@@ -171,13 +271,19 @@ mod tests {
|
||||
"prod/aether-users-backup-20260524010000.json.zst".to_string(),
|
||||
"prod/aether-users-backup-2026052a-010000.json.zst".to_string(),
|
||||
"prod/aether-users-backup-20260524-01000x.json.zst".to_string(),
|
||||
"prod/aether-users-backup-20260524-010000-short.json.zst.aes256gcm".to_string(),
|
||||
];
|
||||
|
||||
let matched = BackupScope::Users.matching_backup_keys("prod/", keys);
|
||||
|
||||
assert_eq!(
|
||||
matched,
|
||||
vec!["prod/aether-users-backup-20260524-010000.json.zst"]
|
||||
vec![
|
||||
"prod/aether-users-backup-20260524-010000.json.zst".to_string(),
|
||||
format!(
|
||||
"prod/aether-users-backup-20260524-010000-{collision_digest}.json.zst.aes256gcm"
|
||||
),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -185,11 +291,11 @@ mod tests {
|
||||
fn backup_key_prefix_boundaries_are_exact() {
|
||||
assert_eq!(
|
||||
BackupScope::Config.object_key("", "20260524-031500"),
|
||||
"aether-config-backup-20260524-031500.json.zst"
|
||||
"aether-config-backup-20260524-031500.json.zst.aes256gcm"
|
||||
);
|
||||
assert_eq!(
|
||||
BackupScope::Config.object_key("prod", "20260524-031500"),
|
||||
"prod/aether-config-backup-20260524-031500.json.zst"
|
||||
"prod/aether-config-backup-20260524-031500.json.zst.aes256gcm"
|
||||
);
|
||||
|
||||
let keys = vec![
|
||||
@@ -208,4 +314,36 @@ mod tests {
|
||||
vec!["prod/aether-config-backup-20260524-010000.json.zst"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encrypted_object_key_parser_binds_scope_and_rejects_path_traversal() {
|
||||
let collision_digest = "a".repeat(64);
|
||||
assert_eq!(
|
||||
BackupScope::from_encrypted_object_key(
|
||||
"prod/aether-config-backup-20260524-010000.json.zst.aes256gcm"
|
||||
),
|
||||
Some(BackupScope::Config)
|
||||
);
|
||||
assert_eq!(
|
||||
BackupScope::from_encrypted_object_key(&format!(
|
||||
"prod/aether-users-backup-20260524-010000-{collision_digest}.json.zst.aes256gcm"
|
||||
)),
|
||||
Some(BackupScope::Users)
|
||||
);
|
||||
for key in [
|
||||
"../aether-data-backup-20260524-010000.json.zst.aes256gcm",
|
||||
"/aether-data-backup-20260524-010000.json.zst.aes256gcm",
|
||||
"prod//aether-data-backup-20260524-010000.json.zst.aes256gcm",
|
||||
"prod/./aether-data-backup-20260524-010000.json.zst.aes256gcm",
|
||||
"prod\\aether-data-backup-20260524-010000.json.zst.aes256gcm",
|
||||
"prod/aether-data-backup-invalid.json.zst.aes256gcm",
|
||||
"prod/unrelated-20260524-010000.json.zst.aes256gcm",
|
||||
] {
|
||||
assert_eq!(
|
||||
BackupScope::from_encrypted_object_key(key),
|
||||
None,
|
||||
"unsafe or unrelated key: {key}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,23 +2,45 @@ use std::collections::BTreeMap;
|
||||
use std::fmt;
|
||||
use std::sync::Arc;
|
||||
|
||||
use bytes::Bytes;
|
||||
use bytes::{Bytes, BytesMut};
|
||||
use futures_util::TryStreamExt;
|
||||
use object_store::aws::AmazonS3Builder;
|
||||
use object_store::path::Path;
|
||||
use object_store::{ClientOptions, ObjectStore};
|
||||
use object_store::{ClientOptions, ObjectStore, ObjectStoreExt, PutMode, PutOptions};
|
||||
use reqwest::header::HeaderValue;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use super::config::S3BackupConfig;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum BackupObjectCreateResult {
|
||||
Created,
|
||||
AlreadyExists,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub(crate) trait BackupObjectStore: Send + Sync {
|
||||
async fn put_object(&self, key: &str, bytes: Bytes) -> Result<(), BackupStoreError>;
|
||||
|
||||
async fn list_keys(&self, prefix: &str) -> Result<Vec<String>, BackupStoreError>;
|
||||
async fn put_object_if_absent(
|
||||
&self,
|
||||
key: &str,
|
||||
bytes: Bytes,
|
||||
) -> Result<BackupObjectCreateResult, BackupStoreError>;
|
||||
|
||||
async fn get_object_limited(
|
||||
&self,
|
||||
key: &str,
|
||||
max_bytes: usize,
|
||||
) -> Result<Bytes, BackupStoreError>;
|
||||
|
||||
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError>;
|
||||
|
||||
async fn list_keys_limited(
|
||||
&self,
|
||||
prefix: &str,
|
||||
max_objects: usize,
|
||||
) -> Result<Vec<String>, BackupStoreError>;
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -60,21 +82,72 @@ impl BackupObjectStore for FakeBackupObjectStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn list_keys(&self, prefix: &str) -> Result<Vec<String>, BackupStoreError> {
|
||||
async fn put_object_if_absent(
|
||||
&self,
|
||||
key: &str,
|
||||
bytes: Bytes,
|
||||
) -> Result<BackupObjectCreateResult, BackupStoreError> {
|
||||
let mut objects = self.objects.write().await;
|
||||
if objects.contains_key(key) {
|
||||
Ok(BackupObjectCreateResult::AlreadyExists)
|
||||
} else {
|
||||
objects.insert(key.to_string(), bytes);
|
||||
Ok(BackupObjectCreateResult::Created)
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_object_limited(
|
||||
&self,
|
||||
key: &str,
|
||||
max_bytes: usize,
|
||||
) -> Result<Bytes, BackupStoreError> {
|
||||
let bytes = self
|
||||
.objects
|
||||
.read()
|
||||
.await
|
||||
.get(key)
|
||||
.cloned()
|
||||
.ok_or_else(|| BackupStoreError::new(format!("backup object `{key}` not found")))?;
|
||||
if bytes.len() > max_bytes {
|
||||
return Err(BackupStoreError::new(format!(
|
||||
"backup object `{key}` exceeds the configured {max_bytes} byte read limit"
|
||||
)));
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError> {
|
||||
self.objects.write().await.remove(key);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn list_keys_limited(
|
||||
&self,
|
||||
prefix: &str,
|
||||
max_objects: usize,
|
||||
) -> Result<Vec<String>, BackupStoreError> {
|
||||
let prefix = directory_list_prefix(prefix);
|
||||
Ok(self
|
||||
let keys: Vec<_> = self
|
||||
.objects
|
||||
.read()
|
||||
.await
|
||||
.keys()
|
||||
.filter(|key| key.starts_with(&prefix))
|
||||
.cloned()
|
||||
.collect())
|
||||
.collect();
|
||||
if keys.len() > max_objects {
|
||||
return Err(BackupStoreError::new(format!(
|
||||
"backup object listing exceeds the configured {max_objects} object limit"
|
||||
)));
|
||||
}
|
||||
Ok(keys)
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError> {
|
||||
self.objects.write().await.remove(key);
|
||||
Ok(())
|
||||
#[cfg(test)]
|
||||
impl FakeBackupObjectStore {
|
||||
pub(crate) async fn object_bytes(&self, key: &str) -> Option<Bytes> {
|
||||
self.objects.read().await.get(key).cloned()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -125,17 +198,68 @@ impl BackupObjectStore for ObjectStoreS3BackupStore {
|
||||
.map_err(|error| BackupStoreError::object_store("put", key, error))
|
||||
}
|
||||
|
||||
async fn list_keys(&self, prefix: &str) -> Result<Vec<String>, BackupStoreError> {
|
||||
let prefix_path = list_prefix_path(prefix);
|
||||
let mut keys = self
|
||||
async fn put_object_if_absent(
|
||||
&self,
|
||||
key: &str,
|
||||
bytes: Bytes,
|
||||
) -> Result<BackupObjectCreateResult, BackupStoreError> {
|
||||
let options = PutOptions {
|
||||
mode: PutMode::Create,
|
||||
..PutOptions::default()
|
||||
};
|
||||
match self
|
||||
.store
|
||||
.list(prefix_path.as_ref())
|
||||
.map_ok(|meta| meta.location.to_string())
|
||||
.try_collect::<Vec<_>>()
|
||||
.put_opts(&Path::from(key), bytes.into(), options)
|
||||
.await
|
||||
.map_err(|error| BackupStoreError::object_store("list", prefix, error))?;
|
||||
keys.sort();
|
||||
Ok(keys)
|
||||
{
|
||||
Ok(_) => Ok(BackupObjectCreateResult::Created),
|
||||
Err(object_store::Error::AlreadyExists { .. }) => {
|
||||
Ok(BackupObjectCreateResult::AlreadyExists)
|
||||
}
|
||||
Err(error) => Err(BackupStoreError::object_store(
|
||||
"conditional put",
|
||||
key,
|
||||
error,
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_object_limited(
|
||||
&self,
|
||||
key: &str,
|
||||
max_bytes: usize,
|
||||
) -> Result<Bytes, BackupStoreError> {
|
||||
let result = self
|
||||
.store
|
||||
.get(&Path::from(key))
|
||||
.await
|
||||
.map_err(|error| BackupStoreError::object_store("get", key, error))?;
|
||||
if result.meta.size > u64::try_from(max_bytes).unwrap_or(u64::MAX) {
|
||||
return Err(BackupStoreError::new(format!(
|
||||
"backup object `{key}` exceeds the configured {max_bytes} byte read limit"
|
||||
)));
|
||||
}
|
||||
let object_size = result.meta.size;
|
||||
let mut stream = result.into_stream();
|
||||
let mut bytes = BytesMut::with_capacity(
|
||||
usize::try_from(object_size)
|
||||
.unwrap_or(max_bytes)
|
||||
.min(max_bytes)
|
||||
.min(8 * 1024 * 1024),
|
||||
);
|
||||
while let Some(chunk) = stream
|
||||
.try_next()
|
||||
.await
|
||||
.map_err(|error| BackupStoreError::object_store("read", key, error))?
|
||||
{
|
||||
if bytes.len().saturating_add(chunk.len()) > max_bytes {
|
||||
return Err(BackupStoreError::new(format!(
|
||||
"backup object `{key}` exceeds the configured {max_bytes} byte read limit"
|
||||
)));
|
||||
}
|
||||
bytes.extend_from_slice(&chunk);
|
||||
}
|
||||
Ok(bytes.freeze())
|
||||
}
|
||||
|
||||
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError> {
|
||||
@@ -144,6 +268,30 @@ impl BackupObjectStore for ObjectStoreS3BackupStore {
|
||||
.await
|
||||
.map_err(|error| BackupStoreError::object_store("delete", key, error))
|
||||
}
|
||||
|
||||
async fn list_keys_limited(
|
||||
&self,
|
||||
prefix: &str,
|
||||
max_objects: usize,
|
||||
) -> Result<Vec<String>, BackupStoreError> {
|
||||
let prefix_path = list_prefix_path(prefix);
|
||||
let mut objects = self.store.list(prefix_path.as_ref());
|
||||
let mut keys = Vec::new();
|
||||
while let Some(meta) = objects
|
||||
.try_next()
|
||||
.await
|
||||
.map_err(|error| BackupStoreError::object_store("list", prefix, error))?
|
||||
{
|
||||
if keys.len() >= max_objects {
|
||||
return Err(BackupStoreError::new(format!(
|
||||
"backup object listing exceeds the configured {max_objects} object limit"
|
||||
)));
|
||||
}
|
||||
keys.push(meta.location.to_string());
|
||||
}
|
||||
keys.sort();
|
||||
Ok(keys)
|
||||
}
|
||||
}
|
||||
|
||||
fn directory_list_prefix(prefix: &str) -> String {
|
||||
@@ -166,10 +314,12 @@ fn list_prefix_path(prefix: &str) -> Option<Path> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{list_prefix_path, BackupObjectStore, FakeBackupObjectStore};
|
||||
use super::{
|
||||
list_prefix_path, BackupObjectCreateResult, BackupObjectStore, FakeBackupObjectStore,
|
||||
};
|
||||
|
||||
#[tokio::test]
|
||||
async fn fake_backup_object_store_puts_lists_and_deletes() {
|
||||
async fn fake_backup_object_store_puts_and_lists() {
|
||||
let store = FakeBackupObjectStore::default();
|
||||
store
|
||||
.put_object(
|
||||
@@ -186,17 +336,59 @@ mod tests {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let keys = store.list_keys("prod/").await.unwrap();
|
||||
assert_eq!(keys.len(), 2);
|
||||
|
||||
store
|
||||
.delete_object("prod/aether-data-backup-20260524-010000.json.zst")
|
||||
.await
|
||||
.unwrap();
|
||||
let keys = store.list_keys("prod/").await.unwrap();
|
||||
let keys = store.list_keys_limited("prod/", 2).await.unwrap();
|
||||
assert_eq!(
|
||||
keys,
|
||||
vec!["prod/aether-data-backup-20260524-020000.json.zst"]
|
||||
vec![
|
||||
"prod/aether-data-backup-20260524-010000.json.zst",
|
||||
"prod/aether-data-backup-20260524-020000.json.zst",
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fake_backup_object_store_enforces_read_and_listing_limits() {
|
||||
let store = FakeBackupObjectStore::default();
|
||||
store
|
||||
.put_object("prod/one", bytes::Bytes::from_static(b"1234"))
|
||||
.await
|
||||
.unwrap();
|
||||
store
|
||||
.put_object("prod/two", bytes::Bytes::from_static(b"5678"))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(store.get_object_limited("prod/one", 3).await.is_err());
|
||||
assert_eq!(
|
||||
store.get_object_limited("prod/one", 4).await.unwrap(),
|
||||
bytes::Bytes::from_static(b"1234")
|
||||
);
|
||||
assert!(store.list_keys_limited("prod/", 1).await.is_err());
|
||||
assert_eq!(store.list_keys_limited("prod/", 2).await.unwrap().len(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fake_backup_object_store_conditional_put_never_overwrites() {
|
||||
let store = FakeBackupObjectStore::default();
|
||||
let key = "prod/aether-data-backup-20260524-010000.json.zst.aes256gcm";
|
||||
|
||||
assert_eq!(
|
||||
store
|
||||
.put_object_if_absent(key, bytes::Bytes::from_static(b"first"))
|
||||
.await
|
||||
.unwrap(),
|
||||
BackupObjectCreateResult::Created
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.put_object_if_absent(key, bytes::Bytes::from_static(b"second"))
|
||||
.await
|
||||
.unwrap(),
|
||||
BackupObjectCreateResult::AlreadyExists
|
||||
);
|
||||
assert_eq!(
|
||||
store.object_bytes(key).await.as_deref(),
|
||||
Some(b"first".as_slice())
|
||||
);
|
||||
}
|
||||
|
||||
@@ -218,7 +410,7 @@ mod tests {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let keys = store.list_keys("prod").await.unwrap();
|
||||
let keys = store.list_keys_limited("prod", 10).await.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
keys,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use std::fmt;
|
||||
use std::future::Future;
|
||||
use std::time::Duration;
|
||||
|
||||
use aether_admin::system::admin_system_config_default_value;
|
||||
@@ -12,14 +13,15 @@ use chrono::Utc;
|
||||
use futures_util::FutureExt;
|
||||
use serde::Serialize;
|
||||
use serde_json::{json, Map, Value};
|
||||
use tokio::task::{JoinError, JoinHandle};
|
||||
use tracing::warn;
|
||||
|
||||
use super::config::S3BackupConfig;
|
||||
use super::executor::{run_backup_with_store, BackupRunResult};
|
||||
use super::scopes::BackupScope;
|
||||
use super::store::ObjectStoreS3BackupStore;
|
||||
use crate::admin_api::AdminAppState;
|
||||
use crate::handlers::shared::decrypt_catalog_secret_with_fallbacks;
|
||||
use crate::admin_api::{AdminAppState, SystemExportMode};
|
||||
use crate::handlers::shared::decrypt_or_migrate_system_config_secret;
|
||||
use crate::task_runtime::{
|
||||
append_event_with_logging, build_task_run_id, now_unix_secs, spawn_fire_and_forget,
|
||||
task_definition, update_run_status, upsert_run_with_logging, TASK_KEY_SYSTEM_S3_BACKUP,
|
||||
@@ -48,6 +50,9 @@ const S3_BACKUP_CONFIG_KEYS: &[&str] = &[
|
||||
];
|
||||
|
||||
const S3_BACKUP_QUEUED_MESSAGE: &str = "S3 备份任务已提交";
|
||||
const S3_BACKUP_INTERNAL_ERROR_DETAIL: &str = "S3 备份服务暂时不可用";
|
||||
const S3_BACKUP_TASK_FAILURE_CODE: &str = "s3_backup_failed";
|
||||
const S3_BACKUP_SLOT_RECORD_FAILURE_CODE: &str = "s3_backup_slot_record_failed";
|
||||
const S3_BACKUP_TASK_LOCK_KEY: &str = "task_runtime:lock:system.s3.backup";
|
||||
const S3_BACKUP_TASK_LOCK_TTL: Duration = Duration::from_secs(60 * 60 * 6);
|
||||
const S3_BACKUP_TASK_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(60 * 5);
|
||||
@@ -67,6 +72,16 @@ pub(crate) struct S3BackupTaskError {
|
||||
detail: String,
|
||||
}
|
||||
|
||||
enum BackupLockRenewalFailure<E> {
|
||||
Lost,
|
||||
Backend(E),
|
||||
}
|
||||
|
||||
enum BackupLockRaceOutcome<T> {
|
||||
BackupCompleted(T),
|
||||
LeaseLost(Result<(), JoinError>),
|
||||
}
|
||||
|
||||
impl S3BackupTaskError {
|
||||
fn bad_request(detail: impl Into<String>) -> Self {
|
||||
Self {
|
||||
@@ -114,8 +129,12 @@ impl fmt::Display for S3BackupTaskError {
|
||||
impl std::error::Error for S3BackupTaskError {}
|
||||
|
||||
impl From<GatewayError> for S3BackupTaskError {
|
||||
fn from(error: GatewayError) -> Self {
|
||||
Self::internal(format!("{error:?}"))
|
||||
fn from(_error: GatewayError) -> Self {
|
||||
warn!(
|
||||
error_category = "dependency_failed",
|
||||
"S3 backup dependency failed"
|
||||
);
|
||||
Self::internal(S3_BACKUP_INTERNAL_ERROR_DETAIL)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -220,8 +239,6 @@ fn s3_backup_task_payload_json(
|
||||
) -> Value {
|
||||
let mut payload = json!({
|
||||
"scope": config.scope.as_config_value(),
|
||||
"bucket": config.bucket.clone(),
|
||||
"prefix": config.prefix.clone(),
|
||||
"compression": config.compression.clone(),
|
||||
"trigger": trigger,
|
||||
});
|
||||
@@ -259,7 +276,7 @@ fn spawn_s3_backup_worker(
|
||||
Some(100),
|
||||
Some("S3 备份任务异常退出".to_string()),
|
||||
None,
|
||||
Some("S3 backup task panicked".to_string()),
|
||||
Some("background_task_panicked".to_string()),
|
||||
None,
|
||||
Some(now_unix_secs()),
|
||||
)
|
||||
@@ -294,16 +311,67 @@ async fn run_s3_backup_worker_inner(
|
||||
.await;
|
||||
append_event_with_logging(&app, &run_id, "running", "S3 backup task started", None).await;
|
||||
|
||||
let heartbeat = spawn_s3_backup_task_heartbeat(app.clone(), run_id.clone(), lock);
|
||||
let result = run_s3_backup_once(&app, &config).await;
|
||||
heartbeat.abort();
|
||||
let _ = heartbeat.await;
|
||||
let heartbeat = spawn_s3_backup_task_heartbeat(app.clone(), run_id.clone(), lock.clone());
|
||||
let result = match race_backup_with_lock_heartbeat(run_s3_backup_once(&app, &config), heartbeat)
|
||||
.await
|
||||
{
|
||||
BackupLockRaceOutcome::BackupCompleted(result) => {
|
||||
match require_successful_backup_lock_renewal(
|
||||
app.runtime_state
|
||||
.lock_renew(&lock, S3_BACKUP_TASK_LOCK_TTL)
|
||||
.await,
|
||||
) {
|
||||
Ok(()) => result,
|
||||
Err(BackupLockRenewalFailure::Lost) => {
|
||||
warn!(
|
||||
run_id = %run_id,
|
||||
lock_key = %lock.key,
|
||||
"S3 backup task lost its distributed lock before publishing completion"
|
||||
);
|
||||
Err(S3BackupTaskError::service_unavailable(
|
||||
"S3 备份任务锁已失效,任务完成状态未发布",
|
||||
))
|
||||
}
|
||||
Err(BackupLockRenewalFailure::Backend(error)) => {
|
||||
warn!(
|
||||
run_id = %run_id,
|
||||
lock_key = %lock.key,
|
||||
error = %error,
|
||||
"S3 backup task could not verify its distributed lock before publishing completion"
|
||||
);
|
||||
Err(S3BackupTaskError::service_unavailable(
|
||||
"无法确认 S3 备份任务锁所有权,任务完成状态未发布",
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
BackupLockRaceOutcome::LeaseLost(heartbeat_result) => {
|
||||
match heartbeat_result {
|
||||
Ok(()) => warn!(
|
||||
run_id = %run_id,
|
||||
"S3 backup task stopped after losing its distributed lock"
|
||||
),
|
||||
Err(error) => warn!(
|
||||
run_id = %run_id,
|
||||
error = %error,
|
||||
"S3 backup lock heartbeat task failed"
|
||||
),
|
||||
}
|
||||
Err(S3BackupTaskError::service_unavailable(
|
||||
"S3 备份任务锁已失效,任务已停止",
|
||||
))
|
||||
}
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(result) => {
|
||||
if let Some(slot) = scheduled_backup_slot_to_record(scheduled_slot.as_deref(), true) {
|
||||
if let Err(error) = record_scheduled_backup_slot(&app, &slot).await {
|
||||
warn!(error = ?error, run_id = %run_id, "S3 backup slot record failed");
|
||||
if record_scheduled_backup_slot(&app, &slot).await.is_err() {
|
||||
warn!(
|
||||
error_category = "slot_record_failed",
|
||||
run_id = %run_id,
|
||||
"S3 backup slot record failed"
|
||||
);
|
||||
let _ = update_run_status(
|
||||
&app,
|
||||
&run_id,
|
||||
@@ -311,7 +379,7 @@ async fn run_s3_backup_worker_inner(
|
||||
Some(100),
|
||||
Some("S3 备份任务完成,但记录调度时间失败".to_string()),
|
||||
None,
|
||||
Some(format!("S3 backup slot record failed: {error:?}")),
|
||||
Some(S3_BACKUP_SLOT_RECORD_FAILURE_CODE.to_string()),
|
||||
None,
|
||||
Some(now_unix_secs()),
|
||||
)
|
||||
@@ -321,7 +389,7 @@ async fn run_s3_backup_worker_inner(
|
||||
&run_id,
|
||||
"failed",
|
||||
"S3 backup slot record failed",
|
||||
Some(json!({ "error": format!("{error:?}") })),
|
||||
Some(json!({ "error_code": S3_BACKUP_SLOT_RECORD_FAILURE_CODE })),
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
@@ -349,8 +417,12 @@ async fn run_s3_backup_worker_inner(
|
||||
)
|
||||
.await;
|
||||
}
|
||||
Err(error) => {
|
||||
warn!(error = %error, run_id = %run_id, "S3 backup task failed");
|
||||
Err(_) => {
|
||||
warn!(
|
||||
error_category = "backup_execution_failed",
|
||||
run_id = %run_id,
|
||||
"S3 backup task failed"
|
||||
);
|
||||
let _ = update_run_status(
|
||||
&app,
|
||||
&run_id,
|
||||
@@ -358,7 +430,7 @@ async fn run_s3_backup_worker_inner(
|
||||
Some(100),
|
||||
Some("S3 备份任务失败".to_string()),
|
||||
None,
|
||||
Some(error.to_string()),
|
||||
Some(S3_BACKUP_TASK_FAILURE_CODE.to_string()),
|
||||
None,
|
||||
Some(now_unix_secs()),
|
||||
)
|
||||
@@ -368,13 +440,44 @@ async fn run_s3_backup_worker_inner(
|
||||
&run_id,
|
||||
"failed",
|
||||
"S3 backup task failed",
|
||||
Some(json!({ "error": error.to_string() })),
|
||||
Some(json!({ "error_code": S3_BACKUP_TASK_FAILURE_CODE })),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn race_backup_with_lock_heartbeat<F, T>(
|
||||
backup: F,
|
||||
mut heartbeat: JoinHandle<()>,
|
||||
) -> BackupLockRaceOutcome<T>
|
||||
where
|
||||
F: Future<Output = T>,
|
||||
{
|
||||
tokio::pin!(backup);
|
||||
tokio::select! {
|
||||
biased;
|
||||
heartbeat_result = &mut heartbeat => {
|
||||
BackupLockRaceOutcome::LeaseLost(heartbeat_result)
|
||||
}
|
||||
result = &mut backup => {
|
||||
heartbeat.abort();
|
||||
let _ = heartbeat.await;
|
||||
BackupLockRaceOutcome::BackupCompleted(result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn require_successful_backup_lock_renewal<E>(
|
||||
result: Result<bool, E>,
|
||||
) -> Result<(), BackupLockRenewalFailure<E>> {
|
||||
match result {
|
||||
Ok(true) => Ok(()),
|
||||
Ok(false) => Err(BackupLockRenewalFailure::Lost),
|
||||
Err(error) => Err(BackupLockRenewalFailure::Backend(error)),
|
||||
}
|
||||
}
|
||||
|
||||
fn spawn_s3_backup_task_heartbeat(
|
||||
app: AppState,
|
||||
run_id: String,
|
||||
@@ -386,10 +489,30 @@ fn spawn_s3_backup_task_heartbeat(
|
||||
interval.tick().await;
|
||||
loop {
|
||||
interval.tick().await;
|
||||
let _ = app
|
||||
.runtime_state
|
||||
.lock_renew(&lock, S3_BACKUP_TASK_LOCK_TTL)
|
||||
.await;
|
||||
match require_successful_backup_lock_renewal(
|
||||
app.runtime_state
|
||||
.lock_renew(&lock, S3_BACKUP_TASK_LOCK_TTL)
|
||||
.await,
|
||||
) {
|
||||
Ok(()) => {}
|
||||
Err(BackupLockRenewalFailure::Lost) => {
|
||||
warn!(
|
||||
run_id = %run_id,
|
||||
lock_key = %lock.key,
|
||||
"S3 backup task distributed lock is no longer owned"
|
||||
);
|
||||
return;
|
||||
}
|
||||
Err(BackupLockRenewalFailure::Backend(error)) => {
|
||||
warn!(
|
||||
run_id = %run_id,
|
||||
lock_key = %lock.key,
|
||||
error = %error,
|
||||
"S3 backup task distributed lock renewal failed"
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
let _ = update_run_status(
|
||||
&app,
|
||||
&run_id,
|
||||
@@ -458,9 +581,15 @@ async fn acquire_s3_backup_task_lock(
|
||||
Ok(None) => Err(S3BackupTaskError::conflict(
|
||||
"已有 S3 备份任务正在执行,请等待当前任务完成后再试",
|
||||
)),
|
||||
Err(error) => Err(S3BackupTaskError::service_unavailable(format!(
|
||||
"无法获取 S3 备份任务锁:{error}"
|
||||
))),
|
||||
Err(_) => {
|
||||
warn!(
|
||||
error_category = "lock_acquisition_failed",
|
||||
"S3 backup task lock acquisition failed"
|
||||
);
|
||||
Err(S3BackupTaskError::service_unavailable(
|
||||
"无法获取 S3 备份任务锁,请稍后重试",
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -509,25 +638,77 @@ async fn run_s3_backup_once(
|
||||
app: &AppState,
|
||||
config: &S3BackupConfig,
|
||||
) -> Result<BackupRunResult, S3BackupTaskError> {
|
||||
let admin_state = AdminAppState::new(app);
|
||||
let payload = match config.scope {
|
||||
BackupScope::Config => {
|
||||
admin_state
|
||||
.build_admin_system_config_export_payload()
|
||||
.await?
|
||||
}
|
||||
BackupScope::Users => {
|
||||
admin_state
|
||||
.build_admin_system_users_export_payload()
|
||||
.await?
|
||||
}
|
||||
BackupScope::Data => admin_state.build_admin_system_data_export_payload().await?,
|
||||
let Some(encryption_secret) = effective_backup_encryption_secret(app) else {
|
||||
return Err(S3BackupTaskError::service_unavailable(
|
||||
"S3 备份需要 AETHER_BACKUP_ENCRYPTION_KEY 或可用的数据加密密钥",
|
||||
));
|
||||
};
|
||||
let store = ObjectStoreS3BackupStore::from_config(config)
|
||||
.map_err(|error| S3BackupTaskError::internal(error.to_string()))?;
|
||||
run_backup_with_store(config, &store, payload, Utc::now())
|
||||
let payload = build_s3_backup_payload_exclusively(app, config.scope).await?;
|
||||
let store = ObjectStoreS3BackupStore::from_config(config).map_err(|_| {
|
||||
warn!(
|
||||
error_category = "object_store_initialization_failed",
|
||||
"S3 backup object store initialization failed"
|
||||
);
|
||||
S3BackupTaskError::internal(S3_BACKUP_INTERNAL_ERROR_DETAIL)
|
||||
})?;
|
||||
run_backup_with_store(config, &store, payload, Utc::now(), &encryption_secret)
|
||||
.await
|
||||
.map_err(|error| S3BackupTaskError::internal(error.to_string()))
|
||||
.map_err(|_| {
|
||||
warn!(
|
||||
error_category = "backup_execution_failed",
|
||||
"S3 backup execution failed"
|
||||
);
|
||||
S3BackupTaskError::internal(S3_BACKUP_INTERNAL_ERROR_DETAIL)
|
||||
})
|
||||
}
|
||||
|
||||
async fn build_s3_backup_payload_exclusively(
|
||||
app: &AppState,
|
||||
scope: BackupScope,
|
||||
) -> Result<Value, S3BackupTaskError> {
|
||||
let admin_state = AdminAppState::new(app);
|
||||
crate::admin_api::execute_admin_system_import_exclusively(app, async {
|
||||
match scope {
|
||||
BackupScope::Config => {
|
||||
admin_state
|
||||
.build_admin_system_config_export_payload(SystemExportMode::RecoveryBackup)
|
||||
.await
|
||||
}
|
||||
BackupScope::Users => {
|
||||
admin_state
|
||||
.build_admin_system_users_export_payload(SystemExportMode::RecoveryBackup)
|
||||
.await
|
||||
}
|
||||
BackupScope::Data => {
|
||||
admin_state
|
||||
.build_admin_system_data_export_payload(SystemExportMode::RecoveryBackup)
|
||||
.await
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|error| {
|
||||
warn!(
|
||||
error_category = "system_import_coordination_failed",
|
||||
lock_error = ?error,
|
||||
"S3 backup snapshot could not acquire or retain the system import lock"
|
||||
);
|
||||
S3BackupTaskError::service_unavailable(S3_BACKUP_INTERNAL_ERROR_DETAIL)
|
||||
})?
|
||||
.map_err(S3BackupTaskError::from)
|
||||
}
|
||||
|
||||
fn effective_backup_encryption_secret(app: &AppState) -> Option<String> {
|
||||
std::env::var("AETHER_BACKUP_ENCRYPTION_KEY")
|
||||
.ok()
|
||||
.map(|value| value.trim().to_string())
|
||||
.filter(|value| !value.is_empty())
|
||||
.or_else(|| {
|
||||
app.encryption_key()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
})
|
||||
}
|
||||
|
||||
async fn load_s3_backup_config_for_run(
|
||||
@@ -551,7 +732,7 @@ pub(crate) async fn load_s3_backup_config_values(
|
||||
.or_else(|| admin_system_config_default_value(key));
|
||||
if let Some(value) = value {
|
||||
let value = if *key == "backup_s3_secret_access_key" {
|
||||
decrypt_s3_secret_access_key(app, value)?
|
||||
decrypt_s3_secret_access_key(app, value).await?
|
||||
} else {
|
||||
value
|
||||
};
|
||||
@@ -561,39 +742,56 @@ pub(crate) async fn load_s3_backup_config_values(
|
||||
Ok(values)
|
||||
}
|
||||
|
||||
fn decrypt_s3_secret_access_key(app: &AppState, value: Value) -> Result<Value, S3BackupTaskError> {
|
||||
let Some(ciphertext) = value
|
||||
async fn decrypt_s3_secret_access_key(
|
||||
app: &AppState,
|
||||
value: Value,
|
||||
) -> Result<Value, S3BackupTaskError> {
|
||||
let Some(stored_value) = value
|
||||
.as_str()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
else {
|
||||
return Ok(value);
|
||||
};
|
||||
let Some(plaintext) = decrypt_catalog_secret_with_fallbacks(app.encryption_key(), ciphertext)
|
||||
else {
|
||||
return Err(S3BackupTaskError::bad_request(
|
||||
let plaintext = decrypt_or_migrate_system_config_secret(
|
||||
app,
|
||||
"backup_s3_secret_access_key",
|
||||
stored_value.to_string(),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
S3BackupTaskError::bad_request(
|
||||
"S3 备份配置无效:Secret Access Key(访问密钥)无法解密,请重新填写",
|
||||
));
|
||||
};
|
||||
)
|
||||
})?;
|
||||
Ok(Value::String(plaintext))
|
||||
}
|
||||
|
||||
fn backup_run_result_json(result: &BackupRunResult) -> Value {
|
||||
json!({
|
||||
"scope": result.scope.as_config_value(),
|
||||
"bucket": result.bucket,
|
||||
"object_key": result.object_key,
|
||||
"bytes": result.bytes,
|
||||
"sha256": result.sha256,
|
||||
"export_version": result.export_version,
|
||||
"exported_at": result.exported_at,
|
||||
"compression": result.compression,
|
||||
"deleted_old_objects": result.deleted_old_objects,
|
||||
"encryption": result.encryption,
|
||||
"legacy_encrypted_copies_created": result.legacy_encrypted_copies_created,
|
||||
"legacy_encrypted_copies_verified": result.legacy_encrypted_copies_verified,
|
||||
"legacy_plaintext_objects_deleted": result.legacy_plaintext_objects_deleted,
|
||||
"legacy_plaintext_objects_retained": result.legacy_plaintext_objects_retained,
|
||||
"retention_cleanup_candidates": result.retention_cleanup_candidates,
|
||||
"automatic_deletions": result.legacy_plaintext_objects_deleted,
|
||||
"object_cleanup_mode": "legacy_plaintext_deleted_after_verified_encryption",
|
||||
"versioned_storage_cleanup_required": result.versioned_storage_cleanup_required,
|
||||
"versioned_storage_cleanup_notice": "legacy_plaintext_versions_require_external_cleanup",
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::convert::Infallible;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
|
||||
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
|
||||
@@ -603,9 +801,78 @@ mod tests {
|
||||
};
|
||||
|
||||
use crate::data::GatewayDataState;
|
||||
use crate::handlers::shared::decrypt_system_config_secret;
|
||||
use crate::state::AppState;
|
||||
use crate::task_runtime::{now_unix_secs, TASK_KEY_SYSTEM_S3_BACKUP};
|
||||
|
||||
#[test]
|
||||
fn backup_lock_renewal_requires_ownership_and_preserves_backend_errors() {
|
||||
assert!(matches!(
|
||||
super::require_successful_backup_lock_renewal::<Infallible>(Ok(true)),
|
||||
Ok(())
|
||||
));
|
||||
assert!(matches!(
|
||||
super::require_successful_backup_lock_renewal::<Infallible>(Ok(false)),
|
||||
Err(super::BackupLockRenewalFailure::Lost)
|
||||
));
|
||||
assert!(matches!(
|
||||
super::require_successful_backup_lock_renewal(Err("redis unavailable")),
|
||||
Err(super::BackupLockRenewalFailure::Backend(
|
||||
"redis unavailable"
|
||||
))
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lost_backup_lock_stops_race_without_publishing_backup_result() {
|
||||
let destructive_stage_reached = Arc::new(AtomicBool::new(false));
|
||||
let destructive_stage_for_backup = Arc::clone(&destructive_stage_reached);
|
||||
let backup = async move {
|
||||
std::future::pending::<()>().await;
|
||||
destructive_stage_for_backup.store(true, Ordering::Release);
|
||||
Ok::<(), super::S3BackupTaskError>(())
|
||||
};
|
||||
let heartbeat = tokio::spawn(async {});
|
||||
let outcome = super::race_backup_with_lock_heartbeat(backup, heartbeat).await;
|
||||
assert!(matches!(
|
||||
outcome,
|
||||
super::BackupLockRaceOutcome::LeaseLost(Ok(()))
|
||||
));
|
||||
assert!(!destructive_stage_reached.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completed_heartbeat_wins_when_backup_completion_is_also_ready() {
|
||||
let heartbeat = tokio::spawn(async {});
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
let outcome = super::race_backup_with_lock_heartbeat(async { 42_u8 }, heartbeat).await;
|
||||
|
||||
assert!(matches!(
|
||||
outcome,
|
||||
super::BackupLockRaceOutcome::LeaseLost(Ok(()))
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn s3_backup_snapshot_refuses_to_overlap_system_import() {
|
||||
let app = AppState::new().expect("app state should build");
|
||||
let lease = crate::admin_api::try_acquire_admin_system_import_lease(&app)
|
||||
.await
|
||||
.expect("test should acquire the system import lease");
|
||||
|
||||
let error = super::build_s3_backup_payload_exclusively(
|
||||
&app,
|
||||
crate::backup::scopes::BackupScope::Config,
|
||||
)
|
||||
.await
|
||||
.expect_err("backup snapshot must not overlap a system import");
|
||||
|
||||
crate::admin_api::release_admin_system_import_lease(&app, &lease).await;
|
||||
assert_eq!(error.status(), axum::http::StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(error.detail(), super::S3_BACKUP_INTERNAL_ERROR_DETAIL);
|
||||
}
|
||||
|
||||
fn valid_s3_backup_config_values() -> Vec<(String, serde_json::Value)> {
|
||||
vec![
|
||||
(
|
||||
@@ -632,6 +899,92 @@ mod tests {
|
||||
]
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_plaintext_s3_secret_is_migrated_when_config_loads() {
|
||||
let plaintext = "legacy-s3-secret-access-key";
|
||||
let mut entries = valid_s3_backup_config_values();
|
||||
entries
|
||||
.iter_mut()
|
||||
.find(|(key, _)| key == "backup_s3_secret_access_key")
|
||||
.expect("secret config fixture should exist")
|
||||
.1 = serde_json::json!(plaintext);
|
||||
let app = AppState::new()
|
||||
.expect("app state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::disabled()
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY)
|
||||
.with_system_config_values_for_tests(entries),
|
||||
);
|
||||
|
||||
let values = super::load_s3_backup_config_values(&app)
|
||||
.await
|
||||
.expect("legacy S3 config should load");
|
||||
assert_eq!(
|
||||
values.get("backup_s3_secret_access_key"),
|
||||
Some(&serde_json::json!(plaintext))
|
||||
);
|
||||
|
||||
let stored = app
|
||||
.read_system_config_json_value_strong("backup_s3_secret_access_key")
|
||||
.await
|
||||
.expect("stored S3 secret should read")
|
||||
.and_then(|value| value.as_str().map(ToOwned::to_owned))
|
||||
.expect("stored S3 secret should remain a string");
|
||||
assert_ne!(stored, plaintext);
|
||||
assert_eq!(
|
||||
decrypt_system_config_secret(&app, "backup_s3_secret_access_key", &stored)
|
||||
.expect("migrated S3 secret should decrypt"),
|
||||
plaintext
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn undecryptable_s3_fernet_secret_fails_closed() {
|
||||
let plaintext = "s3-secret-from-unavailable-key";
|
||||
let ciphertext = encrypt_python_fernet_plaintext("unavailable-s3-key", plaintext)
|
||||
.expect("unknown-key fixture should encrypt");
|
||||
let mut entries = valid_s3_backup_config_values();
|
||||
entries
|
||||
.iter_mut()
|
||||
.find(|(key, _)| key == "backup_s3_secret_access_key")
|
||||
.expect("secret config fixture should exist")
|
||||
.1 = serde_json::json!(ciphertext.clone());
|
||||
let app = AppState::new()
|
||||
.expect("app state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::disabled()
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY)
|
||||
.with_system_config_values_for_tests(entries),
|
||||
);
|
||||
|
||||
let error = super::load_s3_backup_config_values(&app)
|
||||
.await
|
||||
.expect_err("unknown-key S3 ciphertext must fail closed");
|
||||
let error_text = error.to_string();
|
||||
assert!(!error_text.contains(plaintext));
|
||||
assert!(!error_text.contains(&ciphertext));
|
||||
assert_eq!(
|
||||
app.read_system_config_json_value_strong("backup_s3_secret_access_key")
|
||||
.await
|
||||
.expect("stored S3 secret should read"),
|
||||
Some(serde_json::json!(ciphertext))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gateway_dependency_errors_are_not_exposed_to_backup_clients() {
|
||||
let error = super::S3BackupTaskError::from(crate::GatewayError::Internal(
|
||||
"postgresql://admin:database-secret@db.internal/aether".to_string(),
|
||||
));
|
||||
|
||||
assert_eq!(
|
||||
error.status(),
|
||||
axum::http::StatusCode::INTERNAL_SERVER_ERROR
|
||||
);
|
||||
assert_eq!(error.detail(), super::S3_BACKUP_INTERNAL_ERROR_DETAIL);
|
||||
assert!(!error.detail().contains("database-secret"));
|
||||
}
|
||||
|
||||
fn stored_s3_backup_run(status: BackgroundTaskStatus) -> StoredBackgroundTaskRun {
|
||||
let now = now_unix_secs();
|
||||
StoredBackgroundTaskRun {
|
||||
@@ -774,7 +1127,9 @@ mod tests {
|
||||
|
||||
let payload = super::s3_backup_task_payload_json(&config, "manual", None);
|
||||
|
||||
assert!(payload["bucket"].is_string());
|
||||
assert!(payload.get("bucket").is_none());
|
||||
assert!(payload.get("prefix").is_none());
|
||||
assert_eq!(payload["scope"], serde_json::json!("data"));
|
||||
assert_eq!(payload["trigger"], serde_json::json!("manual"));
|
||||
assert!(!payload.to_string().contains("secret"));
|
||||
}
|
||||
|
||||
@@ -29,8 +29,11 @@ pub(crate) fn spawn_s3_backup_worker(app: AppState) -> Option<JoinHandle<()>> {
|
||||
interval.tick().await;
|
||||
loop {
|
||||
interval.tick().await;
|
||||
if let Err(error) = run_s3_backup_schedule_tick(&app, Utc::now()).await {
|
||||
warn!(error = ?error, "S3 backup schedule tick failed");
|
||||
if run_s3_backup_schedule_tick(&app, Utc::now()).await.is_err() {
|
||||
warn!(
|
||||
error_category = "schedule_tick_failed",
|
||||
"S3 backup schedule tick failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -43,15 +46,21 @@ async fn run_s3_backup_schedule_tick(
|
||||
) -> Result<(), GatewayError> {
|
||||
let values = match super::task::load_s3_backup_config_values(app).await {
|
||||
Ok(values) => values,
|
||||
Err(error) => {
|
||||
warn!(error = %error, "S3 backup schedule config load failed");
|
||||
Err(_) => {
|
||||
warn!(
|
||||
error_category = "config_load_failed",
|
||||
"S3 backup schedule config load failed"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
let config = match S3BackupConfig::from_json_map(&values) {
|
||||
Ok(config) => config,
|
||||
Err(error) => {
|
||||
warn!(error = %error, "S3 backup schedule config is invalid");
|
||||
Err(_) => {
|
||||
warn!(
|
||||
error_category = "config_invalid",
|
||||
"S3 backup schedule config is invalid"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
@@ -68,8 +77,11 @@ async fn run_s3_backup_schedule_tick(
|
||||
|
||||
match super::task::start_s3_backup_task_for_schedule(app.clone(), slot).await {
|
||||
Ok(_) => {}
|
||||
Err(error) => {
|
||||
warn!(error = %error, "S3 backup scheduled task submission failed");
|
||||
Err(_) => {
|
||||
warn!(
|
||||
error_category = "task_submission_failed",
|
||||
"S3 backup scheduled task submission failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
use crate::handlers::shared::{
|
||||
decrypt_catalog_secret_with_fallbacks, system_config_bool, system_config_string,
|
||||
bark_device_key_binding, canonical_bark_server_url, decrypt_or_migrate_bark_device_key,
|
||||
system_config_bool, system_config_string,
|
||||
};
|
||||
use crate::{AppState, GatewayError};
|
||||
use serde_json::{json, Value};
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
|
||||
pub(crate) const BARK_PUSH_ENABLED_KEY: &str = "module.bark_push.enabled";
|
||||
pub(crate) const BARK_PUSH_DEVICE_KEY_KEY: &str = "module.bark_push.device_key";
|
||||
@@ -10,8 +12,20 @@ pub(crate) const BARK_PUSH_SERVER_URL_KEY: &str = "module.bark_push.server_url";
|
||||
pub(crate) const BARK_PUSH_TEMPLATE_KEY: &str = "module.bark_push.template";
|
||||
|
||||
const DEFAULT_BARK_API_BASE: &str = "https://api.day.app";
|
||||
const BARK_ALLOW_HTTP_ENV: &str = "AETHER_BARK_ALLOW_HTTP";
|
||||
const BARK_ALLOW_PRIVATE_TARGETS_ENV: &str = "AETHER_BARK_ALLOW_PRIVATE_TARGETS";
|
||||
const MAX_BARK_RESPONSE_BYTES: usize = 64 * 1024;
|
||||
const BARK_CONNECT_TIMEOUT_MS: u64 = 10_000;
|
||||
const BARK_REQUEST_TIMEOUT_MS: u64 = 300_000;
|
||||
const MAX_BARK_SERVER_URL_BYTES: usize = 2 * 1024;
|
||||
const MAX_BARK_DEVICE_KEY_BYTES: usize = 512;
|
||||
const MAX_BARK_TEMPLATE_BYTES: usize = 256 * 1024;
|
||||
const MAX_BARK_TITLE_BYTES: usize = 512;
|
||||
const MAX_BARK_BODY_BYTES: usize = 2 * 1024 * 1024;
|
||||
const MAX_BARK_RENDERED_BODY_BYTES: usize = 2 * 1024 * 1024;
|
||||
const MAX_BARK_RESOLVED_ADDRESSES: usize = 32;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct BarkPushConfig {
|
||||
pub(crate) enabled: bool,
|
||||
pub(crate) device_key: Option<String>,
|
||||
@@ -19,6 +33,21 @@ pub(crate) struct BarkPushConfig {
|
||||
pub(crate) template: Option<String>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for BarkPushConfig {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("BarkPushConfig")
|
||||
.field("enabled", &self.enabled)
|
||||
.field(
|
||||
"device_key",
|
||||
&self.device_key.as_ref().map(|_| "[REDACTED]"),
|
||||
)
|
||||
.field("server_url", &self.server_url)
|
||||
.field("template", &self.template.as_ref().map(|_| "[REDACTED]"))
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn bark_push_module_enabled(state: &AppState) -> Result<bool, GatewayError> {
|
||||
let value = state
|
||||
.read_system_config_json_value(BARK_PUSH_ENABLED_KEY)
|
||||
@@ -35,23 +64,34 @@ pub(crate) async fn read_bark_push_config(
|
||||
state: &AppState,
|
||||
) -> Result<BarkPushConfig, GatewayError> {
|
||||
let enabled = bark_push_module_enabled(state).await?;
|
||||
let device_key = state
|
||||
.read_system_config_json_value(BARK_PUSH_DEVICE_KEY_KEY)
|
||||
.await?
|
||||
.and_then(|value| system_config_string(Some(&value)))
|
||||
.map(|value| {
|
||||
decrypt_catalog_secret_with_fallbacks(state.encryption_key(), &value).unwrap_or(value)
|
||||
});
|
||||
let server_url = state
|
||||
.read_system_config_json_value(BARK_PUSH_SERVER_URL_KEY)
|
||||
.await?
|
||||
.and_then(|value| system_config_string(Some(&value)))
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
.unwrap_or_else(|| DEFAULT_BARK_API_BASE.to_string());
|
||||
validate_bark_config_field("server_url", &server_url, MAX_BARK_SERVER_URL_BYTES)?;
|
||||
let server_url = normalized_bark_server_url(&server_url)?;
|
||||
let binding = bark_device_key_binding(&server_url)
|
||||
.ok_or_else(|| GatewayError::Internal("Bark 服务器地址不是合法 URL".to_string()))?;
|
||||
let device_key = state
|
||||
.read_system_config_json_value(BARK_PUSH_DEVICE_KEY_KEY)
|
||||
.await?
|
||||
.and_then(|value| system_config_string(Some(&value)));
|
||||
let device_key = match device_key {
|
||||
Some(value) => Some(decrypt_or_migrate_bark_device_key(state, &binding, value).await?),
|
||||
None => None,
|
||||
};
|
||||
if let Some(device_key) = device_key.as_deref() {
|
||||
validate_bark_config_field("device_key", device_key, MAX_BARK_DEVICE_KEY_BYTES)?;
|
||||
}
|
||||
let template = state
|
||||
.read_system_config_json_value(BARK_PUSH_TEMPLATE_KEY)
|
||||
.await?
|
||||
.and_then(|value| system_config_string(Some(&value)));
|
||||
if let Some(template) = template.as_deref() {
|
||||
validate_bark_config_field("template", template, MAX_BARK_TEMPLATE_BYTES)?;
|
||||
}
|
||||
|
||||
Ok(BarkPushConfig {
|
||||
enabled,
|
||||
@@ -62,7 +102,7 @@ pub(crate) async fn read_bark_push_config(
|
||||
}
|
||||
|
||||
pub(crate) async fn send_bark_push(
|
||||
state: &AppState,
|
||||
_state: &AppState,
|
||||
config: &BarkPushConfig,
|
||||
title: &str,
|
||||
markdown_body: &str,
|
||||
@@ -76,11 +116,13 @@ pub(crate) async fn send_bark_push(
|
||||
"Bark Device Key 不能为空".to_string(),
|
||||
));
|
||||
}
|
||||
let server_url = normalized_bark_server_url(&config.server_url)?;
|
||||
let body = render_bark_body(config.template.as_deref(), title, markdown_body);
|
||||
let response = state
|
||||
.client
|
||||
.post(format!("{server_url}/push"))
|
||||
validate_bark_config_field("device_key", device_key, MAX_BARK_DEVICE_KEY_BYTES)?;
|
||||
validate_bark_content_field("title", title, MAX_BARK_TITLE_BYTES)?;
|
||||
validate_bark_content_field("body", markdown_body, MAX_BARK_BODY_BYTES)?;
|
||||
let (client, push_url) = build_bark_push_client_and_url(&config.server_url).await?;
|
||||
let body = render_bark_body(config.template.as_deref(), title, markdown_body)?;
|
||||
let response = client
|
||||
.post(push_url)
|
||||
.json(&json!({
|
||||
"device_key": device_key,
|
||||
"title": title,
|
||||
@@ -88,16 +130,14 @@ pub(crate) async fn send_bark_push(
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
.map_err(|err| GatewayError::Internal(bark_request_error_message(&err)))?;
|
||||
let status = response.status();
|
||||
let text = response
|
||||
.text()
|
||||
let body = aether_http::read_response_bytes_with_limit(response, MAX_BARK_RESPONSE_BYTES)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
.map_err(|err| GatewayError::Internal(bark_response_body_error_message(&err)))?;
|
||||
let text = String::from_utf8_lossy(&body);
|
||||
if !status.is_success() {
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"Bark 返回 HTTP {status}: {text}"
|
||||
)));
|
||||
return Err(GatewayError::Internal(format!("Bark 返回 HTTP {status}")));
|
||||
}
|
||||
if let Ok(payload) = serde_json::from_str::<Value>(&text) {
|
||||
let code_is_ok = payload
|
||||
@@ -114,53 +154,249 @@ pub(crate) async fn send_bark_push(
|
||||
})
|
||||
.unwrap_or(true);
|
||||
if !code_is_ok {
|
||||
return Err(GatewayError::Internal(format!("Bark 返回失败: {payload}")));
|
||||
return Err(GatewayError::Internal("Bark 返回失败".to_string()));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn normalized_bark_server_url(server_url: &str) -> Result<String, GatewayError> {
|
||||
let server_url = server_url.trim().trim_end_matches('/');
|
||||
if server_url.is_empty() {
|
||||
return Err(GatewayError::Internal(
|
||||
"Bark 服务器地址不能为空".to_string(),
|
||||
));
|
||||
}
|
||||
if !server_url.starts_with("https://") && !server_url.starts_with("http://") {
|
||||
return Err(GatewayError::Internal(
|
||||
"Bark 服务器地址必须以 http:// 或 https:// 开头".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(server_url.to_string())
|
||||
fn bark_request_error_message(error: &reqwest::Error) -> String {
|
||||
format!("Bark 请求失败 ({})", bark_reqwest_error_kind(error))
|
||||
}
|
||||
|
||||
fn render_bark_body(template: Option<&str>, title: &str, markdown_body: &str) -> String {
|
||||
match template {
|
||||
Some(template) if !template.trim().is_empty() => template
|
||||
.replace("{title}", title)
|
||||
.replace("{body}", markdown_body),
|
||||
_ => markdown_body.to_string(),
|
||||
fn bark_response_body_error_message(error: &aether_http::ResponseBodyReadError) -> String {
|
||||
match error {
|
||||
aether_http::ResponseBodyReadError::TooLarge { max_bytes } => {
|
||||
format!("Bark 响应超过 {max_bytes} 字节")
|
||||
}
|
||||
aether_http::ResponseBodyReadError::Read(error) => {
|
||||
format!("Bark 响应读取失败 ({})", bark_reqwest_error_kind(error))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn bark_reqwest_error_kind(error: &reqwest::Error) -> &'static str {
|
||||
if error.is_timeout() {
|
||||
"timeout"
|
||||
} else if error.is_connect() {
|
||||
"connect"
|
||||
} else if error.is_request() {
|
||||
"request"
|
||||
} else {
|
||||
"transport"
|
||||
}
|
||||
}
|
||||
|
||||
fn normalized_bark_server_url(server_url: &str) -> Result<String, GatewayError> {
|
||||
canonical_bark_server_url(server_url)
|
||||
.ok_or_else(|| GatewayError::Internal("Bark 服务器地址不是合法 URL".to_string()))
|
||||
}
|
||||
|
||||
async fn build_bark_push_client_and_url(
|
||||
server_url: &str,
|
||||
) -> Result<(reqwest::Client, url::Url), GatewayError> {
|
||||
validate_bark_config_field("server_url", server_url, MAX_BARK_SERVER_URL_BYTES)?;
|
||||
let normalized = normalized_bark_server_url(server_url)?;
|
||||
let mut push_url = url::Url::parse(&normalized)
|
||||
.map_err(|_| GatewayError::Internal("Bark 服务器地址不是合法 URL".to_string()))?;
|
||||
validate_bark_transport_policy(&push_url, env_flag_enabled(BARK_ALLOW_HTTP_ENV))?;
|
||||
|
||||
let host = push_url
|
||||
.host_str()
|
||||
.ok_or_else(|| GatewayError::Internal("Bark 服务器地址缺少主机名".to_string()))?
|
||||
.to_string();
|
||||
let port = push_url
|
||||
.port_or_known_default()
|
||||
.ok_or_else(|| GatewayError::Internal("Bark 服务器地址缺少端口".to_string()))?;
|
||||
let addresses = if let Ok(ip) = host.parse::<IpAddr>() {
|
||||
vec![SocketAddr::new(ip, port)]
|
||||
} else {
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_millis(BARK_CONNECT_TIMEOUT_MS),
|
||||
tokio::net::lookup_host((host.as_str(), port)),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| GatewayError::Internal("Bark 服务器 DNS 解析超时".to_string()))?
|
||||
.map_err(|_| GatewayError::Internal("Bark 服务器 DNS 解析失败".to_string()))?
|
||||
.take(MAX_BARK_RESOLVED_ADDRESSES)
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
validate_bark_resolved_addresses(&addresses, env_flag_enabled(BARK_ALLOW_PRIVATE_TARGETS_ENV))?;
|
||||
|
||||
push_url
|
||||
.path_segments_mut()
|
||||
.map_err(|_| GatewayError::Internal("Bark 服务器地址不能作为基础 URL".to_string()))?
|
||||
.pop_if_empty()
|
||||
.push("push");
|
||||
|
||||
let mut builder = aether_http::apply_http_client_config(
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.redirect(reqwest::redirect::Policy::none()),
|
||||
&aether_http::HttpClientConfig {
|
||||
connect_timeout_ms: Some(BARK_CONNECT_TIMEOUT_MS),
|
||||
request_timeout_ms: Some(BARK_REQUEST_TIMEOUT_MS),
|
||||
http2_adaptive_window: true,
|
||||
..aether_http::HttpClientConfig::default()
|
||||
},
|
||||
);
|
||||
if host.parse::<IpAddr>().is_err() {
|
||||
builder = builder.resolve_to_addrs(&host, &addresses);
|
||||
}
|
||||
let client = builder
|
||||
.build()
|
||||
.map_err(|_| GatewayError::Internal("Bark HTTP 客户端初始化失败".to_string()))?;
|
||||
Ok((client, push_url))
|
||||
}
|
||||
|
||||
fn validate_bark_transport_policy(url: &url::Url, allow_http: bool) -> Result<(), GatewayError> {
|
||||
if url.scheme() == "http" && !allow_http {
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"Bark 服务器必须使用 HTTPS;如确需明文 HTTP,请显式设置 {BARK_ALLOW_HTTP_ENV}=true"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_bark_resolved_addresses(
|
||||
addresses: &[SocketAddr],
|
||||
allow_private: bool,
|
||||
) -> Result<(), GatewayError> {
|
||||
if addresses.is_empty() {
|
||||
return Err(GatewayError::Internal(
|
||||
"Bark 服务器 DNS 解析未返回地址".to_string(),
|
||||
));
|
||||
}
|
||||
if !allow_private
|
||||
&& addresses
|
||||
.iter()
|
||||
.any(|address| aether_http::is_private_or_reserved_ip(address.ip()))
|
||||
{
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"Bark 服务器解析到私有或保留地址;如确需内网自建服务,请显式设置 {BARK_ALLOW_PRIVATE_TARGETS_ENV}=true"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn env_flag_enabled(key: &str) -> bool {
|
||||
std::env::var(key).ok().is_some_and(|value| {
|
||||
matches!(
|
||||
value.trim().to_ascii_lowercase().as_str(),
|
||||
"1" | "true" | "yes" | "on"
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_bark_config_field(
|
||||
field: &str,
|
||||
value: &str,
|
||||
max_bytes: usize,
|
||||
) -> Result<(), GatewayError> {
|
||||
if value.len() > max_bytes || value.bytes().any(|byte| byte == 0) {
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"Bark {field} exceeds the allowed size or contains a NUL byte"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_bark_content_field(
|
||||
field: &str,
|
||||
value: &str,
|
||||
max_bytes: usize,
|
||||
) -> Result<(), GatewayError> {
|
||||
validate_bark_config_field(field, value, max_bytes)
|
||||
}
|
||||
|
||||
fn render_bark_body(
|
||||
template: Option<&str>,
|
||||
title: &str,
|
||||
markdown_body: &str,
|
||||
) -> Result<String, GatewayError> {
|
||||
validate_bark_content_field("title", title, MAX_BARK_TITLE_BYTES)?;
|
||||
validate_bark_content_field("body", markdown_body, MAX_BARK_BODY_BYTES)?;
|
||||
let template = template
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
.unwrap_or("{body}");
|
||||
validate_bark_config_field("template", template, MAX_BARK_TEMPLATE_BYTES)?;
|
||||
|
||||
let mut rendered = String::with_capacity(template.len().min(MAX_BARK_RENDERED_BODY_BYTES));
|
||||
let mut cursor = 0usize;
|
||||
while cursor < template.len() {
|
||||
let remaining = &template[cursor..];
|
||||
let title_match = remaining.find("{title}");
|
||||
let body_match = remaining.find("{body}");
|
||||
let next = match (title_match, body_match) {
|
||||
(None, None) => {
|
||||
append_bark_rendered_part(&mut rendered, remaining)?;
|
||||
cursor = template.len();
|
||||
continue;
|
||||
}
|
||||
(Some(index), None) => (index, "{title}", title),
|
||||
(None, Some(index)) => (index, "{body}", markdown_body),
|
||||
(Some(title_index), Some(body_index)) if title_index <= body_index => {
|
||||
(title_index, "{title}", title)
|
||||
}
|
||||
(Some(_), Some(body_index)) => (body_index, "{body}", markdown_body),
|
||||
};
|
||||
append_bark_rendered_part(&mut rendered, &remaining[..next.0])?;
|
||||
append_bark_rendered_part(&mut rendered, next.2)?;
|
||||
cursor += next.0 + next.1.len();
|
||||
}
|
||||
if rendered.is_empty() && template.is_empty() {
|
||||
return Ok(String::new());
|
||||
}
|
||||
Ok(rendered)
|
||||
}
|
||||
|
||||
fn append_bark_rendered_part(output: &mut String, part: &str) -> Result<(), GatewayError> {
|
||||
let next_len = output
|
||||
.len()
|
||||
.checked_add(part.len())
|
||||
.ok_or_else(|| GatewayError::Internal("Bark rendered body is too large".to_string()))?;
|
||||
if next_len > MAX_BARK_RENDERED_BODY_BYTES {
|
||||
return Err(GatewayError::Internal(
|
||||
"Bark rendered body exceeds the allowed size".to_string(),
|
||||
));
|
||||
}
|
||||
output.push_str(part);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{normalized_bark_server_url, render_bark_body};
|
||||
use super::{
|
||||
bark_request_error_message, bark_response_body_error_message, normalized_bark_server_url,
|
||||
render_bark_body, validate_bark_resolved_addresses, validate_bark_transport_policy,
|
||||
};
|
||||
use std::net::SocketAddr;
|
||||
|
||||
#[test]
|
||||
fn bark_body_uses_template_when_provided() {
|
||||
let rendered = render_bark_body(Some("{title}\n\n{body}"), "告警", "原始正文");
|
||||
let rendered = render_bark_body(Some("{title}\n\n{body}"), "告警", "原始正文")
|
||||
.expect("template should render");
|
||||
assert_eq!(rendered, "告警\n\n原始正文");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bark_body_falls_back_to_markdown_body_for_empty_template() {
|
||||
assert_eq!(render_bark_body(None, "告警", "原始正文"), "原始正文");
|
||||
assert_eq!(
|
||||
render_bark_body(Some(" "), "告警", "原始正文"),
|
||||
render_bark_body(None, "告警", "原始正文").expect("fallback should render"),
|
||||
"原始正文"
|
||||
);
|
||||
assert_eq!(
|
||||
render_bark_body(Some(" "), "告警", "原始正文").expect("fallback should render"),
|
||||
"原始正文"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bark_body_rejects_template_expansion_bombs_and_oversized_content() {
|
||||
let template = "x".repeat(super::MAX_BARK_TEMPLATE_BYTES + 1);
|
||||
assert!(render_bark_body(Some(&template), "告警", "正文").is_err());
|
||||
let body = "x".repeat(super::MAX_BARK_BODY_BYTES + 1);
|
||||
assert!(render_bark_body(None, "告警", &body).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -170,4 +406,48 @@ mod tests {
|
||||
"https://api.day.app"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bark_server_url_rejects_credentials_query_and_fragments() {
|
||||
for invalid in [
|
||||
"https://user@example.com",
|
||||
"https://example.com?target=internal",
|
||||
"https://example.com/#fragment",
|
||||
] {
|
||||
assert!(normalized_bark_server_url(invalid).is_err(), "{invalid}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bark_http_transport_requires_explicit_opt_in() {
|
||||
let url = url::Url::parse("http://bark.example.com").unwrap();
|
||||
assert!(validate_bark_transport_policy(&url, false).is_err());
|
||||
assert!(validate_bark_transport_policy(&url, true).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bark_private_targets_require_explicit_opt_in() {
|
||||
let private = [SocketAddr::from(([127, 0, 0, 1], 443))];
|
||||
assert!(validate_bark_resolved_addresses(&private, false).is_err());
|
||||
assert!(validate_bark_resolved_addresses(&private, true).is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bark_transport_errors_do_not_expose_server_url_or_response_body() {
|
||||
let secret = "bark-secret-query";
|
||||
let error = reqwest::Client::new()
|
||||
.post(format!("ftp://bark.example.test/push?token={secret}"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("unsupported URL scheme should fail before network I/O");
|
||||
|
||||
let message = bark_request_error_message(&error);
|
||||
assert!(!message.contains(secret));
|
||||
assert!(!message.contains("bark.example.test"));
|
||||
|
||||
let body_error = aether_http::ResponseBodyReadError::Read(error);
|
||||
let message = bark_response_body_error_message(&body_error);
|
||||
assert!(!message.contains(secret));
|
||||
assert!(!message.contains("bark.example.test"));
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -193,6 +193,7 @@ pub(crate) fn parse_probe_url(raw: &str) -> Result<Url, ProbeFailure> {
|
||||
|| url.password().is_some()
|
||||
|| url.query().is_some()
|
||||
|| url.fragment().is_some()
|
||||
|| (url.scheme() == "ws" && !aether_http::url_has_literal_loopback_host(&url))
|
||||
{
|
||||
return Err(ProbeFailure::InvalidEndpoint);
|
||||
}
|
||||
@@ -210,6 +211,7 @@ pub(crate) const fn turn_timeout(args: &ProbeArgs) -> Duration {
|
||||
|
||||
async fn run_probe(config: &ProbeConfig, started_at: Instant) -> Result<ProbeReport, ProbeFailure> {
|
||||
let client = wreq::Client::builder()
|
||||
.no_proxy()
|
||||
.connect_timeout(config.turn_timeout)
|
||||
.timeout(config.turn_timeout)
|
||||
.build()
|
||||
@@ -434,7 +436,14 @@ mod tests {
|
||||
#[test]
|
||||
fn probe_url_rejects_credentials_and_query_strings() {
|
||||
assert!(parse_probe_url("wss://example.test/v1/responses").is_ok());
|
||||
assert!(parse_probe_url("ws://localhost:8080/v1/responses").is_ok());
|
||||
assert!(parse_probe_url("ws://127.42.0.1:8080/v1/responses").is_ok());
|
||||
assert!(parse_probe_url("ws://[::1]:8080/v1/responses").is_ok());
|
||||
assert!(parse_probe_url("https://example.test/v1/responses").is_err());
|
||||
assert!(parse_probe_url("ws://example.test/v1/responses").is_err());
|
||||
assert!(parse_probe_url("ws://10.0.0.1/v1/responses").is_err());
|
||||
assert!(parse_probe_url("ws://0.0.0.0:8080/v1/responses").is_err());
|
||||
assert!(parse_probe_url("ws://[::ffff:127.0.0.1]:8080/v1/responses").is_err());
|
||||
assert!(parse_probe_url("wss://token@example.test/v1/responses").is_err());
|
||||
assert!(parse_probe_url("wss://example.test/v1/responses?token=secret").is_err());
|
||||
}
|
||||
|
||||
@@ -422,6 +422,7 @@ mod tests {
|
||||
local_rejection: None,
|
||||
allowed_models: None,
|
||||
ip_rules: None,
|
||||
verified_api_key_hash: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -173,6 +173,15 @@ impl SystemConfigCache {
|
||||
self.detach_all_loads();
|
||||
}
|
||||
|
||||
pub(crate) fn invalidate(&self, key: &str) {
|
||||
let Ok(_mutation) = self.mutation.lock() else {
|
||||
return;
|
||||
};
|
||||
self.generation.fetch_add(1, Ordering::AcqRel);
|
||||
self.entries.remove(&key.to_string());
|
||||
self.detach_all_loads();
|
||||
}
|
||||
|
||||
pub(crate) fn insert_if_generation(
|
||||
&self,
|
||||
key: String,
|
||||
|
||||
@@ -18,6 +18,7 @@ pub(crate) const TUNNEL_AFFINITY_FORWARDED_BY_HEADER: &str =
|
||||
"x-aether-tunnel-affinity-forwarded-by";
|
||||
pub(crate) const TUNNEL_AFFINITY_OWNER_INSTANCE_HEADER: &str =
|
||||
"x-aether-tunnel-affinity-owner-instance-id";
|
||||
pub(crate) const TUNNEL_AFFINITY_NODE_ID_HEADER: &str = "x-aether-tunnel-affinity-node-id";
|
||||
pub(crate) const EXECUTION_PATH_PUBLIC_PROXY_PASSTHROUGH: &str = "public_proxy_passthrough";
|
||||
pub(crate) const EXECUTION_PATH_LOCAL_PROXY_PASSTHROUGH_REMOVED: &str =
|
||||
"local_proxy_passthrough_removed";
|
||||
|
||||
@@ -23,7 +23,7 @@ pub(crate) fn extract_requested_model(
|
||||
body: &Bytes,
|
||||
) -> Option<String> {
|
||||
if decision.route_family.as_deref() == Some("gemini") {
|
||||
if let Some(model) = extract_gemini_model_from_path(uri.path()) {
|
||||
if let Some(model) = extract_gemini_requested_model_from_path(uri.path()) {
|
||||
return Some(model);
|
||||
}
|
||||
}
|
||||
@@ -43,23 +43,46 @@ pub(crate) fn extract_requested_model(
|
||||
.filter(|value| !value.is_empty())
|
||||
}
|
||||
|
||||
fn extract_gemini_requested_model_from_path(path: &str) -> Option<String> {
|
||||
let model = extract_gemini_model_from_path(path)?;
|
||||
Some(
|
||||
model
|
||||
.split_once("/operations/")
|
||||
.map(|(model, _)| model)
|
||||
.unwrap_or(model.as_str())
|
||||
.to_string(),
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn extract_request_credentials(
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
auth_endpoint_signature: &str,
|
||||
) -> GatewayExtractedCredentials {
|
||||
extract_request_credentials_with_trusted_auth(headers, uri, auth_endpoint_signature, cfg!(test))
|
||||
}
|
||||
|
||||
pub(super) fn extract_request_credentials_with_trusted_auth(
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
auth_endpoint_signature: &str,
|
||||
trusted_auth_verified: bool,
|
||||
) -> GatewayExtractedCredentials {
|
||||
let bundle = GatewayCredentialBundle {
|
||||
authorization_bearer: header_value_str(headers, http::header::AUTHORIZATION.as_str())
|
||||
.as_deref()
|
||||
.and_then(extract_bearer_token)
|
||||
.map(ToOwned::to_owned),
|
||||
authorization_bearer: unique_header_value_str(
|
||||
headers,
|
||||
http::header::AUTHORIZATION.as_str(),
|
||||
)
|
||||
.as_deref()
|
||||
.and_then(extract_bearer_token)
|
||||
.map(ToOwned::to_owned),
|
||||
x_api_key: header_value_str(headers, "x-api-key"),
|
||||
api_key: header_value_str(headers, "api-key"),
|
||||
x_goog_api_key: header_value_str(headers, "x-goog-api-key"),
|
||||
query_key: extract_query_api_key(uri),
|
||||
cookie_header: header_value_str(headers, http::header::COOKIE.as_str()),
|
||||
};
|
||||
let trusted_headers = extract_trusted_auth_headers(headers);
|
||||
let trusted_headers = extract_trusted_auth_headers(headers, trusted_auth_verified);
|
||||
let trusted_admin_headers = extract_trusted_admin_headers(headers);
|
||||
let primary = select_primary_credential(auth_endpoint_signature, &bundle);
|
||||
|
||||
@@ -71,6 +94,20 @@ pub(super) fn extract_request_credentials(
|
||||
}
|
||||
}
|
||||
|
||||
fn unique_header_value_str(headers: &http::HeaderMap, key: &str) -> Option<String> {
|
||||
let mut values = headers.get_all(key).iter();
|
||||
let value = values.next()?;
|
||||
if values.next().is_some() {
|
||||
return None;
|
||||
}
|
||||
value
|
||||
.to_str()
|
||||
.ok()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
}
|
||||
|
||||
pub(in crate::control) fn resolve_gateway_credential_carrier(
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
@@ -85,6 +122,7 @@ pub(in crate::control) fn resolve_gateway_credential_carrier(
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn has_trusted_gateway_marker(headers: &http::HeaderMap) -> bool {
|
||||
header_value_str(headers, crate::constants::GATEWAY_HEADER)
|
||||
.unwrap_or_default()
|
||||
@@ -97,13 +135,32 @@ pub(super) fn build_auth_context_cache_key(
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
auth_endpoint_signature: &str,
|
||||
) -> Option<String> {
|
||||
build_auth_context_cache_key_with_trusted_auth(
|
||||
headers,
|
||||
uri,
|
||||
auth_endpoint_signature,
|
||||
cfg!(test),
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn build_auth_context_cache_key_with_trusted_auth(
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
auth_endpoint_signature: &str,
|
||||
trusted_auth_verified: bool,
|
||||
) -> Option<String> {
|
||||
let signature = auth_endpoint_signature.trim();
|
||||
if signature.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let extracted = extract_request_credentials(headers, uri, signature);
|
||||
let extracted = extract_request_credentials_with_trusted_auth(
|
||||
headers,
|
||||
uri,
|
||||
signature,
|
||||
trusted_auth_verified,
|
||||
);
|
||||
let trusted_headers = extracted.trusted_headers;
|
||||
let bundle = extracted.bundle;
|
||||
if bundle.authorization_bearer.is_none()
|
||||
@@ -135,7 +192,7 @@ pub(super) fn build_auth_context_cache_key(
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
Some(format!(
|
||||
let raw_cache_identity = format!(
|
||||
"{signature}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}",
|
||||
bundle.authorization_bearer.unwrap_or_default(),
|
||||
bundle.x_api_key.unwrap_or_default(),
|
||||
@@ -147,11 +204,26 @@ pub(super) fn build_auth_context_cache_key(
|
||||
trusted_api_key_id,
|
||||
trusted_balance_remaining,
|
||||
trusted_access_allowed,
|
||||
))
|
||||
);
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(raw_cache_identity.as_bytes());
|
||||
Some(format!("auth-context:sha256:{:x}", hasher.finalize()))
|
||||
}
|
||||
|
||||
fn extract_trusted_auth_headers(headers: &http::HeaderMap) -> Option<GatewayTrustedAuthHeaders> {
|
||||
if !has_trusted_gateway_marker(headers) {
|
||||
fn extract_trusted_auth_headers(
|
||||
headers: &http::HeaderMap,
|
||||
trusted_auth_verified: bool,
|
||||
) -> Option<GatewayTrustedAuthHeaders> {
|
||||
if !trusted_auth_verified {
|
||||
return None;
|
||||
}
|
||||
#[cfg(test)]
|
||||
if !header_value_str(headers, crate::constants::GATEWAY_HEADER)
|
||||
.unwrap_or_default()
|
||||
.trim()
|
||||
.to_ascii_lowercase()
|
||||
.starts_with("rust-phase3")
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let user_id = header_value_str(headers, crate::constants::TRUSTED_AUTH_USER_ID_HEADER)
|
||||
@@ -387,7 +459,7 @@ fn extract_bearer_token(value: &str) -> Option<&str> {
|
||||
return None;
|
||||
}
|
||||
let token = token.trim();
|
||||
if token.is_empty() {
|
||||
if token.is_empty() || token.chars().any(char::is_whitespace) {
|
||||
None
|
||||
} else {
|
||||
Some(token)
|
||||
@@ -472,6 +544,46 @@ mod tests {
|
||||
assert_eq!(requested_model.as_deref(), Some("gpt-5.4"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_requested_model_handles_gemini_generation_and_operation_paths() {
|
||||
let generation_decision = GatewayControlDecision::synthetic(
|
||||
"/v1beta/models/gemini-2.5-pro:generateContent",
|
||||
Some("ai_public".to_string()),
|
||||
Some("gemini".to_string()),
|
||||
Some("generate_content".to_string()),
|
||||
Some("gemini:generate_content".to_string()),
|
||||
);
|
||||
let operation_decision = GatewayControlDecision::synthetic(
|
||||
"/v1beta/models/veo-3/operations/task-123:cancel",
|
||||
Some("ai_public".to_string()),
|
||||
Some("gemini".to_string()),
|
||||
Some("video".to_string()),
|
||||
Some("gemini:video".to_string()),
|
||||
);
|
||||
let headers = http::HeaderMap::new();
|
||||
|
||||
assert_eq!(
|
||||
extract_requested_model(
|
||||
&generation_decision,
|
||||
&uri("/v1beta/models/gemini-2.5-pro:generateContent"),
|
||||
&headers,
|
||||
&Bytes::new(),
|
||||
)
|
||||
.as_deref(),
|
||||
Some("gemini-2.5-pro")
|
||||
);
|
||||
assert_eq!(
|
||||
extract_requested_model(
|
||||
&operation_decision,
|
||||
&uri("/v1beta/models/veo-3/operations/task-123:cancel"),
|
||||
&headers,
|
||||
&Bytes::new(),
|
||||
)
|
||||
.as_deref(),
|
||||
Some("veo-3")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selects_openai_bearer_as_provider_api_key() {
|
||||
let mut headers = http::HeaderMap::new();
|
||||
@@ -491,6 +603,33 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_duplicate_or_combined_authorization_credentials() {
|
||||
let mut duplicate = http::HeaderMap::new();
|
||||
duplicate.append(
|
||||
http::header::AUTHORIZATION,
|
||||
"Bearer first-token".parse().unwrap(),
|
||||
);
|
||||
duplicate.append(
|
||||
http::header::AUTHORIZATION,
|
||||
"Bearer second-token".parse().unwrap(),
|
||||
);
|
||||
let extracted =
|
||||
extract_request_credentials(&duplicate, &uri("/api/admin/system"), "admin:operational");
|
||||
assert!(extracted.bundle.authorization_bearer.is_none());
|
||||
assert!(extracted.primary.is_none());
|
||||
|
||||
let mut combined = http::HeaderMap::new();
|
||||
combined.insert(
|
||||
http::header::AUTHORIZATION,
|
||||
"Bearer first-token, Bearer second-token".parse().unwrap(),
|
||||
);
|
||||
let extracted =
|
||||
extract_request_credentials(&combined, &uri("/api/admin/system"), "admin:operational");
|
||||
assert!(extracted.bundle.authorization_bearer.is_none());
|
||||
assert!(extracted.primary.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selects_codex_live_bearer_as_provider_api_key() {
|
||||
let mut headers = http::HeaderMap::new();
|
||||
@@ -608,7 +747,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cache_key_includes_cookie_header() {
|
||||
fn cache_key_hashes_cookie_header_instead_of_retaining_session_secret() {
|
||||
let mut headers = http::HeaderMap::new();
|
||||
headers.insert(http::header::COOKIE, "session=abc123".parse().unwrap());
|
||||
|
||||
@@ -618,7 +757,8 @@ mod tests {
|
||||
"internal:session",
|
||||
)
|
||||
.expect("cache key should exist");
|
||||
assert!(cache_key.contains("session=abc123"));
|
||||
assert!(cache_key.starts_with("auth-context:sha256:"));
|
||||
assert!(!cache_key.contains("session=abc123"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -669,12 +809,10 @@ mod tests {
|
||||
.expect("trusted cache key should exist");
|
||||
|
||||
assert_ne!(first, second);
|
||||
assert!(first.contains("user-1"));
|
||||
assert!(first.contains("key-1"));
|
||||
assert!(first.contains("1.5"));
|
||||
assert!(first.contains("true"));
|
||||
assert!(second.contains("user-2"));
|
||||
assert!(second.contains("false"));
|
||||
for raw_identity in ["user-1", "key-1", "1.5", "user-2"] {
|
||||
assert!(!first.contains(raw_identity));
|
||||
assert!(!second.contains(raw_identity));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -224,7 +224,7 @@ fn wallet_finite_available_usd(
|
||||
Some(wallet.balance.max(0.0) + wallet.gift_balance.max(0.0))
|
||||
}
|
||||
|
||||
async fn estimate_execution_plan_cost_upper_bound_usd(
|
||||
pub(crate) async fn estimate_execution_plan_cost_upper_bound_usd(
|
||||
state: &AppState,
|
||||
plan: &aether_contracts::ExecutionPlan,
|
||||
report_context: Option<&serde_json::Value>,
|
||||
@@ -925,6 +925,7 @@ mod tests {
|
||||
local_rejection: None,
|
||||
allowed_models: Some(allowed_models),
|
||||
ip_rules: None,
|
||||
verified_api_key_hash: None,
|
||||
});
|
||||
decision
|
||||
}
|
||||
|
||||
@@ -7,13 +7,16 @@ mod types;
|
||||
pub(crate) use credentials::extract_requested_model;
|
||||
pub(super) use credentials::resolve_gateway_credential_carrier;
|
||||
pub(crate) use gate::{
|
||||
execution_plan_balance_capacity_rejection, request_model_local_rejection,
|
||||
should_buffer_request_for_local_auth, trusted_auth_local_rejection, GatewayLocalAuthRejection,
|
||||
estimate_execution_plan_cost_upper_bound_usd, execution_plan_balance_capacity_rejection,
|
||||
request_model_local_rejection, should_buffer_request_for_local_auth,
|
||||
trusted_auth_local_rejection, GatewayLocalAuthRejection,
|
||||
};
|
||||
pub(crate) use resolution::{
|
||||
refresh_execution_runtime_auth_context, refresh_execution_runtime_auth_context_with_snapshot,
|
||||
resolve_execution_runtime_auth_context, GatewayAdminPrincipalContext,
|
||||
GatewayControlAuthContext,
|
||||
resolve_execution_runtime_auth_context, resolve_local_admin_session_principal,
|
||||
GatewayAdminPrincipalContext, GatewayControlAuthContext,
|
||||
};
|
||||
pub(super) use resolution::{
|
||||
resolve_control_decision_auth_with_trusted_auth, ControlDecisionAuthResolution,
|
||||
};
|
||||
pub(super) use resolution::{resolve_control_decision_auth, ControlDecisionAuthResolution};
|
||||
pub(crate) use types::GatewayCredentialCarrier;
|
||||
|
||||
@@ -4,11 +4,9 @@ use aether_data_contracts::repository::provider_catalog::{
|
||||
StoredProviderCatalogEndpoint, StoredProviderCatalogProvider,
|
||||
};
|
||||
use axum::http::Uri;
|
||||
use base64::Engine as _;
|
||||
use hmac::Mac;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use tracing::{debug, info};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::wallet_runtime::{
|
||||
local_rejection_from_wallet_access, resolve_wallet_auth_gate_uncached,
|
||||
@@ -17,7 +15,8 @@ use crate::{AppState, GatewayError};
|
||||
|
||||
use super::super::GatewayControlDecision;
|
||||
use super::credentials::{
|
||||
build_auth_context_cache_key, current_unix_secs, extract_request_credentials,
|
||||
build_auth_context_cache_key, build_auth_context_cache_key_with_trusted_auth,
|
||||
current_unix_secs, extract_request_credentials, extract_request_credentials_with_trusted_auth,
|
||||
extract_trusted_admin_headers, hash_api_key,
|
||||
};
|
||||
use super::gate::GatewayLocalAuthRejection;
|
||||
@@ -27,6 +26,9 @@ use super::types::{
|
||||
};
|
||||
use crate::cache::{AuthContextCacheGeneration, AuthContextInflightRegistration};
|
||||
use crate::headers::header_value_str;
|
||||
use crate::local_auth_token::{
|
||||
decode_local_auth_token, local_auth_token_identity_matches_user, LocalAuthTokenType,
|
||||
};
|
||||
|
||||
const AUTH_CONTEXT_CACHE_TTL: Duration = Duration::from_secs(60);
|
||||
const AUTH_CONTEXT_CACHE_REFRESH_INTERVAL: Duration = Duration::from_secs(10);
|
||||
@@ -93,6 +95,30 @@ pub(crate) struct GatewayControlAuthContext {
|
||||
pub(crate) allowed_models: Option<Vec<String>>,
|
||||
#[serde(skip)]
|
||||
pub(crate) ip_rules: Option<Vec<String>>,
|
||||
/// Credential verifier that established this API-key identity. Long-lived
|
||||
/// executions use it to prove that a later row with the same IDs is still
|
||||
/// the record authenticated by the original request.
|
||||
#[serde(skip)]
|
||||
pub(crate) verified_api_key_hash: Option<VerifiedApiKeyHash>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct VerifiedApiKeyHash(String);
|
||||
|
||||
impl VerifiedApiKeyHash {
|
||||
fn new(value: String) -> Self {
|
||||
Self(value)
|
||||
}
|
||||
|
||||
fn as_str(&self) -> &str {
|
||||
self.0.as_str()
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for VerifiedApiKeyHash {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter.write_str("VerifiedApiKeyHash([REDACTED])")
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -109,11 +135,30 @@ pub(in super::super) enum ControlDecisionAuthResolution {
|
||||
}
|
||||
|
||||
pub(in super::super) async fn resolve_control_decision_auth(
|
||||
state: &AppState,
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
trace_id: &str,
|
||||
decision: GatewayControlDecision,
|
||||
) -> Result<ControlDecisionAuthResolution, GatewayError> {
|
||||
resolve_control_decision_auth_with_trusted_auth(
|
||||
state,
|
||||
headers,
|
||||
uri,
|
||||
trace_id,
|
||||
decision,
|
||||
cfg!(test),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in super::super) async fn resolve_control_decision_auth_with_trusted_auth(
|
||||
state: &AppState,
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
trace_id: &str,
|
||||
mut decision: GatewayControlDecision,
|
||||
trusted_auth_verified: bool,
|
||||
) -> Result<ControlDecisionAuthResolution, GatewayError> {
|
||||
if let Some(admin_principal) =
|
||||
resolve_trusted_admin_principal(headers, decision.auth_endpoint_signature.as_deref())
|
||||
@@ -132,10 +177,18 @@ pub(in super::super) async fn resolve_control_decision_auth(
|
||||
decision.admin_principal = Some(admin_principal);
|
||||
}
|
||||
|
||||
let auth_context_cache_key = decision
|
||||
.auth_endpoint_signature
|
||||
.as_deref()
|
||||
.and_then(|signature| build_auth_context_cache_key(headers, uri, signature));
|
||||
let auth_context_cache_key =
|
||||
decision
|
||||
.auth_endpoint_signature
|
||||
.as_deref()
|
||||
.and_then(|signature| {
|
||||
build_auth_context_cache_key_with_trusted_auth(
|
||||
headers,
|
||||
uri,
|
||||
signature,
|
||||
trusted_auth_verified,
|
||||
)
|
||||
});
|
||||
|
||||
let mut resolved_auth_context = None;
|
||||
if let Some(cache_key) = auth_context_cache_key.as_deref() {
|
||||
@@ -149,6 +202,7 @@ pub(in super::super) async fn resolve_control_decision_auth(
|
||||
decision.auth_endpoint_signature.as_deref(),
|
||||
headers,
|
||||
uri,
|
||||
trusted_auth_verified,
|
||||
)
|
||||
.await?,
|
||||
);
|
||||
@@ -168,6 +222,7 @@ pub(in super::super) async fn resolve_control_decision_auth(
|
||||
uri,
|
||||
decision.auth_endpoint_signature.as_deref(),
|
||||
true,
|
||||
trusted_auth_verified,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -336,7 +391,7 @@ async fn resolve_local_admin_principal(
|
||||
let Some(access_token) = extracted.bundle.authorization_bearer.as_deref() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let claims = match decode_local_auth_token(access_token, "access") {
|
||||
let claims = match decode_local_auth_token(access_token, LocalAuthTokenType::Access) {
|
||||
Ok(claims) => claims,
|
||||
Err(_) => return Ok(None),
|
||||
};
|
||||
@@ -351,6 +406,14 @@ async fn resolve_local_admin_principal(
|
||||
resolve_local_admin_principal_from_claims(state, headers, uri, &claims).await
|
||||
}
|
||||
|
||||
pub(crate) async fn resolve_local_admin_session_principal(
|
||||
state: &AppState,
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
) -> Result<Option<GatewayAdminPrincipalContext>, GatewayError> {
|
||||
resolve_local_admin_principal(state, headers, uri, Some("admin:operational")).await
|
||||
}
|
||||
|
||||
async fn resolve_local_admin_principal_from_claims(
|
||||
state: &AppState,
|
||||
headers: &http::HeaderMap,
|
||||
@@ -373,6 +436,9 @@ async fn resolve_local_admin_principal_from_claims(
|
||||
if !user.is_active || user.is_deleted || !crate::roles::can_access_admin_console(&user.role) {
|
||||
return Ok(None);
|
||||
}
|
||||
if !local_auth_token_identity_matches_user(claims, &user) {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let now = chrono::Utc::now();
|
||||
let Some(session) = state.find_user_session(user_id, session_id).await? else {
|
||||
@@ -380,6 +446,7 @@ async fn resolve_local_admin_principal_from_claims(
|
||||
};
|
||||
if session.is_revoked()
|
||||
|| session.is_expired(now)
|
||||
|| session.security_version != user.security_version
|
||||
|| session.client_device_id != client_device_id
|
||||
{
|
||||
return Ok(None);
|
||||
@@ -431,68 +498,6 @@ fn local_admin_user_agent(headers: &http::HeaderMap) -> Option<String> {
|
||||
.map(|value| value.chars().take(1000).collect())
|
||||
}
|
||||
|
||||
fn local_auth_secret() -> String {
|
||||
std::env::var("JWT_SECRET_KEY")
|
||||
.ok()
|
||||
.map(|value| value.trim().to_string())
|
||||
.filter(|value| !value.is_empty())
|
||||
.unwrap_or_else(|| "aether-rust-dev-jwt-secret".to_string())
|
||||
}
|
||||
|
||||
fn decode_local_auth_token(
|
||||
token: &str,
|
||||
expected_type: &str,
|
||||
) -> Result<serde_json::Map<String, Value>, String> {
|
||||
let mut parts = token.split('.');
|
||||
let Some(header_segment) = parts.next() else {
|
||||
return Err("invalid token".to_string());
|
||||
};
|
||||
let Some(payload_segment) = parts.next() else {
|
||||
return Err("invalid token".to_string());
|
||||
};
|
||||
let Some(signature_segment) = parts.next() else {
|
||||
return Err("invalid token".to_string());
|
||||
};
|
||||
if parts.next().is_some() {
|
||||
return Err("invalid token".to_string());
|
||||
}
|
||||
|
||||
let signing_input = format!("{header_segment}.{payload_segment}");
|
||||
let signature = base64::engine::general_purpose::URL_SAFE_NO_PAD
|
||||
.decode(signature_segment)
|
||||
.map_err(|_| "invalid token".to_string())?;
|
||||
let mut mac = hmac::Hmac::<sha2::Sha256>::new_from_slice(local_auth_secret().as_bytes())
|
||||
.map_err(|_| "invalid token".to_string())?;
|
||||
mac.update(signing_input.as_bytes());
|
||||
mac.verify_slice(&signature)
|
||||
.map_err(|_| "invalid token".to_string())?;
|
||||
|
||||
let payload_bytes = base64::engine::general_purpose::URL_SAFE_NO_PAD
|
||||
.decode(payload_segment)
|
||||
.map_err(|_| "invalid token".to_string())?;
|
||||
let payload =
|
||||
serde_json::from_slice::<Value>(&payload_bytes).map_err(|_| "invalid token".to_string())?;
|
||||
let payload = payload
|
||||
.as_object()
|
||||
.cloned()
|
||||
.ok_or_else(|| "invalid token".to_string())?;
|
||||
let actual_type = payload
|
||||
.get("type")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or_default();
|
||||
if actual_type != expected_type {
|
||||
return Err("invalid token".to_string());
|
||||
}
|
||||
let exp = payload
|
||||
.get("exp")
|
||||
.and_then(Value::as_i64)
|
||||
.ok_or_else(|| "invalid token".to_string())?;
|
||||
if exp <= chrono::Utc::now().timestamp() {
|
||||
return Err("expired token".to_string());
|
||||
}
|
||||
Ok(payload)
|
||||
}
|
||||
|
||||
pub(crate) async fn resolve_execution_runtime_auth_context(
|
||||
state: &AppState,
|
||||
decision: &GatewayControlDecision,
|
||||
@@ -525,6 +530,7 @@ pub(crate) async fn resolve_execution_runtime_auth_context(
|
||||
Some(auth_endpoint_signature),
|
||||
headers,
|
||||
uri,
|
||||
cfg!(test),
|
||||
)
|
||||
.await
|
||||
.map(Some);
|
||||
@@ -539,6 +545,7 @@ pub(crate) async fn resolve_execution_runtime_auth_context(
|
||||
uri,
|
||||
Some(auth_endpoint_signature),
|
||||
true,
|
||||
cfg!(test),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
@@ -558,6 +565,7 @@ async fn revalidate_cached_auth_context(
|
||||
auth_endpoint_signature: Option<&str>,
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
trusted_auth_verified: bool,
|
||||
) -> Result<GatewayControlAuthContext, GatewayError> {
|
||||
if is_negative_auth_context(&auth_context)
|
||||
|| !auth_context.access_allowed
|
||||
@@ -581,6 +589,7 @@ async fn revalidate_cached_auth_context(
|
||||
uri,
|
||||
auth_context.clone(),
|
||||
auth_endpoint_signature,
|
||||
trusted_auth_verified,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -622,6 +631,7 @@ async fn revalidate_cached_auth_context(
|
||||
uri,
|
||||
auth_context,
|
||||
auth_endpoint_signature,
|
||||
trusted_auth_verified,
|
||||
)
|
||||
.await;
|
||||
if refreshed.is_err() {
|
||||
@@ -639,9 +649,16 @@ async fn resolve_security_fresh_auth_context(
|
||||
uri: &Uri,
|
||||
stale: GatewayControlAuthContext,
|
||||
auth_endpoint_signature: Option<&str>,
|
||||
trusted_auth_verified: bool,
|
||||
) -> Result<GatewayControlAuthContext, GatewayError> {
|
||||
if let Some(refreshed) =
|
||||
resolve_data_backed_auth_context(state, headers, uri, auth_endpoint_signature).await?
|
||||
if let Some(refreshed) = resolve_data_backed_auth_context_with_trusted_auth(
|
||||
state,
|
||||
headers,
|
||||
uri,
|
||||
auth_endpoint_signature,
|
||||
trusted_auth_verified,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
return Ok(refreshed);
|
||||
}
|
||||
@@ -660,19 +677,27 @@ async fn resolve_data_backed_auth_context_cached(
|
||||
uri: &Uri,
|
||||
auth_endpoint_signature: Option<&str>,
|
||||
cache_negative: bool,
|
||||
trusted_auth_verified: bool,
|
||||
) -> Result<Option<GatewayControlAuthContext>, GatewayError> {
|
||||
let Some(cache_key) = cache_key else {
|
||||
return resolve_data_backed_auth_context(state, headers, uri, auth_endpoint_signature)
|
||||
.await;
|
||||
return resolve_data_backed_auth_context_with_trusted_auth(
|
||||
state,
|
||||
headers,
|
||||
uri,
|
||||
auth_endpoint_signature,
|
||||
trusted_auth_verified,
|
||||
)
|
||||
.await;
|
||||
};
|
||||
loop {
|
||||
match state.auth_context_cache.register_inflight(cache_key) {
|
||||
AuthContextInflightRegistration::Leader(guard) => {
|
||||
let resolved = match resolve_data_backed_auth_context(
|
||||
let resolved = match resolve_data_backed_auth_context_with_trusted_auth(
|
||||
state,
|
||||
headers,
|
||||
uri,
|
||||
auth_endpoint_signature,
|
||||
trusted_auth_verified,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -708,11 +733,12 @@ async fn resolve_data_backed_auth_context_cached(
|
||||
}
|
||||
}
|
||||
AuthContextInflightRegistration::Bypass => {
|
||||
return resolve_data_backed_auth_context(
|
||||
return resolve_data_backed_auth_context_with_trusted_auth(
|
||||
state,
|
||||
headers,
|
||||
uri,
|
||||
auth_endpoint_signature,
|
||||
trusted_auth_verified,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -768,28 +794,39 @@ pub(crate) async fn refresh_execution_runtime_auth_context_with_snapshot(
|
||||
return Ok((auth_context, None));
|
||||
}
|
||||
|
||||
let verified_api_key_hash = auth_context.verified_api_key_hash.clone();
|
||||
let snapshot = {
|
||||
let _permit = state.acquire_auth_snapshot_load_gate().await?;
|
||||
state
|
||||
.data
|
||||
.read_auth_api_key_snapshot_strong(
|
||||
&auth_context.user_id,
|
||||
&auth_context.api_key_id,
|
||||
current_unix_secs(),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||
if let Some(key_hash) = verified_api_key_hash.as_ref() {
|
||||
state
|
||||
.data
|
||||
.read_auth_api_key_snapshot_by_key_hash_strong(
|
||||
key_hash.as_str(),
|
||||
current_unix_secs(),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||
} else {
|
||||
state
|
||||
.data
|
||||
.read_auth_api_key_snapshot_strong(
|
||||
&auth_context.user_id,
|
||||
&auth_context.api_key_id,
|
||||
current_unix_secs(),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||
}
|
||||
};
|
||||
let Some(snapshot) = snapshot else {
|
||||
let mut denied = auth_context;
|
||||
denied.access_allowed = false;
|
||||
denied.local_rejection = Some(GatewayLocalAuthRejection::InvalidApiKey);
|
||||
denied.balance_remaining = None;
|
||||
return Ok((denied, None));
|
||||
return Ok((deny_refreshed_auth_context(auth_context), None));
|
||||
};
|
||||
if snapshot.user_id != auth_context.user_id || snapshot.api_key_id != auth_context.api_key_id {
|
||||
return Ok((deny_refreshed_auth_context(auth_context), None));
|
||||
};
|
||||
|
||||
let wallet_access = resolve_wallet_auth_gate_uncached(state, &snapshot).await?;
|
||||
let refreshed = build_data_backed_auth_context(
|
||||
let mut refreshed = build_data_backed_auth_context(
|
||||
state,
|
||||
snapshot.clone(),
|
||||
auth_endpoint_signature,
|
||||
@@ -798,9 +835,19 @@ pub(crate) async fn refresh_execution_runtime_auth_context_with_snapshot(
|
||||
wallet_access,
|
||||
)
|
||||
.await;
|
||||
refreshed.verified_api_key_hash = verified_api_key_hash;
|
||||
Ok((refreshed, Some(snapshot)))
|
||||
}
|
||||
|
||||
fn deny_refreshed_auth_context(
|
||||
mut auth_context: GatewayControlAuthContext,
|
||||
) -> GatewayControlAuthContext {
|
||||
auth_context.access_allowed = false;
|
||||
auth_context.local_rejection = Some(GatewayLocalAuthRejection::InvalidApiKey);
|
||||
auth_context.balance_remaining = None;
|
||||
auth_context
|
||||
}
|
||||
|
||||
fn put_cached_auth_context(
|
||||
state: &AppState,
|
||||
cache_key: String,
|
||||
@@ -913,6 +960,23 @@ pub(super) async fn resolve_data_backed_auth_context(
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
auth_endpoint_signature: Option<&str>,
|
||||
) -> Result<Option<GatewayControlAuthContext>, GatewayError> {
|
||||
resolve_data_backed_auth_context_with_trusted_auth(
|
||||
state,
|
||||
headers,
|
||||
uri,
|
||||
auth_endpoint_signature,
|
||||
cfg!(test),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn resolve_data_backed_auth_context_with_trusted_auth(
|
||||
state: &AppState,
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
auth_endpoint_signature: Option<&str>,
|
||||
trusted_auth_verified: bool,
|
||||
) -> Result<Option<GatewayControlAuthContext>, GatewayError> {
|
||||
let Some(signature) = auth_endpoint_signature
|
||||
.map(str::trim)
|
||||
@@ -923,7 +987,12 @@ pub(super) async fn resolve_data_backed_auth_context(
|
||||
if !state.has_auth_api_key_reader() {
|
||||
return Ok(None);
|
||||
}
|
||||
let extracted = extract_request_credentials(headers, uri, signature);
|
||||
let extracted = extract_request_credentials_with_trusted_auth(
|
||||
headers,
|
||||
uri,
|
||||
signature,
|
||||
trusted_auth_verified,
|
||||
);
|
||||
let principal = derive_principal_candidate(&extracted);
|
||||
let now_unix_secs = current_unix_secs();
|
||||
|
||||
@@ -955,6 +1024,7 @@ pub(super) async fn resolve_data_backed_auth_context(
|
||||
local_rejection: Some(GatewayLocalAuthRejection::InvalidApiKey),
|
||||
allowed_models: None,
|
||||
ip_rules: None,
|
||||
verified_api_key_hash: None,
|
||||
}));
|
||||
};
|
||||
|
||||
@@ -963,17 +1033,17 @@ pub(super) async fn resolve_data_backed_auth_context(
|
||||
.await;
|
||||
|
||||
let wallet_access = resolve_wallet_auth_gate_uncached(state, &snapshot).await?;
|
||||
Ok(Some(
|
||||
build_data_backed_auth_context(
|
||||
state,
|
||||
snapshot,
|
||||
signature,
|
||||
None,
|
||||
None,
|
||||
wallet_access,
|
||||
)
|
||||
.await,
|
||||
))
|
||||
let mut auth_context = build_data_backed_auth_context(
|
||||
state,
|
||||
snapshot,
|
||||
signature,
|
||||
None,
|
||||
None,
|
||||
wallet_access,
|
||||
)
|
||||
.await;
|
||||
auth_context.verified_api_key_hash = Some(VerifiedApiKeyHash::new(key_hash));
|
||||
Ok(Some(auth_context))
|
||||
}
|
||||
Some(GatewayPrincipalCandidate::DeferredBearerToken { raw, carrier }) => {
|
||||
if let Some(auth_context) = resolve_antigravity_bearer_bridge_auth_context(
|
||||
@@ -1068,6 +1138,7 @@ async fn resolve_antigravity_bearer_bridge_auth_context(
|
||||
local_rejection: Some(GatewayLocalAuthRejection::InvalidApiKey),
|
||||
allowed_models: None,
|
||||
ip_rules: None,
|
||||
verified_api_key_hash: None,
|
||||
}));
|
||||
};
|
||||
|
||||
@@ -1127,6 +1198,7 @@ async fn resolve_trusted_auth_context(
|
||||
local_rejection: Some(GatewayLocalAuthRejection::InvalidApiKey),
|
||||
allowed_models: None,
|
||||
ip_rules: None,
|
||||
verified_api_key_hash: None,
|
||||
}));
|
||||
};
|
||||
|
||||
@@ -1158,9 +1230,7 @@ async fn build_data_backed_auth_context(
|
||||
let invalid_api_key = !snapshot.user_is_active
|
||||
|| snapshot.user_is_deleted
|
||||
|| !snapshot.api_key_is_active
|
||||
|| snapshot
|
||||
.api_key_expires_at_unix_secs
|
||||
.is_some_and(|expires_at| expires_at < current_unix_secs());
|
||||
|| api_key_is_expired(snapshot.api_key_expires_at_unix_secs, current_unix_secs());
|
||||
let locked_api_key = snapshot.api_key_is_locked && !snapshot.api_key_is_standalone;
|
||||
let key_access_allowed = header_access_allowed
|
||||
.map(|value| value && snapshot.currently_usable)
|
||||
@@ -1225,9 +1295,14 @@ async fn build_data_backed_auth_context(
|
||||
local_rejection,
|
||||
allowed_models,
|
||||
ip_rules: snapshot.api_key_ip_rules,
|
||||
verified_api_key_hash: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn api_key_is_expired(expires_at_unix_secs: Option<u64>, now_unix_secs: u64) -> bool {
|
||||
expires_at_unix_secs.is_some_and(|expires_at| expires_at <= now_unix_secs)
|
||||
}
|
||||
|
||||
fn contains_api_format_or_alias(items: &[String], target: &str) -> bool {
|
||||
items.iter().any(|item| api_format_matches(item, target))
|
||||
}
|
||||
@@ -1282,18 +1357,21 @@ async fn auth_snapshot_allows_requested_provider(
|
||||
return true;
|
||||
}
|
||||
if !state.has_provider_catalog_data_reader() {
|
||||
return true;
|
||||
debug!(
|
||||
"deny requested provider {}: provider catalog is unavailable for allowlist resolution",
|
||||
requested_provider
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
let providers = match state.list_provider_catalog_providers(true).await {
|
||||
Ok(value) => value,
|
||||
Err(err) => {
|
||||
debug!(
|
||||
"skip local provider auth gate for requested provider {}: provider catalog lookup failed: {:?}",
|
||||
requested_provider,
|
||||
err
|
||||
warn!(
|
||||
"deny requested provider {}: provider catalog lookup failed: {:?}",
|
||||
requested_provider, err
|
||||
);
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1331,11 +1409,11 @@ async fn auth_snapshot_allows_requested_provider(
|
||||
{
|
||||
Ok(value) => value,
|
||||
Err(err) => {
|
||||
debug!(
|
||||
"skip local provider auth gate for requested provider {}: provider endpoint lookup failed: {:?}",
|
||||
warn!(
|
||||
"deny requested provider {}: provider endpoint lookup failed: {:?}",
|
||||
requested_provider, err
|
||||
);
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1426,7 +1504,8 @@ mod tests {
|
||||
use std::time::Duration;
|
||||
|
||||
use aether_data::repository::auth::{
|
||||
AuthApiKeyWriteRepository, InMemoryAuthApiKeySnapshotRepository, StoredAuthApiKeySnapshot,
|
||||
AuthApiKeyWriteRepository, CreateUserApiKeyRecord, InMemoryAuthApiKeySnapshotRepository,
|
||||
StoredAuthApiKeySnapshot,
|
||||
};
|
||||
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
||||
use aether_data::repository::wallet::{
|
||||
@@ -1441,9 +1520,10 @@ mod tests {
|
||||
use futures_util::future::join_all;
|
||||
|
||||
use super::{
|
||||
get_cached_auth_context, resolve_control_decision_auth, resolve_data_backed_auth_context,
|
||||
resolve_execution_runtime_auth_context, ControlDecisionAuthResolution,
|
||||
GatewayLocalAuthRejection,
|
||||
api_key_is_expired, get_cached_auth_context,
|
||||
refresh_execution_runtime_auth_context_with_snapshot, resolve_control_decision_auth,
|
||||
resolve_data_backed_auth_context, resolve_execution_runtime_auth_context,
|
||||
ControlDecisionAuthResolution, GatewayLocalAuthRejection,
|
||||
};
|
||||
use crate::control::auth::credentials::{build_auth_context_cache_key, hash_api_key};
|
||||
use crate::control::GatewayControlDecision;
|
||||
@@ -1481,6 +1561,14 @@ mod tests {
|
||||
path.parse().expect("uri should parse")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn api_key_expiry_is_inclusive_at_the_declared_second() {
|
||||
assert!(!api_key_is_expired(None, 100));
|
||||
assert!(!api_key_is_expired(Some(101), 100));
|
||||
assert!(api_key_is_expired(Some(100), 100));
|
||||
assert!(api_key_is_expired(Some(99), 100));
|
||||
}
|
||||
|
||||
fn sample_provider(id: &str, name: &str, provider_type: &str) -> StoredProviderCatalogProvider {
|
||||
StoredProviderCatalogProvider::new(
|
||||
id.to_string(),
|
||||
@@ -1769,6 +1857,97 @@ mod tests {
|
||||
assert_eq!(repository.touch_count("key-1"), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn long_lived_refresh_rejects_same_ids_recreated_with_a_different_credential() {
|
||||
let old_api_key = "sk-old-websocket-credential";
|
||||
let new_api_key = "sk-new-websocket-credential";
|
||||
let old_key_hash = hash_api_key(old_api_key);
|
||||
let new_key_hash = hash_api_key(new_api_key);
|
||||
let mut old_snapshot = sample_snapshot("key-stable-id", "user-stable-id");
|
||||
old_snapshot.user_allowed_api_formats = Some(vec!["openai:responses".to_string()]);
|
||||
old_snapshot.api_key_allowed_api_formats = Some(vec!["openai:responses".to_string()]);
|
||||
let repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
|
||||
Some(old_key_hash.clone()),
|
||||
old_snapshot,
|
||||
)]));
|
||||
let data = GatewayDataState::with_auth_api_key_repository_for_tests(repository.clone());
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(data);
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
http::header::AUTHORIZATION,
|
||||
format!("Bearer {old_api_key}").parse().unwrap(),
|
||||
);
|
||||
|
||||
let original = resolve_data_backed_auth_context(
|
||||
&state,
|
||||
&headers,
|
||||
&uri("/v1/responses"),
|
||||
Some("openai:responses"),
|
||||
)
|
||||
.await
|
||||
.expect("initial auth resolution should succeed")
|
||||
.expect("the old API key should authenticate");
|
||||
assert!(original.access_allowed);
|
||||
assert!(original.verified_api_key_hash.is_some());
|
||||
assert!(
|
||||
!format!("{original:?}").contains(&old_key_hash),
|
||||
"the credential verifier must stay redacted from Debug output"
|
||||
);
|
||||
|
||||
assert!(repository
|
||||
.delete_user_api_key("user-stable-id", "key-stable-id")
|
||||
.await
|
||||
.expect("old API key deletion should succeed"));
|
||||
repository
|
||||
.create_user_api_key(CreateUserApiKeyRecord {
|
||||
user_id: "user-stable-id".to_string(),
|
||||
api_key_id: "key-stable-id".to_string(),
|
||||
key_hash: new_key_hash,
|
||||
key_encrypted: None,
|
||||
name: Some("restored-with-new-secret".to_string()),
|
||||
allowed_providers: Some(vec!["openai".to_string()]),
|
||||
allowed_api_formats: Some(vec!["openai:responses".to_string()]),
|
||||
allowed_models: Some(vec!["gpt-4.1".to_string()]),
|
||||
ip_rules: None,
|
||||
rate_limit: 60,
|
||||
concurrent_limit: Some(5),
|
||||
force_capabilities: None,
|
||||
feature_settings: None,
|
||||
is_active: true,
|
||||
expires_at_unix_secs: Some(4_102_444_800),
|
||||
auto_delete_on_expiry: false,
|
||||
total_requests: 0,
|
||||
total_tokens: 0,
|
||||
total_cost_usd: 0.0,
|
||||
})
|
||||
.await
|
||||
.expect("same-ID API key recreation should resolve")
|
||||
.expect("same-ID API key recreation should persist");
|
||||
|
||||
let (refreshed, snapshot) = refresh_execution_runtime_auth_context_with_snapshot(
|
||||
&state,
|
||||
original,
|
||||
Some("openai:responses"),
|
||||
)
|
||||
.await
|
||||
.expect("long-lived auth refresh should resolve");
|
||||
|
||||
assert!(!refreshed.access_allowed);
|
||||
assert_eq!(
|
||||
refreshed.local_rejection,
|
||||
Some(GatewayLocalAuthRejection::InvalidApiKey)
|
||||
);
|
||||
assert!(snapshot.is_none());
|
||||
assert_eq!(repository.key_hash_lookup_count(&old_key_hash), 1);
|
||||
assert_eq!(
|
||||
repository.snapshot_lookup_count("key-stable-id"),
|
||||
0,
|
||||
"a bound long-lived credential must not fall back to identity-only lookup"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn control_auth_context_singleflights_concurrent_cache_misses() {
|
||||
let api_key = "sk-test-concurrent-auth-miss";
|
||||
@@ -2396,6 +2575,44 @@ mod tests {
|
||||
assert_eq!(auth_context.local_rejection, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn data_backed_auth_context_denies_unresolved_provider_id_without_catalog_reader() {
|
||||
let api_key = "sk-test-provider-no-catalog";
|
||||
let mut snapshot = sample_snapshot("key-no-catalog", "user-no-catalog");
|
||||
snapshot.user_allowed_providers = Some(vec!["provider-custom-claude".to_string()]);
|
||||
snapshot.api_key_allowed_providers = Some(vec!["provider-custom-claude".to_string()]);
|
||||
snapshot.user_allowed_api_formats = None;
|
||||
snapshot.api_key_allowed_api_formats = None;
|
||||
let repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
|
||||
Some(hash_api_key(api_key)),
|
||||
snapshot,
|
||||
)]));
|
||||
let data = GatewayDataState::with_auth_api_key_reader_for_tests(repository);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(data);
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("x-api-key", api_key.parse().unwrap());
|
||||
|
||||
let auth_context = resolve_data_backed_auth_context(
|
||||
&state,
|
||||
&headers,
|
||||
&uri("/v1/messages"),
|
||||
Some("claude:messages"),
|
||||
)
|
||||
.await
|
||||
.expect("resolution should succeed")
|
||||
.expect("auth context should exist");
|
||||
|
||||
assert!(!auth_context.access_allowed);
|
||||
assert_eq!(
|
||||
auth_context.local_rejection,
|
||||
Some(GatewayLocalAuthRejection::ProviderNotAllowed {
|
||||
provider: "claude".to_string(),
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn due_antigravity_bearer_refresh_observes_cross_node_allowlist_revocation() {
|
||||
let raw_bearer = "google-oauth-access-token-revoked-cross-node";
|
||||
|
||||
@@ -44,7 +44,7 @@ pub(super) struct GatewayTrustedAdminHeaders {
|
||||
pub(super) management_token_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
#[derive(Clone, Default, PartialEq, Eq)]
|
||||
pub(super) struct GatewayCredentialBundle {
|
||||
pub(super) authorization_bearer: Option<String>,
|
||||
pub(super) x_api_key: Option<String>,
|
||||
@@ -54,7 +54,25 @@ pub(super) struct GatewayCredentialBundle {
|
||||
pub(super) cookie_header: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
impl std::fmt::Debug for GatewayCredentialBundle {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let redacted = |value: &Option<String>| value.as_ref().map(|_| "[REDACTED]");
|
||||
formatter
|
||||
.debug_struct("GatewayCredentialBundle")
|
||||
.field(
|
||||
"authorization_bearer",
|
||||
&redacted(&self.authorization_bearer),
|
||||
)
|
||||
.field("x_api_key", &redacted(&self.x_api_key))
|
||||
.field("api_key", &redacted(&self.api_key))
|
||||
.field("x_goog_api_key", &redacted(&self.x_goog_api_key))
|
||||
.field("query_key", &redacted(&self.query_key))
|
||||
.field("cookie_header", &redacted(&self.cookie_header))
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub(super) enum GatewayPrimaryCredential {
|
||||
ProviderApiKey {
|
||||
raw: String,
|
||||
@@ -70,6 +88,21 @@ pub(super) enum GatewayPrimaryCredential {
|
||||
},
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for GatewayPrimaryCredential {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let (variant, carrier) = match self {
|
||||
Self::ProviderApiKey { carrier, .. } => ("ProviderApiKey", carrier),
|
||||
Self::BearerToken { carrier, .. } => ("BearerToken", carrier),
|
||||
Self::CookieHeader { carrier, .. } => ("CookieHeader", carrier),
|
||||
};
|
||||
formatter
|
||||
.debug_struct(variant)
|
||||
.field("raw", &"[REDACTED]")
|
||||
.field("carrier", carrier)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(super) struct GatewayExtractedCredentials {
|
||||
pub(super) trusted_headers: Option<GatewayTrustedAuthHeaders>,
|
||||
@@ -78,7 +111,7 @@ pub(super) struct GatewayExtractedCredentials {
|
||||
pub(super) primary: Option<GatewayPrimaryCredential>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
#[derive(Clone, PartialEq)]
|
||||
pub(super) enum GatewayPrincipalCandidate {
|
||||
TrustedHeaders(GatewayTrustedAuthHeaders),
|
||||
ApiKeyHash {
|
||||
@@ -94,3 +127,58 @@ pub(super) enum GatewayPrincipalCandidate {
|
||||
carrier: GatewayCredentialCarrier,
|
||||
},
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for GatewayPrincipalCandidate {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::TrustedHeaders(headers) => formatter
|
||||
.debug_tuple("TrustedHeaders")
|
||||
.field(headers)
|
||||
.finish(),
|
||||
Self::ApiKeyHash { carrier, .. } => formatter
|
||||
.debug_struct("ApiKeyHash")
|
||||
.field("key_hash", &"[REDACTED]")
|
||||
.field("carrier", carrier)
|
||||
.finish(),
|
||||
Self::DeferredBearerToken { carrier, .. } => formatter
|
||||
.debug_struct("DeferredBearerToken")
|
||||
.field("raw", &"[REDACTED]")
|
||||
.field("carrier", carrier)
|
||||
.finish(),
|
||||
Self::DeferredCookieHeader { carrier, .. } => formatter
|
||||
.debug_struct("DeferredCookieHeader")
|
||||
.field("raw", &"[REDACTED]")
|
||||
.field("carrier", carrier)
|
||||
.finish(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod debug_redaction_tests {
|
||||
use super::{GatewayCredentialBundle, GatewayCredentialCarrier, GatewayPrimaryCredential};
|
||||
|
||||
#[test]
|
||||
fn gateway_credential_debug_output_redacts_raw_authorization_values() {
|
||||
let bundle = GatewayCredentialBundle {
|
||||
authorization_bearer: Some("bundle-bearer-canary".to_string()),
|
||||
api_key: Some("bundle-api-key-canary".to_string()),
|
||||
cookie_header: Some("bundle-cookie-canary".to_string()),
|
||||
..GatewayCredentialBundle::default()
|
||||
};
|
||||
let primary = GatewayPrimaryCredential::ProviderApiKey {
|
||||
raw: "primary-api-key-canary".to_string(),
|
||||
carrier: GatewayCredentialCarrier::ApiKey,
|
||||
};
|
||||
let debug = format!("{bundle:?} {primary:?}");
|
||||
assert!(debug.contains("[REDACTED]"));
|
||||
for secret in [
|
||||
"bundle-bearer-canary",
|
||||
"bundle-api-key-canary",
|
||||
"bundle-cookie-canary",
|
||||
"primary-api-key-canary",
|
||||
] {
|
||||
assert!(!debug.contains(secret), "debug output leaked {secret}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -8,9 +8,10 @@ mod public;
|
||||
mod route;
|
||||
|
||||
pub(crate) use auth::{
|
||||
execution_plan_balance_capacity_rejection, extract_requested_model,
|
||||
refresh_execution_runtime_auth_context, refresh_execution_runtime_auth_context_with_snapshot,
|
||||
request_model_local_rejection, resolve_execution_runtime_auth_context,
|
||||
estimate_execution_plan_cost_upper_bound_usd, execution_plan_balance_capacity_rejection,
|
||||
extract_requested_model, refresh_execution_runtime_auth_context,
|
||||
refresh_execution_runtime_auth_context_with_snapshot, request_model_local_rejection,
|
||||
resolve_execution_runtime_auth_context, resolve_local_admin_session_principal,
|
||||
should_buffer_request_for_local_auth, trusted_auth_local_rejection,
|
||||
GatewayAdminPrincipalContext, GatewayControlAuthContext, GatewayCredentialCarrier,
|
||||
GatewayLocalAuthRejection,
|
||||
@@ -18,14 +19,16 @@ pub(crate) use auth::{
|
||||
pub(crate) use execute::{allows_control_execute_emergency, maybe_execute_via_control};
|
||||
pub(crate) use management_token_permissions::{
|
||||
all_assignable_management_token_permissions,
|
||||
audit_admin_read_only_management_token_permissions,
|
||||
audit_admin_read_only_management_token_permissions, legacy_full_management_token_permissions,
|
||||
management_token_permission_catalog_payload, management_token_permission_keys_from_value,
|
||||
management_token_permission_mode_and_summary,
|
||||
management_token_permissions_cover_all_assignable_permissions,
|
||||
management_token_permission_mode_and_summary, management_token_principal_has_permission,
|
||||
management_token_required_permission, normalize_assignable_management_token_permissions,
|
||||
read_only_management_token_permissions, validate_management_token_admin_route_permission,
|
||||
};
|
||||
pub(crate) use public::{resolve_public_request_context, GatewayPublicRequestContext};
|
||||
pub(crate) use public::{
|
||||
resolve_public_request_context, resolve_public_request_context_with_trusted_auth,
|
||||
resolve_public_request_context_without_trusted_auth, GatewayPublicRequestContext,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use route::classify_control_route;
|
||||
pub(crate) use route::{resolve_control_route, GatewayControlDecision};
|
||||
|
||||
@@ -2,7 +2,9 @@ use axum::http::Uri;
|
||||
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
use super::{resolve_control_route, GatewayControlDecision};
|
||||
use super::{
|
||||
resolve_control_route, route::resolve_control_route_with_trusted_auth, GatewayControlDecision,
|
||||
};
|
||||
|
||||
pub(crate) type GatewayPublicRequestContext =
|
||||
aether_gateway_control::PublicRequestContext<GatewayControlDecision>;
|
||||
@@ -23,3 +25,41 @@ pub(crate) async fn resolve_public_request_context(
|
||||
control_decision,
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn resolve_public_request_context_with_trusted_auth(
|
||||
state: &AppState,
|
||||
method: &http::Method,
|
||||
uri: &Uri,
|
||||
headers: &http::HeaderMap,
|
||||
trace_id: &str,
|
||||
) -> Result<GatewayPublicRequestContext, GatewayError> {
|
||||
let control_decision =
|
||||
resolve_control_route_with_trusted_auth(state, method, uri, headers, trace_id, true)
|
||||
.await?;
|
||||
Ok(GatewayPublicRequestContext::from_request_parts(
|
||||
trace_id,
|
||||
method,
|
||||
uri,
|
||||
headers,
|
||||
control_decision,
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn resolve_public_request_context_without_trusted_auth(
|
||||
state: &AppState,
|
||||
method: &http::Method,
|
||||
uri: &Uri,
|
||||
headers: &http::HeaderMap,
|
||||
trace_id: &str,
|
||||
) -> Result<GatewayPublicRequestContext, GatewayError> {
|
||||
let control_decision =
|
||||
resolve_control_route_with_trusted_auth(state, method, uri, headers, trace_id, false)
|
||||
.await?;
|
||||
Ok(GatewayPublicRequestContext::from_request_parts(
|
||||
trace_id,
|
||||
method,
|
||||
uri,
|
||||
headers,
|
||||
control_decision,
|
||||
))
|
||||
}
|
||||
|
||||
@@ -5,19 +5,21 @@ pub(super) fn classify_internal_route(
|
||||
method: &http::Method,
|
||||
normalized_path: &str,
|
||||
) -> Option<ClassifiedRoute> {
|
||||
if method == http::Method::POST && normalized_path.starts_with("/api/internal/gateway/") {
|
||||
let route_kind = match normalized_path {
|
||||
"/api/internal/gateway/resolve" => "resolve",
|
||||
"/api/internal/gateway/auth-context" => "auth_context",
|
||||
"/api/internal/gateway/decision-sync" => "decision_sync",
|
||||
"/api/internal/gateway/decision-stream" => "decision_stream",
|
||||
"/api/internal/gateway/plan-sync" => "plan_sync",
|
||||
"/api/internal/gateway/plan-stream" => "plan_stream",
|
||||
"/api/internal/gateway/report-sync" => "report_sync",
|
||||
"/api/internal/gateway/report-stream" => "report_stream",
|
||||
"/api/internal/gateway/finalize-sync" => "finalize_sync",
|
||||
"/api/internal/gateway/execute-sync" => "execute_sync",
|
||||
"/api/internal/gateway/execute-stream" => "execute_stream",
|
||||
if normalized_path == "/api/internal/gateway"
|
||||
|| normalized_path.starts_with("/api/internal/gateway/")
|
||||
{
|
||||
let route_kind = match (method, normalized_path) {
|
||||
(&http::Method::POST, "/api/internal/gateway/resolve") => "resolve",
|
||||
(&http::Method::POST, "/api/internal/gateway/auth-context") => "auth_context",
|
||||
(&http::Method::POST, "/api/internal/gateway/decision-sync") => "decision_sync",
|
||||
(&http::Method::POST, "/api/internal/gateway/decision-stream") => "decision_stream",
|
||||
(&http::Method::POST, "/api/internal/gateway/plan-sync") => "plan_sync",
|
||||
(&http::Method::POST, "/api/internal/gateway/plan-stream") => "plan_stream",
|
||||
(&http::Method::POST, "/api/internal/gateway/report-sync") => "report_sync",
|
||||
(&http::Method::POST, "/api/internal/gateway/report-stream") => "report_stream",
|
||||
(&http::Method::POST, "/api/internal/gateway/finalize-sync") => "finalize_sync",
|
||||
(&http::Method::POST, "/api/internal/gateway/execute-sync") => "execute_sync",
|
||||
(&http::Method::POST, "/api/internal/gateway/execute-stream") => "execute_stream",
|
||||
_ => "unhandled",
|
||||
};
|
||||
Some(classified(
|
||||
|
||||
@@ -11,7 +11,7 @@ mod oauth;
|
||||
mod public_support;
|
||||
|
||||
use super::auth::{
|
||||
resolve_control_decision_auth, resolve_gateway_credential_carrier,
|
||||
resolve_control_decision_auth_with_trusted_auth, resolve_gateway_credential_carrier,
|
||||
ControlDecisionAuthResolution, GatewayCredentialCarrier,
|
||||
};
|
||||
use super::{GatewayAdminPrincipalContext, GatewayControlAuthContext, GatewayLocalAuthRejection};
|
||||
@@ -175,6 +175,17 @@ pub(crate) async fn resolve_control_route(
|
||||
uri: &Uri,
|
||||
headers: &http::HeaderMap,
|
||||
trace_id: &str,
|
||||
) -> Result<Option<GatewayControlDecision>, GatewayError> {
|
||||
resolve_control_route_with_trusted_auth(state, method, uri, headers, trace_id, cfg!(test)).await
|
||||
}
|
||||
|
||||
pub(crate) async fn resolve_control_route_with_trusted_auth(
|
||||
state: &AppState,
|
||||
method: &http::Method,
|
||||
uri: &Uri,
|
||||
headers: &http::HeaderMap,
|
||||
trace_id: &str,
|
||||
trusted_auth_verified: bool,
|
||||
) -> Result<Option<GatewayControlDecision>, GatewayError> {
|
||||
let Some(mut decision) = classify_control_route(method, uri, headers) else {
|
||||
return Ok(None);
|
||||
@@ -185,7 +196,16 @@ pub(crate) async fn resolve_control_route(
|
||||
crate::system_features::ModelDirectivePolicySnapshot::load(state).await;
|
||||
}
|
||||
|
||||
match resolve_control_decision_auth(state, headers, uri, trace_id, decision).await? {
|
||||
match resolve_control_decision_auth_with_trusted_auth(
|
||||
state,
|
||||
headers,
|
||||
uri,
|
||||
trace_id,
|
||||
decision,
|
||||
trusted_auth_verified,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
ControlDecisionAuthResolution::Resolved(decision) => Ok(Some(decision)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -285,7 +285,7 @@ pub(super) fn classify_oauth_route(
|
||||
"admin_proxy",
|
||||
"provider_oauth_manage",
|
||||
"batch_import_oauth",
|
||||
"admin:pool",
|
||||
"admin:provider_oauth",
|
||||
false,
|
||||
))
|
||||
} else if method == http::Method::POST
|
||||
@@ -296,7 +296,7 @@ pub(super) fn classify_oauth_route(
|
||||
"admin_proxy",
|
||||
"provider_oauth_manage",
|
||||
"start_batch_import_oauth_task",
|
||||
"admin:pool",
|
||||
"admin:provider_oauth",
|
||||
false,
|
||||
))
|
||||
} else if method == http::Method::GET
|
||||
@@ -307,7 +307,7 @@ pub(super) fn classify_oauth_route(
|
||||
"admin_proxy",
|
||||
"provider_oauth_manage",
|
||||
"get_batch_import_task_status",
|
||||
"admin:pool",
|
||||
"admin:provider_oauth",
|
||||
false,
|
||||
))
|
||||
} else if method == http::Method::POST
|
||||
|
||||
@@ -197,18 +197,22 @@ pub(super) fn classify_public_support_route(
|
||||
"public:auth",
|
||||
false,
|
||||
))
|
||||
} else if matches!(method, &http::Method::GET | &http::Method::POST)
|
||||
// Authentication state-changing endpoints must never be dispatched for GET.
|
||||
// Besides violating HTTP method semantics, accepting GET here would allow
|
||||
// browser prefetches/cross-site requests to trigger login, refresh, logout,
|
||||
// registration, or verification side effects. `/me` is the sole read route.
|
||||
} else if (method == http::Method::POST
|
||||
&& matches!(
|
||||
normalized_path,
|
||||
"/api/auth/login"
|
||||
| "/api/auth/refresh"
|
||||
| "/api/auth/register"
|
||||
| "/api/auth/me"
|
||||
| "/api/auth/logout"
|
||||
| "/api/auth/send-verification-code"
|
||||
| "/api/auth/verify-email"
|
||||
| "/api/auth/verification-status"
|
||||
)
|
||||
))
|
||||
|| (method == http::Method::GET && normalized_path == "/api/auth/me")
|
||||
{
|
||||
let route_kind = match normalized_path {
|
||||
"/api/auth/login" => "login",
|
||||
|
||||
@@ -628,7 +628,7 @@ fn classifies_admin_management_token_write_routes_and_permission_catalog() {
|
||||
http::Method::POST,
|
||||
"/api/admin/management-tokens",
|
||||
"create_token",
|
||||
"admin:management_tokens:write",
|
||||
"admin:management_tokens:admin",
|
||||
),
|
||||
(
|
||||
http::Method::PUT,
|
||||
@@ -640,7 +640,7 @@ fn classifies_admin_management_token_write_routes_and_permission_catalog() {
|
||||
http::Method::POST,
|
||||
"/api/admin/management-tokens/token-123/regenerate",
|
||||
"regenerate_token",
|
||||
"admin:management_tokens:write",
|
||||
"admin:management_tokens:admin",
|
||||
),
|
||||
];
|
||||
|
||||
|
||||
@@ -68,11 +68,11 @@ fn classifies_admin_provider_oauth_batch_import_task_status_as_admin_proxy_route
|
||||
);
|
||||
assert_eq!(
|
||||
decision.auth_endpoint_signature.as_deref(),
|
||||
Some("admin:pool")
|
||||
Some("admin:provider_oauth")
|
||||
);
|
||||
assert_eq!(
|
||||
management_token_required_permission(&http::Method::GET, &decision).as_deref(),
|
||||
Some("admin:pool:read")
|
||||
Some("admin:provider_oauth:read")
|
||||
);
|
||||
assert!(!decision.is_execution_runtime_candidate());
|
||||
}
|
||||
@@ -86,42 +86,42 @@ fn classifies_admin_provider_oauth_maintenance_routes_as_admin_proxy_route() {
|
||||
"/api/admin/provider-oauth/keys/key-123/complete",
|
||||
"complete_key_oauth",
|
||||
"admin:provider_oauth",
|
||||
"admin:provider_oauth:write",
|
||||
"admin:provider_oauth:admin",
|
||||
),
|
||||
(
|
||||
http::Method::POST,
|
||||
"/api/admin/provider-oauth/keys/key-123/refresh",
|
||||
"refresh_key_oauth",
|
||||
"admin:provider_oauth",
|
||||
"admin:provider_oauth:write",
|
||||
"admin:provider_oauth:admin",
|
||||
),
|
||||
(
|
||||
http::Method::POST,
|
||||
"/api/admin/provider-oauth/providers/provider-123/complete",
|
||||
"complete_provider_oauth",
|
||||
"admin:provider_oauth",
|
||||
"admin:provider_oauth:write",
|
||||
"admin:provider_oauth:admin",
|
||||
),
|
||||
(
|
||||
http::Method::POST,
|
||||
"/api/admin/provider-oauth/providers/provider-123/import-refresh-token",
|
||||
"import_refresh_token",
|
||||
"admin:provider_oauth",
|
||||
"admin:provider_oauth:write",
|
||||
"admin:provider_oauth:admin",
|
||||
),
|
||||
(
|
||||
http::Method::POST,
|
||||
"/api/admin/provider-oauth/providers/provider-123/cookie-authorize",
|
||||
"cookie_authorize",
|
||||
"admin:provider_oauth",
|
||||
"admin:provider_oauth:write",
|
||||
"admin:provider_oauth:admin",
|
||||
),
|
||||
(
|
||||
http::Method::POST,
|
||||
"/api/admin/provider-oauth/providers/provider-123/cookie-authorize/tasks",
|
||||
"start_cookie_authorize_task",
|
||||
"admin:provider_oauth",
|
||||
"admin:provider_oauth:write",
|
||||
"admin:provider_oauth:admin",
|
||||
),
|
||||
(
|
||||
http::Method::GET,
|
||||
@@ -135,7 +135,7 @@ fn classifies_admin_provider_oauth_maintenance_routes_as_admin_proxy_route() {
|
||||
"/api/admin/provider-oauth/providers/provider-123/agent-identity-import/tasks",
|
||||
"start_agent_identity_import_task",
|
||||
"admin:provider_oauth",
|
||||
"admin:provider_oauth:write",
|
||||
"admin:provider_oauth:admin",
|
||||
),
|
||||
(
|
||||
http::Method::GET,
|
||||
@@ -148,22 +148,22 @@ fn classifies_admin_provider_oauth_maintenance_routes_as_admin_proxy_route() {
|
||||
http::Method::POST,
|
||||
"/api/admin/provider-oauth/providers/provider-123/batch-import",
|
||||
"batch_import_oauth",
|
||||
"admin:pool",
|
||||
"admin:pool:write",
|
||||
"admin:provider_oauth",
|
||||
"admin:provider_oauth:admin",
|
||||
),
|
||||
(
|
||||
http::Method::POST,
|
||||
"/api/admin/provider-oauth/providers/provider-123/batch-import/tasks",
|
||||
"start_batch_import_oauth_task",
|
||||
"admin:pool",
|
||||
"admin:pool:write",
|
||||
"admin:provider_oauth",
|
||||
"admin:provider_oauth:admin",
|
||||
),
|
||||
(
|
||||
http::Method::GET,
|
||||
"/api/admin/provider-oauth/providers/provider-123/batch-import/tasks/task-123",
|
||||
"get_batch_import_task_status",
|
||||
"admin:pool",
|
||||
"admin:pool:read",
|
||||
"admin:provider_oauth",
|
||||
"admin:provider_oauth:read",
|
||||
),
|
||||
(
|
||||
http::Method::POST,
|
||||
@@ -177,7 +177,7 @@ fn classifies_admin_provider_oauth_maintenance_routes_as_admin_proxy_route() {
|
||||
"/api/admin/provider-oauth/providers/provider-123/device-poll",
|
||||
"device_poll",
|
||||
"admin:provider_oauth",
|
||||
"admin:provider_oauth:write",
|
||||
"admin:provider_oauth:admin",
|
||||
),
|
||||
] {
|
||||
let uri: Uri = path.parse().expect("uri should parse");
|
||||
|
||||
@@ -947,6 +947,34 @@ fn classifies_auth_routes_as_public_support_route() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn does_not_classify_state_changing_auth_routes_for_get() {
|
||||
for path in [
|
||||
"/api/auth/login",
|
||||
"/api/auth/refresh",
|
||||
"/api/auth/register",
|
||||
"/api/auth/logout",
|
||||
"/api/auth/send-verification-code",
|
||||
"/api/auth/verify-email",
|
||||
"/api/auth/verification-status",
|
||||
] {
|
||||
let headers = headers(&[]);
|
||||
let uri: Uri = path.parse().expect("uri should parse");
|
||||
assert!(
|
||||
classify_control_route(&http::Method::GET, &uri, &headers).is_none(),
|
||||
"state-changing auth route {path} must not accept GET"
|
||||
);
|
||||
}
|
||||
|
||||
let headers = headers(&[]);
|
||||
let uri: Uri = "/api/auth/me".parse().expect("uri should parse");
|
||||
assert_eq!(
|
||||
classify_control_route(&http::Method::GET, &uri, &headers)
|
||||
.and_then(|decision| decision.route_kind),
|
||||
Some("me".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classifies_oauth_public_providers_route() {
|
||||
let headers = headers(&[]);
|
||||
|
||||
@@ -172,7 +172,7 @@ mod tests {
|
||||
candidates: vec![DecisionTraceCandidate {
|
||||
candidate: sample_candidate("req-1"),
|
||||
provider_name: Some("OpenAI".to_string()),
|
||||
provider_website: Some("https://openai.com".to_string()),
|
||||
provider_website: Some("https://openai.com/".to_string()),
|
||||
provider_type: Some("custom".to_string()),
|
||||
provider_priority: Some(0),
|
||||
provider_keep_priority_on_conversion: Some(false),
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,23 +1,42 @@
|
||||
use super::{
|
||||
ApiKeyLastUsedDelta, DataLayerError, GatewayDataState, GeminiFileMappingListQuery,
|
||||
GeminiFileMappingStats, ProviderCatalogKeyAdaptiveStateUpdate,
|
||||
ProviderCatalogKeyAdminCasUpdate, ProviderCatalogKeyHealthStateUpdate,
|
||||
ProviderCatalogKeyListQuery, ProviderCatalogKeyOAuthCredentialCasDelete,
|
||||
ProviderCatalogKeyOAuthRuntimeStateCasUpdate, ProviderCatalogKeyRuntimeMetadataUpdate,
|
||||
ProviderCatalogKeyStatusSnapshotUpdate, PublicHealthStatusCount, PublicHealthTimelineBucket,
|
||||
StoredGeminiFileMapping, StoredGeminiFileMappingListPage, StoredProviderCatalogEndpoint,
|
||||
StoredProviderCatalogKey, StoredProviderCatalogKeyMaintenanceSummary,
|
||||
StoredProviderCatalogKeyPage, StoredProviderCatalogKeyStats, StoredProviderCatalogProvider,
|
||||
StoredRequestCandidate, UpsertGeminiFileMappingRecord, UpsertRequestCandidateRecord,
|
||||
ProviderCatalogKeyAdminCasUpdate, ProviderCatalogKeyCredentialsCasUpdate,
|
||||
ProviderCatalogKeyHealthStateUpdate, ProviderCatalogKeyListQuery,
|
||||
ProviderCatalogKeyOAuthCredentialCasDelete, ProviderCatalogKeyOAuthRuntimeStateCasUpdate,
|
||||
ProviderCatalogKeyRuntimeMetadataUpdate, ProviderCatalogKeyStatusSnapshotUpdate,
|
||||
ProviderCatalogProviderConfigCasUpdate, ProviderCatalogProxyCasUpdate, PublicHealthStatusCount,
|
||||
PublicHealthTimelineBucket, StoredGeminiFileMapping, StoredGeminiFileMappingListPage,
|
||||
StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
|
||||
StoredProviderCatalogKeyMaintenanceSummary, StoredProviderCatalogKeyPage,
|
||||
StoredProviderCatalogKeyStats, StoredProviderCatalogProvider, StoredRequestCandidate,
|
||||
UpsertGeminiFileMappingRecord, UpsertRequestCandidateRecord,
|
||||
};
|
||||
|
||||
fn sanitize_request_candidate_rows(
|
||||
mut candidates: Vec<StoredRequestCandidate>,
|
||||
) -> Vec<StoredRequestCandidate> {
|
||||
for candidate in &mut candidates {
|
||||
candidate.sanitize_sensitive_diagnostics();
|
||||
}
|
||||
candidates
|
||||
}
|
||||
|
||||
fn sanitize_request_candidate_row(mut candidate: StoredRequestCandidate) -> StoredRequestCandidate {
|
||||
candidate.sanitize_sensitive_diagnostics();
|
||||
candidate
|
||||
}
|
||||
|
||||
impl GatewayDataState {
|
||||
pub(crate) async fn list_request_candidates_by_request_id(
|
||||
&self,
|
||||
request_id: &str,
|
||||
) -> Result<Vec<StoredRequestCandidate>, DataLayerError> {
|
||||
match &self.request_candidate_reader {
|
||||
Some(repository) => repository.list_by_request_id(request_id).await,
|
||||
Some(repository) => repository
|
||||
.list_by_request_id(request_id)
|
||||
.await
|
||||
.map(sanitize_request_candidate_rows),
|
||||
None => Ok(Vec::new()),
|
||||
}
|
||||
}
|
||||
@@ -27,7 +46,10 @@ impl GatewayDataState {
|
||||
request_id: &str,
|
||||
) -> Result<Vec<StoredRequestCandidate>, DataLayerError> {
|
||||
match &self.request_candidate_reader {
|
||||
Some(repository) => repository.list_attempted_by_request_id(request_id).await,
|
||||
Some(repository) => repository
|
||||
.list_attempted_by_request_id(request_id)
|
||||
.await
|
||||
.map(sanitize_request_candidate_rows),
|
||||
None => Ok(Vec::new()),
|
||||
}
|
||||
}
|
||||
@@ -38,7 +60,10 @@ impl GatewayDataState {
|
||||
limit: usize,
|
||||
) -> Result<Vec<StoredRequestCandidate>, DataLayerError> {
|
||||
match &self.request_candidate_reader {
|
||||
Some(repository) => repository.list_by_provider_id(provider_id, limit).await,
|
||||
Some(repository) => repository
|
||||
.list_by_provider_id(provider_id, limit)
|
||||
.await
|
||||
.map(sanitize_request_candidate_rows),
|
||||
None => Ok(Vec::new()),
|
||||
}
|
||||
}
|
||||
@@ -48,7 +73,10 @@ impl GatewayDataState {
|
||||
limit: usize,
|
||||
) -> Result<Vec<StoredRequestCandidate>, DataLayerError> {
|
||||
match &self.request_candidate_reader {
|
||||
Some(repository) => repository.list_recent(limit).await,
|
||||
Some(repository) => repository
|
||||
.list_recent(limit)
|
||||
.await
|
||||
.map(sanitize_request_candidate_rows),
|
||||
None => Ok(Vec::new()),
|
||||
}
|
||||
}
|
||||
@@ -60,11 +88,10 @@ impl GatewayDataState {
|
||||
limit: usize,
|
||||
) -> Result<Vec<StoredRequestCandidate>, DataLayerError> {
|
||||
match &self.request_candidate_reader {
|
||||
Some(repository) => {
|
||||
repository
|
||||
.list_finalized_by_endpoint_ids_since(endpoint_ids, since_unix_secs, limit)
|
||||
.await
|
||||
}
|
||||
Some(repository) => repository
|
||||
.list_finalized_by_endpoint_ids_since(endpoint_ids, since_unix_secs, limit)
|
||||
.await
|
||||
.map(sanitize_request_candidate_rows),
|
||||
None => Ok(Vec::new()),
|
||||
}
|
||||
}
|
||||
@@ -108,14 +135,19 @@ impl GatewayDataState {
|
||||
|
||||
pub(crate) async fn upsert_request_candidate(
|
||||
&self,
|
||||
candidate: UpsertRequestCandidateRecord,
|
||||
mut candidate: UpsertRequestCandidateRecord,
|
||||
) -> Result<Option<StoredRequestCandidate>, DataLayerError> {
|
||||
candidate.sanitize_for_persistence();
|
||||
crate::request_diagnostics::observe_db_operation(
|
||||
"request_candidate_upsert",
|
||||
self.database_pool_summary(),
|
||||
async {
|
||||
match &self.request_candidate_writer {
|
||||
Some(repository) => repository.upsert(candidate).await.map(Some),
|
||||
Some(repository) => repository
|
||||
.upsert(candidate)
|
||||
.await
|
||||
.map(sanitize_request_candidate_row)
|
||||
.map(Some),
|
||||
None => Ok(None),
|
||||
}
|
||||
},
|
||||
@@ -170,6 +202,16 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn upsert_gemini_file_mapping_if_owner_matches(
|
||||
&self,
|
||||
record: UpsertGeminiFileMappingRecord,
|
||||
) -> Result<Option<StoredGeminiFileMapping>, DataLayerError> {
|
||||
match &self.gemini_file_mapping_writer {
|
||||
Some(repository) => repository.upsert_if_owner_matches(record).await,
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn list_gemini_file_mappings(
|
||||
&self,
|
||||
query: &GeminiFileMappingListQuery,
|
||||
@@ -183,6 +225,49 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn find_gemini_file_mapping_by_file_name(
|
||||
&self,
|
||||
file_name: &str,
|
||||
) -> Result<Option<StoredGeminiFileMapping>, DataLayerError> {
|
||||
match &self.gemini_file_mapping_reader {
|
||||
Some(repository) => repository.find_by_file_name(file_name).await,
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn find_active_gemini_file_mapping_for_user(
|
||||
&self,
|
||||
file_name: &str,
|
||||
user_id: &str,
|
||||
now_unix_secs: u64,
|
||||
) -> Result<Option<StoredGeminiFileMapping>, DataLayerError> {
|
||||
match &self.gemini_file_mapping_reader {
|
||||
Some(repository) => {
|
||||
repository
|
||||
.find_active_by_file_name_for_user(file_name, user_id, now_unix_secs)
|
||||
.await
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn find_active_gemini_file_mapping_for_owner(
|
||||
&self,
|
||||
file_name: &str,
|
||||
key_id: &str,
|
||||
user_id: &str,
|
||||
now_unix_secs: u64,
|
||||
) -> Result<Option<StoredGeminiFileMapping>, DataLayerError> {
|
||||
match &self.gemini_file_mapping_reader {
|
||||
Some(repository) => {
|
||||
repository
|
||||
.find_active_by_file_name_for_owner(file_name, key_id, user_id, now_unix_secs)
|
||||
.await
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn summarize_gemini_file_mappings(
|
||||
&self,
|
||||
now_unix_secs: u64,
|
||||
@@ -208,6 +293,37 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_gemini_file_mapping_by_file_name_for_user(
|
||||
&self,
|
||||
file_name: &str,
|
||||
user_id: &str,
|
||||
) -> Result<bool, DataLayerError> {
|
||||
match &self.gemini_file_mapping_writer {
|
||||
Some(repository) => {
|
||||
repository
|
||||
.delete_by_file_name_for_user(file_name, user_id)
|
||||
.await
|
||||
}
|
||||
None => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_gemini_file_mapping_by_file_name_for_owner(
|
||||
&self,
|
||||
file_name: &str,
|
||||
key_id: &str,
|
||||
user_id: &str,
|
||||
) -> Result<bool, DataLayerError> {
|
||||
match &self.gemini_file_mapping_writer {
|
||||
Some(repository) => {
|
||||
repository
|
||||
.delete_by_file_name_for_owner(file_name, key_id, user_id)
|
||||
.await
|
||||
}
|
||||
None => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_gemini_file_mapping_by_id(
|
||||
&self,
|
||||
mapping_id: &str,
|
||||
@@ -357,38 +473,11 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn update_provider_catalog_key_oauth_credentials(
|
||||
&self,
|
||||
key_id: &str,
|
||||
encrypted_api_key: &str,
|
||||
encrypted_auth_config: Option<&str>,
|
||||
expires_at_unix_secs: Option<u64>,
|
||||
) -> Result<bool, DataLayerError> {
|
||||
let updated = match &self.provider_catalog_writer {
|
||||
Some(repository) => {
|
||||
repository
|
||||
.update_key_oauth_credentials(
|
||||
key_id,
|
||||
encrypted_api_key,
|
||||
encrypted_auth_config,
|
||||
expires_at_unix_secs,
|
||||
)
|
||||
.await
|
||||
}
|
||||
None => Ok(false),
|
||||
}?;
|
||||
if updated {
|
||||
self.clear_provider_catalog_cache();
|
||||
}
|
||||
Ok(updated)
|
||||
}
|
||||
|
||||
pub(crate) async fn update_provider_catalog_key_oauth_runtime_state(
|
||||
&self,
|
||||
key_id: &str,
|
||||
oauth_invalid_at_unix_secs: Option<u64>,
|
||||
oauth_invalid_reason: Option<&str>,
|
||||
encrypted_auth_config_update: Option<&str>,
|
||||
updated_at_unix_secs: Option<u64>,
|
||||
) -> Result<bool, DataLayerError> {
|
||||
let updated = match &self.provider_catalog_writer {
|
||||
@@ -398,7 +487,6 @@ impl GatewayDataState {
|
||||
key_id,
|
||||
oauth_invalid_at_unix_secs,
|
||||
oauth_invalid_reason,
|
||||
encrypted_auth_config_update,
|
||||
updated_at_unix_secs,
|
||||
)
|
||||
.await
|
||||
@@ -475,6 +563,30 @@ impl GatewayDataState {
|
||||
Ok(updated)
|
||||
}
|
||||
|
||||
pub(crate) async fn compare_and_swap_provider_catalog_provider_config(
|
||||
&self,
|
||||
update: &ProviderCatalogProviderConfigCasUpdate,
|
||||
) -> Result<bool, DataLayerError> {
|
||||
let updated = match &self.provider_catalog_writer {
|
||||
Some(repository) => repository.compare_and_swap_provider_config(update).await,
|
||||
None => Ok(false),
|
||||
}?;
|
||||
self.clear_provider_catalog_cache();
|
||||
Ok(updated)
|
||||
}
|
||||
|
||||
pub(crate) async fn compare_and_swap_provider_catalog_provider_proxy(
|
||||
&self,
|
||||
update: &ProviderCatalogProxyCasUpdate,
|
||||
) -> Result<bool, DataLayerError> {
|
||||
let updated = match &self.provider_catalog_writer {
|
||||
Some(repository) => repository.compare_and_swap_provider_proxy(update).await,
|
||||
None => Ok(false),
|
||||
}?;
|
||||
self.clear_provider_catalog_cache();
|
||||
Ok(updated)
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_provider_catalog_provider(
|
||||
&self,
|
||||
provider_id: &str,
|
||||
@@ -543,6 +655,18 @@ impl GatewayDataState {
|
||||
Ok(updated)
|
||||
}
|
||||
|
||||
pub(crate) async fn compare_and_swap_provider_catalog_endpoint_proxy(
|
||||
&self,
|
||||
update: &ProviderCatalogProxyCasUpdate,
|
||||
) -> Result<bool, DataLayerError> {
|
||||
let updated = match &self.provider_catalog_writer {
|
||||
Some(repository) => repository.compare_and_swap_endpoint_proxy(update).await,
|
||||
None => Ok(false),
|
||||
}?;
|
||||
self.clear_provider_catalog_cache();
|
||||
Ok(updated)
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_provider_catalog_endpoint(
|
||||
&self,
|
||||
endpoint_id: &str,
|
||||
@@ -571,6 +695,32 @@ impl GatewayDataState {
|
||||
Ok(updated)
|
||||
}
|
||||
|
||||
pub(crate) async fn compare_and_swap_provider_catalog_key_proxy(
|
||||
&self,
|
||||
update: &ProviderCatalogProxyCasUpdate,
|
||||
) -> Result<bool, DataLayerError> {
|
||||
let updated = match &self.provider_catalog_writer {
|
||||
Some(repository) => repository.compare_and_swap_key_proxy(update).await,
|
||||
None => Ok(false),
|
||||
}?;
|
||||
self.clear_provider_catalog_cache();
|
||||
Ok(updated)
|
||||
}
|
||||
|
||||
pub(crate) async fn compare_and_swap_provider_catalog_key_credentials(
|
||||
&self,
|
||||
update: &ProviderCatalogKeyCredentialsCasUpdate,
|
||||
) -> Result<bool, DataLayerError> {
|
||||
let updated = match &self.provider_catalog_writer {
|
||||
Some(repository) => repository.compare_and_swap_key_credentials(update).await,
|
||||
None => Ok(false),
|
||||
}?;
|
||||
// Clear on both outcomes: a CAS miss proves the cached credential
|
||||
// generation was stale and the retry must observe the winning record.
|
||||
self.clear_provider_catalog_cache();
|
||||
Ok(updated)
|
||||
}
|
||||
|
||||
pub(crate) async fn compare_and_update_provider_catalog_key_admin_state(
|
||||
&self,
|
||||
update: &ProviderCatalogKeyAdminCasUpdate,
|
||||
@@ -843,3 +993,81 @@ impl GatewayDataState {
|
||||
Ok(updated)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod request_candidate_security_tests {
|
||||
use serde_json::json;
|
||||
|
||||
use super::{
|
||||
sanitize_request_candidate_row, sanitize_request_candidate_rows, StoredRequestCandidate,
|
||||
};
|
||||
use aether_data_contracts::repository::candidates::RequestCandidateStatus;
|
||||
|
||||
fn untrusted_candidate() -> StoredRequestCandidate {
|
||||
let mut candidate = StoredRequestCandidate::new(
|
||||
"candidate-untrusted".to_string(),
|
||||
"request-1".to_string(),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
0,
|
||||
0,
|
||||
Some("provider-1".to_string()),
|
||||
Some("endpoint-1".to_string()),
|
||||
Some("key-1".to_string()),
|
||||
RequestCandidateStatus::Failed,
|
||||
None,
|
||||
false,
|
||||
Some(500),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
1,
|
||||
None,
|
||||
Some(2),
|
||||
)
|
||||
.expect("candidate should build");
|
||||
candidate.skip_reason = Some("Bearer candidate-secret".to_string());
|
||||
candidate.error_type = Some("candidate-secret".to_string());
|
||||
candidate.error_message = Some("Bearer candidate-secret".to_string());
|
||||
candidate.extra_data = Some(json!({
|
||||
"gateway_execution_runtime": true,
|
||||
"request_body": {"token": "candidate-secret"}
|
||||
}));
|
||||
candidate.required_capabilities = Some(json!({
|
||||
"vision": 1,
|
||||
"tenant_secret": "candidate-secret"
|
||||
}));
|
||||
candidate
|
||||
}
|
||||
|
||||
fn assert_candidate_is_sanitized(candidate: &StoredRequestCandidate) {
|
||||
assert_eq!(candidate.skip_reason.as_deref(), Some("unclassified_skip"));
|
||||
assert_eq!(candidate.error_type.as_deref(), Some("unclassified_error"));
|
||||
assert!(candidate.error_message.is_none());
|
||||
assert_eq!(
|
||||
candidate.extra_data,
|
||||
Some(json!({"gateway_execution_runtime": true}))
|
||||
);
|
||||
assert_eq!(
|
||||
candidate.required_capabilities,
|
||||
Some(json!({"vision": true}))
|
||||
);
|
||||
assert!(!serde_json::to_string(candidate)
|
||||
.expect("candidate should serialize")
|
||||
.contains("candidate-secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gateway_candidate_boundary_sanitizes_repository_rows_and_write_results() {
|
||||
let candidate = sanitize_request_candidate_row(untrusted_candidate());
|
||||
assert_candidate_is_sanitized(&candidate);
|
||||
|
||||
let candidates = sanitize_request_candidate_rows(vec![untrusted_candidate()]);
|
||||
assert_candidate_is_sanitized(&candidates[0]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -895,6 +895,44 @@ impl GatewayDataState {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn compare_and_set_system_config_string_value(
|
||||
&self,
|
||||
key: &str,
|
||||
expected: &str,
|
||||
replacement: &str,
|
||||
) -> Result<bool, DataLayerError> {
|
||||
if let Some(values) = &self.system_config_values {
|
||||
let updated = {
|
||||
let mut values = values.write().expect("system config values lock");
|
||||
match values.get_mut(key) {
|
||||
Some(entry) if entry.value.as_str() == Some(expected) => {
|
||||
entry.value = serde_json::Value::String(replacement.to_string());
|
||||
entry.updated_at_unix_secs =
|
||||
Some(current_system_config_updated_at_unix_secs());
|
||||
true
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
};
|
||||
self.clear_cached_system_config_value(key);
|
||||
return Ok(updated);
|
||||
}
|
||||
|
||||
let result = match self.backends.as_ref() {
|
||||
Some(backends) => {
|
||||
crate::request_diagnostics::observe_db_operation(
|
||||
"system_config_compare_and_set",
|
||||
self.database_pool_summary(),
|
||||
backends.compare_and_set_system_config_string_value(key, expected, replacement),
|
||||
)
|
||||
.await
|
||||
}
|
||||
None => Ok(false),
|
||||
};
|
||||
self.clear_cached_system_config_value(key);
|
||||
result
|
||||
}
|
||||
|
||||
pub(crate) async fn upsert_system_config_value(
|
||||
&self,
|
||||
key: &str,
|
||||
|
||||
@@ -11,7 +11,11 @@ use aether_data_contracts::repository::candidates::DecisionTrace;
|
||||
use aether_data_contracts::repository::provider_catalog::{
|
||||
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
|
||||
};
|
||||
use aether_data_contracts::repository::settlement::{StoredUsageSettlement, UsageSettlementInput};
|
||||
use aether_data_contracts::repository::proxy_nodes::ProxyNodeTrafficMutation;
|
||||
use aether_data_contracts::repository::settlement::{
|
||||
ReconcileUsagePolicyCostInput, StoredUsagePolicyCostReservation, StoredUsageSettlement,
|
||||
UsageSettlementInput,
|
||||
};
|
||||
use aether_data_contracts::repository::usage::{
|
||||
ProxyNodeCounterDelta, StoredRequestUsageAudit, UpsertUsageRecord, UsageWriteRepository,
|
||||
};
|
||||
@@ -34,7 +38,7 @@ const LEGACY_REQUEST_LOG_LEVEL_KEY: &str = "request_log_level";
|
||||
|
||||
fn usage_request_record_level_from_value(value: Option<&Value>) -> UsageRequestRecordLevel {
|
||||
let Some(value) = value.and_then(Value::as_str).map(str::trim) else {
|
||||
return UsageRequestRecordLevel::Full;
|
||||
return UsageRequestRecordLevel::Basic;
|
||||
};
|
||||
|
||||
if value.eq_ignore_ascii_case("basic")
|
||||
@@ -45,7 +49,9 @@ fn usage_request_record_level_from_value(value: Option<&Value>) -> UsageRequestR
|
||||
{
|
||||
UsageRequestRecordLevel::Basic
|
||||
} else {
|
||||
UsageRequestRecordLevel::Full
|
||||
// Raw HTTP payload capture is disabled at the runtime boundary. The setting remains
|
||||
// accepted for compatibility, but no longer authorizes collecting request/response data.
|
||||
UsageRequestRecordLevel::Basic
|
||||
}
|
||||
}
|
||||
|
||||
@@ -95,6 +101,14 @@ impl StoredVideoTaskReadSide for GatewayDataState {
|
||||
) -> Result<Option<StoredVideoTask>, DataLayerError> {
|
||||
GatewayDataState::find_video_task(self, key).await
|
||||
}
|
||||
|
||||
async fn find_stored_video_task_for_user(
|
||||
&self,
|
||||
key: VideoTaskLookupKey<'_>,
|
||||
user_id: &str,
|
||||
) -> Result<Option<StoredVideoTask>, DataLayerError> {
|
||||
GatewayDataState::find_video_task_for_user(self, key, user_id).await
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -219,6 +233,13 @@ impl UsageSettlementWriter for GatewayDataState {
|
||||
GatewayDataState::has_settlement_writer(self)
|
||||
}
|
||||
|
||||
async fn reconcile_usage_policy_cost(
|
||||
&self,
|
||||
input: ReconcileUsagePolicyCostInput,
|
||||
) -> Result<Option<StoredUsagePolicyCostReservation>, DataLayerError> {
|
||||
GatewayDataState::reconcile_usage_policy_cost(self, input).await
|
||||
}
|
||||
|
||||
async fn settle_usage(
|
||||
&self,
|
||||
input: UsageSettlementInput,
|
||||
@@ -285,10 +306,26 @@ impl aether_usage_runtime::ManualProxyNodeCounter for GatewayDataState {
|
||||
failed_delta: i64,
|
||||
latency_ms: Option<i64>,
|
||||
) -> Result<(), DataLayerError> {
|
||||
// This API predates incarnation fences and only receives a node id. Read
|
||||
// the selected node first so every durable path can bind the delta to
|
||||
// the observed generation. If the node disappeared, fail closed rather
|
||||
// than allowing a bare id to target a replacement node.
|
||||
let Some(node) = self.find_proxy_node(node_id).await? else {
|
||||
return Ok(());
|
||||
};
|
||||
if !node.is_manual {
|
||||
return Ok(());
|
||||
}
|
||||
let expected_tunnel_generation = node.tunnel_generation.trim().to_string();
|
||||
if expected_tunnel_generation.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if let Some(repository) = &self.usage_writer {
|
||||
let enqueued = repository
|
||||
.enqueue_proxy_node_counter_delta(ProxyNodeCounterDelta {
|
||||
node_id: node_id.to_string(),
|
||||
expected_tunnel_generation: Some(expected_tunnel_generation.clone()),
|
||||
total_requests_delta: total_delta,
|
||||
failed_requests_delta: failed_delta,
|
||||
dns_failures_delta: 0,
|
||||
@@ -300,14 +337,25 @@ impl aether_usage_runtime::ManualProxyNodeCounter for GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
match &self.proxy_node_writer {
|
||||
Some(repository) => {
|
||||
repository
|
||||
.increment_manual_node_requests(node_id, total_delta, failed_delta, latency_ms)
|
||||
.await
|
||||
}
|
||||
None => Ok(()),
|
||||
// The legacy increment method has no generation argument and would
|
||||
// re-read the current row, which is vulnerable to an id reuse between
|
||||
// the read above and the write. Use the fenced traffic mutation as the
|
||||
// only fallback. It intentionally omits the legacy latency-only field;
|
||||
// preserving counters safely is more important than an unfenced write.
|
||||
if let Some(repository) = &self.proxy_node_writer {
|
||||
let _ = repository
|
||||
.record_traffic(&ProxyNodeTrafficMutation {
|
||||
node_id: node_id.to_string(),
|
||||
expected_tunnel_generation: Some(expected_tunnel_generation),
|
||||
total_requests_delta: total_delta,
|
||||
failed_requests_delta: failed_delta,
|
||||
dns_failures_delta: 0,
|
||||
stream_errors_delta: 0,
|
||||
})
|
||||
.await?;
|
||||
}
|
||||
let _ = latency_ms;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -452,6 +500,20 @@ mod tests {
|
||||
assert_eq!(level, UsageRequestRecordLevel::Basic);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_runtime_access_disables_full_http_capture() {
|
||||
let state = GatewayDataState::disabled().with_system_config_values_for_tests([(
|
||||
"request_record_level".to_string(),
|
||||
json!("full"),
|
||||
)]);
|
||||
|
||||
let level = UsageRuntimeAccess::request_record_level(&state)
|
||||
.await
|
||||
.expect("request record level should read");
|
||||
|
||||
assert_eq!(level, UsageRequestRecordLevel::Basic);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_runtime_access_falls_back_to_legacy_request_log_level_alias() {
|
||||
let state = GatewayDataState::disabled().with_system_config_values_for_tests([(
|
||||
@@ -467,14 +529,14 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_runtime_access_defaults_missing_request_record_level_to_full() {
|
||||
async fn usage_runtime_access_defaults_missing_request_record_level_to_basic() {
|
||||
let state = GatewayDataState::disabled();
|
||||
|
||||
let level = UsageRuntimeAccess::request_record_level(&state)
|
||||
.await
|
||||
.expect("missing request record level should fall back");
|
||||
|
||||
assert_eq!(level, UsageRequestRecordLevel::Full);
|
||||
assert_eq!(level, UsageRequestRecordLevel::Basic);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -488,6 +550,20 @@ mod tests {
|
||||
.await
|
||||
.expect("body capture policy should read");
|
||||
|
||||
assert_eq!(policy.record_level, UsageRequestRecordLevel::Full);
|
||||
assert_eq!(policy.record_level, UsageRequestRecordLevel::Basic);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_runtime_access_fails_closed_for_unknown_record_level() {
|
||||
let state = GatewayDataState::disabled().with_system_config_values_for_tests([(
|
||||
"request_record_level".to_string(),
|
||||
json!("everything"),
|
||||
)]);
|
||||
|
||||
let level = UsageRuntimeAccess::request_record_level(&state)
|
||||
.await
|
||||
.expect("request record level should read");
|
||||
|
||||
assert_eq!(level, UsageRequestRecordLevel::Basic);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,8 +27,8 @@ use aether_data::repository::auth::{
|
||||
StoredAuthApiKeyExportRecord, StoredAuthApiKeySnapshot,
|
||||
};
|
||||
use aether_data::repository::auth_modules::{
|
||||
AuthModuleReadRepository, AuthModuleWriteRepository, StoredLdapModuleConfig,
|
||||
StoredOAuthProviderModuleConfig,
|
||||
AuthModuleReadRepository, AuthModuleWriteRepository, CompareAndSwapLdapConfigResult,
|
||||
LdapBindPasswordUpdate, StoredLdapModuleConfig, StoredOAuthProviderModuleConfig,
|
||||
};
|
||||
use aether_data::repository::gemini_file_mappings::{
|
||||
GeminiFileMappingListQuery, GeminiFileMappingReadRepository, GeminiFileMappingStats,
|
||||
@@ -36,9 +36,10 @@ use aether_data::repository::gemini_file_mappings::{
|
||||
UpsertGeminiFileMappingRecord,
|
||||
};
|
||||
use aether_data::repository::management_tokens::{
|
||||
CreateManagementTokenRecord, ManagementTokenListQuery, ManagementTokenReadRepository,
|
||||
ManagementTokenWriteRepository, RegenerateManagementTokenSecret, StoredManagementToken,
|
||||
StoredManagementTokenListPage, StoredManagementTokenWithUser, UpdateManagementTokenRecord,
|
||||
ActivateManagementTokenIfMatches, CreateManagementTokenRecord, ManagementTokenListQuery,
|
||||
ManagementTokenReadRepository, ManagementTokenWriteRepository, RegenerateManagementTokenSecret,
|
||||
StoredManagementToken, StoredManagementTokenListPage, StoredManagementTokenWithUser,
|
||||
UpdateManagementTokenRecord,
|
||||
};
|
||||
use aether_data::repository::oauth_providers::{
|
||||
OAuthProviderReadRepository, OAuthProviderWriteRepository, StoredOAuthProviderConfig,
|
||||
@@ -63,21 +64,24 @@ pub(crate) use aether_data::repository::users::{
|
||||
use aether_data::repository::wallet::{
|
||||
AdjustWalletBalanceInput, AdminPaymentOrderListQuery, AdminRedeemCodeBatchListQuery,
|
||||
AdminRedeemCodeListQuery, AdminWalletLedgerQuery, AdminWalletListQuery,
|
||||
AdminWalletRefundRequestListQuery, CompleteAdminWalletRefundInput,
|
||||
CreateAdminRedeemCodeBatchInput, CreateAdminRedeemCodeBatchResult,
|
||||
CreateManualWalletRechargeInput, CreatePlanPurchaseOrderInput, CreatePlanPurchaseOrderOutcome,
|
||||
CreateWalletRechargeOrderInput, CreateWalletRechargeOrderOutcome,
|
||||
CreateWalletRefundRequestInput, CreateWalletRefundRequestOutcome, CreditAdminPaymentOrderInput,
|
||||
AdminWalletRefundRequestListQuery, CompareAndSwapPaymentOrderStripeClientSecretInput,
|
||||
CompleteAdminWalletRefundInput, CreateAdminRedeemCodeBatchInput,
|
||||
CreateAdminRedeemCodeBatchResult, CreateManualWalletRechargeInput,
|
||||
CreatePlanPurchaseOrderInput, CreatePlanPurchaseOrderOutcome, CreateWalletRechargeOrderInput,
|
||||
CreateWalletRechargeOrderOutcome, CreateWalletRefundRequestInput,
|
||||
CreateWalletRefundRequestOutcome, CreditAdminPaymentOrderInput,
|
||||
DeleteAdminRedeemCodeBatchInput, DisableAdminRedeemCodeBatchInput, DisableAdminRedeemCodeInput,
|
||||
FailAdminWalletRefundInput, ProcessAdminWalletRefundInput, ProcessPaymentCallbackInput,
|
||||
ProcessPaymentCallbackOutcome, RedeemWalletCodeInput, RedeemWalletCodeOutcome,
|
||||
FailAdminWalletRefundInput, FailWalletRechargeCheckoutInput, InitializeAuthWalletOutcome,
|
||||
ProcessAdminWalletRefundInput, ProcessPaymentCallbackInput, ProcessPaymentCallbackOutcome,
|
||||
ReclaimWalletRechargeCheckoutInput, RedeemWalletCodeInput, RedeemWalletCodeOutcome,
|
||||
StoredAdminPaymentCallback, StoredAdminPaymentCallbackPage, StoredAdminPaymentOrder,
|
||||
StoredAdminPaymentOrderPage, StoredAdminRedeemCode, StoredAdminRedeemCodeBatch,
|
||||
StoredAdminRedeemCodeBatchPage, StoredAdminRedeemCodePage, StoredAdminWalletLedgerPage,
|
||||
StoredAdminWalletListPage, StoredAdminWalletRefund, StoredAdminWalletRefundPage,
|
||||
StoredAdminWalletRefundRequestPage, StoredAdminWalletTransaction,
|
||||
StoredAdminWalletTransactionPage, StoredWalletDailyUsageLedger,
|
||||
StoredWalletDailyUsageLedgerPage, StoredWalletSnapshot, WalletLookupKey, WalletMutationOutcome,
|
||||
StoredWalletDailyUsageLedgerPage, StoredWalletSnapshot, UpdateAdminWalletRefundGatewayInput,
|
||||
UpdateWalletRechargeCheckoutInput, WalletLookupKey, WalletMutationOutcome,
|
||||
WalletReadRepository, WalletWriteRepository,
|
||||
};
|
||||
use aether_data::{
|
||||
@@ -92,8 +96,9 @@ use aether_data_contracts::repository::background_tasks::{
|
||||
use aether_data_contracts::repository::billing::{
|
||||
AdminBillingCollectorRecord, AdminBillingCollectorWriteInput, AdminBillingMutationOutcome,
|
||||
AdminBillingPresetApplyResult, AdminBillingRuleRecord, AdminBillingRuleWriteInput,
|
||||
BillingPlanRecord, BillingPlanWriteInput, BillingReadRepository, PaymentGatewayConfigRecord,
|
||||
PaymentGatewayConfigWriteInput, StoredBillingModelContext, UserDailyQuotaAvailabilityRecord,
|
||||
BillingPlanRecord, BillingPlanWriteInput, BillingReadRepository,
|
||||
PaymentGatewayConfigCasWriteInput, PaymentGatewayConfigRecord, PaymentGatewayConfigWriteInput,
|
||||
PaymentGatewaySecretCasUpdate, StoredBillingModelContext, UserDailyQuotaAvailabilityRecord,
|
||||
UserPlanEntitlementRecord,
|
||||
};
|
||||
use aether_data_contracts::repository::candidate_selection::{
|
||||
@@ -122,12 +127,14 @@ use aether_data_contracts::repository::pool_scores::{
|
||||
};
|
||||
use aether_data_contracts::repository::provider_catalog::{
|
||||
ProviderCatalogKeyAdaptiveStateUpdate, ProviderCatalogKeyAdminCasUpdate,
|
||||
ProviderCatalogKeyHealthStateUpdate, ProviderCatalogKeyListQuery,
|
||||
ProviderCatalogKeyOAuthCredentialCasDelete, ProviderCatalogKeyOAuthRuntimeStateCasUpdate,
|
||||
ProviderCatalogKeyRuntimeMetadataUpdate, ProviderCatalogKeyStatusSnapshotUpdate,
|
||||
ProviderCatalogReadRepository, ProviderCatalogWriteRepository, StoredProviderCatalogEndpoint,
|
||||
StoredProviderCatalogKey, StoredProviderCatalogKeyMaintenanceSummary,
|
||||
StoredProviderCatalogKeyPage, StoredProviderCatalogKeyStats, StoredProviderCatalogProvider,
|
||||
ProviderCatalogKeyCredentialsCasUpdate, ProviderCatalogKeyHealthStateUpdate,
|
||||
ProviderCatalogKeyListQuery, ProviderCatalogKeyOAuthCredentialCasDelete,
|
||||
ProviderCatalogKeyOAuthRuntimeStateCasUpdate, ProviderCatalogKeyRuntimeMetadataUpdate,
|
||||
ProviderCatalogKeyStatusSnapshotUpdate, ProviderCatalogProviderConfigCasUpdate,
|
||||
ProviderCatalogProxyCasUpdate, ProviderCatalogReadRepository, ProviderCatalogWriteRepository,
|
||||
StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
|
||||
StoredProviderCatalogKeyMaintenanceSummary, StoredProviderCatalogKeyPage,
|
||||
StoredProviderCatalogKeyStats, StoredProviderCatalogProvider,
|
||||
};
|
||||
use aether_data_contracts::repository::quota::{
|
||||
ProviderQuotaReadRepository, ProviderQuotaWriteRepository, StoredProviderQuotaSnapshot,
|
||||
@@ -136,7 +143,10 @@ use aether_data_contracts::repository::routing_profiles::{
|
||||
RoutingGroupReadRepository, RoutingGroupWriteRepository,
|
||||
};
|
||||
use aether_data_contracts::repository::settlement::{
|
||||
SettlementWriteRepository, StoredUsageSettlement, UsageSettlementInput,
|
||||
ReconcileUsagePolicyCostInput, ReleaseUsagePolicyRequestAdmissionInput,
|
||||
ReserveUsagePolicyCostInput, ReserveUsagePolicyCostOutcome, ReserveUsagePolicyRequestInput,
|
||||
ReserveUsagePolicyRequestOutcome, SettlementWriteRepository, StoredUsagePolicyCostReservation,
|
||||
StoredUsagePolicyRequestAdmission, StoredUsageSettlement, UsageSettlementInput,
|
||||
};
|
||||
use aether_data_contracts::repository::usage::{
|
||||
ApiKeyLastUsedDelta, ManagementTokenCounterDelta, PendingUsageCleanupSummary,
|
||||
@@ -150,9 +160,9 @@ use aether_data_contracts::repository::video_tasks::{
|
||||
use aether_runtime_state::RuntimeQueueStore;
|
||||
|
||||
pub(crate) use self::referrals::{
|
||||
ReferralAdminStats, ReferralMutationStatus, ReferralRelationshipListQuery,
|
||||
ReferralRelationshipRecord, ReferralRewardConfig, ReferralRewardListQuery,
|
||||
ReferralRewardRecord, ReferralUserDashboard,
|
||||
ReferralAdminStats, ReferralMutationStatus, ReferralReconciliationSummary,
|
||||
ReferralRelationshipListQuery, ReferralRelationshipRecord, ReferralRewardConfig,
|
||||
ReferralRewardListQuery, ReferralRewardRecord, ReferralUserDashboard,
|
||||
};
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
|
||||
@@ -4,9 +4,9 @@ use aether_data::DataLayerError;
|
||||
use super::GatewayDataState;
|
||||
|
||||
pub(crate) use aether_data::backend::{
|
||||
ReferralAdminStats, ReferralMutationStatus, ReferralRelationshipListQuery,
|
||||
ReferralRelationshipRecord, ReferralRewardConfig, ReferralRewardListQuery,
|
||||
ReferralRewardRecord, ReferralUserDashboard,
|
||||
ReferralAdminStats, ReferralMutationStatus, ReferralReconciliationSummary,
|
||||
ReferralRelationshipListQuery, ReferralRelationshipRecord, ReferralRewardConfig,
|
||||
ReferralRewardListQuery, ReferralRewardRecord, ReferralUserDashboard,
|
||||
};
|
||||
|
||||
impl GatewayDataState {
|
||||
@@ -115,4 +115,13 @@ impl GatewayDataState {
|
||||
.reverse_referral_rewards_for_order(order_id, amount_usd)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn reconcile_referral_rewards_once(
|
||||
&self,
|
||||
reward_config: Option<ReferralRewardConfig>,
|
||||
) -> Result<ReferralReconciliationSummary, DataLayerError> {
|
||||
self.referrals()
|
||||
.reconcile_referral_rewards_once(reward_config)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,33 +7,39 @@ use super::{
|
||||
AdminWalletRefundRequestListQuery, AnnouncementListQuery, AuditLogListQuery,
|
||||
BackgroundTaskListQuery, BackgroundTaskSummary, BillingModelContextCacheKey,
|
||||
BillingModelContextCacheState, BillingModelContextInflightState, BillingPlanRecord,
|
||||
BillingPlanWriteInput, CompleteAdminWalletRefundInput, CreateAdminRedeemCodeBatchInput,
|
||||
BillingPlanWriteInput, CompareAndSwapPaymentOrderStripeClientSecretInput,
|
||||
CompleteAdminWalletRefundInput, CreateAdminRedeemCodeBatchInput,
|
||||
CreateAdminRedeemCodeBatchResult, CreateAnnouncementRecord, CreateManualWalletRechargeInput,
|
||||
CreatePlanPurchaseOrderInput, CreatePlanPurchaseOrderOutcome, CreateWalletRechargeOrderInput,
|
||||
CreateWalletRechargeOrderOutcome, CreateWalletRefundRequestInput,
|
||||
CreateWalletRefundRequestOutcome, CreditAdminPaymentOrderInput, DataLayerError,
|
||||
DatabaseMaintenanceSummary, DecisionTrace, DeleteAdminRedeemCodeBatchInput,
|
||||
DisableAdminRedeemCodeBatchInput, DisableAdminRedeemCodeInput, FailAdminWalletRefundInput,
|
||||
GatewayDataState, GatewayProviderTransportSnapshot, LocalVideoTaskReadResponse,
|
||||
PaymentGatewayConfigRecord, PaymentGatewayConfigWriteInput, ProcessAdminWalletRefundInput,
|
||||
ProcessPaymentCallbackInput, ProcessPaymentCallbackOutcome, RedeemWalletCodeInput,
|
||||
RedeemWalletCodeOutcome, RequestAuditBundle, RequestCandidateTrace, StoredAdminAuditLogPage,
|
||||
StoredAdminPaymentCallbackPage, StoredAdminPaymentOrder, StoredAdminPaymentOrderPage,
|
||||
StoredAdminRedeemCodeBatch, StoredAdminRedeemCodeBatchPage, StoredAdminRedeemCodePage,
|
||||
StoredAdminWalletLedgerPage, StoredAdminWalletListPage, StoredAdminWalletRefund,
|
||||
StoredAdminWalletRefundPage, StoredAdminWalletRefundRequestPage, StoredAdminWalletTransaction,
|
||||
FailWalletRechargeCheckoutInput, GatewayDataState, GatewayProviderTransportSnapshot,
|
||||
LocalVideoTaskReadResponse, PaymentGatewayConfigCasWriteInput, PaymentGatewayConfigRecord,
|
||||
PaymentGatewayConfigWriteInput, PaymentGatewaySecretCasUpdate, ProcessAdminWalletRefundInput,
|
||||
ProcessPaymentCallbackInput, ProcessPaymentCallbackOutcome, ReclaimWalletRechargeCheckoutInput,
|
||||
ReconcileUsagePolicyCostInput, RedeemWalletCodeInput, RedeemWalletCodeOutcome,
|
||||
ReleaseUsagePolicyRequestAdmissionInput, RequestAuditBundle, RequestCandidateTrace,
|
||||
ReserveUsagePolicyCostInput, ReserveUsagePolicyCostOutcome, ReserveUsagePolicyRequestInput,
|
||||
ReserveUsagePolicyRequestOutcome, StoredAdminAuditLogPage, StoredAdminPaymentCallbackPage,
|
||||
StoredAdminPaymentOrder, StoredAdminPaymentOrderPage, StoredAdminRedeemCodeBatch,
|
||||
StoredAdminRedeemCodeBatchPage, StoredAdminRedeemCodePage, StoredAdminWalletLedgerPage,
|
||||
StoredAdminWalletListPage, StoredAdminWalletRefund, StoredAdminWalletRefundPage,
|
||||
StoredAdminWalletRefundRequestPage, StoredAdminWalletTransaction,
|
||||
StoredAdminWalletTransactionPage, StoredAnnouncement, StoredAnnouncementPage,
|
||||
StoredBackgroundTaskEvent, StoredBackgroundTaskRun, StoredBackgroundTaskRunPage,
|
||||
StoredBillingModelContext, StoredProviderQuotaSnapshot, StoredProviderUsageSummary,
|
||||
StoredRequestUsageAudit, StoredSuspiciousActivity, StoredUsageSettlement,
|
||||
StoredUserAuditLogPage, StoredUserAuthRecord, StoredUserExportRow, StoredUserSummary,
|
||||
StoredVideoTask, StoredWalletDailyUsageLedger, StoredWalletDailyUsageLedgerPage,
|
||||
StoredWalletSnapshot, UpdateAnnouncementRecord, UpsertBackgroundTaskEvent,
|
||||
UpsertBackgroundTaskRun, UpsertUsageRecord, UpsertVideoTask, UsageSettlementInput,
|
||||
UserDailyQuotaAvailabilityRecord, UserPlanEntitlementRecord, VideoTaskLookupKey,
|
||||
VideoTaskModelCount, VideoTaskQueryFilter, VideoTaskStatusCount,
|
||||
WalletDailyUsageAggregationInput, WalletDailyUsageAggregationResult, WalletLookupKey,
|
||||
WalletMutationOutcome,
|
||||
StoredRequestUsageAudit, StoredSuspiciousActivity, StoredUsagePolicyCostReservation,
|
||||
StoredUsagePolicyRequestAdmission, StoredUsageSettlement, StoredUserAuditLogPage,
|
||||
StoredUserAuthRecord, StoredUserExportRow, StoredUserSummary, StoredVideoTask,
|
||||
StoredWalletDailyUsageLedger, StoredWalletDailyUsageLedgerPage, StoredWalletSnapshot,
|
||||
UpdateAdminWalletRefundGatewayInput, UpdateAnnouncementRecord,
|
||||
UpdateWalletRechargeCheckoutInput, UpsertBackgroundTaskEvent, UpsertBackgroundTaskRun,
|
||||
UpsertUsageRecord, UpsertVideoTask, UsageSettlementInput, UserDailyQuotaAvailabilityRecord,
|
||||
UserPlanEntitlementRecord, VideoTaskLookupKey, VideoTaskModelCount, VideoTaskQueryFilter,
|
||||
VideoTaskStatusCount, WalletDailyUsageAggregationInput, WalletDailyUsageAggregationResult,
|
||||
WalletLookupKey, WalletMutationOutcome,
|
||||
};
|
||||
use aether_data_contracts::repository::usage::{
|
||||
PendingUsageCleanupSummary, ProviderApiKeyWindowUsageRequest,
|
||||
@@ -43,7 +49,9 @@ use aether_data_contracts::repository::usage::{
|
||||
UsageDailyHeatmapQuery,
|
||||
};
|
||||
use aether_runtime_state::RuntimeQueueStore;
|
||||
use aether_video_tasks_core::read_data_backed_video_task_response;
|
||||
use aether_video_tasks_core::{
|
||||
read_data_backed_video_task_response, read_data_backed_video_task_response_for_user,
|
||||
};
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::time::timeout;
|
||||
|
||||
@@ -558,6 +566,17 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn find_video_task_for_user(
|
||||
&self,
|
||||
key: VideoTaskLookupKey<'_>,
|
||||
user_id: &str,
|
||||
) -> Result<Option<StoredVideoTask>, DataLayerError> {
|
||||
match &self.video_task_reader {
|
||||
Some(repository) => repository.find_for_user(key, user_id).await,
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn list_video_task_page(
|
||||
&self,
|
||||
filter: &VideoTaskQueryFilter,
|
||||
@@ -894,6 +913,21 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn find_wallet_recharge_order_by_order_no(
|
||||
&self,
|
||||
user_id: &str,
|
||||
order_no: &str,
|
||||
) -> Result<Option<StoredAdminPaymentOrder>, DataLayerError> {
|
||||
match &self.wallet_reader {
|
||||
Some(repository) => {
|
||||
repository
|
||||
.find_wallet_recharge_order_by_order_no(user_id, order_no)
|
||||
.await
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn find_pending_plan_purchase_order_by_user_id(
|
||||
&self,
|
||||
user_id: &str,
|
||||
@@ -909,6 +943,16 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn find_payment_order_by_order_no(
|
||||
&self,
|
||||
order_no: &str,
|
||||
) -> Result<Option<StoredAdminPaymentOrder>, DataLayerError> {
|
||||
match &self.wallet_reader {
|
||||
Some(repository) => repository.find_payment_order_by_order_no(order_no).await,
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn find_wallet_refund(
|
||||
&self,
|
||||
wallet_id: &str,
|
||||
@@ -934,6 +978,58 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn update_wallet_recharge_checkout(
|
||||
&self,
|
||||
input: UpdateWalletRechargeCheckoutInput,
|
||||
) -> Result<Option<WalletMutationOutcome<StoredAdminPaymentOrder>>, DataLayerError> {
|
||||
match &self.wallet_writer {
|
||||
Some(repository) => repository
|
||||
.update_wallet_recharge_checkout(input)
|
||||
.await
|
||||
.map(Some),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn compare_and_swap_payment_order_stripe_client_secret(
|
||||
&self,
|
||||
input: CompareAndSwapPaymentOrderStripeClientSecretInput,
|
||||
) -> Result<Option<bool>, DataLayerError> {
|
||||
match &self.wallet_writer {
|
||||
Some(repository) => repository
|
||||
.compare_and_swap_payment_order_stripe_client_secret(input)
|
||||
.await
|
||||
.map(Some),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn fail_wallet_recharge_checkout(
|
||||
&self,
|
||||
input: FailWalletRechargeCheckoutInput,
|
||||
) -> Result<Option<WalletMutationOutcome<StoredAdminPaymentOrder>>, DataLayerError> {
|
||||
match &self.wallet_writer {
|
||||
Some(repository) => repository
|
||||
.fail_wallet_recharge_checkout(input)
|
||||
.await
|
||||
.map(Some),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn reclaim_wallet_recharge_checkout(
|
||||
&self,
|
||||
input: ReclaimWalletRechargeCheckoutInput,
|
||||
) -> Result<Option<WalletMutationOutcome<StoredAdminPaymentOrder>>, DataLayerError> {
|
||||
match &self.wallet_writer {
|
||||
Some(repository) => repository
|
||||
.reclaim_wallet_recharge_checkout(input)
|
||||
.await
|
||||
.map(Some),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn create_plan_purchase_order(
|
||||
&self,
|
||||
input: CreatePlanPurchaseOrderInput,
|
||||
@@ -1009,6 +1105,19 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn update_admin_wallet_refund_gateway(
|
||||
&self,
|
||||
input: UpdateAdminWalletRefundGatewayInput,
|
||||
) -> Result<Option<WalletMutationOutcome<StoredAdminWalletRefund>>, DataLayerError> {
|
||||
match &self.wallet_writer {
|
||||
Some(repository) => repository
|
||||
.update_admin_wallet_refund_gateway(input)
|
||||
.await
|
||||
.map(Some),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn complete_admin_wallet_refund(
|
||||
&self,
|
||||
input: CompleteAdminWalletRefundInput,
|
||||
@@ -1151,6 +1260,83 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn reserve_usage_policy_cost(
|
||||
&self,
|
||||
input: ReserveUsagePolicyCostInput,
|
||||
) -> Result<Option<ReserveUsagePolicyCostOutcome>, DataLayerError> {
|
||||
match &self.settlement_writer {
|
||||
Some(repository) => repository.reserve_usage_policy_cost(input).await.map(Some),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn reserve_usage_policy_request(
|
||||
&self,
|
||||
input: ReserveUsagePolicyRequestInput,
|
||||
) -> Result<Option<ReserveUsagePolicyRequestOutcome>, DataLayerError> {
|
||||
match &self.settlement_writer {
|
||||
Some(repository) => repository
|
||||
.reserve_usage_policy_request(input)
|
||||
.await
|
||||
.map(Some),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn release_usage_policy_request_admission(
|
||||
&self,
|
||||
input: ReleaseUsagePolicyRequestAdmissionInput,
|
||||
) -> Result<Option<StoredUsagePolicyRequestAdmission>, DataLayerError> {
|
||||
match &self.settlement_writer {
|
||||
Some(repository) => {
|
||||
repository
|
||||
.release_usage_policy_request_admission(input)
|
||||
.await
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn cleanup_usage_policy_request_admissions(
|
||||
&self,
|
||||
now_unix_secs: u64,
|
||||
batch_size: usize,
|
||||
) -> Result<usize, DataLayerError> {
|
||||
match &self.settlement_writer {
|
||||
Some(repository) => {
|
||||
repository
|
||||
.cleanup_usage_policy_request_admissions(now_unix_secs, batch_size)
|
||||
.await
|
||||
}
|
||||
None => Ok(0),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn reconcile_usage_policy_cost(
|
||||
&self,
|
||||
input: ReconcileUsagePolicyCostInput,
|
||||
) -> Result<Option<StoredUsagePolicyCostReservation>, DataLayerError> {
|
||||
match &self.settlement_writer {
|
||||
Some(repository) => repository.reconcile_usage_policy_cost(input).await,
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn cleanup_usage_policy_cost_reservations(
|
||||
&self,
|
||||
now_unix_secs: u64,
|
||||
batch_size: usize,
|
||||
) -> Result<usize, DataLayerError> {
|
||||
match &self.settlement_writer {
|
||||
Some(repository) => {
|
||||
repository
|
||||
.cleanup_usage_policy_cost_reservations(now_unix_secs, batch_size)
|
||||
.await
|
||||
}
|
||||
None => Ok(0),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn reset_due_provider_quotas(
|
||||
&self,
|
||||
now_unix_secs: u64,
|
||||
@@ -2496,6 +2682,48 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn find_payment_gateway_config_strong(
|
||||
&self,
|
||||
provider: &str,
|
||||
) -> Result<Option<PaymentGatewayConfigRecord>, DataLayerError> {
|
||||
match &self.billing_reader {
|
||||
Some(repository) => {
|
||||
repository
|
||||
.find_payment_gateway_config_strong(provider)
|
||||
.await
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn compare_and_swap_payment_gateway_secret(
|
||||
&self,
|
||||
update: &PaymentGatewaySecretCasUpdate,
|
||||
) -> Result<bool, DataLayerError> {
|
||||
match &self.billing_reader {
|
||||
Some(repository) => {
|
||||
repository
|
||||
.compare_and_swap_payment_gateway_secret(update)
|
||||
.await
|
||||
}
|
||||
None => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn compare_and_swap_payment_gateway_config(
|
||||
&self,
|
||||
input: &PaymentGatewayConfigCasWriteInput,
|
||||
) -> Result<AdminBillingMutationOutcome<PaymentGatewayConfigRecord>, DataLayerError> {
|
||||
match &self.billing_reader {
|
||||
Some(repository) => {
|
||||
repository
|
||||
.compare_and_swap_payment_gateway_config(input)
|
||||
.await
|
||||
}
|
||||
None => Ok(AdminBillingMutationOutcome::Unavailable),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn upsert_payment_gateway_config(
|
||||
&self,
|
||||
input: &PaymentGatewayConfigWriteInput,
|
||||
@@ -2667,6 +2895,16 @@ impl GatewayDataState {
|
||||
read_data_backed_video_task_response(self, route_family, request_path).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_video_task_response_for_user(
|
||||
&self,
|
||||
route_family: Option<&str>,
|
||||
request_path: &str,
|
||||
user_id: &str,
|
||||
) -> Result<Option<LocalVideoTaskReadResponse>, DataLayerError> {
|
||||
read_data_backed_video_task_response_for_user(self, route_family, request_path, user_id)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn find_background_task_run(
|
||||
&self,
|
||||
run_id: &str,
|
||||
|
||||
@@ -1790,6 +1790,18 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn attach_auth_api_key_repository_for_tests<T>(mut self, repository: Arc<T>) -> Self
|
||||
where
|
||||
T: aether_data::repository::auth::AuthRepository + 'static,
|
||||
{
|
||||
let auth_api_key_reader: Arc<dyn AuthApiKeyReadRepository> = repository.clone();
|
||||
let auth_api_key_writer: Arc<dyn AuthApiKeyWriteRepository> = repository;
|
||||
self.auth_api_key_reader = Some(auth_api_key_reader);
|
||||
self.auth_api_key_writer = Some(auth_api_key_writer);
|
||||
self
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn with_decision_trace_readers_for_tests(
|
||||
request_candidate_repository: Arc<dyn RequestCandidateReadRepository>,
|
||||
@@ -2379,6 +2391,36 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn with_auth_candidate_selection_provider_catalog_request_candidate_and_gemini_file_mapping_repositories_for_tests<
|
||||
T,
|
||||
U,
|
||||
V,
|
||||
>(
|
||||
auth_api_key_repository: Arc<dyn AuthApiKeyReadRepository>,
|
||||
candidate_selection_repository: Arc<dyn MinimalCandidateSelectionReadRepository>,
|
||||
provider_catalog_repository: Arc<U>,
|
||||
request_candidate_repository: Arc<T>,
|
||||
gemini_file_mapping_repository: Arc<V>,
|
||||
encryption_key: impl Into<String>,
|
||||
) -> Self
|
||||
where
|
||||
T: RequestCandidateRepository + 'static,
|
||||
U: ProviderCatalogReadRepository + ProviderCatalogWriteRepository + 'static,
|
||||
V: aether_data::repository::gemini_file_mappings::GeminiFileMappingRepository + 'static,
|
||||
{
|
||||
let mut state = Self::with_auth_candidate_selection_provider_catalog_and_request_candidate_repository_for_tests(
|
||||
auth_api_key_repository,
|
||||
candidate_selection_repository,
|
||||
provider_catalog_repository,
|
||||
request_candidate_repository,
|
||||
encryption_key,
|
||||
);
|
||||
state.gemini_file_mapping_reader = Some(gemini_file_mapping_repository.clone());
|
||||
state.gemini_file_mapping_writer = Some(gemini_file_mapping_repository);
|
||||
state
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn with_auth_candidate_selection_provider_catalog_request_candidates_for_tests<
|
||||
T,
|
||||
|
||||
@@ -12,10 +12,113 @@ use aether_data_contracts::repository::usage::{
|
||||
};
|
||||
|
||||
use aether_data::repository::auth::AuthApiKeyReadRepository;
|
||||
use aether_data::repository::management_tokens::{
|
||||
InMemoryManagementTokenRepository, ManagementTokenReadRepository,
|
||||
ManagementTokenWriteRepository, StoredManagementToken, StoredManagementTokenUserSummary,
|
||||
StoredManagementTokenWithUser,
|
||||
};
|
||||
use aether_data::repository::proxy_nodes::{InMemoryProxyNodeRepository, StoredProxyNode};
|
||||
use aether_data::repository::proxy_nodes::{ProxyNodeReadRepository, ProxyNodeWriteRepository};
|
||||
use aether_data::repository::users::{
|
||||
InMemoryUserReadRepository, StoredUserAuthRecord, UserReadRepository,
|
||||
};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::{GatewayDataConfig, GatewayDataState};
|
||||
|
||||
impl GatewayDataState {
|
||||
pub(crate) fn with_tunnel_management_auth_for_testkit(
|
||||
node_id: &str,
|
||||
tunnel_generation: &str,
|
||||
raw_token: &str,
|
||||
encryption_key: impl Into<String>,
|
||||
) -> Result<Self, aether_data::DataLayerError> {
|
||||
const TOKEN_ID: &str = "token-tunnel-harness";
|
||||
const USER_ID: &str = "user-tunnel-harness";
|
||||
|
||||
let node = StoredProxyNode::new(
|
||||
node_id.to_string(),
|
||||
"tunnel harness node".to_string(),
|
||||
"127.0.0.1".to_string(),
|
||||
0,
|
||||
false,
|
||||
"offline".to_string(),
|
||||
30,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
true,
|
||||
false,
|
||||
0,
|
||||
)?
|
||||
.with_tunnel_generation(tunnel_generation.to_string());
|
||||
let proxy_repository = Arc::new(InMemoryProxyNodeRepository::seed([node]));
|
||||
|
||||
let user_summary = StoredManagementTokenUserSummary::new(
|
||||
USER_ID.to_string(),
|
||||
Some("tunnel-harness@example.com".to_string()),
|
||||
"tunnel_harness_admin".to_string(),
|
||||
"admin".to_string(),
|
||||
)?;
|
||||
let token = StoredManagementToken::new(
|
||||
TOKEN_ID.to_string(),
|
||||
USER_ID.to_string(),
|
||||
"tunnel harness token".to_string(),
|
||||
)?
|
||||
.with_permissions(Some(serde_json::json!(["admin:proxy_nodes:admin"])));
|
||||
let token_hash = format!("{:x}", Sha256::digest(raw_token.as_bytes()));
|
||||
let token_repository = Arc::new(InMemoryManagementTokenRepository::seed_with_hashes(
|
||||
[StoredManagementTokenWithUser::new(token, user_summary)],
|
||||
[(token_hash, TOKEN_ID.to_string())],
|
||||
));
|
||||
let token_reader: Arc<dyn ManagementTokenReadRepository> = token_repository.clone();
|
||||
let token_writer: Arc<dyn ManagementTokenWriteRepository> = token_repository;
|
||||
|
||||
let user = StoredUserAuthRecord::new(
|
||||
USER_ID.to_string(),
|
||||
Some("tunnel-harness@example.com".to_string()),
|
||||
true,
|
||||
"tunnel_harness_admin".to_string(),
|
||||
None,
|
||||
"admin".to_string(),
|
||||
"local".to_string(),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
true,
|
||||
false,
|
||||
None,
|
||||
None,
|
||||
)?;
|
||||
let user_reader: Arc<dyn UserReadRepository> =
|
||||
Arc::new(InMemoryUserReadRepository::seed_auth_users([user]));
|
||||
|
||||
let mut state =
|
||||
Self::with_proxy_node_repository_for_testkit(proxy_repository, encryption_key);
|
||||
state.management_token_reader = Some(token_reader);
|
||||
state.management_token_writer = Some(token_writer);
|
||||
state.user_reader = Some(user_reader);
|
||||
Ok(state)
|
||||
}
|
||||
|
||||
pub(crate) fn with_proxy_node_repository_for_testkit<T>(
|
||||
repository: Arc<T>,
|
||||
encryption_key: impl Into<String>,
|
||||
) -> Self
|
||||
where
|
||||
T: ProxyNodeReadRepository + ProxyNodeWriteRepository + 'static,
|
||||
{
|
||||
let proxy_node_reader: Arc<dyn ProxyNodeReadRepository> = repository.clone();
|
||||
let proxy_node_writer: Arc<dyn ProxyNodeWriteRepository> = repository;
|
||||
let mut state = Self::disabled();
|
||||
state.config = GatewayDataConfig::disabled().with_encryption_key(encryption_key);
|
||||
state.proxy_node_reader = Some(proxy_node_reader);
|
||||
state.proxy_node_writer = Some(proxy_node_writer);
|
||||
state
|
||||
}
|
||||
|
||||
pub(crate) fn with_openai_chat_pressure_repositories_for_testkit<T, U, V>(
|
||||
auth_api_key_repository: Arc<dyn AuthApiKeyReadRepository>,
|
||||
candidate_selection_repository: Arc<dyn MinimalCandidateSelectionReadRepository>,
|
||||
|
||||
@@ -312,7 +312,7 @@ async fn data_state_checks_user_uniqueness_through_user_reader() {
|
||||
Some("admin@example.com".to_string()),
|
||||
true,
|
||||
"admin".to_string(),
|
||||
Some(format!("$2b$12${}", "a".repeat(53))),
|
||||
Some("$2b$12$4qL4tdcsFwVaDTw5Ck3xzu8GpNdre56DiNR6Dnw7t6gCXaEnqAe7G".to_string()),
|
||||
"admin".to_string(),
|
||||
"local".to_string(),
|
||||
None,
|
||||
|
||||
@@ -4806,6 +4806,15 @@ mod tests {
|
||||
))
|
||||
}
|
||||
|
||||
fn provider_catalog_credential_state() -> AppState {
|
||||
AppState::new()
|
||||
.expect("credential state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::disabled()
|
||||
.with_encryption_key_for_tests(aether_crypto::DEVELOPMENT_ENCRYPTION_KEY),
|
||||
)
|
||||
}
|
||||
|
||||
fn large_pool_fixture(
|
||||
key_count: usize,
|
||||
provider_config: Option<serde_json::Value>,
|
||||
@@ -4856,10 +4865,18 @@ mod tests {
|
||||
)
|
||||
.expect("endpoint transport should build");
|
||||
|
||||
let credential_state = provider_catalog_credential_state();
|
||||
let mut keys = Vec::with_capacity(key_count);
|
||||
let mut rows = Vec::with_capacity(key_count);
|
||||
for index in 0..key_count {
|
||||
let key_id = format!("key-{index:05}");
|
||||
let encrypted_api_key = credential_state
|
||||
.seal_provider_catalog_key_api_key(
|
||||
"provider-pool",
|
||||
&key_id,
|
||||
&format!("secret-{index}"),
|
||||
)
|
||||
.expect("api key should encrypt");
|
||||
let mut key = StoredProviderCatalogKey::new(
|
||||
key_id.clone(),
|
||||
"provider-pool".to_string(),
|
||||
@@ -4871,7 +4888,7 @@ mod tests {
|
||||
.expect("key should build")
|
||||
.with_transport_fields(
|
||||
Some(json!(["openai:chat"])),
|
||||
Some(format!("secret-{index}")),
|
||||
encrypted_api_key,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
@@ -5007,6 +5024,9 @@ mod tests {
|
||||
}
|
||||
|
||||
fn sample_codex_pool_key(provider_id: &str, key_id: &str) -> StoredProviderCatalogKey {
|
||||
let encrypted_api_key = provider_catalog_credential_state()
|
||||
.seal_provider_catalog_key_api_key(provider_id, key_id, &format!("secret-{key_id}"))
|
||||
.expect("api key should encrypt");
|
||||
let mut key = StoredProviderCatalogKey::new(
|
||||
key_id.to_string(),
|
||||
provider_id.to_string(),
|
||||
@@ -5018,7 +5038,7 @@ mod tests {
|
||||
.expect("key should build")
|
||||
.with_transport_fields(
|
||||
Some(json!(["openai:responses"])),
|
||||
Some(format!("secret-{key_id}")),
|
||||
encrypted_api_key,
|
||||
None,
|
||||
None,
|
||||
Some(json!({"openai:responses": 1})),
|
||||
|
||||
@@ -1,13 +1,28 @@
|
||||
use base64::Engine;
|
||||
|
||||
use crate::handlers::shared::{
|
||||
decrypt_catalog_secret_with_fallbacks, system_config_bool, system_config_string,
|
||||
decrypt_or_migrate_smtp_password, smtp_password_binding, system_config_bool,
|
||||
};
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
const SMTP_TIMEOUT_SECS: u64 = 30;
|
||||
const SMTP_MAX_HOST_BYTES: usize = 255;
|
||||
const SMTP_MAX_ADDRESS_BYTES: usize = 320;
|
||||
const SMTP_MAX_HEADER_VALUE_BYTES: usize = 512;
|
||||
const SMTP_MAX_USERNAME_BYTES: usize = 320;
|
||||
const SMTP_MAX_PASSWORD_BYTES: usize = 16 * 1024;
|
||||
const SMTP_MAX_STORED_PASSWORD_BYTES: usize = 64 * 1024;
|
||||
const SMTP_MAX_BODY_BYTES: usize = 2 * 1024 * 1024;
|
||||
const SMTP_MAX_MESSAGE_BYTES: usize = 8 * 1024 * 1024;
|
||||
const SMTP_MAX_DIAGNOSTIC_BYTES: usize = 4096;
|
||||
// SMTP servers normally emit short ASCII status lines. Keep parser buffers
|
||||
// bounded even when the peer is untrusted or compromised; these limits apply
|
||||
// only to control responses, not to the message body being submitted.
|
||||
const SMTP_MAX_RESPONSE_LINE_BYTES: usize = 16 * 1024;
|
||||
const SMTP_MAX_RESPONSE_BYTES: usize = 256 * 1024;
|
||||
const SMTP_MAX_RESPONSE_LINES: usize = 128;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct SmtpDeliveryConfig {
|
||||
pub(crate) host: String,
|
||||
pub(crate) port: u16,
|
||||
@@ -19,7 +34,7 @@ pub(crate) struct SmtpDeliveryConfig {
|
||||
pub(crate) from_name: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct ComposedEmail {
|
||||
pub(crate) to_email: String,
|
||||
pub(crate) subject: String,
|
||||
@@ -27,6 +42,26 @@ pub(crate) struct ComposedEmail {
|
||||
pub(crate) text_body: String,
|
||||
}
|
||||
|
||||
fn bounded_system_config_string(
|
||||
field: &str,
|
||||
value: Option<&serde_json::Value>,
|
||||
max_bytes: usize,
|
||||
) -> Result<Option<String>, GatewayError> {
|
||||
let Some(serde_json::Value::String(raw)) = value else {
|
||||
return Ok(None);
|
||||
};
|
||||
let value = raw.trim();
|
||||
if value.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
if value.len() > max_bytes {
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"smtp {field} exceeds the allowed size"
|
||||
)));
|
||||
}
|
||||
Ok(Some(value.to_string()))
|
||||
}
|
||||
|
||||
pub(crate) async fn read_smtp_delivery_config(
|
||||
state: &AppState,
|
||||
) -> Result<Option<SmtpDeliveryConfig>, GatewayError> {
|
||||
@@ -34,10 +69,16 @@ pub(crate) async fn read_smtp_delivery_config(
|
||||
let smtp_from_email = state
|
||||
.read_system_config_json_value("smtp_from_email")
|
||||
.await?;
|
||||
let Some(host) = system_config_string(smtp_host.as_ref()) else {
|
||||
let Some(host) = bounded_system_config_string("host", smtp_host.as_ref(), SMTP_MAX_HOST_BYTES)?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(from_email) = system_config_string(smtp_from_email.as_ref()) else {
|
||||
let Some(from_email) = bounded_system_config_string(
|
||||
"from_email",
|
||||
smtp_from_email.as_ref(),
|
||||
SMTP_MAX_ADDRESS_BYTES,
|
||||
)?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
@@ -50,20 +91,43 @@ pub(crate) async fn read_smtp_delivery_config(
|
||||
.read_system_config_json_value("smtp_from_name")
|
||||
.await?;
|
||||
|
||||
let password = system_config_string(smtp_password.as_ref()).map(|value| {
|
||||
decrypt_catalog_secret_with_fallbacks(state.encryption_key(), &value).unwrap_or(value)
|
||||
});
|
||||
let port = system_config_u16(smtp_port.as_ref(), 587);
|
||||
let user = bounded_system_config_string("user", smtp_user.as_ref(), SMTP_MAX_USERNAME_BYTES)?;
|
||||
let use_tls = system_config_bool(smtp_use_tls.as_ref(), true);
|
||||
let use_ssl = system_config_bool(smtp_use_ssl.as_ref(), false);
|
||||
let password = match (
|
||||
bounded_system_config_string(
|
||||
"stored_password",
|
||||
smtp_password.as_ref(),
|
||||
SMTP_MAX_STORED_PASSWORD_BYTES,
|
||||
)?,
|
||||
smtp_password_binding(&host, port, user.as_deref(), use_tls, use_ssl),
|
||||
) {
|
||||
(Some(value), Some(binding)) => {
|
||||
Some(decrypt_or_migrate_smtp_password(state, &binding, value).await?)
|
||||
}
|
||||
(Some(_), None) => {
|
||||
return Err(GatewayError::Internal(
|
||||
"SMTP password binding is invalid".to_string(),
|
||||
));
|
||||
}
|
||||
(None, _) => None,
|
||||
};
|
||||
|
||||
Ok(Some(SmtpDeliveryConfig {
|
||||
host,
|
||||
port: system_config_u16(smtp_port.as_ref(), 587),
|
||||
user: system_config_string(smtp_user.as_ref()),
|
||||
port,
|
||||
user,
|
||||
password,
|
||||
use_tls: system_config_bool(smtp_use_tls.as_ref(), true),
|
||||
use_ssl: system_config_bool(smtp_use_ssl.as_ref(), false),
|
||||
use_tls,
|
||||
use_ssl,
|
||||
from_email,
|
||||
from_name: system_config_string(smtp_from_name.as_ref())
|
||||
.unwrap_or_else(|| "Aether".to_string()),
|
||||
from_name: bounded_system_config_string(
|
||||
"from_name",
|
||||
smtp_from_name.as_ref(),
|
||||
SMTP_MAX_HEADER_VALUE_BYTES,
|
||||
)?
|
||||
.unwrap_or_else(|| "Aether".to_string()),
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -71,17 +135,150 @@ pub(crate) async fn send_smtp_email(
|
||||
config: SmtpDeliveryConfig,
|
||||
email: ComposedEmail,
|
||||
) -> Result<(), GatewayError> {
|
||||
validate_smtp_delivery_inputs(&config, &email)?;
|
||||
tokio::task::spawn_blocking(move || send_smtp_email_blocking(config, email))
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||
}
|
||||
|
||||
pub(crate) async fn probe_smtp_connection(config: SmtpDeliveryConfig) -> Result<(), GatewayError> {
|
||||
validate_smtp_config(&config)?;
|
||||
tokio::task::spawn_blocking(move || probe_smtp_connection_blocking(config))
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||
}
|
||||
|
||||
fn validate_smtp_control_field(field: &str, value: &str) -> Result<(), GatewayError> {
|
||||
if value.bytes().any(|byte| byte < 0x20 || byte == 0x7f) {
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"smtp {field} contains forbidden control characters"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_smtp_bounded_field(
|
||||
field: &str,
|
||||
value: &str,
|
||||
max_bytes: usize,
|
||||
) -> Result<(), GatewayError> {
|
||||
validate_smtp_control_field(field, value)?;
|
||||
if value.len() > max_bytes {
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"smtp {field} exceeds the allowed size"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_smtp_body_field(field: &str, value: &str) -> Result<(), GatewayError> {
|
||||
// Bodies are base64 encoded before DATA is written, so line breaks and
|
||||
// tabs are valid content. NUL is still rejected because it is not valid
|
||||
// textual mail content and can confuse downstream gateways.
|
||||
if value.bytes().any(|byte| byte == 0) {
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"smtp {field} contains a forbidden NUL byte"
|
||||
)));
|
||||
}
|
||||
if value.len() > SMTP_MAX_BODY_BYTES {
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"smtp {field} exceeds the allowed size"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_smtp_address(field: &str, value: &str) -> Result<(), GatewayError> {
|
||||
validate_smtp_bounded_field(field, value, SMTP_MAX_ADDRESS_BYTES)?;
|
||||
if value.is_empty() || value.trim() != value || value.chars().any(char::is_whitespace) {
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"smtp {field} must be a single mailbox address"
|
||||
)));
|
||||
}
|
||||
// Addresses are inserted inside SMTP angle brackets. Reject delimiters
|
||||
// that could turn one envelope/header value into multiple fields.
|
||||
if value
|
||||
.bytes()
|
||||
.any(|byte| matches!(byte, b'<' | b'>' | b',' | b';' | b'"' | b'\\'))
|
||||
{
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"smtp {field} contains invalid mailbox delimiters"
|
||||
)));
|
||||
}
|
||||
let Some((local, domain)) = value.split_once('@') else {
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"smtp {field} must contain a mailbox domain"
|
||||
)));
|
||||
};
|
||||
if local.is_empty() || domain.is_empty() || domain.contains('@') {
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"smtp {field} must contain a valid mailbox domain"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_smtp_auth_config(config: &SmtpDeliveryConfig) -> Result<(), GatewayError> {
|
||||
let username = config.user.as_deref().map(str::trim);
|
||||
let has_username = username.is_some_and(|value| !value.is_empty());
|
||||
let has_password = config.password.is_some();
|
||||
if has_username != has_password {
|
||||
return Err(GatewayError::Internal(
|
||||
"smtp username and password must be configured together".to_string(),
|
||||
));
|
||||
}
|
||||
if let Some(username) = username.filter(|value| !value.is_empty()) {
|
||||
validate_smtp_bounded_field("user", username, SMTP_MAX_USERNAME_BYTES)?;
|
||||
if !config.use_tls && !config.use_ssl {
|
||||
return Err(GatewayError::Internal(
|
||||
"smtp authentication requires TLS or SSL encryption".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
if let Some(password) = config.password.as_deref() {
|
||||
validate_smtp_bounded_field("password", password, SMTP_MAX_PASSWORD_BYTES)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_smtp_config(config: &SmtpDeliveryConfig) -> Result<(), GatewayError> {
|
||||
validate_smtp_bounded_field("host", &config.host, SMTP_MAX_HOST_BYTES)?;
|
||||
if config.host.is_empty() || config.host.trim() != config.host {
|
||||
return Err(GatewayError::Internal(
|
||||
"smtp host must not be empty or padded".to_string(),
|
||||
));
|
||||
}
|
||||
if config.host.chars().any(char::is_whitespace) {
|
||||
return Err(GatewayError::Internal(
|
||||
"smtp host contains invalid whitespace".to_string(),
|
||||
));
|
||||
}
|
||||
if config.port == 0 {
|
||||
return Err(GatewayError::Internal(
|
||||
"smtp port must be non-zero".to_string(),
|
||||
));
|
||||
}
|
||||
if config.use_tls && config.use_ssl {
|
||||
return Err(GatewayError::Internal(
|
||||
"smtp TLS and SSL modes cannot both be enabled".to_string(),
|
||||
));
|
||||
}
|
||||
validate_smtp_address("from_email", &config.from_email)?;
|
||||
validate_smtp_bounded_field("from_name", &config.from_name, SMTP_MAX_HEADER_VALUE_BYTES)?;
|
||||
validate_smtp_auth_config(config)
|
||||
}
|
||||
|
||||
fn validate_smtp_delivery_inputs(
|
||||
config: &SmtpDeliveryConfig,
|
||||
email: &ComposedEmail,
|
||||
) -> Result<(), GatewayError> {
|
||||
validate_smtp_config(config)?;
|
||||
validate_smtp_address("to_email", &email.to_email)?;
|
||||
validate_smtp_bounded_field("subject", &email.subject, SMTP_MAX_HEADER_VALUE_BYTES)?;
|
||||
validate_smtp_body_field("html_body", &email.html_body)?;
|
||||
validate_smtp_body_field("text_body", &email.text_body)
|
||||
}
|
||||
|
||||
pub(crate) fn system_config_u16(value: Option<&serde_json::Value>, default: u16) -> u16 {
|
||||
match value {
|
||||
Some(serde_json::Value::Number(value)) => value
|
||||
@@ -132,8 +329,42 @@ fn resolve_server_name(host: &str) -> Result<rustls::pki_types::ServerName<'stat
|
||||
}
|
||||
|
||||
fn connect_tcp_stream(config: &SmtpDeliveryConfig) -> Result<std::net::TcpStream, GatewayError> {
|
||||
let stream = std::net::TcpStream::connect((config.host.as_str(), config.port))
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
use std::net::ToSocketAddrs;
|
||||
let addresses = (config.host.as_str(), config.port)
|
||||
.to_socket_addrs()
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||
.take(16)
|
||||
.collect::<Vec<_>>();
|
||||
if addresses.is_empty() {
|
||||
return Err(GatewayError::Internal(
|
||||
"smtp host did not resolve to an address".to_string(),
|
||||
));
|
||||
}
|
||||
let deadline = std::time::Instant::now()
|
||||
.checked_add(std::time::Duration::from_secs(SMTP_TIMEOUT_SECS))
|
||||
.unwrap_or_else(std::time::Instant::now);
|
||||
let mut last_error = None;
|
||||
let mut stream = None;
|
||||
for address in addresses {
|
||||
let remaining = deadline.saturating_duration_since(std::time::Instant::now());
|
||||
if remaining.is_zero() {
|
||||
break;
|
||||
}
|
||||
match std::net::TcpStream::connect_timeout(&address, remaining) {
|
||||
Ok(candidate) => {
|
||||
stream = Some(candidate);
|
||||
break;
|
||||
}
|
||||
Err(err) => last_error = Some(err),
|
||||
}
|
||||
}
|
||||
let stream = stream.ok_or_else(|| {
|
||||
GatewayError::Internal(
|
||||
last_error
|
||||
.map(|err| err.to_string())
|
||||
.unwrap_or_else(|| "smtp connection timed out".to_string()),
|
||||
)
|
||||
})?;
|
||||
stream
|
||||
.set_read_timeout(Some(std::time::Duration::from_secs(SMTP_TIMEOUT_SECS)))
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
@@ -155,38 +386,109 @@ fn wrap_tls_stream(
|
||||
|
||||
fn smtp_read_response<T: std::io::BufRead>(reader: &mut T) -> Result<(u16, String), GatewayError> {
|
||||
let mut message = String::new();
|
||||
let code = loop {
|
||||
let parsed_code;
|
||||
let continuation;
|
||||
let trimmed;
|
||||
{
|
||||
let mut line = String::new();
|
||||
let bytes = reader
|
||||
.read_line(&mut line)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
if bytes == 0 {
|
||||
let mut expected_code = None;
|
||||
for line_number in 0..SMTP_MAX_RESPONSE_LINES {
|
||||
let mut line = Vec::new();
|
||||
let bytes = read_smtp_response_line(reader, &mut line)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
if bytes == 0 {
|
||||
return Err(GatewayError::Internal(
|
||||
"smtp connection closed unexpectedly".to_string(),
|
||||
));
|
||||
}
|
||||
if line.len() > SMTP_MAX_RESPONSE_LINE_BYTES {
|
||||
return Err(GatewayError::Internal(
|
||||
"smtp response line exceeds the allowed size".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
// Strip only the protocol line ending. The remaining bytes are kept
|
||||
// for diagnostics after validating that they are UTF-8.
|
||||
while matches!(line.last(), Some(b'\r' | b'\n')) {
|
||||
line.pop();
|
||||
}
|
||||
if line.len() < 3 || !line[..3].iter().all(|byte| byte.is_ascii_digit()) {
|
||||
return Err(GatewayError::Internal("invalid smtp response".to_string()));
|
||||
}
|
||||
let parsed_code = u16::from(line[0] - b'0') * 100
|
||||
+ u16::from(line[1] - b'0') * 10
|
||||
+ u16::from(line[2] - b'0');
|
||||
if let Some(expected_code) = expected_code {
|
||||
if parsed_code != expected_code {
|
||||
return Err(GatewayError::Internal(
|
||||
"smtp connection closed unexpectedly".to_string(),
|
||||
"smtp response continuation code changed".to_string(),
|
||||
));
|
||||
}
|
||||
trimmed = line.trim_end_matches(['\r', '\n']).to_string();
|
||||
if trimmed.len() < 3 {
|
||||
return Err(GatewayError::Internal("invalid smtp response".to_string()));
|
||||
}
|
||||
parsed_code = trimmed[..3]
|
||||
.parse::<u16>()
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
continuation = trimmed.as_bytes().get(3).copied() == Some(b'-');
|
||||
} else {
|
||||
expected_code = Some(parsed_code);
|
||||
}
|
||||
let separator = line.get(3).copied().unwrap_or(b' ');
|
||||
if separator != b'-' && separator != b' ' {
|
||||
return Err(GatewayError::Internal("invalid smtp response".to_string()));
|
||||
}
|
||||
|
||||
let trimmed = std::str::from_utf8(&line)
|
||||
.map_err(|_| GatewayError::Internal("smtp response is not valid UTF-8".to_string()))?;
|
||||
let additional = trimmed.len() + usize::from(!message.is_empty());
|
||||
if message
|
||||
.len()
|
||||
.checked_add(additional)
|
||||
.map_or(true, |length| length > SMTP_MAX_RESPONSE_BYTES)
|
||||
{
|
||||
return Err(GatewayError::Internal(
|
||||
"smtp response exceeds the allowed size".to_string(),
|
||||
));
|
||||
}
|
||||
if !message.is_empty() {
|
||||
message.push('\n');
|
||||
}
|
||||
message.push_str(&trimmed);
|
||||
if !continuation {
|
||||
break parsed_code;
|
||||
message.push_str(trimmed);
|
||||
|
||||
if separator != b'-' {
|
||||
return Ok((parsed_code, message));
|
||||
}
|
||||
};
|
||||
Ok((code, message))
|
||||
|
||||
if line_number + 1 == SMTP_MAX_RESPONSE_LINES {
|
||||
return Err(GatewayError::Internal(
|
||||
"smtp response has too many continuation lines".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
Err(GatewayError::Internal(
|
||||
"smtp response has too many continuation lines".to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
/// Read one SMTP response line without allowing `BufRead::read_until` to
|
||||
/// allocate an attacker-controlled amount of memory before a size check.
|
||||
fn read_smtp_response_line<T: std::io::BufRead>(
|
||||
reader: &mut T,
|
||||
line: &mut Vec<u8>,
|
||||
) -> std::io::Result<usize> {
|
||||
loop {
|
||||
let buffered = reader.fill_buf()?;
|
||||
if buffered.is_empty() {
|
||||
return Ok(line.len());
|
||||
}
|
||||
let newline = buffered.iter().position(|byte| *byte == b'\n');
|
||||
let take = newline.map_or(buffered.len(), |index| index + 1);
|
||||
if line
|
||||
.len()
|
||||
.checked_add(take)
|
||||
.map_or(true, |length| length > SMTP_MAX_RESPONSE_LINE_BYTES)
|
||||
{
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"smtp response line exceeds the allowed size",
|
||||
));
|
||||
}
|
||||
line.extend_from_slice(&buffered[..take]);
|
||||
reader.consume(take);
|
||||
if newline.is_some() {
|
||||
return Ok(line.len());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn smtp_expect<T: std::io::BufRead>(
|
||||
@@ -197,11 +499,42 @@ fn smtp_expect<T: std::io::BufRead>(
|
||||
if allowed_codes.contains(&code) {
|
||||
return Ok(message);
|
||||
}
|
||||
let message = sanitize_smtp_diagnostic(&message);
|
||||
Err(GatewayError::Internal(format!(
|
||||
"unexpected smtp response {code}: {message}"
|
||||
)))
|
||||
}
|
||||
|
||||
/// SMTP responses are controlled by a remote server. Keep diagnostics useful
|
||||
/// while preventing terminal escapes, log/UI line injection, and oversized
|
||||
/// error payloads from crossing the API boundary.
|
||||
fn sanitize_smtp_diagnostic(message: &str) -> String {
|
||||
let mut sanitized = String::new();
|
||||
let mut previous_space = false;
|
||||
for character in message.chars() {
|
||||
if character == '\u{1b}' || character.is_control() {
|
||||
if !previous_space {
|
||||
sanitized.push(' ');
|
||||
previous_space = true;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if sanitized.len() + character.len_utf8() > SMTP_MAX_DIAGNOSTIC_BYTES {
|
||||
break;
|
||||
}
|
||||
if character.is_whitespace() {
|
||||
if !previous_space {
|
||||
sanitized.push(' ');
|
||||
previous_space = true;
|
||||
}
|
||||
} else {
|
||||
sanitized.push(character);
|
||||
previous_space = false;
|
||||
}
|
||||
}
|
||||
sanitized.trim().to_string()
|
||||
}
|
||||
|
||||
fn smtp_write_line<T: std::io::Write>(writer: &mut T, line: &str) -> Result<(), GatewayError> {
|
||||
writer
|
||||
.write_all(line.as_bytes())
|
||||
@@ -223,7 +556,11 @@ fn smtp_send_command<S: std::io::Read + std::io::Write>(
|
||||
smtp_expect(reader, allowed_codes)
|
||||
}
|
||||
|
||||
fn build_email_message(config: &SmtpDeliveryConfig, email: &ComposedEmail) -> String {
|
||||
fn build_email_message(
|
||||
config: &SmtpDeliveryConfig,
|
||||
email: &ComposedEmail,
|
||||
) -> Result<String, GatewayError> {
|
||||
validate_smtp_delivery_inputs(config, email)?;
|
||||
let boundary = format!("aether-{}", uuid::Uuid::new_v4().simple());
|
||||
let text_body =
|
||||
wrap_base64(&base64::engine::general_purpose::STANDARD.encode(email.text_body.as_bytes()));
|
||||
@@ -238,11 +575,17 @@ fn build_email_message(config: &SmtpDeliveryConfig, email: &ComposedEmail) -> St
|
||||
config.from_email
|
||||
)
|
||||
};
|
||||
format!(
|
||||
let message = format!(
|
||||
"From: {from_header}\r\nTo: <{to_email}>\r\nSubject: {subject}\r\nMIME-Version: 1.0\r\nContent-Type: multipart/alternative; boundary=\"{boundary}\"\r\n\r\n--{boundary}\r\nContent-Type: text/plain; charset=\"utf-8\"\r\nContent-Transfer-Encoding: base64\r\n\r\n{text_body}--{boundary}\r\nContent-Type: text/html; charset=\"utf-8\"\r\nContent-Transfer-Encoding: base64\r\n\r\n{html_body}--{boundary}--\r\n",
|
||||
to_email = email.to_email,
|
||||
subject = encode_mime_header(&email.subject),
|
||||
)
|
||||
);
|
||||
if message.len() > SMTP_MAX_MESSAGE_BYTES {
|
||||
return Err(GatewayError::Internal(
|
||||
"smtp message exceeds the allowed size".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(message)
|
||||
}
|
||||
|
||||
fn smtp_authenticate<S: std::io::Read + std::io::Write>(
|
||||
@@ -257,6 +600,11 @@ fn smtp_authenticate<S: std::io::Read + std::io::Write>(
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
if !config.use_tls && !config.use_ssl {
|
||||
return Err(GatewayError::Internal(
|
||||
"smtp authentication requires TLS or SSL encryption".to_string(),
|
||||
));
|
||||
}
|
||||
let password = config.password.as_deref().unwrap_or("");
|
||||
smtp_send_command(reader, "AUTH LOGIN", &[334])?;
|
||||
smtp_send_command(
|
||||
@@ -277,6 +625,9 @@ fn smtp_deliver_message<S: std::io::Read + std::io::Write>(
|
||||
config: &SmtpDeliveryConfig,
|
||||
email: &ComposedEmail,
|
||||
) -> Result<(), GatewayError> {
|
||||
// Keep this check next to command construction for callers that bypass
|
||||
// the async delivery wrapper.
|
||||
validate_smtp_delivery_inputs(config, email)?;
|
||||
smtp_send_command(
|
||||
reader,
|
||||
&format!("MAIL FROM:<{}>", config.from_email),
|
||||
@@ -288,7 +639,7 @@ fn smtp_deliver_message<S: std::io::Read + std::io::Write>(
|
||||
&[250, 251],
|
||||
)?;
|
||||
smtp_send_command(reader, "DATA", &[354])?;
|
||||
let message = build_email_message(config, email);
|
||||
let message = build_email_message(config, email)?;
|
||||
reader
|
||||
.get_mut()
|
||||
.write_all(message.as_bytes())
|
||||
@@ -379,3 +730,135 @@ fn probe_smtp_connection_blocking(config: SmtpDeliveryConfig) -> Result<(), Gate
|
||||
let _ = smtp_send_command(&mut reader, "QUIT", &[221]);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn config() -> SmtpDeliveryConfig {
|
||||
SmtpDeliveryConfig {
|
||||
host: "smtp.example.com".to_string(),
|
||||
port: 587,
|
||||
user: Some("user@example.com".to_string()),
|
||||
password: Some("password".to_string()),
|
||||
use_tls: true,
|
||||
use_ssl: false,
|
||||
from_email: "sender@example.com".to_string(),
|
||||
from_name: "Aether".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn email() -> ComposedEmail {
|
||||
ComposedEmail {
|
||||
to_email: "recipient@example.com".to_string(),
|
||||
subject: "Subject".to_string(),
|
||||
html_body: "<p>hello</p>".to_string(),
|
||||
text_body: "hello".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_crlf_in_smtp_envelope_and_header_fields() {
|
||||
let mut malicious_config = config();
|
||||
malicious_config.from_email =
|
||||
"sender@example.com\r\nRCPT TO:<attacker@example.com>".to_string();
|
||||
let error = validate_smtp_delivery_inputs(&malicious_config, &email())
|
||||
.expect_err("CRLF in an envelope address must be rejected");
|
||||
assert!(format!("{error:?}").contains("from_email"));
|
||||
|
||||
let mut malicious_email = email();
|
||||
malicious_email.subject = "Subject\nX-Injected: yes".to_string();
|
||||
let error = validate_smtp_delivery_inputs(&config(), &malicious_email)
|
||||
.expect_err("CRLF in a header value must be rejected");
|
||||
assert!(format!("{error:?}").contains("subject"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn allows_normal_smtp_values() {
|
||||
assert!(validate_smtp_delivery_inputs(&config(), &email()).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_authentication_over_plaintext_smtp() {
|
||||
let mut insecure = config();
|
||||
insecure.use_tls = false;
|
||||
insecure.use_ssl = false;
|
||||
let error = validate_smtp_delivery_inputs(&insecure, &email())
|
||||
.expect_err("SMTP credentials must never be sent over plaintext");
|
||||
assert!(format!("{error:?}").contains("requires TLS or SSL"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_malformed_mailboxes_and_non_protocol_delimiters() {
|
||||
let mut malicious = email();
|
||||
malicious.to_email = "recipient@example.com>\x01RCPT TO:<attacker@example.com>".to_string();
|
||||
let error = validate_smtp_delivery_inputs(&config(), &malicious)
|
||||
.expect_err("control characters and envelope delimiters must be rejected");
|
||||
assert!(format!("{error:?}").contains("to_email"));
|
||||
|
||||
let mut malformed = email();
|
||||
malformed.to_email = "not-an-email".to_string();
|
||||
assert!(validate_smtp_delivery_inputs(&config(), &malformed).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounds_message_bodies_before_smtp_submission() {
|
||||
let mut oversized = email();
|
||||
oversized.html_body = "x".repeat(SMTP_MAX_BODY_BYTES + 1);
|
||||
let error = validate_smtp_delivery_inputs(&config(), &oversized)
|
||||
.expect_err("oversized message bodies must be rejected");
|
||||
assert!(format!("{error:?}").contains("html_body"));
|
||||
|
||||
let mut textual = email();
|
||||
textual.text_body = "line one\nline two\t✓".to_string();
|
||||
assert!(validate_smtp_delivery_inputs(&config(), &textual).is_ok());
|
||||
|
||||
textual.text_body.push('\0');
|
||||
assert!(validate_smtp_delivery_inputs(&config(), &textual).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sanitizes_remote_response_diagnostics() {
|
||||
let mut reader = std::io::BufReader::new("550 bad\u{1b}[31m\r\n".as_bytes());
|
||||
let error = smtp_expect(&mut reader, &[250]).expect_err("unexpected response must fail");
|
||||
let GatewayError::Internal(message) = error else {
|
||||
panic!("expected internal SMTP error");
|
||||
};
|
||||
assert!(!message.contains('\u{1b}'));
|
||||
assert!(!message.contains('\n'));
|
||||
assert!(message.len() < SMTP_MAX_DIAGNOSTIC_BYTES);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn smtp_response_rejects_non_ascii_status_prefix_without_panicking() {
|
||||
let mut reader = std::io::BufReader::new("é00 greeting\r\n".as_bytes());
|
||||
let error = smtp_read_response(&mut reader)
|
||||
.expect_err("a non-ASCII status prefix must be rejected");
|
||||
assert!(format!("{error:?}").contains("invalid smtp response"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn smtp_response_rejects_oversized_lines_before_allocating_unbounded_memory() {
|
||||
let mut input = vec![b'2'; SMTP_MAX_RESPONSE_LINE_BYTES + 1];
|
||||
input.push(b'\n');
|
||||
let mut reader = std::io::BufReader::new(input.as_slice());
|
||||
let error = smtp_read_response(&mut reader).expect_err("oversized line must be rejected");
|
||||
assert!(format!("{error:?}").contains("response line exceeds"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn smtp_response_bounds_continuation_lines_and_code_changes() {
|
||||
let repeated = (0..=SMTP_MAX_RESPONSE_LINES)
|
||||
.map(|_| "250-more\r\n")
|
||||
.collect::<String>();
|
||||
let mut reader = std::io::BufReader::new(repeated.as_bytes());
|
||||
let error = smtp_read_response(&mut reader)
|
||||
.expect_err("too many continuation lines must be rejected");
|
||||
assert!(format!("{error:?}").contains("too many continuation lines"));
|
||||
|
||||
let mut reader = std::io::BufReader::new("250-more\r\n550 done\r\n".as_bytes());
|
||||
let error = smtp_read_response(&mut reader)
|
||||
.expect_err("continuation response code changes must be rejected");
|
||||
assert!(format!("{error:?}").contains("continuation code changed"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ use axum::http::{Response, StatusCode};
|
||||
use axum::response::IntoResponse;
|
||||
use axum::Json;
|
||||
use serde_json::json;
|
||||
use sha2::{Digest, Sha256};
|
||||
use tracing::warn;
|
||||
|
||||
use crate::ai_serving::AiSurfaceFinalizeError;
|
||||
@@ -33,6 +34,9 @@ pub(crate) enum GatewayError {
|
||||
status: StatusCode,
|
||||
message: String,
|
||||
},
|
||||
PlanUsageLimited(crate::plan_usage_policy::PlanUsagePolicyRejection),
|
||||
LastActiveAdminUpdateDenied,
|
||||
LastActiveAdminDeleteDenied,
|
||||
Internal(String),
|
||||
}
|
||||
|
||||
@@ -43,6 +47,10 @@ impl GatewayError {
|
||||
| Self::ControlUnavailable { message, .. }
|
||||
| Self::Client { message, .. }
|
||||
| Self::Internal(message) => message,
|
||||
Self::PlanUsageLimited(rejection) => format!(
|
||||
"subscription plan {} limit {} reached for {} window; retry after {} seconds",
|
||||
rejection.metric, rejection.limit, rejection.window, rejection.retry_after
|
||||
),
|
||||
Self::LocalExecutionPlanningTimeout {
|
||||
phase, timeout_ms, ..
|
||||
} => {
|
||||
@@ -55,6 +63,8 @@ impl GatewayError {
|
||||
} => {
|
||||
format!("gateway admission gate {gate} timed out after {queue_budget_ms}ms")
|
||||
}
|
||||
Self::LastActiveAdminUpdateDenied => "不能降级或停用最后一个管理员账户".to_string(),
|
||||
Self::LastActiveAdminDeleteDenied => "不能删除最后一个管理员账户".to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -63,7 +73,13 @@ impl IntoResponse for GatewayError {
|
||||
fn into_response(self) -> Response<Body> {
|
||||
match self {
|
||||
Self::UpstreamUnavailable { trace_id, message } => {
|
||||
warn!(trace_id = %trace_id, error = %message, "gateway proxy unavailable");
|
||||
let error_fingerprint = gateway_error_fingerprint(&message);
|
||||
warn!(
|
||||
trace_id = %trace_id,
|
||||
error_fingerprint,
|
||||
error_length = message.len(),
|
||||
"gateway proxy unavailable"
|
||||
);
|
||||
let body = Json(json!({
|
||||
"error": {
|
||||
"message": "gateway proxy unavailable",
|
||||
@@ -81,7 +97,13 @@ impl IntoResponse for GatewayError {
|
||||
response
|
||||
}
|
||||
Self::ControlUnavailable { trace_id, message } => {
|
||||
warn!(trace_id = %trace_id, error = %message, "gateway control unavailable");
|
||||
let error_fingerprint = gateway_error_fingerprint(&message);
|
||||
warn!(
|
||||
trace_id = %trace_id,
|
||||
error_fingerprint,
|
||||
error_length = message.len(),
|
||||
"gateway control unavailable"
|
||||
);
|
||||
let body = Json(json!({
|
||||
"error": {
|
||||
"message": "gateway control unavailable",
|
||||
@@ -162,19 +184,59 @@ impl IntoResponse for GatewayError {
|
||||
})),
|
||||
)
|
||||
.into_response(),
|
||||
Self::Internal(message) => (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
Self::PlanUsageLimited(rejection) => (
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
Json(json!({
|
||||
"error": {
|
||||
"message": message,
|
||||
"type": "plan_usage_limit_exceeded",
|
||||
"message": "套餐使用限制已达到上限,请稍后重试",
|
||||
"details": {
|
||||
"metric": rejection.metric,
|
||||
"window": rejection.window,
|
||||
"limit": rejection.limit,
|
||||
"retry_after": rejection.retry_after,
|
||||
}
|
||||
}
|
||||
})),
|
||||
)
|
||||
.into_response(),
|
||||
Self::LastActiveAdminUpdateDenied => (
|
||||
StatusCode::BAD_REQUEST,
|
||||
Json(json!({ "detail": "不能降级或停用最后一个管理员账户" })),
|
||||
)
|
||||
.into_response(),
|
||||
Self::LastActiveAdminDeleteDenied => (
|
||||
StatusCode::BAD_REQUEST,
|
||||
Json(json!({ "detail": "不能删除最后一个管理员账户" })),
|
||||
)
|
||||
.into_response(),
|
||||
Self::Internal(message) => {
|
||||
let error_fingerprint = gateway_error_fingerprint(&message);
|
||||
tracing::error!(
|
||||
event_name = "gateway_internal_error",
|
||||
error_fingerprint,
|
||||
error_length = message.len(),
|
||||
"internal gateway error hidden from client"
|
||||
);
|
||||
(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
Json(json!({
|
||||
"error": {
|
||||
"message": "internal server error",
|
||||
}
|
||||
})),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn gateway_error_fingerprint(message: &str) -> String {
|
||||
let digest = Sha256::digest(message.as_bytes());
|
||||
format!("{:x}", digest)[..16].to_string()
|
||||
}
|
||||
|
||||
impl From<AiSurfaceFinalizeError> for GatewayError {
|
||||
fn from(error: AiSurfaceFinalizeError) -> Self {
|
||||
GatewayError::Internal(error.0)
|
||||
@@ -183,12 +245,41 @@ impl From<AiSurfaceFinalizeError> for GatewayError {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use axum::body::to_bytes;
|
||||
use axum::http::{header::RETRY_AFTER, StatusCode};
|
||||
use axum::response::IntoResponse;
|
||||
|
||||
use crate::constants::TRACE_ID_HEADER;
|
||||
|
||||
use super::GatewayError;
|
||||
use super::{gateway_error_fingerprint, GatewayError};
|
||||
|
||||
#[tokio::test]
|
||||
async fn internal_errors_do_not_expose_internal_details() {
|
||||
let response = GatewayError::Internal(
|
||||
"database connection failed: password=internal-secret".to_string(),
|
||||
)
|
||||
.into_response();
|
||||
|
||||
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("internal error response body should read");
|
||||
let payload: serde_json::Value =
|
||||
serde_json::from_slice(&body).expect("internal error response should be JSON");
|
||||
assert_eq!(payload["error"]["message"], "internal server error");
|
||||
assert!(!String::from_utf8_lossy(&body).contains("internal-secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_fingerprints_are_stable_without_containing_source_text() {
|
||||
let secret_error = "postgresql://admin:database-secret@db.internal/aether";
|
||||
let fingerprint = gateway_error_fingerprint(secret_error);
|
||||
|
||||
assert_eq!(fingerprint, gateway_error_fingerprint(secret_error));
|
||||
assert_eq!(fingerprint.len(), 16);
|
||||
assert!(fingerprint.bytes().all(|byte| byte.is_ascii_hexdigit()));
|
||||
assert!(!fingerprint.contains("database-secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn admission_timeout_returns_429_with_retry_after_without_panicking() {
|
||||
|
||||
@@ -433,7 +433,8 @@ impl AttemptBodyCapture {
|
||||
if bytes.is_empty() || self.truncated {
|
||||
return;
|
||||
}
|
||||
let max_bytes = DEFAULT_USAGE_RESPONSE_BODY_CAPTURE_LIMIT_BYTES;
|
||||
let max_bytes = DEFAULT_USAGE_RESPONSE_BODY_CAPTURE_LIMIT_BYTES
|
||||
.min(crate::execution_runtime::MAX_STREAM_BODY_CAPTURE_BYTES);
|
||||
if self.buffer.len() >= max_bytes {
|
||||
self.truncated = true;
|
||||
return;
|
||||
@@ -447,11 +448,22 @@ impl AttemptBodyCapture {
|
||||
}
|
||||
|
||||
pub(crate) fn encode(&self) -> (Option<String>, Option<UsageBodyCaptureState>) {
|
||||
let body = (!self.buffer.is_empty())
|
||||
.then(|| base64::engine::general_purpose::STANDARD.encode(&self.buffer));
|
||||
let state = if self.truncated {
|
||||
self.encode_with_limit(
|
||||
DEFAULT_USAGE_RESPONSE_BODY_CAPTURE_LIMIT_BYTES
|
||||
.min(crate::execution_runtime::MAX_STREAM_BODY_CAPTURE_BYTES),
|
||||
)
|
||||
}
|
||||
|
||||
fn encode_with_limit(
|
||||
&self,
|
||||
max_bytes: usize,
|
||||
) -> (Option<String>, Option<UsageBodyCaptureState>) {
|
||||
let captured = &self.buffer[..self.buffer.len().min(max_bytes)];
|
||||
let body = (!captured.is_empty())
|
||||
.then(|| base64::engine::general_purpose::STANDARD.encode(captured));
|
||||
let state = if self.truncated || captured.len() < self.buffer.len() {
|
||||
UsageBodyCaptureState::Truncated
|
||||
} else if self.buffer.is_empty() {
|
||||
} else if captured.is_empty() {
|
||||
UsageBodyCaptureState::None
|
||||
} else {
|
||||
UsageBodyCaptureState::Inline
|
||||
@@ -1418,15 +1430,14 @@ mod stage_tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// body capture 的编码状态。截断分支这里到不了:共享的
|
||||
/// `DEFAULT_USAGE_RESPONSE_BODY_CAPTURE_LIMIT_BYTES` 是 `usize::MAX`,
|
||||
/// 也就是默认不限长;截断只在 usage 侧把上限调低后才可能发生。
|
||||
/// body capture 的编码状态。Full 记录级别仍受 gateway 的硬上限约束,
|
||||
/// 这样长连接不会把审计副本无限累积。
|
||||
#[test]
|
||||
fn body_capture_encodes_inline_and_empty_states() {
|
||||
assert_eq!(
|
||||
super::DEFAULT_USAGE_RESPONSE_BODY_CAPTURE_LIMIT_BYTES,
|
||||
usize::MAX,
|
||||
"the default capture limit is unbounded; truncation is not reachable here"
|
||||
super::DEFAULT_USAGE_RESPONSE_BODY_CAPTURE_LIMIT_BYTES
|
||||
.min(crate::execution_runtime::MAX_STREAM_BODY_CAPTURE_BYTES),
|
||||
crate::execution_runtime::MAX_STREAM_BODY_CAPTURE_BYTES
|
||||
);
|
||||
|
||||
let mut capture = AttemptBodyCapture::default();
|
||||
@@ -1454,6 +1465,20 @@ mod stage_tests {
|
||||
state,
|
||||
Some(aether_data_contracts::repository::usage::UsageBodyCaptureState::None)
|
||||
);
|
||||
|
||||
let defensive = AttemptBodyCapture {
|
||||
buffer: b"abcdef".to_vec(),
|
||||
truncated: false,
|
||||
};
|
||||
let (body, state) = defensive.encode_with_limit(3);
|
||||
let decoded = base64::engine::general_purpose::STANDARD
|
||||
.decode(body.expect("bounded capture should be encoded"))
|
||||
.expect("capture is valid base64");
|
||||
assert_eq!(decoded, b"abc");
|
||||
assert_eq!(
|
||||
state,
|
||||
Some(aether_data_contracts::repository::usage::UsageBodyCaptureState::Truncated)
|
||||
);
|
||||
}
|
||||
|
||||
/// candidate 行的 error_type 映射:投递失败与供应商侧失败必须各有名字。
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -3,3 +3,13 @@
|
||||
pub(crate) use aether_gateway_execution::{
|
||||
MAX_ERROR_BODY_BYTES, MAX_STREAM_PREFETCH_BYTES, MAX_STREAM_PREFETCH_FRAMES,
|
||||
};
|
||||
|
||||
// Usage/audit captures are secondary copies of the stream. Keep a hard
|
||||
// ceiling even when the configurable "full" record level is otherwise
|
||||
// unbounded; this does not limit bytes forwarded to the client.
|
||||
pub(crate) const MAX_STREAM_BODY_CAPTURE_BYTES: usize = 64 * 1024 * 1024;
|
||||
|
||||
// Stream frames are newline-delimited JSON. Binary response chunks are base64
|
||||
// encoded before framing, so this must be larger than the normal 64 MiB raw
|
||||
// response limit while still bounding an attacker-controlled unterminated line.
|
||||
pub(crate) const MAX_EXECUTION_STREAM_FRAME_LINE_BYTES: usize = 128 * 1024 * 1024;
|
||||
|
||||
@@ -355,6 +355,7 @@ pub(crate) fn resolve_core_stream_direct_finalize_report_kind(plan_kind: &str) -
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use aether_contracts::{ExecutionError, ExecutionErrorKind, ExecutionPhase, ExecutionResult};
|
||||
@@ -435,6 +436,15 @@ mod tests {
|
||||
}
|
||||
|
||||
fn sample_key() -> StoredProviderCatalogKey {
|
||||
let credential_state = AppState::new()
|
||||
.expect("credential state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::disabled()
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY),
|
||||
);
|
||||
let encrypted_api_key = credential_state
|
||||
.seal_provider_catalog_key_api_key("provider-1", "key-1", "plain-upstream-key")
|
||||
.expect("api key should encrypt");
|
||||
StoredProviderCatalogKey::new(
|
||||
"key-1".to_string(),
|
||||
"provider-1".to_string(),
|
||||
@@ -446,7 +456,7 @@ mod tests {
|
||||
.expect("key should build")
|
||||
.with_transport_fields(
|
||||
Some(serde_json::json!(["openai:chat"])),
|
||||
"plain-upstream-key".to_string(),
|
||||
encrypted_api_key,
|
||||
None,
|
||||
None,
|
||||
Some(serde_json::json!({"openai:chat": 1})),
|
||||
@@ -466,7 +476,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
AppState::new()
|
||||
.expect("state should build")
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -21,11 +21,13 @@ use crate::execution_runtime::kiro_cache::{
|
||||
};
|
||||
use crate::execution_runtime::ndjson::encode_stream_frame_ndjson;
|
||||
use crate::execution_runtime::transport::{
|
||||
DirectSyncExecutionRuntime, ExecutionRuntimeTransportError,
|
||||
apply_upstream_response_body_limit, decode_base64_body_with_limit,
|
||||
json_value_fits_serialized_limit, DirectSyncExecutionRuntime, ExecutionRuntimeTransportError,
|
||||
};
|
||||
use crate::AppState;
|
||||
|
||||
const WEB_SEARCH_TOOL_NAME: &str = "web_search";
|
||||
const MAX_KIRO_MCP_JSON_BYTES: usize = 8 * 1024 * 1024;
|
||||
const WEB_SEARCH_TOOL_TYPE_PREFIX: &str = "web_search";
|
||||
const WEB_SEARCH_QUERY_PREFIX: &str = "Perform a web search for the query: ";
|
||||
|
||||
@@ -73,8 +75,6 @@ struct McpResponse {
|
||||
struct McpError {
|
||||
#[serde(default)]
|
||||
code: Option<i64>,
|
||||
#[serde(default)]
|
||||
message: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
@@ -146,14 +146,14 @@ pub(crate) async fn maybe_execute_kiro_web_search_stream(
|
||||
request_id = %plan.request_id,
|
||||
candidate_id = ?plan.candidate_id,
|
||||
status_code = mcp_execution.result.status_code,
|
||||
mcp_url = %mcp_execution.url,
|
||||
mcp_origin = %crate::handlers::shared::security_log_url_origin(&mcp_execution.url),
|
||||
profile_arn_present = mcp_execution.profile_arn_present,
|
||||
"gateway executed Kiro web_search through MCP endpoint"
|
||||
);
|
||||
|
||||
if !(200..300).contains(&mcp_execution.result.status_code) {
|
||||
return Ok(Some(KiroWebSearchStream {
|
||||
frame_stream: execution_result_frame_stream(&mcp_execution.result),
|
||||
frame_stream: execution_result_failure_frame_stream(&mcp_execution.result),
|
||||
report_context: report_context.cloned(),
|
||||
}));
|
||||
}
|
||||
@@ -183,7 +183,7 @@ pub(crate) async fn maybe_execute_kiro_web_search_stream(
|
||||
cache_usage,
|
||||
)
|
||||
.map_err(ExecutionRuntimeTransportError::BodyEncode)?;
|
||||
let mut synthetic_context = synthetic_report_context(report_context, mcp_execution.url);
|
||||
let mut synthetic_context = synthetic_report_context(report_context);
|
||||
if let Some(context) = synthetic_context.as_mut().and_then(Value::as_object_mut) {
|
||||
context.insert("kiro_web_search_mcp".to_string(), Value::Bool(true));
|
||||
}
|
||||
@@ -214,13 +214,13 @@ async fn kiro_simulated_cache_enabled(state: &AppState, plan: &ExecutionPlan) ->
|
||||
.is_some_and(|provider| {
|
||||
kiro_simulated_cache_enabled_from_provider_config(provider.config.as_ref())
|
||||
}),
|
||||
Err(err) => {
|
||||
Err(_) => {
|
||||
warn!(
|
||||
event_name = "kiro_simulated_cache_config_read_failed",
|
||||
log_type = "event",
|
||||
request_id = %plan.request_id,
|
||||
provider_id = %plan.provider_id,
|
||||
error = ?err,
|
||||
error_category = "provider_catalog_read_failed",
|
||||
"failed to read Kiro simulated cache provider config; defaulting disabled"
|
||||
);
|
||||
false
|
||||
@@ -228,30 +228,28 @@ async fn kiro_simulated_cache_enabled(state: &AppState, plan: &ExecutionPlan) ->
|
||||
}
|
||||
}
|
||||
|
||||
fn execute_result_body_bytes(result: &ExecutionResult) -> Vec<u8> {
|
||||
let Some(body) = result.body.as_ref() else {
|
||||
return Vec::new();
|
||||
};
|
||||
if let Some(json_body) = body.json_body.as_ref() {
|
||||
return serde_json::to_vec(json_body).unwrap_or_default();
|
||||
}
|
||||
body.body_bytes_b64
|
||||
.as_deref()
|
||||
.and_then(|body| base64::engine::general_purpose::STANDARD.decode(body).ok())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn execution_result_frame_stream(
|
||||
fn execution_result_failure_frame_stream(
|
||||
result: &ExecutionResult,
|
||||
) -> BoxStream<'static, Result<Bytes, IoError>> {
|
||||
let body = serde_json::to_vec(&kiro_mcp_failure_body(result.status_code)).unwrap_or_default();
|
||||
raw_response_frame_stream(
|
||||
result.status_code,
|
||||
result.headers.clone(),
|
||||
Bytes::from(execute_result_body_bytes(result)),
|
||||
BTreeMap::from([("content-type".to_string(), "application/json".to_string())]),
|
||||
Bytes::from(body),
|
||||
result.telemetry.clone(),
|
||||
)
|
||||
}
|
||||
|
||||
fn kiro_mcp_failure_body(status_code: u16) -> Value {
|
||||
json!({
|
||||
"error": {
|
||||
"type": "kiro_web_search_error",
|
||||
"message": "Kiro web search request failed",
|
||||
"code": status_code,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn sse_frame_stream(body: Bytes) -> BoxStream<'static, Result<Bytes, IoError>> {
|
||||
raw_response_frame_stream(
|
||||
200,
|
||||
@@ -328,12 +326,7 @@ async fn execute_mcp_request(
|
||||
request: &McpRequest,
|
||||
) -> Result<KiroMcpExecution, ExecutionRuntimeTransportError> {
|
||||
let mcp_url = aether_provider_transport::kiro::build_kiro_mcp_url_from_resolved_url(&plan.url)
|
||||
.ok_or_else(|| {
|
||||
ExecutionRuntimeTransportError::UpstreamRequest(format!(
|
||||
"failed to build Kiro MCP url from {}",
|
||||
plan.url
|
||||
))
|
||||
})?;
|
||||
.ok_or_else(kiro_mcp_url_build_error)?;
|
||||
let mut request_context = build_mcp_request_context(state, plan).await;
|
||||
if !request_context.profile_arn_present {
|
||||
if let Some(profile_arn) = discover_kiro_profile_arn(state, plan, &request_context).await? {
|
||||
@@ -346,7 +339,7 @@ async fn execute_mcp_request(
|
||||
}
|
||||
let body_json =
|
||||
serde_json::to_value(request).map_err(ExecutionRuntimeTransportError::BodyEncode)?;
|
||||
let mcp_plan = ExecutionPlan {
|
||||
let mut mcp_plan = ExecutionPlan {
|
||||
request_id: plan.request_id.clone(),
|
||||
candidate_id: plan.candidate_id.clone(),
|
||||
provider_name: plan.provider_name.clone(),
|
||||
@@ -367,6 +360,7 @@ async fn execute_mcp_request(
|
||||
transport_profile: plan.transport_profile.clone(),
|
||||
timeouts: plan.timeouts.clone(),
|
||||
};
|
||||
apply_upstream_response_body_limit(&mut mcp_plan, MAX_KIRO_MCP_JSON_BYTES);
|
||||
let result = DirectSyncExecutionRuntime::new()
|
||||
.execute_sync(&mcp_plan)
|
||||
.await?;
|
||||
@@ -390,7 +384,7 @@ async fn build_mcp_request_context(
|
||||
{
|
||||
Ok(Some(transport)) => transport,
|
||||
Ok(None) => return fallback(),
|
||||
Err(err) => {
|
||||
Err(_) => {
|
||||
warn!(
|
||||
event_name = "kiro_web_search_transport_snapshot_unavailable",
|
||||
log_type = "ops",
|
||||
@@ -399,7 +393,7 @@ async fn build_mcp_request_context(
|
||||
provider_id = %plan.provider_id,
|
||||
endpoint_id = %plan.endpoint_id,
|
||||
key_id = %plan.key_id,
|
||||
error = ?err,
|
||||
error_category = "transport_snapshot_read_failed",
|
||||
"gateway could not read Kiro transport snapshot for web_search MCP"
|
||||
);
|
||||
return fallback();
|
||||
@@ -574,7 +568,7 @@ async fn discover_kiro_profile_arn_in_region(
|
||||
if let Some(token) = next_token.as_deref() {
|
||||
body.insert("nextToken".to_string(), Value::String(token.to_string()));
|
||||
}
|
||||
let list_plan = ExecutionPlan {
|
||||
let mut list_plan = ExecutionPlan {
|
||||
request_id: plan.request_id.clone(),
|
||||
candidate_id: plan.candidate_id.clone(),
|
||||
provider_name: plan.provider_name.clone(),
|
||||
@@ -598,6 +592,7 @@ async fn discover_kiro_profile_arn_in_region(
|
||||
transport_profile: plan.transport_profile.clone(),
|
||||
timeouts: plan.timeouts.clone(),
|
||||
};
|
||||
apply_upstream_response_body_limit(&mut list_plan, MAX_KIRO_MCP_JSON_BYTES);
|
||||
let result = DirectSyncExecutionRuntime::new()
|
||||
.execute_sync(&list_plan)
|
||||
.await?;
|
||||
@@ -676,6 +671,10 @@ fn kiro_runtime_base_url_for_region(region: &str) -> String {
|
||||
}
|
||||
|
||||
fn kiro_runtime_host_for_region(region: &str) -> String {
|
||||
// Region values can originate in persisted OAuth metadata. Normalize
|
||||
// before interpolation so a crafted value cannot turn the host header or
|
||||
// URL into an attacker-controlled origin.
|
||||
let region = aether_provider_transport::kiro::normalize_kiro_region(region);
|
||||
match region {
|
||||
"us-gov-east-1" | "us-gov-west-1" => format!("q-fips.{region}.amazonaws.com"),
|
||||
"us-iso-east-1" => "q.us-iso-east-1.c2s.ic.gov".to_string(),
|
||||
@@ -959,7 +958,6 @@ fn parse_mcp_search_results(result: &ExecutionResult) -> Option<WebSearchResults
|
||||
event_name = "kiro_web_search_mcp_error",
|
||||
log_type = "event",
|
||||
code = error.code.unwrap_or_default(),
|
||||
message = error.message.as_deref().unwrap_or("unknown"),
|
||||
"Kiro MCP web_search returned JSON-RPC error"
|
||||
);
|
||||
return None;
|
||||
@@ -983,12 +981,18 @@ fn parse_mcp_search_results(result: &ExecutionResult) -> Option<WebSearchResults
|
||||
fn execution_result_body_json(result: &ExecutionResult) -> Option<Value> {
|
||||
let body = result.body.as_ref()?;
|
||||
if let Some(json_body) = body.json_body.as_ref() {
|
||||
return Some(json_body.clone());
|
||||
return json_value_fits_serialized_limit(json_body, MAX_KIRO_MCP_JSON_BYTES)
|
||||
.then(|| json_body.clone());
|
||||
}
|
||||
let body = body.body_bytes_b64.as_deref()?;
|
||||
let bytes = base64::engine::general_purpose::STANDARD
|
||||
.decode(body)
|
||||
.ok()?;
|
||||
// MCP responses are parsed into an owned JSON tree. Keep a fixed ceiling
|
||||
// here even if the operator disables the general internal body cap, so a
|
||||
// forged execution result cannot trigger an unbounded base64 allocation.
|
||||
let bytes = decode_base64_body_with_limit(
|
||||
body,
|
||||
crate::headers::max_internal_buffered_body_bytes().min(MAX_KIRO_MCP_JSON_BYTES),
|
||||
)
|
||||
.ok()?;
|
||||
serde_json::from_slice(&bytes).ok()
|
||||
}
|
||||
|
||||
@@ -1198,13 +1202,14 @@ fn generate_search_summary(query: &str, results: Option<&WebSearchResults>) -> S
|
||||
}
|
||||
summary.push_str(&format!(" Source: {}\n\n", result.url));
|
||||
}
|
||||
if let Some(error) = results
|
||||
if results
|
||||
.error
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.is_some()
|
||||
{
|
||||
summary.push_str(&format!("Search warning: {error}\n\n"));
|
||||
summary.push_str("Search warning: the provider reported an incomplete result.\n\n");
|
||||
}
|
||||
}
|
||||
_ => summary.push_str("No results found.\n"),
|
||||
@@ -1237,12 +1242,15 @@ fn estimate_text_tokens(text: &str) -> u64 {
|
||||
((text.len() as u64 + 3) / 4).max(1)
|
||||
}
|
||||
|
||||
fn synthetic_report_context(report_context: Option<&Value>, mcp_url: String) -> Option<Value> {
|
||||
fn kiro_mcp_url_build_error() -> ExecutionRuntimeTransportError {
|
||||
ExecutionRuntimeTransportError::UpstreamRequest("failed to build Kiro MCP URL".to_string())
|
||||
}
|
||||
|
||||
fn synthetic_report_context(report_context: Option<&Value>) -> Option<Value> {
|
||||
let mut context = report_context.cloned()?;
|
||||
if let Some(object) = context.as_object_mut() {
|
||||
object.insert("has_envelope".to_string(), Value::Bool(false));
|
||||
object.insert("needs_conversion".to_string(), Value::Bool(false));
|
||||
object.insert("upstream_url".to_string(), Value::String(mcp_url));
|
||||
object.remove("envelope_name");
|
||||
}
|
||||
Some(context)
|
||||
@@ -1257,7 +1265,9 @@ mod tests {
|
||||
|
||||
use super::{
|
||||
build_mcp_headers_from_plan, build_web_search_sse_body, detect_kiro_web_search_request,
|
||||
parse_mcp_search_results, strip_search_query_prefix, KiroPromptCacheUsage,
|
||||
execution_result_body_json, generate_search_summary, kiro_mcp_failure_body,
|
||||
kiro_mcp_url_build_error, parse_mcp_search_results, strip_search_query_prefix,
|
||||
KiroPromptCacheUsage, WebSearchResult, WebSearchResults,
|
||||
};
|
||||
|
||||
fn sample_plan(body: serde_json::Value) -> ExecutionPlan {
|
||||
@@ -1443,6 +1453,43 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kiro_mcp_url_build_error_omits_resolved_url() {
|
||||
let sensitive_url = "https://token:secret-kiro-url@internal.example.invalid/path";
|
||||
let message = kiro_mcp_url_build_error().to_string();
|
||||
|
||||
assert_eq!(
|
||||
message,
|
||||
"failed to execute upstream request: failed to build Kiro MCP URL"
|
||||
);
|
||||
assert!(!message.contains(sensitive_url));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kiro_mcp_errors_and_search_warnings_do_not_copy_upstream_secrets() {
|
||||
let secret = "authorization=Bearer upstream-secret";
|
||||
let failure = kiro_mcp_failure_body(502).to_string();
|
||||
let summary = generate_search_summary(
|
||||
"test",
|
||||
Some(&WebSearchResults {
|
||||
results: vec![WebSearchResult {
|
||||
title: "Safe result".to_string(),
|
||||
url: "https://example.test/result".to_string(),
|
||||
snippet: None,
|
||||
published_date: None,
|
||||
}],
|
||||
total_results: Some(1),
|
||||
query: None,
|
||||
error: Some(secret.to_string()),
|
||||
}),
|
||||
);
|
||||
|
||||
assert!(failure.contains("Kiro web search request failed"));
|
||||
assert!(summary.contains("provider reported an incomplete result"));
|
||||
assert!(!failure.contains("upstream-secret"));
|
||||
assert!(!summary.contains("upstream-secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_mcp_search_result_text_payload() {
|
||||
let result = aether_contracts::ExecutionResult {
|
||||
@@ -1474,6 +1521,29 @@ mod tests {
|
||||
assert_eq!(parsed.results[0].title, "Example");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kiro_mcp_result_rejects_oversized_base64_before_decode() {
|
||||
let encoded_limit =
|
||||
crate::execution_runtime::transport::maximum_base64_len_for_decoded_limit(
|
||||
super::MAX_KIRO_MCP_JSON_BYTES,
|
||||
);
|
||||
let result = aether_contracts::ExecutionResult {
|
||||
request_id: "req-oversized".to_string(),
|
||||
candidate_id: None,
|
||||
status_code: 200,
|
||||
headers: BTreeMap::new(),
|
||||
response_observation: None,
|
||||
body: Some(aether_contracts::ResponseBody {
|
||||
json_body: None,
|
||||
body_bytes_b64: Some("A".repeat(encoded_limit + 1)),
|
||||
}),
|
||||
telemetry: None,
|
||||
error: None,
|
||||
};
|
||||
|
||||
assert!(execution_result_body_json(&result).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_anthropic_web_search_sse() {
|
||||
let sse = build_web_search_sse_body(
|
||||
|
||||
@@ -30,7 +30,8 @@ pub(crate) use self::admission::{
|
||||
};
|
||||
pub(crate) use self::chatgpt_web_image::maybe_execute_chatgpt_web_image_sync;
|
||||
pub(crate) use self::constants::{
|
||||
MAX_ERROR_BODY_BYTES, MAX_STREAM_PREFETCH_BYTES, MAX_STREAM_PREFETCH_FRAMES,
|
||||
MAX_ERROR_BODY_BYTES, MAX_EXECUTION_STREAM_FRAME_LINE_BYTES, MAX_STREAM_BODY_CAPTURE_BYTES,
|
||||
MAX_STREAM_PREFETCH_BYTES, MAX_STREAM_PREFETCH_FRAMES,
|
||||
};
|
||||
pub(crate) use self::fallback::{
|
||||
analyze_local_candidate_failover_sync, local_failover_response_text,
|
||||
|
||||
@@ -3,21 +3,56 @@ use aether_contracts::{ExecutionPlan, ExecutionResult};
|
||||
use crate::constants::TRACE_ID_HEADER;
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
fn remote_runtime_request_error_kind(error: &reqwest::Error) -> &'static str {
|
||||
if error.is_timeout() {
|
||||
"timeout"
|
||||
} else if error.is_connect() {
|
||||
"connect"
|
||||
} else if error.is_request() {
|
||||
"request"
|
||||
} else if error.is_body() {
|
||||
"body"
|
||||
} else if error.is_decode() {
|
||||
"decode"
|
||||
} else {
|
||||
"transport"
|
||||
}
|
||||
}
|
||||
|
||||
fn build_remote_execution_runtime_request(
|
||||
state: &AppState,
|
||||
remote_execution_runtime_base_url: &str,
|
||||
path: &str,
|
||||
trace_id: Option<&str>,
|
||||
plan: &ExecutionPlan,
|
||||
) -> reqwest::RequestBuilder {
|
||||
) -> Result<reqwest::RequestBuilder, GatewayError> {
|
||||
let envelope_limit = crate::execution_runtime::transport::execution_result_envelope_limit_bytes(
|
||||
crate::headers::max_internal_buffered_body_bytes(),
|
||||
);
|
||||
let body = crate::execution_runtime::transport::serialize_serializable_with_limit(
|
||||
plan,
|
||||
envelope_limit,
|
||||
)
|
||||
.map_err(|error| {
|
||||
let kind = match error {
|
||||
crate::execution_runtime::transport::ExecutionRuntimeTransportError::BodyTooLarge {
|
||||
..
|
||||
} => "too_large",
|
||||
_ => "encode",
|
||||
};
|
||||
GatewayError::Internal(format!(
|
||||
"remote execution runtime request body failed ({kind})"
|
||||
))
|
||||
})?;
|
||||
let mut request = state
|
||||
.client
|
||||
.post(format!("{remote_execution_runtime_base_url}{path}"))
|
||||
.json(plan);
|
||||
.header(reqwest::header::CONTENT_TYPE, "application/json")
|
||||
.body(body);
|
||||
if let Some(trace_id) = trace_id.map(str::trim).filter(|value| !value.is_empty()) {
|
||||
request = request.header(TRACE_ID_HEADER, trace_id);
|
||||
}
|
||||
request
|
||||
Ok(request)
|
||||
}
|
||||
|
||||
pub(crate) async fn post_sync_plan_to_remote_execution_runtime(
|
||||
@@ -32,10 +67,15 @@ pub(crate) async fn post_sync_plan_to_remote_execution_runtime(
|
||||
"/v1/execute/sync",
|
||||
trace_id,
|
||||
plan,
|
||||
)
|
||||
)?
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))
|
||||
.map_err(|err| {
|
||||
GatewayError::Internal(format!(
|
||||
"remote execution runtime request failed ({})",
|
||||
remote_runtime_request_error_kind(&err)
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn post_stream_plan_to_remote_execution_runtime(
|
||||
@@ -50,10 +90,15 @@ pub(crate) async fn post_stream_plan_to_remote_execution_runtime(
|
||||
"/v1/execute/stream",
|
||||
trace_id,
|
||||
plan,
|
||||
)
|
||||
)?
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))
|
||||
.map_err(|err| {
|
||||
GatewayError::Internal(format!(
|
||||
"remote execution runtime request failed ({})",
|
||||
remote_runtime_request_error_kind(&err)
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn execute_sync_plan_via_remote_execution_runtime(
|
||||
@@ -76,8 +121,27 @@ pub(crate) async fn execute_sync_plan_via_remote_execution_runtime(
|
||||
)));
|
||||
}
|
||||
|
||||
response
|
||||
.json::<ExecutionResult>()
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))
|
||||
let body = aether_http::read_response_bytes_with_limit(
|
||||
response,
|
||||
crate::execution_runtime::transport::execution_result_envelope_limit_bytes(
|
||||
crate::headers::max_internal_buffered_body_bytes(),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
GatewayError::Internal(format!(
|
||||
"remote execution runtime response body failed ({})",
|
||||
match err {
|
||||
aether_http::ResponseBodyReadError::TooLarge { .. } => "too_large",
|
||||
aether_http::ResponseBodyReadError::Read(error) => {
|
||||
remote_runtime_request_error_kind(&error)
|
||||
}
|
||||
}
|
||||
))
|
||||
})?;
|
||||
serde_json::from_slice::<ExecutionResult>(&body).map_err(|_| {
|
||||
GatewayError::Internal(
|
||||
"remote execution runtime returned invalid execution JSON".to_string(),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -191,6 +191,46 @@ pub(super) fn inspect_prefetched_stream_body(
|
||||
}
|
||||
}
|
||||
|
||||
fn append_error_frame_payload(
|
||||
body: &mut Vec<u8>,
|
||||
chunk_b64: Option<&str>,
|
||||
text: Option<&str>,
|
||||
) -> Result<bool, GatewayError> {
|
||||
let remaining = MAX_ERROR_BODY_BYTES.saturating_sub(body.len());
|
||||
if remaining == 0 {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
if let Some(chunk_b64) = chunk_b64 {
|
||||
// Do not decode an attacker-controlled megabyte-scale base64 value
|
||||
// merely to retain the first 16 KiB of an error body. A standard
|
||||
// base64 value representing at most `remaining` bytes cannot exceed
|
||||
// this length.
|
||||
let max_encoded_len = remaining
|
||||
.saturating_add(2)
|
||||
.checked_div(3)
|
||||
.unwrap_or(usize::MAX)
|
||||
.saturating_mul(4);
|
||||
if chunk_b64.len() > max_encoded_len {
|
||||
warn!(
|
||||
encoded_bytes = chunk_b64.len(),
|
||||
max_encoded_bytes = max_encoded_len,
|
||||
"execution runtime error frame body exceeded capture limit"
|
||||
);
|
||||
return Ok(false);
|
||||
}
|
||||
let chunk = base64::engine::general_purpose::STANDARD
|
||||
.decode(chunk_b64)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
body.extend_from_slice(&chunk[..chunk.len().min(remaining)]);
|
||||
} else if let Some(text) = text {
|
||||
let text_bytes = text.as_bytes();
|
||||
body.extend_from_slice(&text_bytes[..text_bytes.len().min(remaining)]);
|
||||
}
|
||||
|
||||
Ok(body.len() < MAX_ERROR_BODY_BYTES)
|
||||
}
|
||||
|
||||
pub(super) async fn collect_error_body<R>(
|
||||
lines: &mut FramedRead<R, LinesCodec>,
|
||||
) -> Result<Vec<u8>, GatewayError>
|
||||
@@ -201,23 +241,19 @@ where
|
||||
while let Some(frame) = read_next_frame(lines).await? {
|
||||
match frame.payload {
|
||||
StreamFramePayload::Data { chunk_b64, text } => {
|
||||
let chunk = if let Some(chunk_b64) = chunk_b64 {
|
||||
base64::engine::general_purpose::STANDARD
|
||||
.decode(chunk_b64)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||
} else {
|
||||
text.unwrap_or_default().into_bytes()
|
||||
};
|
||||
body.extend_from_slice(&chunk);
|
||||
if body.len() >= MAX_ERROR_BODY_BYTES {
|
||||
body.truncate(MAX_ERROR_BODY_BYTES);
|
||||
if !append_error_frame_payload(&mut body, chunk_b64.as_deref(), text.as_deref())? {
|
||||
break;
|
||||
}
|
||||
}
|
||||
StreamFramePayload::Telemetry { .. } => {}
|
||||
StreamFramePayload::Eof { .. } => break,
|
||||
StreamFramePayload::Error { error } => {
|
||||
warn!(error = %error.message, "execution runtime stream emitted error frame while collecting error body");
|
||||
warn!(
|
||||
error_kind = ?error.kind,
|
||||
error_phase = ?error.phase,
|
||||
upstream_status = ?error.upstream_status,
|
||||
"execution runtime stream emitted error frame while collecting error body"
|
||||
);
|
||||
break;
|
||||
}
|
||||
StreamFramePayload::Headers { .. } => {}
|
||||
@@ -242,3 +278,32 @@ where
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{append_error_frame_payload, MAX_ERROR_BODY_BYTES};
|
||||
|
||||
#[test]
|
||||
fn oversized_base64_error_frame_is_rejected_before_decode() {
|
||||
let mut body = Vec::new();
|
||||
let encoded = "x".repeat((MAX_ERROR_BODY_BYTES + 2) / 3 * 4 + 1);
|
||||
|
||||
let keep_reading = append_error_frame_payload(&mut body, Some(&encoded), None)
|
||||
.expect("oversized frame should be handled without a decode error");
|
||||
|
||||
assert!(!keep_reading);
|
||||
assert!(body.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_frame_payload_is_capped_to_remaining_capture_budget() {
|
||||
let mut body = vec![b'a'; MAX_ERROR_BODY_BYTES - 2];
|
||||
|
||||
let keep_reading = append_error_frame_payload(&mut body, None, Some("hello"))
|
||||
.expect("text payload should append");
|
||||
|
||||
assert!(!keep_reading);
|
||||
assert_eq!(body.len(), MAX_ERROR_BODY_BYTES);
|
||||
assert_eq!(&body[MAX_ERROR_BODY_BYTES - 2..], b"he");
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -68,6 +68,60 @@ enum StreamFailureHandling {
|
||||
HonorLocalFailover,
|
||||
}
|
||||
|
||||
const UPSTREAM_STREAM_FAILURE_MESSAGE: &str = "Upstream response stream failed";
|
||||
const EXECUTION_STREAM_PROTOCOL_FAILURE_MESSAGE: &str = "Execution runtime stream protocol failed";
|
||||
const EXECUTION_STREAM_PROCESSING_FAILURE_MESSAGE: &str =
|
||||
"Execution runtime stream processing failed";
|
||||
|
||||
fn encode_bounded_stream_capture(body: &[u8]) -> Option<String> {
|
||||
encode_stream_capture_with_limit(
|
||||
body,
|
||||
crate::execution_runtime::MAX_STREAM_BODY_CAPTURE_BYTES,
|
||||
)
|
||||
}
|
||||
|
||||
fn encode_stream_capture_with_limit(body: &[u8], max_bytes: usize) -> Option<String> {
|
||||
let captured = &body[..body.len().min(max_bytes)];
|
||||
(!captured.is_empty()).then(|| base64::engine::general_purpose::STANDARD.encode(captured))
|
||||
}
|
||||
|
||||
fn stable_stream_failure_message(error_type: &str) -> &'static str {
|
||||
match error_type {
|
||||
"first_byte_timeout" => "Upstream response timed out before the first byte",
|
||||
"read_timeout" => "Upstream response stream timed out",
|
||||
"execution_runtime_stream_read_error" => UPSTREAM_STREAM_FAILURE_MESSAGE,
|
||||
"execution_runtime_stream_frame_decode_error"
|
||||
| "execution_runtime_stream_chunk_decode_error" => {
|
||||
EXECUTION_STREAM_PROTOCOL_FAILURE_MESSAGE
|
||||
}
|
||||
"execution_runtime_sync_json_stream_bridge_error"
|
||||
| "execution_runtime_stream_rewrite_error"
|
||||
| "execution_runtime_stream_rewrite_flush_error" => {
|
||||
EXECUTION_STREAM_PROCESSING_FAILURE_MESSAGE
|
||||
}
|
||||
_ => "Execution runtime stream failed",
|
||||
}
|
||||
}
|
||||
|
||||
fn public_execution_error_message(error: &ExecutionError) -> String {
|
||||
match &error.kind {
|
||||
ExecutionErrorKind::ConnectTimeout => "Upstream connection timed out".to_string(),
|
||||
ExecutionErrorKind::FirstByteTimeout => {
|
||||
"Upstream response timed out before the first byte".to_string()
|
||||
}
|
||||
ExecutionErrorKind::ReadTimeout => "Upstream response stream timed out".to_string(),
|
||||
ExecutionErrorKind::Upstream4xx | ExecutionErrorKind::Upstream5xx => error
|
||||
.upstream_status
|
||||
.map(|status| format!("Upstream request returned HTTP {status}"))
|
||||
.unwrap_or_else(|| "Upstream request failed".to_string()),
|
||||
ExecutionErrorKind::TlsError => "Upstream TLS connection failed".to_string(),
|
||||
ExecutionErrorKind::ProxyError => "Upstream proxy request failed".to_string(),
|
||||
ExecutionErrorKind::ProtocolError => UPSTREAM_STREAM_FAILURE_MESSAGE.to_string(),
|
||||
ExecutionErrorKind::Cancelled => "Request was cancelled".to_string(),
|
||||
ExecutionErrorKind::Internal => "Execution runtime stream failed".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
impl StreamFailureReport {
|
||||
fn into_body_jsons(self) -> (Value, Option<Value>) {
|
||||
let Self {
|
||||
@@ -110,11 +164,11 @@ impl StreamFailureReport {
|
||||
|
||||
pub(super) fn build_stream_failure_report(
|
||||
error_type: impl Into<String>,
|
||||
error_message: impl Into<String>,
|
||||
_error_message: impl Into<String>,
|
||||
status_code: u16,
|
||||
) -> StreamFailureReport {
|
||||
let error_type = error_type.into();
|
||||
let error_message = error_message.into();
|
||||
let error_message = stable_stream_failure_message(error_type.as_str()).to_string();
|
||||
StreamFailureReport {
|
||||
status_code,
|
||||
error_type,
|
||||
@@ -129,13 +183,14 @@ pub(super) fn build_stream_failure_report(
|
||||
|
||||
pub(super) fn build_stream_transport_failure_report(
|
||||
error_type: impl Into<String>,
|
||||
error_message: impl Into<String>,
|
||||
_error_message: impl Into<String>,
|
||||
status_code: u16,
|
||||
) -> StreamFailureReport {
|
||||
let error_type = error_type.into();
|
||||
StreamFailureReport {
|
||||
status_code,
|
||||
error_type: error_type.into(),
|
||||
error_message: error_message.into(),
|
||||
error_message: stable_stream_failure_message(error_type.as_str()).to_string(),
|
||||
error_type,
|
||||
upstream_status_code: None,
|
||||
transport_error: true,
|
||||
honor_http_failover: false,
|
||||
@@ -163,7 +218,7 @@ pub(super) fn build_stream_failure_from_execution_error(
|
||||
.ok()
|
||||
.and_then(|value| value.as_str().map(ToOwned::to_owned))
|
||||
.unwrap_or_else(|| "internal".to_string());
|
||||
let error_message = error.message.trim().to_string();
|
||||
let error_message = public_execution_error_message(error);
|
||||
let phase = serde_json::to_value(&error.phase).unwrap_or(Value::Null);
|
||||
let mut error_object = Map::from_iter([
|
||||
("phase".to_string(), phase),
|
||||
@@ -323,8 +378,7 @@ fn build_stream_failure_sync_payload(
|
||||
headers,
|
||||
body_json: Some(body),
|
||||
client_body_json: client_body,
|
||||
body_base64: (!provider_buffered_body.is_empty())
|
||||
.then(|| base64::engine::general_purpose::STANDARD.encode(provider_buffered_body)),
|
||||
body_base64: encode_bounded_stream_capture(provider_buffered_body),
|
||||
telemetry,
|
||||
}
|
||||
}
|
||||
@@ -524,8 +578,7 @@ pub(super) async fn handle_prefetch_provider_private_stream_error(
|
||||
headers,
|
||||
body_json: Some(body_json),
|
||||
client_body_json: None,
|
||||
body_base64: (!buffered_body.is_empty())
|
||||
.then(|| base64::engine::general_purpose::STANDARD.encode(buffered_body)),
|
||||
body_base64: encode_bounded_stream_capture(buffered_body),
|
||||
telemetry,
|
||||
};
|
||||
let failure_analysis = record_stream_sync_failure(
|
||||
@@ -813,11 +866,11 @@ pub(super) async fn submit_midstream_stream_failure(
|
||||
started_at_unix_ms: u64,
|
||||
failure: StreamFailureReport,
|
||||
) {
|
||||
let Some(report_kind) =
|
||||
direct_stream_finalize_kind.and_then(resolve_core_error_background_report_kind)
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let background_report_kind =
|
||||
direct_stream_finalize_kind.and_then(resolve_core_error_background_report_kind);
|
||||
let submit_background_report = background_report_kind.is_some();
|
||||
let report_kind =
|
||||
background_report_kind.unwrap_or_else(|| "execution_runtime_stream_error".to_string());
|
||||
|
||||
let candidate_status_code = failure.upstream_status_code;
|
||||
let payload = build_stream_failure_sync_payload(
|
||||
@@ -839,7 +892,10 @@ pub(super) async fn submit_midstream_stream_failure(
|
||||
StreamFailureHandling::Terminal,
|
||||
)
|
||||
.await;
|
||||
if let Err(err) = submit_sync_report(state, payload).await {
|
||||
if !submit_background_report {
|
||||
return;
|
||||
}
|
||||
if let Err(_err) = submit_sync_report(state, payload).await {
|
||||
let request_id = short_request_id(plan.request_id.as_str());
|
||||
warn!(
|
||||
event_name = "execution_report_submit_failed",
|
||||
@@ -848,7 +904,7 @@ pub(super) async fn submit_midstream_stream_failure(
|
||||
request_id = %request_id,
|
||||
candidate_id = ?plan.candidate_id,
|
||||
report_scope = "stream_failure",
|
||||
error = ?err,
|
||||
error_category = "stream_report_submit_failed",
|
||||
"gateway failed to submit sync execution report for terminal stream failure"
|
||||
);
|
||||
}
|
||||
@@ -864,9 +920,22 @@ mod tests {
|
||||
|
||||
use super::{
|
||||
build_stream_failure_from_execution_error, build_stream_failure_from_provider_error_body,
|
||||
build_stream_failure_sync_payload, build_stream_transport_failure_report,
|
||||
build_stream_failure_report, build_stream_failure_sync_payload,
|
||||
build_stream_transport_failure_report, encode_stream_capture_with_limit,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn failure_capture_encoding_defensively_caps_an_oversized_slice() {
|
||||
let encoded = encode_stream_capture_with_limit(b"abcdef", 3)
|
||||
.expect("bounded failure capture should be encoded");
|
||||
let decoded = base64::engine::general_purpose::STANDARD
|
||||
.decode(encoded)
|
||||
.expect("capture should be valid base64");
|
||||
|
||||
assert_eq!(decoded, b"abc");
|
||||
assert!(encode_stream_capture_with_limit(b"abc", 0).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn committed_transport_failure_has_no_upstream_status() {
|
||||
for status_code in [502, 504] {
|
||||
@@ -913,6 +982,61 @@ mod tests {
|
||||
assert!(!failure.transport_error);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn execution_error_details_are_not_projected_to_stream_clients() {
|
||||
let secret = "Bearer stream-secret https://user:password@example.test/private";
|
||||
let failure = build_stream_failure_from_execution_error(&ExecutionError {
|
||||
kind: ExecutionErrorKind::ProtocolError,
|
||||
phase: ExecutionPhase::StreamRead,
|
||||
message: secret.to_string(),
|
||||
upstream_status: None,
|
||||
retryable: true,
|
||||
failover_recommended: true,
|
||||
});
|
||||
|
||||
assert_eq!(failure.error_message, "Upstream response stream failed");
|
||||
let client_body = failure
|
||||
.to_json_string()
|
||||
.expect("stream failure should serialize");
|
||||
assert!(!client_body.contains(secret));
|
||||
assert!(!client_body.contains("stream-secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upstream_execution_error_keeps_only_http_status_diagnostics() {
|
||||
let secret = "authorization=Bearer upstream-secret";
|
||||
let failure = build_stream_failure_from_execution_error(&ExecutionError {
|
||||
kind: ExecutionErrorKind::Upstream4xx,
|
||||
phase: ExecutionPhase::FirstByte,
|
||||
message: secret.to_string(),
|
||||
upstream_status: Some(429),
|
||||
retryable: true,
|
||||
failover_recommended: true,
|
||||
});
|
||||
|
||||
assert_eq!(failure.error_message, "Upstream request returned HTTP 429");
|
||||
assert!(!failure
|
||||
.to_json_string()
|
||||
.expect("stream failure should serialize")
|
||||
.contains(secret));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn internal_stream_failure_details_are_replaced_with_stable_text() {
|
||||
let secret = "failed near /Users/admin/.config with token=stream-secret";
|
||||
let failure =
|
||||
build_stream_failure_report("execution_runtime_stream_rewrite_error", secret, 502);
|
||||
|
||||
assert_eq!(
|
||||
failure.error_message,
|
||||
"Execution runtime stream processing failed"
|
||||
);
|
||||
assert!(!failure
|
||||
.to_json_string()
|
||||
.expect("stream failure should serialize")
|
||||
.contains(secret));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn midstream_failure_trace_uses_terminal_error_instead_of_buffered_sse() {
|
||||
let provider_buffered_body = concat!(
|
||||
|
||||
@@ -22,13 +22,14 @@ use crate::ai_serving::api::{
|
||||
};
|
||||
use crate::execution_runtime::ndjson::encode_stream_frame_ndjson;
|
||||
use crate::execution_runtime::transport::{
|
||||
append_upstream_response_body_chunk, decode_response_body_bytes, format_hyper_error_chain,
|
||||
format_wreq_upstream_request_error, stream_first_byte_timeout_message, DirectUpstreamResponse,
|
||||
append_upstream_response_body_chunk, decode_response_body_bytes,
|
||||
stream_first_byte_timeout_message, DirectUpstreamResponse,
|
||||
};
|
||||
use crate::execution_runtime::DirectUpstreamStreamExecution;
|
||||
use crate::GatewayError;
|
||||
|
||||
const STREAM_USAGE_OBSERVER_MAX_LINE_BYTES: usize = 1024 * 1024;
|
||||
const UPSTREAM_STREAM_READ_ERROR_MESSAGE: &str = "Upstream response stream failed";
|
||||
|
||||
pub(crate) fn build_direct_execution_frame_stream(
|
||||
execution: DirectUpstreamStreamExecution,
|
||||
@@ -39,6 +40,7 @@ pub(crate) fn build_direct_execution_frame_stream(
|
||||
candidate_id: _,
|
||||
status_code,
|
||||
headers,
|
||||
upstream_content_length,
|
||||
provider_api_format,
|
||||
stream_summary_report_context,
|
||||
prefetched_body,
|
||||
@@ -74,7 +76,11 @@ pub(crate) fn build_direct_execution_frame_stream(
|
||||
let mut stream_terminal_observer = StreamingStandardTerminalObserver::default();
|
||||
let mut observer_buffered = Vec::new();
|
||||
|
||||
if should_buffer_non_stream_response(&headers, &observer_context) {
|
||||
if should_buffer_non_stream_response(
|
||||
&headers,
|
||||
upstream_content_length,
|
||||
&observer_context,
|
||||
) {
|
||||
let original_headers = headers.clone();
|
||||
match buffer_non_sse_upstream_body(
|
||||
prefetched_body,
|
||||
@@ -104,8 +110,10 @@ pub(crate) fn build_direct_execution_frame_stream(
|
||||
summary = outcome.terminal_summary;
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(err) => {
|
||||
yield Err(IoError::other(format!("{err:?}")));
|
||||
Err(_err) => {
|
||||
yield Err(IoError::other(
|
||||
"Execution runtime stream conversion failed",
|
||||
));
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -250,16 +258,16 @@ pub(crate) fn build_direct_execution_frame_stream(
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(message) => {
|
||||
Err(_message) => {
|
||||
warn!(
|
||||
event_name = "stream_pump_body_read_error",
|
||||
log_type = "ops",
|
||||
status_code,
|
||||
upstream_bytes,
|
||||
error = %message,
|
||||
error_category = "prefetched_body_read_failed",
|
||||
"upstream body stream read error"
|
||||
);
|
||||
match encode_error_frame(message) {
|
||||
match encode_error_frame(UPSTREAM_STREAM_READ_ERROR_MESSAGE.to_string()) {
|
||||
Ok(frame) => yield Ok(frame),
|
||||
Err(encode_err) => {
|
||||
yield Err(encode_err);
|
||||
@@ -333,14 +341,14 @@ pub(crate) fn build_direct_execution_frame_stream(
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
let message = format_error_chain(&err);
|
||||
Err(_err) => {
|
||||
let message = UPSTREAM_STREAM_READ_ERROR_MESSAGE.to_string();
|
||||
warn!(
|
||||
event_name = "stream_pump_body_read_error",
|
||||
log_type = "ops",
|
||||
status_code,
|
||||
upstream_bytes,
|
||||
error = %message,
|
||||
error_category = "reqwest_body_read_failed",
|
||||
"upstream body stream read error"
|
||||
);
|
||||
match encode_error_frame(message) {
|
||||
@@ -415,14 +423,14 @@ pub(crate) fn build_direct_execution_frame_stream(
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
let message = format_hyper_error_chain(&err);
|
||||
Err(_err) => {
|
||||
let message = UPSTREAM_STREAM_READ_ERROR_MESSAGE.to_string();
|
||||
warn!(
|
||||
event_name = "stream_pump_body_read_error",
|
||||
log_type = "ops",
|
||||
status_code,
|
||||
upstream_bytes,
|
||||
error = %message,
|
||||
error_category = "hyper_body_read_failed",
|
||||
"upstream body stream read error"
|
||||
);
|
||||
match encode_error_frame(message) {
|
||||
@@ -497,14 +505,14 @@ pub(crate) fn build_direct_execution_frame_stream(
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
let message = format_wreq_upstream_request_error(&err);
|
||||
Err(_err) => {
|
||||
let message = UPSTREAM_STREAM_READ_ERROR_MESSAGE.to_string();
|
||||
warn!(
|
||||
event_name = "stream_pump_body_read_error",
|
||||
log_type = "ops",
|
||||
status_code,
|
||||
upstream_bytes,
|
||||
error = %message,
|
||||
error_category = "browser_body_read_failed",
|
||||
"upstream body stream read error"
|
||||
);
|
||||
match encode_error_frame(message) {
|
||||
@@ -575,16 +583,16 @@ pub(crate) fn build_direct_execution_frame_stream(
|
||||
}
|
||||
}
|
||||
Ok(None) => break,
|
||||
Err(message) => {
|
||||
Err(_message) => {
|
||||
warn!(
|
||||
event_name = "stream_pump_body_read_error",
|
||||
log_type = "ops",
|
||||
status_code,
|
||||
upstream_bytes,
|
||||
error = %message,
|
||||
error_category = "tunnel_body_read_failed",
|
||||
"upstream body stream read error"
|
||||
);
|
||||
match encode_error_frame(message) {
|
||||
match encode_error_frame(UPSTREAM_STREAM_READ_ERROR_MESSAGE.to_string()) {
|
||||
Ok(frame) => yield Ok(frame),
|
||||
Err(encode_err) => {
|
||||
yield Err(encode_err);
|
||||
@@ -664,14 +672,14 @@ fn encode_data_frame(chunk: &Bytes) -> Result<Bytes, IoError> {
|
||||
})
|
||||
}
|
||||
|
||||
fn encode_error_frame(message: String) -> Result<Bytes, IoError> {
|
||||
fn encode_error_frame(_message: String) -> Result<Bytes, IoError> {
|
||||
encode_stream_frame_ndjson(&StreamFrame {
|
||||
frame_type: StreamFrameType::Error,
|
||||
payload: StreamFramePayload::Error {
|
||||
error: ExecutionError {
|
||||
kind: ExecutionErrorKind::ProtocolError,
|
||||
phase: ExecutionPhase::StreamRead,
|
||||
message,
|
||||
message: UPSTREAM_STREAM_READ_ERROR_MESSAGE.to_string(),
|
||||
upstream_status: None,
|
||||
retryable: true,
|
||||
failover_recommended: true,
|
||||
@@ -770,6 +778,7 @@ fn should_treat_upstream_response_as_stream(
|
||||
|
||||
fn should_buffer_non_stream_response(
|
||||
headers: &BTreeMap<String, String>,
|
||||
upstream_content_length: Option<u64>,
|
||||
report_context: &Value,
|
||||
) -> bool {
|
||||
if should_treat_upstream_response_as_stream(headers, report_context) {
|
||||
@@ -784,6 +793,22 @@ fn should_buffer_non_stream_response(
|
||||
return true;
|
||||
}
|
||||
|
||||
// `content-length` is intentionally removed from the response header map
|
||||
// before it reaches the execution stream. Retain its parsed value as
|
||||
// internal metadata so only a declared fixed-length JSON response is
|
||||
// converted to the client's SSE contract.
|
||||
if report_context
|
||||
.get("upstream_is_stream")
|
||||
.and_then(Value::as_bool)
|
||||
== Some(true)
|
||||
&& upstream_content_length.is_some()
|
||||
&& headers
|
||||
.get("content-type")
|
||||
.is_some_and(|value| value.to_ascii_lowercase().contains("json"))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
headers
|
||||
.get("content-length")
|
||||
.and_then(|value| value.trim().parse::<u64>().ok())
|
||||
@@ -813,9 +838,9 @@ async fn buffer_non_sse_upstream_body(
|
||||
&mut upstream_bytes,
|
||||
)?;
|
||||
}
|
||||
Err(message) => {
|
||||
Err(_message) => {
|
||||
return Err(BufferedUpstreamBodyError {
|
||||
message,
|
||||
message: UPSTREAM_STREAM_READ_ERROR_MESSAGE.to_string(),
|
||||
ttfb_ms,
|
||||
upstream_bytes,
|
||||
first_byte_timeout: None,
|
||||
@@ -864,13 +889,13 @@ async fn buffer_non_sse_upstream_body(
|
||||
&mut upstream_bytes,
|
||||
)?;
|
||||
}
|
||||
Err(err) => {
|
||||
let message = format_error_chain(&err);
|
||||
Err(_err) => {
|
||||
let message = UPSTREAM_STREAM_READ_ERROR_MESSAGE.to_string();
|
||||
warn!(
|
||||
event_name = "stream_pump_body_read_error",
|
||||
log_type = "ops",
|
||||
upstream_bytes,
|
||||
error = %message,
|
||||
error_category = "reqwest_body_read_failed",
|
||||
"upstream body stream read error"
|
||||
);
|
||||
return Err(BufferedUpstreamBodyError {
|
||||
@@ -922,13 +947,13 @@ async fn buffer_non_sse_upstream_body(
|
||||
&mut upstream_bytes,
|
||||
)?;
|
||||
}
|
||||
Err(err) => {
|
||||
let message = format_hyper_error_chain(&err);
|
||||
Err(_err) => {
|
||||
let message = UPSTREAM_STREAM_READ_ERROR_MESSAGE.to_string();
|
||||
warn!(
|
||||
event_name = "stream_pump_body_read_error",
|
||||
log_type = "ops",
|
||||
upstream_bytes,
|
||||
error = %message,
|
||||
error_category = "hyper_body_read_failed",
|
||||
"upstream body stream read error"
|
||||
);
|
||||
return Err(BufferedUpstreamBodyError {
|
||||
@@ -980,13 +1005,13 @@ async fn buffer_non_sse_upstream_body(
|
||||
&mut upstream_bytes,
|
||||
)?;
|
||||
}
|
||||
Err(err) => {
|
||||
let message = format_wreq_upstream_request_error(&err);
|
||||
Err(_err) => {
|
||||
let message = UPSTREAM_STREAM_READ_ERROR_MESSAGE.to_string();
|
||||
warn!(
|
||||
event_name = "stream_pump_body_read_error",
|
||||
log_type = "ops",
|
||||
upstream_bytes,
|
||||
error = %message,
|
||||
error_category = "browser_body_read_failed",
|
||||
"upstream body stream read error"
|
||||
);
|
||||
return Err(BufferedUpstreamBodyError {
|
||||
@@ -1034,16 +1059,16 @@ async fn buffer_non_sse_upstream_body(
|
||||
)?;
|
||||
}
|
||||
Ok(None) => break,
|
||||
Err(message) => {
|
||||
Err(_message) => {
|
||||
warn!(
|
||||
event_name = "stream_pump_body_read_error",
|
||||
log_type = "ops",
|
||||
upstream_bytes,
|
||||
error = %message,
|
||||
error_category = "tunnel_body_read_failed",
|
||||
"upstream body stream read error"
|
||||
);
|
||||
return Err(BufferedUpstreamBodyError {
|
||||
message,
|
||||
message: UPSTREAM_STREAM_READ_ERROR_MESSAGE.to_string(),
|
||||
ttfb_ms,
|
||||
upstream_bytes,
|
||||
first_byte_timeout: None,
|
||||
@@ -1071,14 +1096,16 @@ fn maybe_bridge_non_sse_sync_json_to_stream(
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let decoded_body_bytes = decode_response_body_bytes(headers, body_bytes)
|
||||
.map_err(|error| GatewayError::Internal(error.to_string()))?;
|
||||
let decoded_body_bytes = decode_response_body_bytes(headers, body_bytes).map_err(|_error| {
|
||||
GatewayError::Internal("execution runtime response decode failed".to_string())
|
||||
})?;
|
||||
if !response_body_is_json(headers, decoded_body_bytes.as_ref()) {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let body_json: Value = serde_json::from_slice(decoded_body_bytes.as_ref())
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
let body_json: Value = serde_json::from_slice(decoded_body_bytes.as_ref()).map_err(|_err| {
|
||||
GatewayError::Internal("execution runtime response JSON decode failed".to_string())
|
||||
})?;
|
||||
let client_api_format = report_context
|
||||
.get("client_api_format")
|
||||
.and_then(Value::as_str)
|
||||
@@ -1114,17 +1141,6 @@ fn response_body_is_json(headers: &BTreeMap<String, String>, body_bytes: &[u8])
|
||||
serde_json::from_slice::<Value>(body_bytes).is_ok()
|
||||
}
|
||||
|
||||
fn format_error_chain(err: &(dyn std::error::Error + 'static)) -> String {
|
||||
let mut message = err.to_string();
|
||||
let mut source = err.source();
|
||||
while let Some(cause) = source {
|
||||
message.push_str(": ");
|
||||
message.push_str(&cause.to_string());
|
||||
source = cause.source();
|
||||
}
|
||||
message
|
||||
}
|
||||
|
||||
fn observe_stream_chunk(
|
||||
observer: &mut StreamingStandardTerminalObserver,
|
||||
report_context: &Value,
|
||||
@@ -1135,10 +1151,8 @@ fn observe_stream_chunk(
|
||||
let normalized = if let Some(normalizer) = private_stream_normalizer {
|
||||
match normalizer.push_chunk(chunk) {
|
||||
Ok(normalized) => normalized,
|
||||
Err(err) => {
|
||||
observer.disable_with_error(format!(
|
||||
"failed to normalize provider private stream chunk: {err:?}"
|
||||
));
|
||||
Err(_err) => {
|
||||
observer.disable_with_error("provider stream normalization failed");
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -1160,23 +1174,21 @@ fn finalize_stream_terminal_summary(
|
||||
Ok(flushed) => {
|
||||
observe_normalized_bytes(observer, report_context, observer_buffered, &flushed)
|
||||
}
|
||||
Err(err) => observer.disable_with_error(format!(
|
||||
"failed to flush provider private stream normalization: {err:?}"
|
||||
)),
|
||||
Err(_err) => observer.disable_with_error("provider stream normalization failed"),
|
||||
}
|
||||
}
|
||||
|
||||
if !observer_buffered.is_empty() {
|
||||
let line = std::mem::take(observer_buffered);
|
||||
if let Err(err) = observer.push_line(report_context, line) {
|
||||
observer.disable_with_error(err.to_string());
|
||||
if let Err(_err) = observer.push_line(report_context, line) {
|
||||
observer.disable_with_error("stream usage parsing failed");
|
||||
}
|
||||
}
|
||||
|
||||
match observer.finish(report_context) {
|
||||
Ok(summary) => summary,
|
||||
Err(err) => {
|
||||
observer.disable_with_error(err.to_string());
|
||||
Err(_err) => {
|
||||
observer.disable_with_error("stream usage parsing failed");
|
||||
observer.latest_summary().cloned()
|
||||
}
|
||||
}
|
||||
@@ -1216,8 +1228,8 @@ fn observe_normalized_bytes(
|
||||
remaining = &remaining[line_part_len..];
|
||||
if observer_buffered.last() == Some(&b'\n') {
|
||||
let line = std::mem::take(observer_buffered);
|
||||
if let Err(err) = observer.push_line(report_context, line) {
|
||||
observer.disable_with_error(err.to_string());
|
||||
if let Err(_err) = observer.push_line(report_context, line) {
|
||||
observer.disable_with_error("stream usage parsing failed");
|
||||
observer_buffered.clear();
|
||||
return;
|
||||
}
|
||||
@@ -1232,7 +1244,10 @@ mod tests {
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use aether_contracts::tunnel_security::TUNNEL_SECURITY_NON_TLS_REQUIRED;
|
||||
use aether_contracts::{ExecutionPlan, ExecutionTimeouts, RequestBody};
|
||||
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||
use aether_data::repository::proxy_nodes::{InMemoryProxyNodeRepository, StoredProxyNode};
|
||||
use async_stream::stream;
|
||||
use axum::body::{Body, Bytes};
|
||||
use axum::extract::ws::Message;
|
||||
@@ -1245,9 +1260,9 @@ mod tests {
|
||||
use tokio::sync::watch;
|
||||
|
||||
use super::{
|
||||
build_direct_execution_frame_stream, observe_normalized_bytes,
|
||||
build_direct_execution_frame_stream, encode_error_frame, observe_normalized_bytes,
|
||||
should_buffer_non_stream_response, should_treat_upstream_response_as_stream,
|
||||
STREAM_USAGE_OBSERVER_MAX_LINE_BYTES,
|
||||
STREAM_USAGE_OBSERVER_MAX_LINE_BYTES, UPSTREAM_STREAM_READ_ERROR_MESSAGE,
|
||||
};
|
||||
use crate::ai_serving::api::StreamingStandardTerminalObserver;
|
||||
use crate::execution_runtime::transport::{
|
||||
@@ -1267,6 +1282,66 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
const LOCAL_TUNNEL_TEST_PSK: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc=";
|
||||
const LOCAL_TUNNEL_TEST_GENERATION: &str = "stream-pump-test-generation-1";
|
||||
|
||||
fn authenticated_local_tunnel_test_state() -> AppState {
|
||||
let node = StoredProxyNode::new(
|
||||
"node-1".to_string(),
|
||||
"Node 1".to_string(),
|
||||
"127.0.0.1".to_string(),
|
||||
0,
|
||||
false,
|
||||
"online".to_string(),
|
||||
30,
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
true,
|
||||
true,
|
||||
1,
|
||||
)
|
||||
.expect("tunnel node should build")
|
||||
.with_runtime_fields(
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
Some(serde_json::json!({
|
||||
"tunnel_security": {
|
||||
"mode": TUNNEL_SECURITY_NON_TLS_REQUIRED,
|
||||
"encryption_key": LOCAL_TUNNEL_TEST_PSK,
|
||||
}
|
||||
})),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.with_tunnel_generation(LOCAL_TUNNEL_TEST_GENERATION.to_string());
|
||||
let data = crate::data::GatewayDataState::with_proxy_node_repository_for_tests(Arc::new(
|
||||
InMemoryProxyNodeRepository::seed([node]),
|
||||
))
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY);
|
||||
AppState::new()
|
||||
.expect("app state should build")
|
||||
.with_data_state_for_tests(data)
|
||||
}
|
||||
|
||||
async fn recv_tunnel_test_frame(
|
||||
proxy_rx: &mut aether_runtime::BoundedQueueReceiver<Message>,
|
||||
description: &str,
|
||||
) -> Message {
|
||||
tokio::time::timeout(Duration::from_secs(5), proxy_rx.recv())
|
||||
.await
|
||||
.unwrap_or_else(|_| panic!("timed out waiting for {description}"))
|
||||
.unwrap_or_else(|| panic!("proxy channel closed before {description}"))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn treats_kiro_eventstream_envelope_as_stream_even_when_content_type_is_json() {
|
||||
let headers = BTreeMap::from([("content-type".into(), "application/json".into())]);
|
||||
@@ -1295,10 +1370,12 @@ mod tests {
|
||||
|
||||
assert!(!should_buffer_non_stream_response(
|
||||
&BTreeMap::from([("content-type".into(), "application/json".into())]),
|
||||
None,
|
||||
&streaming_context
|
||||
));
|
||||
assert!(should_buffer_non_stream_response(
|
||||
&BTreeMap::from([("content-type".into(), "application/json".into())]),
|
||||
None,
|
||||
&non_stream_context
|
||||
));
|
||||
assert!(should_buffer_non_stream_response(
|
||||
@@ -1306,14 +1383,27 @@ mod tests {
|
||||
("content-type".into(), "application/json".into()),
|
||||
("content-length".into(), "128".into()),
|
||||
]),
|
||||
Some(128),
|
||||
&streaming_context
|
||||
));
|
||||
assert!(!should_buffer_non_stream_response(
|
||||
&BTreeMap::from([("content-type".into(), "text/event-stream".into())]),
|
||||
None,
|
||||
&non_stream_context
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_frames_do_not_include_transport_details() {
|
||||
let secret = "Bearer stream-secret https://user:password@example.test/private";
|
||||
let frame = encode_error_frame(secret.to_string()).expect("error frame should encode");
|
||||
let frame = String::from_utf8(frame.to_vec()).expect("error frame should be utf8");
|
||||
|
||||
assert!(frame.contains(UPSTREAM_STREAM_READ_ERROR_MESSAGE));
|
||||
assert!(!frame.contains(secret));
|
||||
assert!(!frame.contains("stream-secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversized_usage_line_disables_observation_without_retaining_the_line() {
|
||||
let mut observer = StreamingStandardTerminalObserver::default();
|
||||
@@ -1920,20 +2010,24 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn direct_execution_frame_stream_preserves_local_tunnel_stream_error_message() {
|
||||
let state = AppState::new().expect("app state should build");
|
||||
async fn direct_execution_frame_stream_sanitizes_local_tunnel_stream_error_message() {
|
||||
let state = authenticated_local_tunnel_test_state();
|
||||
let tunnel_app = state.tunnel.app_state();
|
||||
let (proxy_tx, mut proxy_rx) = aether_runtime::bounded_queue(8);
|
||||
let (proxy_close_tx, _) = watch::channel(false);
|
||||
tunnel_app.hub.register_proxy(Arc::new(TunnelProxyConn::new(
|
||||
801,
|
||||
"node-1".to_string(),
|
||||
"Node 1".to_string(),
|
||||
proxy_tx,
|
||||
proxy_close_tx,
|
||||
16,
|
||||
2,
|
||||
)));
|
||||
tunnel_app.hub.register_proxy(Arc::new(
|
||||
TunnelProxyConn::new(
|
||||
801,
|
||||
"node-1".to_string(),
|
||||
"Node 1".to_string(),
|
||||
proxy_tx,
|
||||
proxy_close_tx,
|
||||
16,
|
||||
2,
|
||||
)
|
||||
.with_tunnel_generation(LOCAL_TUNNEL_TEST_GENERATION.to_string())
|
||||
.with_authenticated_key(LOCAL_TUNNEL_TEST_PSK.to_string()),
|
||||
));
|
||||
|
||||
let plan = ExecutionPlan {
|
||||
request_id: "req-local-stream-error-1".into(),
|
||||
@@ -1967,7 +2061,7 @@ mod tests {
|
||||
execute_stream_plan_via_local_tunnel(&state_for_task, &plan_for_task).await
|
||||
});
|
||||
|
||||
let request_headers = match proxy_rx.recv().await.expect("headers frame should arrive") {
|
||||
let request_headers = match recv_tunnel_test_frame(&mut proxy_rx, "headers frame").await {
|
||||
Message::Binary(data) => data,
|
||||
other => panic!("unexpected message: {other:?}"),
|
||||
};
|
||||
@@ -1975,7 +2069,7 @@ mod tests {
|
||||
.expect("request header frame should parse");
|
||||
assert_eq!(request_header.msg_type, tunnel_protocol::REQUEST_HEADERS);
|
||||
|
||||
let request_body = match proxy_rx.recv().await.expect("body frame should arrive") {
|
||||
let request_body = match recv_tunnel_test_frame(&mut proxy_rx, "body frame").await {
|
||||
Message::Binary(data) => data,
|
||||
other => panic!("unexpected message: {other:?}"),
|
||||
};
|
||||
@@ -2057,28 +2151,29 @@ mod tests {
|
||||
.and_then(Value::as_str)
|
||||
.expect("error frame should include a message");
|
||||
|
||||
assert_eq!(error_message, original_error);
|
||||
assert!(
|
||||
!error_message.contains("unexpected EOF during chunk size line"),
|
||||
"local tunnel path should preserve the original proxy error text"
|
||||
);
|
||||
assert_eq!(error_message, UPSTREAM_STREAM_READ_ERROR_MESSAGE);
|
||||
assert!(!error_message.contains(original_error));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn second_local_tunnel_request_works_after_first_completes() {
|
||||
let state = AppState::new().expect("app state should build");
|
||||
let state = authenticated_local_tunnel_test_state();
|
||||
let tunnel_app = state.tunnel.app_state();
|
||||
let (proxy_tx, mut proxy_rx) = aether_runtime::bounded_queue(8);
|
||||
let (proxy_close_tx, _) = watch::channel(false);
|
||||
tunnel_app.hub.register_proxy(Arc::new(TunnelProxyConn::new(
|
||||
900,
|
||||
"node-1".to_string(),
|
||||
"Node 1".to_string(),
|
||||
proxy_tx,
|
||||
proxy_close_tx,
|
||||
16,
|
||||
2,
|
||||
)));
|
||||
tunnel_app.hub.register_proxy(Arc::new(
|
||||
TunnelProxyConn::new(
|
||||
900,
|
||||
"node-1".to_string(),
|
||||
"Node 1".to_string(),
|
||||
proxy_tx,
|
||||
proxy_close_tx,
|
||||
16,
|
||||
2,
|
||||
)
|
||||
.with_tunnel_generation(LOCAL_TUNNEL_TEST_GENERATION.to_string())
|
||||
.with_authenticated_key(LOCAL_TUNNEL_TEST_PSK.to_string()),
|
||||
));
|
||||
|
||||
let plan = ExecutionPlan {
|
||||
request_id: "req-reuse-1".into(),
|
||||
@@ -2115,13 +2210,13 @@ mod tests {
|
||||
);
|
||||
|
||||
// Read request frames from proxy side
|
||||
let req1_headers = match proxy_rx.recv().await.expect("req1 headers") {
|
||||
let req1_headers = match recv_tunnel_test_frame(&mut proxy_rx, "req1 headers").await {
|
||||
Message::Binary(data) => data,
|
||||
other => panic!("unexpected: {other:?}"),
|
||||
};
|
||||
let req1_header =
|
||||
tunnel_protocol::FrameHeader::parse(&req1_headers).expect("req1 header parse");
|
||||
let _req1_body = proxy_rx.recv().await.expect("req1 body");
|
||||
let _req1_body = recv_tunnel_test_frame(&mut proxy_rx, "req1 body").await;
|
||||
|
||||
// Simulate proxy response
|
||||
let resp_meta = serde_json::to_vec(&tunnel_protocol::ResponseMeta {
|
||||
@@ -2188,10 +2283,7 @@ mod tests {
|
||||
);
|
||||
|
||||
// Read second request's frames
|
||||
let req2_headers = tokio::time::timeout(Duration::from_secs(2), proxy_rx.recv())
|
||||
.await
|
||||
.expect("second request should arrive within 2s")
|
||||
.expect("req2 headers");
|
||||
let req2_headers = recv_tunnel_test_frame(&mut proxy_rx, "req2 headers").await;
|
||||
let req2_data = match req2_headers {
|
||||
Message::Binary(data) => data,
|
||||
other => panic!("unexpected: {other:?}"),
|
||||
|
||||
@@ -9,9 +9,9 @@ use crate::api::response::build_client_response_from_parts;
|
||||
use crate::control::GatewayControlDecision;
|
||||
use crate::usage::spawn_sync_report;
|
||||
use crate::{usage::GatewaySyncReportRequest, AppState, GatewayError};
|
||||
use aether_usage_runtime::decode_internal_report_body_base64;
|
||||
use axum::body::Body;
|
||||
use axum::http::{Response, StatusCode};
|
||||
use base64::Engine as _;
|
||||
use tracing::warn;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
@@ -144,9 +144,8 @@ fn build_local_core_sync_finalize_fallback_response(
|
||||
}
|
||||
|
||||
if let Some(body_base64) = payload.body_base64.as_ref() {
|
||||
let body_bytes = base64::engine::general_purpose::STANDARD
|
||||
.decode(body_base64)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
let body_bytes =
|
||||
decode_internal_report_body_base64(body_base64).map_err(GatewayError::Internal)?;
|
||||
return build_local_sync_response_from_bytes(trace_id, decision, payload, body_bytes);
|
||||
}
|
||||
|
||||
@@ -299,9 +298,8 @@ fn resolve_local_sync_source_body_json(
|
||||
let body_json = if let Some(body_json) = payload.body_json.clone() {
|
||||
body_json
|
||||
} else if let Some(body_base64) = payload.body_base64.as_deref() {
|
||||
let body_bytes = base64::engine::general_purpose::STANDARD
|
||||
.decode(body_base64)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
let body_bytes =
|
||||
decode_internal_report_body_base64(body_base64).map_err(GatewayError::Internal)?;
|
||||
let stripped = strip_utf8_bom_and_ws(&body_bytes);
|
||||
let Ok(body_json) = serde_json::from_slice::<serde_json::Value>(stripped) else {
|
||||
return Ok(None);
|
||||
@@ -328,9 +326,8 @@ fn decode_local_sync_body_text(
|
||||
let Some(body_base64) = payload.body_base64.as_deref() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let body_bytes = base64::engine::general_purpose::STANDARD
|
||||
.decode(body_base64)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
let body_bytes =
|
||||
decode_internal_report_body_base64(body_base64).map_err(GatewayError::Internal)?;
|
||||
let stripped = strip_utf8_bom_and_ws(&body_bytes);
|
||||
let body_text = String::from_utf8_lossy(stripped).trim().to_string();
|
||||
if body_text.is_empty() {
|
||||
|
||||
@@ -60,8 +60,8 @@ use crate::execution_runtime::transport::{
|
||||
build_request_body, collect_response_headers, decode_response_body_bytes_with_limit,
|
||||
execution_plan_response_body_limit_bytes, execution_response_body_mode,
|
||||
format_hyper_error_chain, format_upstream_request_error, format_wreq_upstream_request_error,
|
||||
response_body_is_json, send_request, DirectHttpResponse, DirectSyncExecutionRuntime,
|
||||
ExecutionRuntimeTransportError,
|
||||
response_body_is_json, safe_transport_error_message, send_request, DirectHttpResponse,
|
||||
DirectSyncExecutionRuntime, ExecutionRuntimeTransportError,
|
||||
};
|
||||
use crate::execution_runtime::windsurf::maybe_execute_windsurf_sync;
|
||||
use crate::execution_runtime::{
|
||||
@@ -101,7 +101,7 @@ mod policy;
|
||||
#[path = "execution/response.rs"]
|
||||
mod response;
|
||||
|
||||
use policy::decode_execution_result_body;
|
||||
use policy::{decode_execution_result_body, decode_execution_result_body_with_limit};
|
||||
pub(crate) use response::{
|
||||
maybe_build_local_sync_finalize_response, maybe_build_local_video_error_response,
|
||||
maybe_build_local_video_success_outcome, resolve_local_sync_error_background_report_kind,
|
||||
@@ -115,6 +115,11 @@ const SYNC_EXECUTION_IDLE_LOG_INTERVAL: Duration = Duration::from_secs(60);
|
||||
const OPENAI_IMAGE_SYNC_JSON_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(15);
|
||||
const OPENAI_IMAGE_SYNC_JSON_HEARTBEAT_BYTES: &[u8] = b"\n";
|
||||
const OPENAI_IMAGE_SYNC_PROGRESS_WRITE_INTERVAL: Duration = Duration::from_secs(5);
|
||||
// Progress parsing must retain only the incomplete SSE record. This is not a
|
||||
// response-body limit: the full upstream body is still handled by the normal
|
||||
// execution body policy, while malformed streams cannot grow telemetry state
|
||||
// forever by withholding a record separator.
|
||||
const OPENAI_IMAGE_SYNC_PROGRESS_MAX_BUFFER_BYTES: usize = 16 * 1024 * 1024;
|
||||
const INVALID_GEMINI_PROVIDER_SUCCESS_MESSAGE: &str = "Provider returned HTTP 200 but the Gemini response did not contain visible model output; refusing to finalize it as a successful response.";
|
||||
|
||||
fn elapsed_ms_since(started_at: Instant) -> u64 {
|
||||
@@ -174,6 +179,39 @@ struct SyncAttemptTerminalGuard {
|
||||
armed: bool,
|
||||
}
|
||||
|
||||
/// Keep forced-terminal records useful for operations without copying an
|
||||
/// arbitrary `GatewayError` into the candidate/usage stores. Gateway errors
|
||||
/// can wrap provider URLs, credentials, query strings, or database details;
|
||||
/// those values belong in the internal logging path only.
|
||||
fn persisted_sync_abort_message(error: &GatewayError) -> &'static str {
|
||||
match error {
|
||||
GatewayError::UpstreamUnavailable { .. } => {
|
||||
"local sync attempt aborted before terminal finalization: upstream unavailable"
|
||||
}
|
||||
GatewayError::ControlUnavailable { .. } => {
|
||||
"local sync attempt aborted before terminal finalization: control unavailable"
|
||||
}
|
||||
GatewayError::LocalExecutionPlanningTimeout { .. } => {
|
||||
"local sync attempt aborted before terminal finalization: planning timeout"
|
||||
}
|
||||
GatewayError::AdmissionTimeout { .. } => {
|
||||
"local sync attempt aborted before terminal finalization: admission timeout"
|
||||
}
|
||||
GatewayError::Client { .. } => {
|
||||
"local sync attempt aborted before terminal finalization: client error"
|
||||
}
|
||||
GatewayError::PlanUsageLimited(_) => {
|
||||
"local sync attempt aborted before terminal finalization: usage limit"
|
||||
}
|
||||
GatewayError::LastActiveAdminUpdateDenied | GatewayError::LastActiveAdminDeleteDenied => {
|
||||
"local sync attempt aborted before terminal finalization: policy denied"
|
||||
}
|
||||
GatewayError::Internal(_) => {
|
||||
"local sync attempt aborted before terminal finalization: internal error"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SyncAttemptTerminalGuard {
|
||||
fn new(
|
||||
state: &AppState,
|
||||
@@ -213,7 +251,7 @@ impl SyncAttemptTerminalGuard {
|
||||
RequestCandidateStatus::Failed,
|
||||
StatusCode::INTERNAL_SERVER_ERROR.as_u16(),
|
||||
"local_sync_attempt_aborted",
|
||||
format!("Local sync attempt failed before terminal finalization: {error:?}"),
|
||||
persisted_sync_abort_message(error),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -346,7 +384,7 @@ impl SyncExecutionFailure {
|
||||
error_type: fallback_kind
|
||||
.map(SyncExecutionFailureFallbackKind::error_type)
|
||||
.unwrap_or("execution_runtime_unavailable"),
|
||||
message: err.to_string(),
|
||||
message: safe_transport_error_message(&err),
|
||||
status_code: fallback_kind.map(|_| StatusCode::BAD_GATEWAY.as_u16()),
|
||||
latency_ms: None,
|
||||
fallback_kind,
|
||||
@@ -1132,10 +1170,42 @@ impl<'a> OpenAiImageSyncProgressRecorder<'a> {
|
||||
if chunk.is_empty() {
|
||||
return;
|
||||
}
|
||||
self.buffer.extend_from_slice(chunk);
|
||||
let mut blocks = Vec::new();
|
||||
let mut remaining = chunk;
|
||||
let mut parser_overflowed = false;
|
||||
loop {
|
||||
while let Some(block_end) = find_sse_block_end(&self.buffer) {
|
||||
blocks.push(self.buffer.drain(..block_end).collect::<Vec<_>>());
|
||||
}
|
||||
if remaining.is_empty() {
|
||||
break;
|
||||
}
|
||||
|
||||
let capacity =
|
||||
OPENAI_IMAGE_SYNC_PROGRESS_MAX_BUFFER_BYTES.saturating_sub(self.buffer.len());
|
||||
if capacity == 0 {
|
||||
// The progress recorder is observational. Drop an incomplete
|
||||
// oversized record and keep the client-facing response alive.
|
||||
self.buffer.clear();
|
||||
parser_overflowed = true;
|
||||
break;
|
||||
}
|
||||
let take = find_sse_block_end(remaining)
|
||||
.map_or(remaining.len(), |block_end| block_end)
|
||||
.min(capacity);
|
||||
if take == 0 {
|
||||
self.buffer.clear();
|
||||
parser_overflowed = true;
|
||||
break;
|
||||
}
|
||||
self.buffer.extend_from_slice(&remaining[..take]);
|
||||
remaining = &remaining[take..];
|
||||
}
|
||||
if parser_overflowed {
|
||||
debug!("openai image sync progress parser dropped an oversized incomplete SSE record");
|
||||
}
|
||||
let mut force_persist = false;
|
||||
while let Some(block_end) = find_sse_block_end(&self.buffer) {
|
||||
let block = self.buffer.drain(..block_end).collect::<Vec<_>>();
|
||||
for block in blocks {
|
||||
let Some(frame) = parse_openai_image_sync_sse_frame(&block) else {
|
||||
continue;
|
||||
};
|
||||
@@ -1813,12 +1883,16 @@ async fn openai_image_sync_json_heartbeat_final_bytes(
|
||||
{
|
||||
Ok(bytes) if !bytes.is_empty() => bytes.to_vec(),
|
||||
Ok(_) => openai_image_sync_json_heartbeat_error_body("empty sync image response"),
|
||||
Err(err) => openai_image_sync_json_heartbeat_error_body(&err.to_string()),
|
||||
Err(_err) => {
|
||||
openai_image_sync_json_heartbeat_error_body("sync image response read failed")
|
||||
}
|
||||
},
|
||||
Ok(None) => openai_image_sync_json_heartbeat_error_body(
|
||||
"sync image execution ended without a local response",
|
||||
),
|
||||
Err(err) => openai_image_sync_json_heartbeat_error_body(&format!("{err:?}")),
|
||||
// Do not serialize the internal error: Debug output can contain
|
||||
// upstream URLs, credentials, or other request-specific details.
|
||||
Err(_err) => openai_image_sync_json_heartbeat_error_body("sync image execution failed"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1840,7 +1914,7 @@ async fn apply_sync_success_effects(
|
||||
) {
|
||||
if let Some(report_context) = report_context {
|
||||
crate::ai_serving::persist_converted_response_history(
|
||||
state.runtime_state(),
|
||||
state,
|
||||
report_context,
|
||||
payload
|
||||
.client_body_json
|
||||
@@ -2155,7 +2229,7 @@ async fn execute_execution_runtime_sync_impl(
|
||||
}
|
||||
},
|
||||
Err(err) => {
|
||||
let transport_error_message = err.to_string();
|
||||
let transport_error_message = safe_transport_error_message(&err);
|
||||
warn!(
|
||||
event_name = "chatgpt_web_image_execution_unavailable",
|
||||
log_type = "ops",
|
||||
@@ -2167,7 +2241,7 @@ async fn execute_execution_runtime_sync_impl(
|
||||
key_id,
|
||||
model_name,
|
||||
candidate_index = candidate_index.as_str(),
|
||||
error = %err,
|
||||
error = %transport_error_message,
|
||||
"gateway ChatGPT-Web image execution unavailable"
|
||||
);
|
||||
let terminal_unix_secs = current_request_candidate_unix_ms();
|
||||
@@ -2207,7 +2281,7 @@ async fn execute_execution_runtime_sync_impl(
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
let transport_error_message = err.to_string();
|
||||
let transport_error_message = safe_transport_error_message(&err);
|
||||
warn!(
|
||||
event_name = "grok_execution_unavailable",
|
||||
log_type = "ops",
|
||||
@@ -2219,7 +2293,7 @@ async fn execute_execution_runtime_sync_impl(
|
||||
key_id,
|
||||
model_name,
|
||||
candidate_index = candidate_index.as_str(),
|
||||
error = %err,
|
||||
error = %transport_error_message,
|
||||
"gateway Grok execution unavailable"
|
||||
);
|
||||
let terminal_unix_secs = current_request_candidate_unix_ms();
|
||||
@@ -2262,7 +2336,7 @@ async fn execute_execution_runtime_sync_impl(
|
||||
match (override_fn.0)(&plan) {
|
||||
Ok(result) => result,
|
||||
Err(err) => {
|
||||
let transport_error_message = format!("{err:?}");
|
||||
let transport_error_message = persisted_sync_abort_message(&err).to_string();
|
||||
warn!(
|
||||
event_name = "sync_execution_runtime_test_override_failed",
|
||||
log_type = "ops",
|
||||
@@ -2408,7 +2482,7 @@ async fn execute_execution_runtime_sync_impl(
|
||||
}
|
||||
},
|
||||
Err(err) => {
|
||||
let transport_error_message = err.to_string();
|
||||
let transport_error_message = safe_transport_error_message(&err);
|
||||
warn!(
|
||||
event_name = "chatgpt_web_image_execution_unavailable",
|
||||
log_type = "ops",
|
||||
@@ -2420,7 +2494,7 @@ async fn execute_execution_runtime_sync_impl(
|
||||
key_id,
|
||||
model_name,
|
||||
candidate_index = candidate_index.as_str(),
|
||||
error = %err,
|
||||
error = %transport_error_message,
|
||||
"gateway ChatGPT-Web image execution unavailable"
|
||||
);
|
||||
let terminal_unix_secs = current_request_candidate_unix_ms();
|
||||
@@ -2459,7 +2533,7 @@ async fn execute_execution_runtime_sync_impl(
|
||||
}
|
||||
},
|
||||
Err(err) => {
|
||||
let transport_error_message = err.to_string();
|
||||
let transport_error_message = safe_transport_error_message(&err);
|
||||
warn!(
|
||||
event_name = "grok_execution_unavailable",
|
||||
log_type = "ops",
|
||||
@@ -2471,7 +2545,7 @@ async fn execute_execution_runtime_sync_impl(
|
||||
key_id,
|
||||
model_name,
|
||||
candidate_index = candidate_index.as_str(),
|
||||
error = %err,
|
||||
error = %transport_error_message,
|
||||
"gateway Grok execution unavailable"
|
||||
);
|
||||
let terminal_unix_secs = current_request_candidate_unix_ms();
|
||||
@@ -2572,8 +2646,26 @@ async fn execute_execution_runtime_sync_impl(
|
||||
.as_ref()
|
||||
.and_then(|telemetry| telemetry.elapsed_ms);
|
||||
let mut headers = std::mem::take(&mut result.headers);
|
||||
let (body_bytes, mut body_json, body_base64) =
|
||||
decode_execution_result_body(result.body.take(), &mut headers)?;
|
||||
let result_body = result.body.take();
|
||||
let chatgpt_web_image_result = plan
|
||||
.provider_api_format
|
||||
.eq_ignore_ascii_case("openai:image")
|
||||
&& (plan.headers.iter().any(|(name, value)| {
|
||||
name.eq_ignore_ascii_case("x-aether-chatgpt-web-image") && value == "1"
|
||||
}) || report_context
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("chatgpt_web_image"))
|
||||
.and_then(Value::as_bool)
|
||||
.unwrap_or(false));
|
||||
let (body_bytes, mut body_json, body_base64) = if chatgpt_web_image_result {
|
||||
decode_execution_result_body_with_limit(
|
||||
result_body,
|
||||
&mut headers,
|
||||
crate::execution_runtime::chatgpt_web_image::chatgpt_web_image_sse_envelope_limit_bytes(),
|
||||
)?
|
||||
} else {
|
||||
decode_execution_result_body(result_body, &mut headers)?
|
||||
};
|
||||
if let Some(message) = invalid_gemini_provider_success_message(
|
||||
&plan,
|
||||
report_context.as_ref(),
|
||||
@@ -3382,7 +3474,7 @@ async fn execute_sync_via_remote_execution_runtime(
|
||||
status: RequestCandidateStatus::Failed,
|
||||
status_code: None,
|
||||
error_type: Some("execution_runtime_unavailable".to_string()),
|
||||
error_message: Some(format!("{err:?}")),
|
||||
error_message: Some(persisted_sync_abort_message(&err).to_string()),
|
||||
latency_ms: Some(elapsed_ms_since(candidate_started_at)),
|
||||
started_at_unix_ms: Some(candidate_started_unix_secs),
|
||||
finished_at_unix_ms: Some(terminal_unix_secs),
|
||||
@@ -3423,9 +3515,15 @@ async fn execute_sync_via_remote_execution_runtime(
|
||||
}
|
||||
|
||||
let remote_response_observed_at_unix_ms = current_request_candidate_unix_ms();
|
||||
let mut result = response
|
||||
.json::<ExecutionResult>()
|
||||
.await
|
||||
let response_body = aether_http::read_response_bytes_with_limit(
|
||||
response,
|
||||
crate::execution_runtime::transport::execution_result_envelope_limit_bytes(
|
||||
crate::headers::max_internal_buffered_body_bytes(),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
let mut result = serde_json::from_slice::<ExecutionResult>(&response_body)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
result
|
||||
.response_observation
|
||||
@@ -3893,6 +3991,20 @@ mod tests {
|
||||
assert!(message.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn forced_sync_abort_message_does_not_copy_internal_error_details() {
|
||||
let secret = "https://user:password@example.test/v1?api_key=should-not-persist";
|
||||
let error = GatewayError::Internal(secret.to_string());
|
||||
|
||||
let message = persisted_sync_abort_message(&error);
|
||||
assert_eq!(
|
||||
message,
|
||||
"local sync attempt aborted before terminal finalization: internal error"
|
||||
);
|
||||
assert!(!message.contains("password"));
|
||||
assert!(!message.contains("api_key"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sync_attempt_terminal_guard_marks_dropped_pending_attempt_cancelled() {
|
||||
let usage_repository = Arc::new(InMemoryUsageReadRepository::default());
|
||||
@@ -4482,4 +4594,22 @@ mod tests {
|
||||
Some(&report_context),
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn openai_image_sync_json_heartbeat_hides_internal_error_details() {
|
||||
let secret = "https://user:password@example.test/private?api_key=top-secret";
|
||||
let bytes = openai_image_sync_json_heartbeat_final_bytes(Err(GatewayError::Internal(
|
||||
secret.to_string(),
|
||||
)))
|
||||
.await;
|
||||
|
||||
let body: Value = serde_json::from_slice(&bytes).expect("heartbeat error body is JSON");
|
||||
assert_eq!(
|
||||
body.pointer("/error/message").and_then(Value::as_str),
|
||||
Some("sync image execution failed")
|
||||
);
|
||||
let body_text = String::from_utf8(bytes).expect("heartbeat body is UTF-8");
|
||||
assert!(!body_text.contains(secret));
|
||||
assert!(!body_text.contains("top-secret"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,15 +1,35 @@
|
||||
use aether_contracts::ResponseBody;
|
||||
use serde_json::Value;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use aether_contracts::ResponseBody;
|
||||
use base64::Engine as _;
|
||||
|
||||
use crate::execution_runtime::transport::{
|
||||
decode_base64_body_with_limit,
|
||||
serialize_json_body_with_limit as serialize_transport_json_body_with_limit,
|
||||
};
|
||||
use crate::GatewayError;
|
||||
|
||||
type DecodedBody = (Vec<u8>, Option<serde_json::Value>, Option<String>);
|
||||
|
||||
fn serialize_json_body_with_limit(body: &Value, limit: usize) -> Result<Vec<u8>, GatewayError> {
|
||||
serialize_transport_json_body_with_limit(body, limit)
|
||||
.map_err(|error| GatewayError::Internal(error.to_string()))
|
||||
}
|
||||
|
||||
pub(super) fn decode_execution_result_body(
|
||||
body: Option<ResponseBody>,
|
||||
headers: &mut BTreeMap<String, String>,
|
||||
) -> Result<DecodedBody, GatewayError> {
|
||||
decode_execution_result_body_with_limit(
|
||||
body,
|
||||
headers,
|
||||
crate::headers::max_internal_buffered_body_bytes(),
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn decode_execution_result_body_with_limit(
|
||||
body: Option<ResponseBody>,
|
||||
headers: &mut BTreeMap<String, String>,
|
||||
body_limit: usize,
|
||||
) -> Result<DecodedBody, GatewayError> {
|
||||
let Some(body) = body else {
|
||||
return Ok((Vec::new(), None, None));
|
||||
@@ -18,10 +38,8 @@ pub(super) fn decode_execution_result_body(
|
||||
json_body,
|
||||
body_bytes_b64,
|
||||
} = body;
|
||||
|
||||
if let Some(body_bytes_b64) = body_bytes_b64 {
|
||||
let bytes = base64::engine::general_purpose::STANDARD
|
||||
.decode(&body_bytes_b64)
|
||||
let bytes = decode_base64_body_with_limit(&body_bytes_b64, body_limit)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
return Ok((bytes, json_body, Some(body_bytes_b64)));
|
||||
}
|
||||
@@ -32,8 +50,7 @@ pub(super) fn decode_execution_result_body(
|
||||
headers
|
||||
.entry("content-type".to_string())
|
||||
.or_insert_with(|| "application/json".to_string());
|
||||
let bytes = serde_json::to_vec(&json_body)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
let bytes = serialize_json_body_with_limit(&json_body, body_limit)?;
|
||||
headers.insert("content-length".to_string(), bytes.len().to_string());
|
||||
return Ok((bytes, Some(json_body), None));
|
||||
}
|
||||
@@ -118,4 +135,34 @@ mod tests {
|
||||
Some(raw_len.as_str())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scoped_decode_limit_can_cover_a_bounded_synthetic_envelope() {
|
||||
let raw = vec![b'x'; 65 * 1024];
|
||||
let encoded = base64::engine::general_purpose::STANDARD.encode(&raw);
|
||||
let mut headers = BTreeMap::new();
|
||||
|
||||
let (decoded, json, retained) = super::decode_execution_result_body_with_limit(
|
||||
Some(ResponseBody {
|
||||
json_body: None,
|
||||
body_bytes_b64: Some(encoded.clone()),
|
||||
}),
|
||||
&mut headers,
|
||||
raw.len(),
|
||||
)
|
||||
.expect("body at the scoped limit should decode");
|
||||
assert_eq!(decoded, raw);
|
||||
assert_eq!(json, None);
|
||||
assert_eq!(retained.as_deref(), Some(encoded.as_str()));
|
||||
|
||||
assert!(super::decode_execution_result_body_with_limit(
|
||||
Some(ResponseBody {
|
||||
json_body: None,
|
||||
body_bytes_b64: Some(encoded),
|
||||
}),
|
||||
&mut headers,
|
||||
raw.len() - 1,
|
||||
)
|
||||
.is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,13 +2,10 @@ use std::collections::BTreeMap;
|
||||
|
||||
use aether_contracts::ExecutionPlan;
|
||||
use axum::body::Body;
|
||||
use axum::http::header::HeaderValue;
|
||||
use axum::http::Response;
|
||||
use serde_json::json;
|
||||
|
||||
use crate::api::response::{
|
||||
build_client_response_from_parts, build_client_response_from_parts_with_mutator,
|
||||
};
|
||||
use crate::api::response::build_client_response_from_parts;
|
||||
use crate::async_task::VideoTaskService;
|
||||
use crate::control::GatewayControlDecision;
|
||||
use crate::video_tasks::{
|
||||
@@ -156,35 +153,55 @@ pub(crate) fn maybe_build_local_video_error_response(
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let empty_body = json!({});
|
||||
let response_body = payload.body_json.as_ref().unwrap_or(&empty_body);
|
||||
let body_bytes =
|
||||
serde_json::to_vec(response_body).map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
let body_len = body_bytes.len().to_string();
|
||||
let response_body =
|
||||
local_video_error_response_body(payload.report_kind.as_str(), payload.status_code);
|
||||
let body_bytes = serde_json::to_vec(&response_body)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
let headers = BTreeMap::from([
|
||||
("content-type".to_string(), "application/json".to_string()),
|
||||
("content-length".to_string(), body_bytes.len().to_string()),
|
||||
]);
|
||||
|
||||
Ok(Some(build_client_response_from_parts_with_mutator(
|
||||
Ok(Some(build_client_response_from_parts(
|
||||
payload.status_code,
|
||||
&payload.headers,
|
||||
&headers,
|
||||
Body::from(body_bytes),
|
||||
trace_id,
|
||||
Some(decision),
|
||||
|headers| {
|
||||
headers.remove(http::header::CONTENT_ENCODING);
|
||||
headers.remove(http::header::CONTENT_LENGTH);
|
||||
headers.insert(
|
||||
http::header::CONTENT_TYPE,
|
||||
HeaderValue::from_static("application/json"),
|
||||
);
|
||||
headers.insert(
|
||||
http::header::CONTENT_LENGTH,
|
||||
HeaderValue::from_str(body_len.as_str())
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?,
|
||||
);
|
||||
Ok(())
|
||||
},
|
||||
)?))
|
||||
}
|
||||
|
||||
fn local_video_error_response_body(report_kind: &str, status_code: u16) -> serde_json::Value {
|
||||
let (code, gemini_status) = match status_code {
|
||||
400 => ("invalid_request", "INVALID_ARGUMENT"),
|
||||
401 => ("authentication_error", "UNAUTHENTICATED"),
|
||||
403 => ("permission_denied", "PERMISSION_DENIED"),
|
||||
404 => ("not_found", "NOT_FOUND"),
|
||||
429 => ("rate_limit_exceeded", "RESOURCE_EXHAUSTED"),
|
||||
503 => ("server_error", "UNAVAILABLE"),
|
||||
500..=599 => ("server_error", "INTERNAL"),
|
||||
_ => ("provider_error", "UNKNOWN"),
|
||||
};
|
||||
|
||||
if report_kind.starts_with("gemini_video_") {
|
||||
json!({
|
||||
"error": {
|
||||
"code": status_code,
|
||||
"message": "Video generation failed",
|
||||
"status": gemini_status,
|
||||
}
|
||||
})
|
||||
} else {
|
||||
json!({
|
||||
"error": {
|
||||
"message": "Video generation failed",
|
||||
"type": code,
|
||||
"code": code,
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -193,7 +210,7 @@ mod tests {
|
||||
use serde_json::json;
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_video_error_response_rewrites_headers_without_mutating_payload() {
|
||||
async fn local_video_error_response_does_not_expose_upstream_payload_or_headers() {
|
||||
let decision = GatewayControlDecision::synthetic(
|
||||
"/v1/videos",
|
||||
Some("ai_public".to_string()),
|
||||
@@ -212,12 +229,15 @@ mod tests {
|
||||
headers: BTreeMap::from([
|
||||
("content-encoding".to_string(), "gzip".to_string()),
|
||||
("content-length".to_string(), "999".to_string()),
|
||||
("x-upstream-id".to_string(), "video-123".to_string()),
|
||||
(
|
||||
"x-upstream-debug".to_string(),
|
||||
"Authorization: Bearer header-secret".to_string(),
|
||||
),
|
||||
]),
|
||||
body_json: Some(json!({
|
||||
"error": {
|
||||
"type": "video_backend_error",
|
||||
"message": "backend failed",
|
||||
"message": "Authorization: Bearer body-secret at https://internal.test/?key=secret",
|
||||
}
|
||||
})),
|
||||
client_body_json: None,
|
||||
@@ -239,13 +259,7 @@ mod tests {
|
||||
Some("application/json")
|
||||
);
|
||||
assert_eq!(response.headers().get(http::header::CONTENT_ENCODING), None);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get("x-upstream-id")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("video-123")
|
||||
);
|
||||
assert_eq!(response.headers().get("x-upstream-debug"), None);
|
||||
assert_eq!(
|
||||
payload.headers.get("content-encoding").map(String::as_str),
|
||||
Some("gzip")
|
||||
@@ -258,12 +272,36 @@ mod tests {
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("response body should read");
|
||||
let response_body =
|
||||
serde_json::from_slice::<serde_json::Value>(&body).expect("response body should parse");
|
||||
assert_eq!(
|
||||
serde_json::from_slice::<serde_json::Value>(&body).expect("response body should parse"),
|
||||
payload
|
||||
.body_json
|
||||
.clone()
|
||||
.expect("payload body should exist")
|
||||
response_body,
|
||||
json!({
|
||||
"error": {
|
||||
"message": "Video generation failed",
|
||||
"type": "server_error",
|
||||
"code": "server_error",
|
||||
}
|
||||
})
|
||||
);
|
||||
let encoded = response_body.to_string();
|
||||
for sensitive in ["Bearer", "body-secret", "header-secret", "internal.test"] {
|
||||
assert!(!encoded.contains(sensitive));
|
||||
}
|
||||
assert!(payload.body_json.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gemini_video_error_response_uses_fixed_schema() {
|
||||
assert_eq!(
|
||||
local_video_error_response_body("gemini_video_create_sync_finalize", 429),
|
||||
json!({
|
||||
"error": {
|
||||
"code": 429,
|
||||
"message": "Video generation failed",
|
||||
"status": "RESOURCE_EXHAUSTED",
|
||||
}
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user