mirror of
https://github.com/fawney19/Aether.git
synced 2026-10-05 00:47:48 +08:00
Compare commits
121
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1b01b08c31 | ||
|
|
2281f2b754 | ||
|
|
b5ed802277 | ||
|
|
d1b5eb08ee | ||
|
|
dba5e6e9e9 | ||
|
|
c125e78c5f | ||
|
|
d1cb0ebecf | ||
|
|
9d7a0665c0 | ||
|
|
882bb43125 | ||
|
|
db6c522d60 | ||
|
|
e29442a06a | ||
|
|
e15ea0d5d3 | ||
|
|
2f374d6af2 | ||
|
|
4a356f4ea5 | ||
|
|
c7676d567d | ||
|
|
5ca4f87951 | ||
|
|
1fee8954cc | ||
|
|
f69b770f5e | ||
|
|
856accdced | ||
|
|
92749b4d6e | ||
|
|
f08c2e6729 | ||
|
|
e420bc6324 | ||
|
|
d723fb92d3 | ||
|
|
5b1de5f921 | ||
|
|
7ed48e7b58 | ||
|
|
af712ebdbf | ||
|
|
33d5cd5993 | ||
|
|
f5e1420ee6 | ||
|
|
b37b252b14 | ||
|
|
0097ea89ad | ||
|
|
10e63507f0 | ||
|
|
9ff4d73d5c | ||
|
|
0e3bd7eff4 | ||
|
|
1c89b5f9ab | ||
|
|
cdbbda40a6 | ||
|
|
29a9d608d9 | ||
|
|
a6dc43d5f6 | ||
|
|
c6718754d3 | ||
|
|
afdd033745 | ||
|
|
d5f54ffe8b | ||
|
|
f5ec76c5c8 | ||
|
|
784a1e0611 | ||
|
|
507cb33089 | ||
|
|
b08fa3bdb6 | ||
|
|
018af84d7d | ||
|
|
27b0381a9a | ||
|
|
57cdef4b8d | ||
|
|
36e9d21e3f | ||
|
|
b72b6ab137 | ||
|
|
30b2c8548a | ||
|
|
7c5cce4b3c | ||
|
|
9362c34fcd | ||
|
|
344b3031e9 | ||
|
|
e89c3aa674 | ||
|
|
ddbbf835af | ||
|
|
cb58a63ee3 | ||
|
|
6b1074cfcd | ||
|
|
d6894b5532 | ||
|
|
635c6765d9 | ||
|
|
86f7cc0d58 | ||
|
|
206995645b | ||
|
|
9282cce1d6 | ||
|
|
c005700a7e | ||
|
|
14744abd57 | ||
|
|
66d6c17d2d | ||
|
|
c142d39951 | ||
|
|
1eb2d10dec | ||
|
|
dabaeb8dfa | ||
|
|
2d17d4b73f | ||
|
|
18d78dd6c9 | ||
|
|
499942e3e7 | ||
|
|
ba11a72214 | ||
|
|
12571764bc | ||
|
|
1e13fa032c | ||
|
|
47b21a25d3 | ||
|
|
45a3ba8829 | ||
|
|
03f2914044 | ||
|
|
c8d1ae3e7e | ||
|
|
c5ae9c2c77 | ||
|
|
4e47c00154 | ||
|
|
313a637982 | ||
|
|
fe8ff268df | ||
|
|
bac6d6866a | ||
|
|
579f2c7cc1 | ||
|
|
ddcbeb3ae9 | ||
|
|
e25fc984af | ||
|
|
4cf47b1dee | ||
|
|
95cbd43097 | ||
|
|
09005939bf | ||
|
|
7b612b8b5a | ||
|
|
670d5e8d33 | ||
|
|
1de2e70d41 | ||
|
|
89b57464d2 | ||
|
|
09ef3adf70 | ||
|
|
3dfc15963c | ||
|
|
f6884eb8c4 | ||
|
|
f8b4382a54 | ||
|
|
d78b5a81fb | ||
|
|
89fe9e9f0a | ||
|
|
4291a91dc0 | ||
|
|
4c6bafe255 | ||
|
|
979dbc4b33 | ||
|
|
9309ad844f | ||
|
|
d672ba2068 | ||
|
|
587486ab0c | ||
|
|
40a5e1470d | ||
|
|
058660ec2e | ||
|
|
668bf5e40f | ||
|
|
77f93c638d | ||
|
|
d0c0996b9f | ||
|
|
76fb8905c9 | ||
|
|
f822df6cce | ||
|
|
45c840b8d3 | ||
|
|
214f3d6406 | ||
|
|
cc6f5e89b6 | ||
|
|
2ed2cc66ef | ||
|
|
b1bf7837cf | ||
|
|
77229943d1 | ||
|
|
a0369cf49a | ||
|
|
dbbe7b22ab | ||
|
|
d5f34b2ee2 |
+41
-8
@@ -15,6 +15,11 @@ APP_PORT=8084
|
||||
# APP_IMAGE=ghcr.io/fawney19/aether:beta
|
||||
# APP_IMAGE=ghcr.io/fawney19/aether:0.7.0-rc.1
|
||||
|
||||
# Compose 应用容器的非 root 数字身份。
|
||||
# install.sh 会自动写入安装用户的 UID/GID。
|
||||
AETHER_CONTAINER_UID=65532
|
||||
AETHER_CONTAINER_GID=65532
|
||||
|
||||
# API Key 前缀(默认 sk)
|
||||
API_KEY_PREFIX=sk
|
||||
|
||||
@@ -30,17 +35,17 @@ RUST_LOG=aether_gateway=info
|
||||
# AUTH_REFRESH_COOKIE_SAMESITE=None
|
||||
# AUTH_REFRESH_COOKIE_SECURE=true
|
||||
|
||||
# 数据库配置
|
||||
# 数据库配置(仅支持 PostgreSQL)
|
||||
DB_HOST=localhost
|
||||
DB_PORT=5432
|
||||
DB_USER=postgres
|
||||
DB_NAME=aether
|
||||
DB_PASSWORD=aether
|
||||
DB_PASSWORD=
|
||||
|
||||
# Redis 配置
|
||||
REDIS_HOST=localhost
|
||||
REDIS_PORT=6379
|
||||
REDIS_PASSWORD=aether
|
||||
REDIS_PASSWORD=
|
||||
|
||||
# JWT密钥(使用 ./generate_keys.sh 生成)
|
||||
# 用于用户登录 token 签名,更换后所有用户需重新登录
|
||||
@@ -50,6 +55,10 @@ JWT_SECRET_KEY=change-this-to-a-secure-random-string
|
||||
# 注意:更换此密钥后需要在管理面板重新配置所有 Provider API Key
|
||||
ENCRYPTION_KEY=change-this-to-another-secure-random-string
|
||||
|
||||
# S3 备份的独立加密密钥(推荐)。未配置时为兼容旧部署,会回退到 ENCRYPTION_KEY。
|
||||
# 密钥轮换前必须保留旧值,离线恢复工具需要它解密历史备份。
|
||||
# AETHER_BACKUP_ENCRYPTION_KEY=change-this-to-a-dedicated-secure-random-string
|
||||
|
||||
# 启动自举管理员(仅在当前库里还没有活动管理员时生效)
|
||||
# 首次启动前必须设置 ADMIN_PASSWORD;install.sh 首次生成配置时会提示输入。
|
||||
ADMIN_EMAIL=[email protected]
|
||||
@@ -77,13 +86,37 @@ ADMIN_USERNAME=admin123456
|
||||
# AETHER_GATEWAY_DATA_POSTGRES_MAX_CONNECTIONS=80
|
||||
# AETHER_GATEWAY_MAX_IN_FLIGHT_REQUESTS=2048
|
||||
# AETHER_GATEWAY_REQUEST_BODY_BUFFER_BUDGET_MB=256
|
||||
# AETHER_GATEWAY_REQUEST_BODY_READ_TIMEOUT_MS=120000
|
||||
# 可选的 Payload 上限(MiB);默认及 0 均表示不限制。
|
||||
# AETHER_MAX_REQUEST_BODY_MB=0
|
||||
# 请求体完整读取总超时默认关闭;确需限制时配置 1000-600000 毫秒的非零值。
|
||||
# AETHER_GATEWAY_REQUEST_BODY_READ_TIMEOUT_MS=0
|
||||
# 单请求解压后 Payload 上限(MiB),默认 256;显式设为 0 才表示不限制。
|
||||
# AETHER_MAX_REQUEST_BODY_MB=256
|
||||
# AETHER_GATEWAY_SECURITY_CACHE_TTL_MS=1000
|
||||
# AETHER_MAX_REDACTED_SYNC_RESPONSE_BODY_MB=0
|
||||
# AETHER_MAX_INTERNAL_BUFFERED_BODY_MB=0
|
||||
# AETHER_MAX_REDACTED_SYNC_RESPONSE_BODY_MB=64
|
||||
# AETHER_MAX_INTERNAL_BUFFERED_BODY_MB=64
|
||||
# AETHER_TUNNEL_NODE_STATUS_QUEUE_CAPACITY=1024
|
||||
# Tunnel relay 使用的独立 HMAC 密钥。启用 HTTP tunnel relay 或多网关 owner 转发时必须配置,
|
||||
# 所有网关实例必须使用同一个至少 32 字节的随机值;不要复用 JWT 或数据加密密钥。
|
||||
# AETHER_TUNNEL_RELAY_AUTH_SECRET=
|
||||
# 旧版 /api/internal/gateway/* 控制面默认关闭。确需独立服务调用时,配置至少 32 字节的
|
||||
# 独立 HMAC 密钥;不要复用 JWT、数据加密或 tunnel relay 密钥。多节点必须使用相同值和共享 Redis。
|
||||
# AETHER_INTERNAL_GATEWAY_AUTH_SECRET=
|
||||
# 远程 relay 地址必须使用 HTTPS;HTTP 仅允许 localhost 或回环 IP。
|
||||
# AETHER_TUNNEL_RELAY_BASE_URL=https://gateway-a.example.com
|
||||
# 跨网关 relay 解析到受控私有地址时才显式开启;默认关闭以防止被篡改的 attachment
|
||||
# 记录诱导网关向内网转发 relay 凭据。该开关不放宽普通 provider 的目标地址策略。
|
||||
# AETHER_TUNNEL_RELAY_ALLOW_PRIVATE_TARGETS=false
|
||||
# 更推荐按 relay 主机名精确放行私网部署(逗号分隔,大小写不敏感);不支持通配符/后缀。
|
||||
# AETHER_TUNNEL_RELAY_PRIVATE_HOST_ALLOWLIST=gateway-a.internal,gateway-b.internal
|
||||
# Bark 自建服务默认仅允许公网 HTTPS。确需明文 HTTP 或内网目标时分别显式开启:
|
||||
# AETHER_BARK_ALLOW_HTTP=false
|
||||
# AETHER_BARK_ALLOW_PRIVATE_TARGETS=false
|
||||
|
||||
# 可选 Provider OAuth 客户端。使用 Gemini CLI / Antigravity 浏览器授权时必须配置
|
||||
# 对应的 client secret;client ID 未配置时使用内置的公开 native-app client ID。
|
||||
# AETHER_GEMINI_CLI_OAUTH_CLIENT_ID=
|
||||
# AETHER_GEMINI_CLI_OAUTH_CLIENT_SECRET=
|
||||
# AETHER_ANTIGRAVITY_OAUTH_CLIENT_ID=
|
||||
# AETHER_ANTIGRAVITY_OAUTH_CLIENT_SECRET=
|
||||
|
||||
# PostgreSQL 容器调优:docker-compose.yml 已内置通用默认值,通常不用配置。
|
||||
# 只有在 Postgres 独占大内存、或压测显示 DB 缓存/排序/维护任务成为瓶颈时再覆盖。
|
||||
|
||||
@@ -6,7 +6,8 @@ on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: build-tunnel-${{ github.ref }}
|
||||
@@ -17,7 +18,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Ensure tunnel tag matches Cargo version
|
||||
shell: bash
|
||||
@@ -78,10 +79,10 @@ jobs:
|
||||
use_cross: false
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
@@ -89,14 +90,14 @@ jobs:
|
||||
run: rustup target add ${{ matrix.target }}
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
workspaces: apps/aether-tunnel -> target
|
||||
key: ${{ matrix.target }}
|
||||
|
||||
- name: Install cross
|
||||
if: matrix.use_cross
|
||||
uses: taiki-e/install-action@cross
|
||||
uses: taiki-e/install-action@1ae7257be536a92d9218a6b343dc6e6ba650f7e1 # cross
|
||||
|
||||
- name: Build
|
||||
working-directory: apps/aether-tunnel
|
||||
@@ -122,9 +123,10 @@ jobs:
|
||||
run: |
|
||||
cd target/${{ matrix.target }}/release
|
||||
7z a ../../../aether-tunnel-${{ matrix.name }}.zip aether-tunnel.exe
|
||||
tar czf ../../../aether-tunnel-${{ matrix.name }}.tar.gz aether-tunnel.exe
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: aether-tunnel-${{ matrix.name }}
|
||||
path: |
|
||||
@@ -137,9 +139,14 @@ jobs:
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
permissions:
|
||||
actions: read
|
||||
attestations: write
|
||||
contents: write
|
||||
id-token: write
|
||||
steps:
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
merge-multiple: true
|
||||
path: artifacts
|
||||
@@ -148,6 +155,20 @@ jobs:
|
||||
working-directory: artifacts
|
||||
run: sha256sum aether-tunnel-* > SHA256SUMS.txt
|
||||
|
||||
- name: Attest tunnel release provenance
|
||||
id: attest-release
|
||||
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||
with:
|
||||
subject-path: |
|
||||
artifacts/aether-tunnel-*.tar.gz
|
||||
artifacts/aether-tunnel-*.zip
|
||||
artifacts/SHA256SUMS.txt
|
||||
|
||||
- name: Bundle tunnel release provenance
|
||||
env:
|
||||
ATTESTATION_BUNDLE: ${{ steps.attest-release.outputs.bundle-path }}
|
||||
run: install -m 0644 "${ATTESTATION_BUNDLE}" artifacts/AETHER_TUNNEL_RELEASE_PROVENANCE.sigstore.json
|
||||
|
||||
- name: Delete stale draft releases for tag
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
@@ -170,12 +191,13 @@ jobs:
|
||||
done <<< "${draft_ids}"
|
||||
|
||||
- name: Create GitHub Release
|
||||
uses: softprops/action-gh-release@v2
|
||||
uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2
|
||||
with:
|
||||
name: "${{ github.ref_name }}"
|
||||
generate_release_notes: true
|
||||
files: |
|
||||
artifacts/aether-tunnel-*
|
||||
artifacts/AETHER_TUNNEL_RELEASE_PROVENANCE.sigstore.json
|
||||
artifacts/SHA256SUMS.txt
|
||||
fail_on_unmatched_files: true
|
||||
|
||||
@@ -183,8 +205,10 @@ jobs:
|
||||
needs: release
|
||||
runs-on: ubuntu-latest
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
ref: main
|
||||
|
||||
|
||||
@@ -7,8 +7,6 @@ on:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pages: write
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: pages
|
||||
@@ -46,10 +44,10 @@ jobs:
|
||||
if: needs.preflight.outputs.deploy_pages == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v5
|
||||
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: 'npm'
|
||||
@@ -77,10 +75,10 @@ jobs:
|
||||
run: cp frontend/dist/index.html frontend/dist/404.html
|
||||
|
||||
- name: Setup Pages
|
||||
uses: actions/configure-pages@v5
|
||||
uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-pages-artifact@v3
|
||||
uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3
|
||||
with:
|
||||
path: frontend/dist
|
||||
|
||||
@@ -90,7 +88,10 @@ jobs:
|
||||
url: ${{ steps.deployment.outputs.page_url }}
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
permissions:
|
||||
id-token: write
|
||||
pages: write
|
||||
steps:
|
||||
- name: Deploy to GitHub Pages
|
||||
id: deployment
|
||||
uses: actions/deploy-pages@v4
|
||||
uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4
|
||||
|
||||
@@ -25,7 +25,6 @@ env:
|
||||
CARGO_PROFILE_TEST_DEBUG: '0'
|
||||
CARGO_TERM_COLOR: always
|
||||
RUST_BACKTRACE: '1'
|
||||
GHCR_IMAGE: ghcr.io/fawney19/aether
|
||||
|
||||
jobs:
|
||||
source:
|
||||
@@ -36,6 +35,7 @@ jobs:
|
||||
sha: ${{ steps.snapshot.outputs.sha }}
|
||||
short_sha: ${{ steps.snapshot.outputs.short_sha }}
|
||||
date: ${{ steps.snapshot.outputs.date }}
|
||||
ghcr_image: ${{ steps.snapshot.outputs.ghcr_image }}
|
||||
steps:
|
||||
- name: Require main branch
|
||||
id: snapshot
|
||||
@@ -49,9 +49,13 @@ jobs:
|
||||
fi
|
||||
|
||||
sha="${GITHUB_SHA}"
|
||||
# Docker 镜像仓库名必须全小写;GitHub owner 可能保留大写,先统一规范化。
|
||||
repository_owner="${GITHUB_REPOSITORY%%/*}"
|
||||
repository_owner="${repository_owner,,}"
|
||||
echo "sha=${sha}" >> "${GITHUB_OUTPUT}"
|
||||
echo "short_sha=${sha:0:7}" >> "${GITHUB_OUTPUT}"
|
||||
echo "date=$(date -u +'%Y-%m-%d')" >> "${GITHUB_OUTPUT}"
|
||||
echo "ghcr_image=ghcr.io/${repository_owner}/aether" >> "${GITHUB_OUTPUT}"
|
||||
echo "Building main at ${sha}."
|
||||
|
||||
# Keep the scheduled backend coverage in one place so it cannot drift from PR CI.
|
||||
@@ -66,12 +70,12 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 90
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
ref: ${{ needs.source.outputs.sha }}
|
||||
|
||||
- name: Install pinned Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
|
||||
@@ -79,13 +83,13 @@ jobs:
|
||||
run: rustc -Vv
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: nightly-rust-1.95-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Check all workspace targets
|
||||
env:
|
||||
@@ -112,12 +116,12 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
ref: ${{ needs.source.outputs.sha }}
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v5
|
||||
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: npm
|
||||
@@ -157,7 +161,7 @@ jobs:
|
||||
run: npm run build
|
||||
|
||||
- name: Upload frontend artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: nightly-frontend-dist
|
||||
path: frontend/dist/
|
||||
@@ -171,12 +175,12 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
ref: ${{ needs.source.outputs.sha }}
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v5
|
||||
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5
|
||||
with:
|
||||
node-version: '22'
|
||||
|
||||
@@ -260,25 +264,25 @@ jobs:
|
||||
os: macos-15
|
||||
use_cross: false
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
ref: ${{ needs.source.outputs.sha }}
|
||||
|
||||
- name: Install pinned Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: nightly-release-${{ matrix.target }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Install cross
|
||||
if: matrix.use_cross
|
||||
uses: taiki-e/install-action@cross
|
||||
uses: taiki-e/install-action@1ae7257be536a92d9218a6b343dc6e6ba650f7e1 # cross
|
||||
|
||||
- name: Build release binary
|
||||
env:
|
||||
@@ -295,7 +299,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Upload binary artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: nightly-gateway-${{ matrix.platform }}-${{ matrix.arch }}
|
||||
path: target/${{ matrix.target }}/release/aether-gateway
|
||||
@@ -308,17 +312,19 @@ jobs:
|
||||
needs: [source, checks, build]
|
||||
if: ${{ needs.checks.result == 'success' && needs.build.result == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
GHCR_IMAGE: ${{ needs.source.outputs.ghcr_image }}
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
ref: ${{ needs.source.outputs.sha }}
|
||||
|
||||
- name: Download Linux binaries and frontend
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
pattern: nightly-*
|
||||
path: artifacts
|
||||
@@ -335,20 +341,20 @@ jobs:
|
||||
cp -R artifacts/nightly-frontend-dist/. dist/frontend/
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6 # v3
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build and push nightly image
|
||||
uses: docker/build-push-action@v6
|
||||
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile.app
|
||||
@@ -372,12 +378,12 @@ jobs:
|
||||
actions: read
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
ref: ${{ needs.source.outputs.sha }}
|
||||
|
||||
- name: Download nightly artifacts
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
pattern: nightly-*
|
||||
path: artifacts
|
||||
@@ -434,7 +440,7 @@ jobs:
|
||||
done
|
||||
|
||||
- name: Upload nightly package artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: nightly-release-assets
|
||||
path: release-assets/*
|
||||
@@ -452,7 +458,7 @@ jobs:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Download nightly package artifact
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
name: nightly-release-assets
|
||||
path: release-assets
|
||||
@@ -466,6 +472,7 @@ jobs:
|
||||
SOURCE_SHA: ${{ needs.source.outputs.sha }}
|
||||
SOURCE_SHORT_SHA: ${{ needs.source.outputs.short_sha }}
|
||||
RELEASE_DATE: ${{ needs.source.outputs.date }}
|
||||
GHCR_IMAGE: ${{ needs.source.outputs.ghcr_image }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
|
||||
@@ -6,8 +6,8 @@ on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
packages: write
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: release-aether-${{ github.ref }}
|
||||
@@ -70,10 +70,10 @@ jobs:
|
||||
needs: preflight
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: npm
|
||||
@@ -94,7 +94,7 @@ jobs:
|
||||
npm run build
|
||||
|
||||
- name: Upload frontend artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: frontend-dist
|
||||
path: frontend/dist/
|
||||
@@ -106,10 +106,10 @@ jobs:
|
||||
needs: preflight
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: npm
|
||||
@@ -159,7 +159,7 @@ jobs:
|
||||
unzip -l "aether-vscodex-${version}.vsix" | grep 'extension/node_modules/ws/index.js' >/dev/null
|
||||
|
||||
- name: Upload VSIX artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: aether-vscodex-vsix
|
||||
path: aether-vscodex/vscode-extension/aether-vscodex-*.vsix
|
||||
@@ -199,22 +199,22 @@ jobs:
|
||||
os: macos-15
|
||||
use_cross: false
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: release-${{ matrix.target }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Install cross
|
||||
if: matrix.use_cross
|
||||
uses: taiki-e/install-action@cross
|
||||
uses: taiki-e/install-action@1ae7257be536a92d9218a6b343dc6e6ba650f7e1 # cross
|
||||
|
||||
- name: Build
|
||||
env:
|
||||
@@ -230,7 +230,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Upload binary artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: aether-gateway-${{ matrix.platform }}-${{ matrix.arch }}
|
||||
path: target/${{ matrix.target }}/release/aether-gateway
|
||||
@@ -242,11 +242,17 @@ jobs:
|
||||
needs: [preflight, frontend, build]
|
||||
if: needs.preflight.outputs.publish == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: read
|
||||
attestations: write
|
||||
contents: read
|
||||
id-token: write
|
||||
packages: write
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
path: artifacts
|
||||
|
||||
@@ -259,27 +265,27 @@ jobs:
|
||||
cp -r artifacts/frontend-dist dist/frontend
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5
|
||||
with:
|
||||
images: |
|
||||
${{ env.REGISTRY }}/${{ env.GHCR_IMAGE }}
|
||||
@@ -295,7 +301,8 @@ jobs:
|
||||
latest=false
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
id: push
|
||||
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile.app
|
||||
@@ -304,15 +311,36 @@ jobs:
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
platforms: linux/amd64,linux/arm64
|
||||
|
||||
- name: Attest GHCR image provenance
|
||||
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||
with:
|
||||
subject-name: ${{ env.REGISTRY }}/${{ env.GHCR_IMAGE }}
|
||||
subject-digest: ${{ steps.push.outputs.digest }}
|
||||
push-to-registry: true
|
||||
create-storage-record: false
|
||||
|
||||
- name: Attest Docker Hub image provenance
|
||||
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||
with:
|
||||
subject-name: docker.io/${{ env.DOCKERHUB_IMAGE }}
|
||||
subject-digest: ${{ steps.push.outputs.digest }}
|
||||
push-to-registry: true
|
||||
create-storage-record: false
|
||||
|
||||
package:
|
||||
name: Release tarballs
|
||||
needs: [preflight, frontend, build]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: read
|
||||
attestations: write
|
||||
contents: read
|
||||
id-token: write
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
path: artifacts
|
||||
|
||||
@@ -362,8 +390,24 @@ jobs:
|
||||
chmod +x release-assets/install.sh
|
||||
(cd release-assets && sha256sum *.tar.gz > SHA256SUMS)
|
||||
|
||||
- name: Attest release package provenance
|
||||
id: attest-release
|
||||
if: needs.preflight.outputs.publish == 'true'
|
||||
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||
with:
|
||||
subject-path: |
|
||||
release-assets/*.tar.gz
|
||||
release-assets/install.sh
|
||||
release-assets/SHA256SUMS
|
||||
|
||||
- name: Bundle release package provenance
|
||||
if: needs.preflight.outputs.publish == 'true'
|
||||
env:
|
||||
ATTESTATION_BUNDLE: ${{ steps.attest-release.outputs.bundle-path }}
|
||||
run: install -m 0644 "${ATTESTATION_BUNDLE}" release-assets/AETHER_RELEASE_PROVENANCE.sigstore.json
|
||||
|
||||
- name: Upload release package artifact
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||
with:
|
||||
name: release-assets
|
||||
path: release-assets/*
|
||||
@@ -375,15 +419,18 @@ jobs:
|
||||
needs: [preflight, docker, package, vscodex]
|
||||
if: needs.preflight.outputs.publish == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: read
|
||||
contents: write
|
||||
steps:
|
||||
- name: Download release package artifact
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
name: release-assets
|
||||
path: release-assets
|
||||
|
||||
- name: Download VSIX artifact
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||
with:
|
||||
name: aether-vscodex-vsix
|
||||
path: release-assets
|
||||
@@ -410,13 +457,14 @@ jobs:
|
||||
done <<< "${draft_ids}"
|
||||
|
||||
- name: Publish GitHub Release assets
|
||||
uses: softprops/action-gh-release@v2
|
||||
uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2
|
||||
with:
|
||||
generate_release_notes: true
|
||||
prerelease: ${{ needs.preflight.outputs.prerelease }}
|
||||
make_latest: ${{ needs.preflight.outputs.make_latest }}
|
||||
files: |
|
||||
release-assets/*.tar.gz
|
||||
release-assets/AETHER_RELEASE_PROVENANCE.sigstore.json
|
||||
release-assets/SHA256SUMS
|
||||
release-assets/install.sh
|
||||
release-assets/*.vsix
|
||||
|
||||
+122
-197
@@ -11,6 +11,26 @@ on:
|
||||
- "Cargo.lock"
|
||||
- "crates/**"
|
||||
- "apps/**"
|
||||
- "install.sh"
|
||||
- "deploy.sh"
|
||||
- "update.sh"
|
||||
- "generate_keys.sh"
|
||||
- ".env.example"
|
||||
- "README.md"
|
||||
- "Dockerfile.app"
|
||||
- "docker-compose.yml"
|
||||
- "docker-compose.single-node.yml"
|
||||
- "docker-compose.local.yml"
|
||||
- "docker-compose.release-local.yml"
|
||||
- "tests/compose_database_config_test.py"
|
||||
- "tests/install_*_test.sh"
|
||||
- "tests/deploy_*_test.sh"
|
||||
- "tests/update_*_test.sh"
|
||||
- "tests/release_supply_chain_test.sh"
|
||||
- "tests/tunnel_installer_config_security_test.sh"
|
||||
- ".github/workflows/build-tunnel.yml"
|
||||
- ".github/workflows/deploy-pages.yml"
|
||||
- ".github/workflows/release.yml"
|
||||
- ".github/workflows/rust-ci.yml"
|
||||
- ".github/workflows/nightly.yml"
|
||||
pull_request:
|
||||
@@ -19,6 +39,26 @@ on:
|
||||
- "Cargo.lock"
|
||||
- "crates/**"
|
||||
- "apps/**"
|
||||
- "install.sh"
|
||||
- "deploy.sh"
|
||||
- "update.sh"
|
||||
- "generate_keys.sh"
|
||||
- ".env.example"
|
||||
- "README.md"
|
||||
- "Dockerfile.app"
|
||||
- "docker-compose.yml"
|
||||
- "docker-compose.single-node.yml"
|
||||
- "docker-compose.local.yml"
|
||||
- "docker-compose.release-local.yml"
|
||||
- "tests/compose_database_config_test.py"
|
||||
- "tests/install_*_test.sh"
|
||||
- "tests/deploy_*_test.sh"
|
||||
- "tests/update_*_test.sh"
|
||||
- "tests/release_supply_chain_test.sh"
|
||||
- "tests/tunnel_installer_config_security_test.sh"
|
||||
- ".github/workflows/build-tunnel.yml"
|
||||
- ".github/workflows/deploy-pages.yml"
|
||||
- ".github/workflows/release.yml"
|
||||
- ".github/workflows/rust-ci.yml"
|
||||
- ".github/workflows/nightly.yml"
|
||||
|
||||
@@ -36,14 +76,35 @@ env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
jobs:
|
||||
shell_security:
|
||||
name: Shell security fixtures
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Run installer and supply-chain fixtures
|
||||
shell: bash
|
||||
run: |
|
||||
python3 tests/compose_database_config_test.py
|
||||
bash tests/deploy_state_safety_test.sh
|
||||
bash tests/install_archive_safety_test.sh
|
||||
bash tests/install_container_runtime_security_test.sh
|
||||
bash tests/install_current_release_link_test.sh
|
||||
bash tests/install_local_bundle_safety_test.sh
|
||||
bash tests/install_privileged_write_safety_test.sh
|
||||
bash tests/install_source_trust_test.sh
|
||||
bash tests/release_supply_chain_test.sh
|
||||
bash tests/update_compose_safety_test.sh
|
||||
bash tests/tunnel_installer_config_security_test.sh
|
||||
|
||||
fmt:
|
||||
name: Format
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
components: rustfmt
|
||||
@@ -55,22 +116,22 @@ jobs:
|
||||
name: Clippy (Gateway)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
components: clippy
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Clippy
|
||||
env:
|
||||
@@ -89,22 +150,22 @@ jobs:
|
||||
name: Clippy (Data)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
components: clippy
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Clippy
|
||||
env:
|
||||
@@ -123,22 +184,22 @@ jobs:
|
||||
name: Clippy (Workspace Rest)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
components: clippy
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Clippy
|
||||
env:
|
||||
@@ -175,28 +236,31 @@ jobs:
|
||||
name: Test (Gateway)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
|
||||
- name: Show Rust toolchain
|
||||
run: rustup show active-toolchain
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Setup mold
|
||||
uses: rui314/setup-mold@v1
|
||||
uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1
|
||||
|
||||
- name: Install nextest
|
||||
uses: taiki-e/install-action@nextest
|
||||
uses: taiki-e/install-action@d5f9268ff7620505a81ada10ddf18cdd72240185 # nextest
|
||||
|
||||
- name: Expose PostgreSQL test binaries
|
||||
run: pg_config --bindir >> "$GITHUB_PATH"
|
||||
|
||||
- name: Test lib
|
||||
env:
|
||||
@@ -225,30 +289,34 @@ jobs:
|
||||
name: Test (Data)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
|
||||
- name: Show Rust toolchain
|
||||
run: rustup show active-toolchain
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Install nextest
|
||||
uses: taiki-e/install-action@nextest
|
||||
uses: taiki-e/install-action@d5f9268ff7620505a81ada10ddf18cdd72240185 # nextest
|
||||
|
||||
- name: Expose PostgreSQL test binaries
|
||||
run: pg_config --bindir >> "$GITHUB_PATH"
|
||||
|
||||
- name: Test
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_REQUIRE_LOCAL_POSTGRES_TESTS: "true"
|
||||
run: cargo nextest run -p aether-data
|
||||
|
||||
- name: Show sccache stats
|
||||
@@ -266,23 +334,21 @@ jobs:
|
||||
matrix:
|
||||
feature:
|
||||
- postgres
|
||||
- mysql
|
||||
- sqlite
|
||||
- all-drivers
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Check selected data driver
|
||||
env:
|
||||
@@ -301,25 +367,25 @@ jobs:
|
||||
name: Test (Workspace Rest)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
|
||||
- name: Show Rust toolchain
|
||||
run: rustup show active-toolchain
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Install nextest
|
||||
uses: taiki-e/install-action@nextest
|
||||
uses: taiki-e/install-action@d5f9268ff7620505a81ada10ddf18cdd72240185 # nextest
|
||||
|
||||
- name: Test
|
||||
env:
|
||||
@@ -342,25 +408,23 @@ jobs:
|
||||
matrix:
|
||||
package:
|
||||
- aether-data-postgres
|
||||
- aether-data-mysql
|
||||
- aether-data-sqlite
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Install nextest
|
||||
uses: taiki-e/install-action@nextest
|
||||
uses: taiki-e/install-action@d5f9268ff7620505a81ada10ddf18cdd72240185 # nextest
|
||||
|
||||
- name: Test adapter
|
||||
env:
|
||||
@@ -379,19 +443,22 @@ jobs:
|
||||
name: Test (Integration Scenarios)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Expose PostgreSQL test binaries
|
||||
run: pg_config --bindir >> "$GITHUB_PATH"
|
||||
|
||||
- name: Test scenario binaries and end-to-end suites
|
||||
env:
|
||||
@@ -430,40 +497,6 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
data_db_smoke_sqlite:
|
||||
name: Data DB Smoke (SQLite)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Show Rust toolchain
|
||||
run: rustup show active-toolchain
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
|
||||
- name: Run SQLite data smoke tests
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
run: cargo test -p aether-data --all-features sqlite --lib
|
||||
|
||||
- name: Show sccache stats
|
||||
if: always()
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
run: sccache --show-stats
|
||||
|
||||
data_db_smoke_postgres:
|
||||
name: Data DB Smoke (Postgres)
|
||||
runs-on: ubuntu-latest
|
||||
@@ -482,22 +515,22 @@ jobs:
|
||||
--health-timeout=5s
|
||||
--health-retries=20
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
|
||||
- name: Show Rust toolchain
|
||||
run: rustup show active-toolchain
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
|
||||
- name: Add PostgreSQL server binaries to PATH
|
||||
run: echo "$(pg_config --bindir)" >> "$GITHUB_PATH"
|
||||
@@ -535,112 +568,6 @@ jobs:
|
||||
AETHER_TEST_POSTGRES_URL: postgres://aether:[email protected]:5432/aether_test
|
||||
run: cargo test -p aether-data --all-features postgres_core_export_reads_migrated_database_rows_when_url_is_set --lib -- --nocapture
|
||||
|
||||
- name: Run SQLite-to-Postgres import smoke test
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_POSTGRES_URL: postgres://aether:[email protected]:5432/aether_test
|
||||
run: cargo test -p aether-data --all-features sqlite_core_export_reads_migrated_database_rows --lib -- --nocapture
|
||||
|
||||
- name: Show sccache stats
|
||||
if: always()
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
run: sccache --show-stats
|
||||
|
||||
data_db_smoke_mysql:
|
||||
name: Data DB Smoke (MySQL)
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
mysql:
|
||||
image: mysql:8.0
|
||||
env:
|
||||
MYSQL_DATABASE: aether_test
|
||||
MYSQL_USER: aether
|
||||
MYSQL_PASSWORD: aether
|
||||
MYSQL_ROOT_PASSWORD: aether_root
|
||||
ports:
|
||||
- 3306:3306
|
||||
options: >-
|
||||
--health-cmd="mysqladmin ping -h 127.0.0.1 -uaether -paether --silent"
|
||||
--health-interval=5s
|
||||
--health-timeout=5s
|
||||
--health-retries=20
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Show Rust toolchain
|
||||
run: rustup show active-toolchain
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/[email protected]
|
||||
|
||||
- name: Run MySQL migration smoke test
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
||||
run: cargo test -p aether-data --all-features mysql_migrations_create_core_config_tables_when_url_is_set --lib -- --nocapture
|
||||
|
||||
- name: Run MySQL usage write smoke test
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
||||
run: cargo test -p aether-data-mysql mysql_usage_write_repository_upserts_and_flushes_counters_when_url_is_set --lib -- --nocapture
|
||||
|
||||
- name: Run MySQL usage read smoke test
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
||||
run: cargo test -p aether-data-mysql mysql_usage_read_repository_reads_usage_contract_views_when_url_is_set --lib -- --nocapture
|
||||
|
||||
- name: Run MySQL provider catalog smoke test
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
||||
run: cargo test -p aether-data-mysql mysql_provider_catalog_repository_round_trips_when_url_is_set --lib -- --nocapture
|
||||
|
||||
- name: Run MySQL core export smoke test
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
||||
run: cargo test -p aether-data --all-features mysql_core_export_reads_migrated_database_rows_when_url_is_set --lib -- --nocapture
|
||||
|
||||
- name: Run MySQL wallet read smoke test
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
||||
run: cargo test -p aether-data-mysql mysql_wallet_read_repository_reads_wallet_contract_views --lib -- --nocapture
|
||||
|
||||
- name: Run MySQL wallet daily usage aggregation smoke test
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
||||
run: cargo test -p aether-data --all-features mysql_wallet_daily_usage_aggregation_uses_settlement_wallets_when_url_is_set --lib -- --nocapture
|
||||
|
||||
- name: Run MySQL stats aggregation smoke test
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
||||
run: cargo test -p aether-data --all-features mysql_stats_aggregation_runs_after_mysql_migrations_when_url_is_set --lib -- --nocapture
|
||||
|
||||
- name: Show sccache stats
|
||||
if: always()
|
||||
env:
|
||||
@@ -652,16 +579,12 @@ jobs:
|
||||
name: Data DB Smoke
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- data_db_smoke_sqlite
|
||||
- data_db_smoke_postgres
|
||||
- data_db_smoke_mysql
|
||||
if: ${{ always() }}
|
||||
steps:
|
||||
- name: Verify database smoke jobs
|
||||
run: |
|
||||
if [ "${{ needs.data_db_smoke_sqlite.result }}" != "success" ] || \
|
||||
[ "${{ needs.data_db_smoke_postgres.result }}" != "success" ] || \
|
||||
[ "${{ needs.data_db_smoke_mysql.result }}" != "success" ]; then
|
||||
if [ "${{ needs.data_db_smoke_postgres.result }}" != "success" ]; then
|
||||
echo "Data DB smoke failed"
|
||||
exit 1
|
||||
fi
|
||||
@@ -674,6 +597,7 @@ jobs:
|
||||
- clippy
|
||||
- test
|
||||
- data_db_smoke
|
||||
- shell_security
|
||||
if: ${{ always() }}
|
||||
steps:
|
||||
- name: Verify required jobs
|
||||
@@ -681,7 +605,8 @@ jobs:
|
||||
if [ "${{ needs.fmt.result }}" != "success" ] || \
|
||||
[ "${{ needs.clippy.result }}" != "success" ] || \
|
||||
[ "${{ needs.test.result }}" != "success" ] || \
|
||||
[ "${{ needs.data_db_smoke.result }}" != "success" ]; then
|
||||
[ "${{ needs.data_db_smoke.result }}" != "success" ] || \
|
||||
[ "${{ needs.shell_security.result }}" != "success" ]; then
|
||||
echo "Rust CI failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
Generated
+594
-385
File diff suppressed because it is too large
Load Diff
+3
-5
@@ -11,8 +11,6 @@ members = [
|
||||
"crates/aether-routing-core",
|
||||
"crates/aether-data/contracts",
|
||||
"crates/aether-data/adapters/postgres",
|
||||
"crates/aether-data/adapters/mysql",
|
||||
"crates/aether-data/adapters/sqlite",
|
||||
"crates/aether-data/query",
|
||||
"crates/aether-data/schema",
|
||||
"crates/aether-dispatch-core",
|
||||
@@ -66,8 +64,6 @@ aether-provider-pool = { path = "crates/aether-provider/pool" }
|
||||
aether-routing-core = { path = "crates/aether-routing-core" }
|
||||
aether-data-contracts = { path = "crates/aether-data/contracts" }
|
||||
aether-data-postgres = { path = "crates/aether-data/adapters/postgres" }
|
||||
aether-data-mysql = { path = "crates/aether-data/adapters/mysql" }
|
||||
aether-data-sqlite = { path = "crates/aether-data/adapters/sqlite" }
|
||||
aether-data-query = { path = "crates/aether-data/query" }
|
||||
aether-data-schema = { path = "crates/aether-data/schema" }
|
||||
aether-dispatch-core = { path = "crates/aether-dispatch-core" }
|
||||
@@ -101,6 +97,7 @@ aether-runtime = { path = "crates/aether-runtime/base" }
|
||||
aether-testkit = { path = "crates/aether-testing/testkit" }
|
||||
aes = "0.8"
|
||||
aes-gcm = "0.10"
|
||||
aws-lc-rs = { version = "1.16.2", default-features = false, features = ["alloc", "aws-lc-sys"] }
|
||||
async-stream = "0.3"
|
||||
async-trait = "0.1"
|
||||
axum = "0.8"
|
||||
@@ -117,8 +114,9 @@ flate2 = "1"
|
||||
futures-util = "0.3"
|
||||
hmac = "0.12"
|
||||
http = "1"
|
||||
object_store = { version = "0.12", default-features = false, features = ["aws"] }
|
||||
object_store = { version = "0.14.1", default-features = false, features = ["aws"] }
|
||||
pbkdf2 = { version = "0.12", default-features = false, features = ["hmac"] }
|
||||
percent-encoding = "2"
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "rustls-tls", "http2", "socks"] }
|
||||
redis = { version = "0.28", default-features = false, features = ["tokio-comp", "script", "streams", "connection-manager"] }
|
||||
regex = "1"
|
||||
|
||||
+9
-4
@@ -10,20 +10,24 @@
|
||||
|
||||
# --- layout stage: create /opt/aether directory structure with symlink ---
|
||||
# distroless has no shell, so we use busybox to set up the symlink.
|
||||
FROM busybox:1.37-musl AS layout
|
||||
FROM busybox:1.37.0-musl@sha256:fc6dddc4c44b1bfe37f41cae8e67d1693828e8f42a91862816d7953e2c9d3f23 AS layout
|
||||
|
||||
ARG TARGETARCH
|
||||
|
||||
RUN mkdir -p /opt/aether/releases/image/bin /opt/aether/releases/image/frontend /opt/aether/logs
|
||||
|
||||
COPY dist/aether-gateway-${TARGETARCH} /opt/aether/releases/image/bin/aether-gateway
|
||||
RUN chmod 0755 /opt/aether/releases/image/bin/aether-gateway
|
||||
COPY dist/frontend/ /opt/aether/releases/image/frontend/
|
||||
|
||||
# Keep the immutable release root-owned while guaranteeing that the runtime
|
||||
# identity can traverse and read every packaged asset.
|
||||
RUN chmod -R u=rwX,go=rX /opt/aether/releases/image \
|
||||
&& chmod 0755 /opt/aether/releases/image/bin/aether-gateway
|
||||
|
||||
RUN ln -s /opt/aether/releases/image /opt/aether/current
|
||||
|
||||
# --- final stage: distroless runtime ---
|
||||
FROM gcr.io/distroless/static-debian12
|
||||
FROM gcr.io/distroless/static-debian12@sha256:6447365a6337c3732f412d1b74357b30a633831955b2bc45552b0086be907687
|
||||
|
||||
COPY --from=layout /opt/aether /opt/aether
|
||||
|
||||
@@ -31,6 +35,7 @@ WORKDIR /opt/aether
|
||||
|
||||
ENV RUST_LOG=aether_gateway=info \
|
||||
APP_PORT=8084 \
|
||||
HOME=/tmp/aether-home \
|
||||
AETHER_UPDATE_STRATEGY=docker \
|
||||
AETHER_GATEWAY_STATIC_DIR=/opt/aether/current/frontend
|
||||
|
||||
@@ -39,5 +44,5 @@ EXPOSE 8084
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
||||
CMD ["/opt/aether/current/bin/aether-gateway", "--healthcheck"]
|
||||
|
||||
USER root
|
||||
USER 65532:65532
|
||||
ENTRYPOINT ["/opt/aether/current/bin/aether-gateway"]
|
||||
|
||||
@@ -36,15 +36,6 @@ lowercase() {
|
||||
printf '%s' "$$1" | tr '[:upper:]' '[:lower:]'
|
||||
}
|
||||
|
||||
dev_uses_sqlite_database() {
|
||||
local driver
|
||||
local url
|
||||
driver="$$(lowercase "$${AETHER_DATABASE_DRIVER:-}")"
|
||||
url="$${AETHER_DATABASE_URL:-$${DATABASE_URL:-}}"
|
||||
|
||||
[[ "$${driver}" == "sqlite" || "$${url}" == sqlite:* ]]
|
||||
}
|
||||
|
||||
dev_uses_postgres_database() {
|
||||
local driver
|
||||
local url
|
||||
@@ -68,9 +59,6 @@ dev_uses_redis_runtime() {
|
||||
if [[ "$${backend}" == "redis" ]]; then
|
||||
return 0
|
||||
fi
|
||||
if dev_uses_sqlite_database; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
@@ -356,6 +344,9 @@ if ! ensure_dev_infra; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=> 编译 aether-gateway..."
|
||||
cargo build -p aether-gateway --bin aether-gateway
|
||||
|
||||
GATEWAY_PID=""
|
||||
GATEWAY_LOG_DIR=""
|
||||
GATEWAY_LOG_FILE=""
|
||||
@@ -364,8 +355,8 @@ create_gateway_log_file
|
||||
|
||||
echo "=> 启动 aether-gateway (Rust frontdoor: 0.0.0.0:$${APP_PORT})..."
|
||||
echo "=> 日志过滤: $${RUST_LOG}"
|
||||
echo "=> 执行命令: cargo run -p aether-gateway --bin aether-gateway -- --app-port $${APP_PORT}"
|
||||
cargo run -p aether-gateway --bin aether-gateway -- --app-port "$${APP_PORT}" > >(
|
||||
echo "=> 执行命令: target/debug/aether-gateway --app-port $${APP_PORT}"
|
||||
target/debug/aether-gateway --app-port "$${APP_PORT}" > >(
|
||||
tee -a "$${GATEWAY_LOG_FILE}"
|
||||
) 2>&1 &
|
||||
GATEWAY_PID=$$!
|
||||
@@ -456,7 +447,7 @@ if [ -f .env ]; then
|
||||
fi
|
||||
export APP_PORT="$${APP_PORT:-8084}"
|
||||
|
||||
echo "=> 启动后端: RUST_LOG=$${DEV_RUST_LOG} cargo run -p aether-gateway --bin aether-gateway -- --app-port $${APP_PORT:-8084}"
|
||||
echo "=> 启动后端: 先编译 aether-gateway,再运行 target/debug/aether-gateway --app-port $${APP_PORT:-8084}"
|
||||
/bin/bash -euo pipefail -c "$$DEV_BACKEND_SCRIPT" &
|
||||
backend_pid=$$!
|
||||
|
||||
|
||||
@@ -44,17 +44,22 @@ cd Aether
|
||||
|
||||
# 2. 配置环境变量
|
||||
cp .env.example .env
|
||||
# 生成 JWT_SECRET_KEY / ENCRYPTION_KEY, 并填入 .env
|
||||
# .env 包含数据库、JWT 和数据加密密钥,先限制为仅当前用户可读写
|
||||
chmod 600 .env
|
||||
# 生成 JWT / 加密 / Postgres / Redis 独立随机密钥,并填入 .env
|
||||
./generate_keys.sh
|
||||
# 编辑 .env 设置 ADMIN_PASSWORD
|
||||
|
||||
# 3. 首次部署 / 更新 (从以下部署形态任选其一)
|
||||
# Postgres + Redis (适用于企业或多人使用)
|
||||
# Postgres + Redis (推荐)
|
||||
docker compose pull && docker compose up -d
|
||||
# Single Node (适用于个人用户或朋友分享)
|
||||
# Single Node:同样使用 PostgreSQL + Redis,无需挂载本地数据库文件
|
||||
docker compose -f docker-compose.single-node.yml pull && docker compose -f docker-compose.single-node.yml up -d
|
||||
```
|
||||
|
||||
应用镜像默认以固定非 root 身份 `65532:65532` 运行;Compose 移除全部 Linux capabilities、禁止提权、启用只读根文件系统,并提供带 `nosuid,nodev,noexec` 的 `/tmp`。如需使用其他身份,可在 `.env` 中设置非零的 `AETHER_CONTAINER_UID` / `AETHER_CONTAINER_GID`。数据库使用独立 PostgreSQL 容器和 named volume,不再需要调整应用数据库目录的权限。
|
||||
|
||||
|
||||
### 一键更新
|
||||
|
||||
Docker Compose 部署后,可在部署目录直接执行:
|
||||
@@ -69,13 +74,29 @@ Docker Compose 部署后,可在部署目录直接执行:
|
||||
./update.sh --mode single-node
|
||||
```
|
||||
|
||||
仓库自带的 Docker Compose 默认把应用日志输出到容器 `stdout/stderr`,直接用 `docker compose logs -f app` 查看,并由 Docker 轮转日志,避免正式发布镜像切换到非 root 用户后再被宿主机挂载日志目录的权限问题拖垮启动。如果你确实需要文件日志,需要在 compose 里把 `AETHER_LOG_DESTINATION` 改成 `file|both`,并额外挂载一个容器用户可写的目录到 `/opt/aether/logs`。
|
||||
现在仅支持 PostgreSQL。标准和单节点 Docker Compose 均部署 PostgreSQL + Redis;原生 systemd / launchd 安装需要显式提供 PostgreSQL `DATABASE_URL`,例如 `DATABASE_URL=postgresql://user:password@host:5432/aether`。旧数据库不会自动迁移或清空。升级时保留原有 PostgreSQL 密码、`JWT_SECRET_KEY` 和 `ENCRYPTION_KEY`,不要重新生成整个 `.env`。
|
||||
|
||||
仓库自带的 Docker Compose 默认把应用日志输出到容器 `stdout/stderr`,直接用 `docker compose logs -f app` 查看,并由 Docker 轮转日志,避免非 root 用户被宿主机日志目录权限拖垮启动。如果你确实需要文件日志,需要在 compose 里把 `AETHER_LOG_DESTINATION` 改成 `file|both`,额外挂载目录到 `/opt/aether/logs`,并让它归 `.env` 中配置的容器 UID/GID 所有;只读根文件系统不会阻止显式可写挂载。
|
||||
|
||||
管理后台右上角“版本信息”会检测新版本。Docker Compose 部署只提示版本,实际更新继续执行 `./update.sh`;systemd / launchd / 二进制部署才使用后台自更新,流程是下载对应平台的 GitHub Release 包、强制校验 `SHA256SUMS`、解压到 `/opt/aether/releases/<version>`,再切换 `/opt/aether/current` 并退出进程,交给 systemd / launchd 拉起新版本。
|
||||
|
||||
正式 Release 还会发布由 GitHub Actions OIDC / Sigstore 签发的 SLSA build provenance。需要验证发布者身份时,下载目标 tarball 和 `AETHER_RELEASE_PROVENANCE.sigstore.json`,并把 `TAG` 设置为对应 Release tag:
|
||||
|
||||
```bash
|
||||
gh attestation verify "aether-${TAG}-linux-amd64.tar.gz" \
|
||||
--repo fawney19/Aether \
|
||||
--signer-workflow fawney19/Aether/.github/workflows/release.yml \
|
||||
--source-ref "refs/tags/${TAG}" \
|
||||
--bundle AETHER_RELEASE_PROVENANCE.sigstore.json
|
||||
```
|
||||
|
||||
`docker-compose.yml` 中的官方 PostgreSQL 和 Redis 镜像均固定到多架构 OCI index digest。升级这些依赖时应在发布变更中显式更新 digest,避免同名 tag 在无人审查的情况下改变部署内容。
|
||||
|
||||
正式发布到 GHCR 和 Docker Hub 的多架构 Aether 镜像也带有同一 GitHub Actions OIDC / Sigstore provenance;生产 `Dockerfile.app` 的 BusyBox 与 Distroless 基础镜像同样固定到多架构 OCI index digest。
|
||||
|
||||
源码或本地构建版本不会启用后台在线更新,请继续使用源码更新流程。Docker Compose 用户如果希望“容器重建后也保持镜像层面的新版本”,仍建议定期运行 `./update.sh` 拉取并重建 app 镜像。服务器访问 GitHub 需要代理时,可设置 `AETHER_UPDATE_PROXY_URL`,也兼容 `UPDATE_PROXY_URL`、`HTTPS_PROXY`、`ALL_PROXY`、`HTTP_PROXY` 以及 `NO_PROXY`。共享出口触发 GitHub API 限流时,可设置只读 `AETHER_UPDATE_GITHUB_TOKEN`,也兼容 `GITHUB_TOKEN` / `GH_TOKEN`。下载总超时默认 600 秒,连续无响应/无数据默认 30 秒,可通过 `AETHER_UPDATE_DOWNLOAD_TIMEOUT_SECS` 和 `AETHER_UPDATE_DOWNLOAD_IDLE_TIMEOUT_SECS` 调整。
|
||||
|
||||
标准 Docker Compose 使用 Docker named volumes 存放 Postgres/Redis/MySQL 数据;Single Node 使用部署目录下的 `./data` 存放 SQLite 数据。
|
||||
标准和 Single Node Docker Compose 均使用 Docker named volume 存放 PostgreSQL 数据。
|
||||
|
||||
如果是本地源码构建镜像的部署,继续使用:
|
||||
|
||||
@@ -105,25 +126,27 @@ docker compose -f docker-compose.release-local.yml down -v
|
||||
- `AETHER_RELEASE_LOCAL_PORT`:本地联调端口,默认 `18085`
|
||||
- `LOCAL_RELEASE_APP_IMAGE`:本地联调镜像名,默认 `aether-app:release-local`
|
||||
|
||||
### 一键安装(默认 Single Node:Linux systemd / macOS launchd + SQLite)
|
||||
### 一键安装(PostgreSQL + Redis)
|
||||
|
||||
```bash
|
||||
git clone https://github.com/fawney19/Aether.git
|
||||
cd Aether
|
||||
curl -fsSL https://raw.githubusercontent.com/fawney19/Aether/main/install.sh | sudo bash
|
||||
curl -fsSL https://raw.githubusercontent.com/fawney19/Aether/main/install.sh | sudo bash -s -- --mode compose
|
||||
```
|
||||
|
||||
原生 Linux systemd / macOS launchd 安装需先准备 PostgreSQL,将连接串通过 `DATABASE_URL` 传给安装进程,并选择 `--mode single-node`;不再自动创建本地数据库文件。
|
||||
|
||||
### Nightly(每日 main 构建)
|
||||
|
||||
Nightly workflow 每天从 `main` 的固定 commit 构建并发布滚动的 GitHub Release `nightly`,同时推送多架构 GHCR 镜像 `ghcr.io/fawney19/aether:nightly`。Nightly 是预发布版本,适合验证最新代码,不保证与正式版相同的稳定性。滚动 Release 需要仓库保持关闭 GitHub Release immutability。
|
||||
|
||||
安装最新 nightly(Linux systemd / macOS launchd + SQLite):
|
||||
安装最新 nightly(PostgreSQL + Redis):
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/fawney19/Aether/main/install.sh | sudo bash -s -- --channel nightly
|
||||
curl -fsSL https://raw.githubusercontent.com/fawney19/Aether/main/install.sh | sudo bash -s -- --mode compose --channel nightly
|
||||
```
|
||||
|
||||
Docker Compose 用户可在部署目录的 `.env` 中设置 `APP_IMAGE=ghcr.io/fawney19/aether:nightly`,然后运行 `./update.sh` 获取下一次 nightly。二进制方式可重新执行上述安装命令升级;当前管理后台的在线更新列表只跟踪正式版/RC/Beta,不会自动提示下一次 nightly。
|
||||
Docker Compose 用户可在部署目录的 `.env` 中设置 `APP_IMAGE=ghcr.io/fawney19/aether:nightly`,然后运行 `./update.sh` 获取下一次 nightly。二进制部署请沿用已有 PostgreSQL 环境配置,并使用 `--mode single-node --channel nightly` 重新运行安装脚本升级;当前管理后台的在线更新列表只跟踪正式版/RC/Beta,不会自动提示下一次 nightly。
|
||||
|
||||
## 本地开发
|
||||
|
||||
@@ -136,7 +159,7 @@ make dev
|
||||
|
||||
`make dev` 会同时启动后端 `aether-gateway` 和前端 `frontend` 的 Vite dev server。需要单独启动时可使用 `make dev-backend` 或 `make dev-frontend`。
|
||||
Postgres / Redis 本地依赖未就绪时,`make dev` 会自动执行 `docker compose up -d postgres redis`。
|
||||
数据库 schema 和历史数据准备也会在启动时自动完成;通常不需要手动区分 migration 与 backfill。排查或部署前预执行时可使用:
|
||||
`make dev` 会先完成后端编译,再开始计算服务健康检查超时。数据库 schema 和必要的派生数据准备也会在启动时自动完成;通常不需要手动区分 migration 与 backfill。升级不会主动重写或清除已有业务历史记录,新写入会直接遵循当前的数据持久化策略。排查或部署前预执行时可使用:
|
||||
|
||||
```bash
|
||||
make db-status
|
||||
@@ -168,26 +191,46 @@ Aether Tunnel 是配套的正向代理节点,部署在海外 VPS 上,为墙
|
||||
## 环境变量
|
||||
|
||||
- `APP_PORT`:`aether-gateway` 唯一监听端口,固定绑定 `0.0.0.0:${APP_PORT}`
|
||||
- `DATABASE_URL`:数据库连接串;SQLite 例如 `sqlite:///opt/aether/data/aether.db`,Postgres 例如 `postgresql://postgres:aether@postgres:5432/aether`
|
||||
- `AETHER_GATEWAY_DATA_POSTGRES_MIN_CONNECTIONS` / `AETHER_GATEWAY_DATA_POSTGRES_MAX_CONNECTIONS`:数据库连接池手动覆盖值;未配置时 SQLite 固定 `1/1`,Postgres/MySQL 按每核 `4` 条自动推导,总池范围为 `32-100`。该预算按进程计算,多实例部署应按数据库连接上限显式分配
|
||||
- `DATABASE_URL`:PostgreSQL 连接串,例如 `postgresql://USER:PASSWORD@HOST:5432/aether`
|
||||
- `AETHER_GATEWAY_DATA_POSTGRES_MIN_CONNECTIONS` / `AETHER_GATEWAY_DATA_POSTGRES_MAX_CONNECTIONS`:数据库连接池手动覆盖值;未配置时 PostgreSQL 按每核 `4` 条自动推导,总池范围为 `32-100`。该预算按进程计算,多实例部署应按数据库连接上限显式分配
|
||||
- `AETHER_GATEWAY_MAX_IN_FLIGHT_REQUESTS`:单实例请求并发上限;未配置时按 CPU 自动推导(基础范围 `512-65536`),低文件描述符预算时会进一步下调
|
||||
- `AETHER_GATEWAY_REQUEST_BODY_BUFFER_BUDGET_MB`:单实例同时读取和解压请求体的加权内存预算,默认 `256MB`
|
||||
- `AETHER_GATEWAY_REQUEST_BODY_READ_TIMEOUT_MS`:请求体完整读取超时,默认 `120000ms`
|
||||
- `AETHER_MAX_REQUEST_BODY_MB`:可选的单请求解压后请求体上限;未配置或设为 `0` 时不限制
|
||||
- `AETHER_MAX_INTERNAL_BUFFERED_BODY_MB`:可选的 heartbeat、管理探测等内部整包响应体上限;未配置或设为 `0` 时不限制
|
||||
- `AETHER_GATEWAY_REQUEST_BODY_READ_TIMEOUT_MS`:可选的请求体完整读取超时;默认或显式设为 `0` 时关闭,非零值限制在 `1000-600000ms`
|
||||
- `AETHER_MAX_REQUEST_BODY_MB`:单请求解压后请求体上限,默认 `256MB`;显式设为 `0` 表示不再收紧默认值,但仍受 `256MB` 安全硬上限约束
|
||||
- `AETHER_MAX_INTERNAL_BUFFERED_BODY_MB`:heartbeat、管理探测等内部整包响应体上限,默认 `64MB`;显式设为 `0` 表示不再收紧默认值,但仍受 `256MB` 安全硬上限约束
|
||||
- `AETHER_TUNNEL_NODE_STATUS_QUEUE_CAPACITY`:隧道节点状态上报队列容量,默认 `1024`;满载时拒绝新事件,避免控制面故障导致无界内存增长
|
||||
- `AETHER_TUNNEL_RELAY_ALLOW_PRIVATE_TARGETS`:跨网关 owner relay 解析到私有/保留地址时的显式运维开关,默认关闭;仅当多网关 relay URL 是受控的内网 HTTPS 地址时设置为 `true`。它不改变普通 provider 请求的 DNS/代理策略,也不允许明文 HTTP 非 loopback relay
|
||||
- `AETHER_TUNNEL_RELAY_PRIVATE_HOST_ALLOWLIST`:更窄的 owner relay 私网例外,填写逗号分隔的精确主机名(例如 `gateway-a.internal,gateway-b.internal`,忽略大小写和末尾点);仅这些主机解析出的私有地址会被允许,并且请求仍使用解析后地址 pin。不要填写通配符或 `.internal` 这类后缀
|
||||
- `AETHER_INTERNAL_GATEWAY_AUTH_SECRET`:旧版 `/api/internal/gateway/*` 高权限控制面的独立 HMAC 密钥,至少 `32` 字节;未配置时该控制面返回 `404`。不要复用 JWT、数据加密或 tunnel relay 密钥,多节点必须使用同一值及共享 Redis 防重放
|
||||
- `AETHER_GATEWAY_SECURITY_CACHE_TTL_MS`:IP 黑白名单本地缓存时间,默认 `1000ms`,写操作会主动失效相关缓存
|
||||
- `AETHER_MAX_REDACTED_SYNC_RESPONSE_BODY_MB`:可选的 PII 恢复同步响应缓冲上限;未配置或设为 `0` 时不限制
|
||||
- `AETHER_MAX_REDACTED_SYNC_RESPONSE_BODY_MB`:PII 恢复同步响应缓冲上限,默认 `64MB`;显式设为 `0` 表示不再收紧默认值,但仍受 `256MB` 安全硬上限约束
|
||||
- `REDIS_URL`:Redis 连接串;仅 Postgres + Redis 的 Docker Compose 部署需要配置
|
||||
- `AETHER_RUNTIME_BACKEND=memory|redis`:运行时缓存/协调后端。SQLite 默认用 `memory`,不会连接 Redis;多节点部署和需要跨 gateway 重启恢复 OpenAI Responses continuation history 的部署必须使用共享 Redis
|
||||
- `AETHER_RUNTIME_BACKEND=memory|redis`:运行时缓存/协调后端。配置 Redis 时使用 `redis`,否则使用 `memory`;多节点部署和需要跨 gateway 重启恢复 OpenAI Responses continuation history 的部署必须使用共享 Redis
|
||||
- `AETHER_GATEWAY_DATABASE_MODE=auto|verify-only`:数据库启动策略,默认 `auto`,自动完成挂起的 schema migration 和 backfill;`verify-only` 仅检查并在数据库落后时拒绝启动
|
||||
- `AETHER_GATEWAY_AUTO_PREPARE_DATABASE`:旧版兼容开关;新配置请使用 `AETHER_GATEWAY_DATABASE_MODE`
|
||||
- `JWT_SECRET_KEY` / `ENCRYPTION_KEY`:认证和敏感数据加密所需密钥
|
||||
- `AETHER_BACKUP_ENCRYPTION_KEY`:推荐的 S3 备份独立加密密钥;缺省回退到 `ENCRYPTION_KEY`。新备份使用带 key ID 的 AES-256-GCM v2 envelope,轮换前必须保留旧密钥
|
||||
- `API_KEY_PREFIX`:用户和管理员新建 API Key 时使用的前缀,默认 `sk`
|
||||
- `ADMIN_USERNAME` / `ADMIN_PASSWORD` / `ADMIN_EMAIL`:首次启动时自举首个本地管理员;`install.sh` 会提示输入管理员密码
|
||||
- `CORS_ORIGINS` / `CORS_ALLOW_CREDENTIALS`:前端跨域来源控制;如果要跨域带登录 Cookie,`CORS_ORIGINS` 不能写 `*`
|
||||
- `RUST_LOG`:Rust 日志过滤,例如 `aether_gateway=info`、`aether_gateway=debug,sqlx=warn`
|
||||
- Docker Compose 的 `DB_PASSWORD` / `REDIS_PASSWORD` 默认使用 `aether`
|
||||
- `DB_PASSWORD` / `REDIS_PASSWORD`:Docker Compose 后端密码,首次安装时分别随机生成;手工部署必须替换示例占位值,不要互相复用
|
||||
|
||||
### S3 备份离线恢复
|
||||
|
||||
先从 S3 下载完整的 `.json.zst.aes256gcm` 对象,再使用原始的完整 S3 object key 做认证解密。恢复工具只验证并输出本地 JSON,不会直接写数据库;数据库导入仍应在维护窗口通过管理端完成。
|
||||
|
||||
```bash
|
||||
AETHER_BACKUP_ENCRYPTION_KEY='原备份密钥' \
|
||||
cargo run -p aether-gateway --bin aether-backup-restore -- \
|
||||
--input ./backup.json.zst.aes256gcm \
|
||||
--object-key 'aether/backups/aether-data-backup-20260822-010000.json.zst.aes256gcm' \
|
||||
--output ./restored-backup.json
|
||||
```
|
||||
|
||||
工具默认拒绝覆盖,输出采用原子写并在 Unix 上设置为 `0600`;Unix 可用 `--overwrite` 原子替换,Windows 为避免非原子删除窗口会要求选择新输出路径。密钥不能作为命令行参数。可使用 `AETHER_BACKUP_ENCRYPTION_KEY`、兼容用 `AETHER_GATEWAY_DATA_ENCRYPTION_KEY` / `ENCRYPTION_KEY`、受保护的 `--key-file`,或 `AETHER_BACKUP_KEYRING_FILE`。Keyring JSON 格式为 `{"version":1,"keys":["当前或历史 v2 secret"],"legacy_v1":["旧 v1 secret"]}`;条目也可写成 `{"secret":"..."}`(兼容字段名 `key`)。也可由 `AETHER_BACKUP_HISTORICAL_KEYS_JSON` 提供同一结构。密钥文件必须是非符号链接的普通文件,Unix 下权限需为 `0600` 或更严格。
|
||||
|
||||
默认限制密文为 `512MiB`、解压后 JSON 为 `1GiB`,可通过受限的 `--max-encrypted-mib` / `--max-json-mib` 调整。网关最多扫描同一备份前缀下 10,000 个对象,并且不会自动删除 S3 对象:`backup_s3_retention_count` 只用于报告超出保留数量的清理候选。旧明文备份在创建并验证加密副本后仍会保留,必须通过 bucket lifecycle 或支持版本条件的外部清理工具移除;启用 Versioning 时还需清理 noncurrent versions,Object Lock/retention 可能阻止物理删除。
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -92,7 +92,7 @@ npm --prefix aether-vscodex/web run build
|
||||
`vscodex_data`,短期配对码、60 秒一次性浏览器票据和在线房间保存在内存。不要在未引入
|
||||
共享连接目录前横向扩容 sidecar。
|
||||
|
||||
登录 Aether 后打开“Codex 远程控制”,生成一次性配对码。然后在 VS Code 命令面板执行
|
||||
登录 Aether 后打开“远程控制”,生成一次性配对码。然后在 VS Code 命令面板执行
|
||||
**Codex Remote: Pair with Aether**,填写 Aether 地址和配对码。插件会把设备凭据写入
|
||||
VS Code SecretStorage,并同时保持本机控制台连接。
|
||||
|
||||
|
||||
@@ -65,14 +65,14 @@ http.workspace = true
|
||||
http-body-util = "0.1"
|
||||
hyper = { version = "1", features = ["client", "server", "http1", "http2"] }
|
||||
hyper-util = { version = "0.1", features = ["client-legacy", "client-pool", "server-auto", "service", "tokio"] }
|
||||
ldap3 = { version = "0.11", default-features = false, features = ["sync", "tls-rustls"] }
|
||||
ldap3 = { version = "0.12.1", default-features = false, features = ["sync", "tls-rustls-ring"] }
|
||||
libc = "0.2"
|
||||
md-5 = "0.10"
|
||||
object_store.workspace = true
|
||||
parking_lot = "0.12"
|
||||
percent-encoding.workspace = true
|
||||
regex.workspace = true
|
||||
reqwest.workspace = true
|
||||
rsa = "0.9.10"
|
||||
rustls.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
@@ -81,7 +81,7 @@ sha1 = "0.10"
|
||||
sha2 = { workspace = true, features = ["oid"] }
|
||||
socket2.workspace = true
|
||||
tar.workspace = true
|
||||
sqlx = { workspace = true, features = ["postgres", "mysql", "sqlite", "migrate"] }
|
||||
sqlx = { workspace = true, features = ["postgres", "migrate"] }
|
||||
sysinfo = "0.32"
|
||||
thiserror.workspace = true
|
||||
tokio.workspace = true
|
||||
@@ -102,5 +102,7 @@ tikv-jemallocator = { version = "0.6", optional = true }
|
||||
tikv-jemalloc-sys = { version = "0.6", optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
aether-testkit = { workspace = true, features = ["postgres"] }
|
||||
aether-test-support.workspace = true
|
||||
aws-lc-rs.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
|
||||
@@ -30,7 +30,7 @@ struct Args {
|
||||
#[arg(
|
||||
long,
|
||||
env = "AETHER_EXECUTION_RUNTIME_UNIX_SOCKET",
|
||||
default_value = "/tmp/aether-execution-runtime.sock"
|
||||
default_value = "/tmp/aether-execution-runtime/aether-execution-runtime.sock"
|
||||
)]
|
||||
unix_socket: PathBuf,
|
||||
|
||||
|
||||
@@ -1,17 +1,18 @@
|
||||
pub(crate) use crate::handlers::admin::{
|
||||
admin_provider_ops_local_action_response, admin_provider_pool_config,
|
||||
build_internal_control_error_response, create_provider_oauth_catalog_key,
|
||||
find_duplicate_provider_oauth_key, maybe_build_local_admin_pool_response,
|
||||
maybe_build_local_admin_response, persist_provider_quota_refresh_state,
|
||||
provider_oauth_maintenance_endpoint_for_provider, provider_oauth_runtime_endpoint_for_provider,
|
||||
provider_quota_refresh_endpoint_for_provider, provider_type_supports_quota_refresh,
|
||||
reconcile_admin_fixed_provider_template_endpoints,
|
||||
execute_admin_system_import_exclusively, find_duplicate_provider_oauth_key,
|
||||
maybe_build_local_admin_pool_response, maybe_build_local_admin_response,
|
||||
persist_provider_quota_refresh_state, provider_oauth_maintenance_endpoint_for_provider,
|
||||
provider_oauth_runtime_endpoint_for_provider, provider_quota_refresh_endpoint_for_provider,
|
||||
provider_type_supports_quota_refresh, reconcile_admin_fixed_provider_template_endpoints,
|
||||
refresh_provider_oauth_account_state_after_update, refresh_provider_pool_quota_locally,
|
||||
store_admin_provider_ops_balance_cache, update_existing_provider_oauth_catalog_key,
|
||||
release_admin_system_import_lease, store_admin_provider_ops_balance_cache,
|
||||
try_acquire_admin_system_import_lease, update_existing_provider_oauth_catalog_key,
|
||||
AdminAppState, AdminGatewayProviderTransportSnapshot, AdminLocalOAuthRefreshError,
|
||||
AdminRequestContext, AdminRouteRequest, AdminRouteResponse, AdminRouteResult,
|
||||
AdminStatsTimeRange, AdminStatsUsageFilter, OAUTH_ACCOUNT_BLOCK_PREFIX,
|
||||
OAUTH_REQUEST_FAILED_PREFIX,
|
||||
AdminStatsTimeRange, AdminStatsUsageFilter, AdminSystemImportLockError, SystemExportMode,
|
||||
OAUTH_ACCOUNT_BLOCK_PREFIX, OAUTH_REQUEST_FAILED_PREFIX,
|
||||
};
|
||||
|
||||
use crate::handlers::admin::{
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use aether_usage_runtime::decode_internal_report_body_base64;
|
||||
use base64::Engine as _;
|
||||
use serde_json::Value;
|
||||
|
||||
@@ -43,9 +44,8 @@ pub(crate) fn maybe_normalize_provider_private_sync_report_payload(
|
||||
}
|
||||
|
||||
if let Some(body_base64) = payload.body_base64.as_deref() {
|
||||
let body_bytes = base64::engine::general_purpose::STANDARD
|
||||
.decode(body_base64)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
let body_bytes =
|
||||
decode_internal_report_body_base64(body_base64).map_err(GatewayError::Internal)?;
|
||||
let Some(normalized_bytes) =
|
||||
normalize_provider_private_stream_bytes(report_context, &body_bytes)?
|
||||
else {
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use aether_provider_transport::CodexFingerprintConvergenceContext;
|
||||
use http::{request::Parts, HeaderMap};
|
||||
use serde_json::Value;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::ai_serving::transport::ProviderOutboundRequestContext;
|
||||
use crate::client_session_affinity::codex_request_signals_from_request;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct CodexFingerprintContextSlot(Arc<OnceLock<CodexFingerprintConvergenceContext>>);
|
||||
pub(crate) struct CodexFingerprintContextSlot(Arc<OnceLock<ProviderOutboundRequestContext>>);
|
||||
|
||||
impl Default for CodexFingerprintContextSlot {
|
||||
fn default() -> Self {
|
||||
@@ -18,11 +18,7 @@ impl Default for CodexFingerprintContextSlot {
|
||||
}
|
||||
|
||||
impl CodexFingerprintContextSlot {
|
||||
fn resolve(
|
||||
&self,
|
||||
headers: &HeaderMap,
|
||||
body_json: &Value,
|
||||
) -> CodexFingerprintConvergenceContext {
|
||||
fn resolve(&self, headers: &HeaderMap, body_json: &Value) -> ProviderOutboundRequestContext {
|
||||
self.0
|
||||
.get_or_init(|| {
|
||||
build_codex_fingerprint_context(headers, body_json, Uuid::now_v7().to_string())
|
||||
@@ -34,10 +30,10 @@ impl CodexFingerprintContextSlot {
|
||||
pub(crate) fn resolve_codex_fingerprint_context(
|
||||
parts: &Parts,
|
||||
body_json: &Value,
|
||||
) -> CodexFingerprintConvergenceContext {
|
||||
) -> ProviderOutboundRequestContext {
|
||||
if let Some(context) = parts
|
||||
.extensions
|
||||
.get::<CodexFingerprintConvergenceContext>()
|
||||
.get::<ProviderOutboundRequestContext>()
|
||||
.cloned()
|
||||
{
|
||||
return context;
|
||||
@@ -51,7 +47,7 @@ pub(crate) fn resolve_codex_fingerprint_context(
|
||||
pub(crate) fn install_codex_fingerprint_context_slot(parts: &mut Parts) {
|
||||
if parts
|
||||
.extensions
|
||||
.get::<CodexFingerprintConvergenceContext>()
|
||||
.get::<ProviderOutboundRequestContext>()
|
||||
.is_none()
|
||||
&& parts
|
||||
.extensions
|
||||
@@ -67,11 +63,11 @@ pub(crate) fn install_codex_fingerprint_context_slot(parts: &mut Parts) {
|
||||
pub(crate) fn ensure_codex_fingerprint_context(
|
||||
parts: &mut Parts,
|
||||
body_json: &Value,
|
||||
) -> CodexFingerprintConvergenceContext {
|
||||
) -> ProviderOutboundRequestContext {
|
||||
let context = resolve_codex_fingerprint_context(parts, body_json);
|
||||
if parts
|
||||
.extensions
|
||||
.get::<CodexFingerprintConvergenceContext>()
|
||||
.get::<ProviderOutboundRequestContext>()
|
||||
.is_none()
|
||||
{
|
||||
parts.extensions.remove::<CodexFingerprintContextSlot>();
|
||||
@@ -84,7 +80,7 @@ pub(crate) fn attach_codex_logical_turn_context(
|
||||
parts: &mut Parts,
|
||||
body_json: &Value,
|
||||
logical_turn_id: &str,
|
||||
) -> CodexFingerprintConvergenceContext {
|
||||
) -> ProviderOutboundRequestContext {
|
||||
let context =
|
||||
build_codex_fingerprint_context(&parts.headers, body_json, logical_turn_id.to_string());
|
||||
parts.extensions.remove::<CodexFingerprintContextSlot>();
|
||||
@@ -94,7 +90,7 @@ pub(crate) fn attach_codex_logical_turn_context(
|
||||
|
||||
pub(crate) fn restore_codex_logical_turn_context(
|
||||
parts: &mut Parts,
|
||||
context: &CodexFingerprintConvergenceContext,
|
||||
context: &ProviderOutboundRequestContext,
|
||||
) {
|
||||
parts.extensions.remove::<CodexFingerprintContextSlot>();
|
||||
parts.extensions.insert(context.clone());
|
||||
@@ -104,10 +100,9 @@ fn build_codex_fingerprint_context(
|
||||
headers: &HeaderMap,
|
||||
body_json: &Value,
|
||||
logical_turn_id: String,
|
||||
) -> CodexFingerprintConvergenceContext {
|
||||
) -> ProviderOutboundRequestContext {
|
||||
let signals = codex_request_signals_from_request(headers, Some(body_json));
|
||||
let mut context =
|
||||
CodexFingerprintConvergenceContext::new(logical_turn_id, current_unix_millis());
|
||||
let mut context = ProviderOutboundRequestContext::new(logical_turn_id, current_unix_millis());
|
||||
|
||||
if let Some(turn_id) = signals.turn_id {
|
||||
context = context.with_original_turn_id(turn_id);
|
||||
@@ -160,14 +155,14 @@ mod tests {
|
||||
assert_eq!(context.original_client_session_id(), Some("header-thread"));
|
||||
assert_eq!(context.original_prompt_cache_key(), Some("client-cache"));
|
||||
assert_eq!(
|
||||
parts.extensions.get::<CodexFingerprintConvergenceContext>(),
|
||||
parts.extensions.get::<ProviderOutboundRequestContext>(),
|
||||
Some(&context)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restored_context_wins_over_retry_request_signals() {
|
||||
let original = CodexFingerprintConvergenceContext::new("logical-turn", 1234)
|
||||
let original = ProviderOutboundRequestContext::new("logical-turn", 1234)
|
||||
.with_original_turn_id("original-turn")
|
||||
.with_original_client_session_id("original-thread")
|
||||
.with_original_prompt_cache_key("original-cache");
|
||||
|
||||
@@ -11,10 +11,10 @@ use super::{
|
||||
convert_gemini_chat_response_to_openai_chat, convert_gemini_response_to_openai_responses,
|
||||
maybe_build_local_core_sync_finalize_response,
|
||||
};
|
||||
use crate::ai_serving::GatewayControlDecision;
|
||||
use crate::ai_serving::{
|
||||
convert_openai_chat_response_to_openai_responses,
|
||||
convert_openai_responses_response_to_openai_chat,
|
||||
convert_openai_responses_response_to_openai_chat, openai_responses_message_item_id,
|
||||
GatewayControlDecision,
|
||||
};
|
||||
use crate::usage::GatewaySyncReportRequest;
|
||||
|
||||
@@ -192,7 +192,7 @@ fn aggregates_openai_responses_stream_completed_event_to_final_response() {
|
||||
"output_text": "Hello",
|
||||
"output": [{
|
||||
"type": "message",
|
||||
"id": "resp_123_msg",
|
||||
"id": openai_responses_message_item_id("resp_123", 0),
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{
|
||||
@@ -843,7 +843,7 @@ fn converts_claude_cli_response_to_openai_responses_response() {
|
||||
"output_text": "Hello Claude CLI",
|
||||
"output": [{
|
||||
"type": "message",
|
||||
"id": "msg_cli_123_msg",
|
||||
"id": openai_responses_message_item_id("msg_cli_123", 0),
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{
|
||||
@@ -907,7 +907,7 @@ fn converts_claude_cli_tool_use_to_openai_responses_function_call() {
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_cli_tool_123_msg",
|
||||
"id": openai_responses_message_item_id("msg_cli_tool_123", 0),
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{
|
||||
@@ -977,7 +977,7 @@ fn converts_gemini_cli_response_to_openai_responses_response() {
|
||||
"output_text": "Hello Gemini CLI",
|
||||
"output": [{
|
||||
"type": "message",
|
||||
"id": "resp_cli_123_msg",
|
||||
"id": openai_responses_message_item_id("resp_cli_123", 0),
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{
|
||||
@@ -1046,7 +1046,7 @@ fn converts_gemini_cli_function_call_to_openai_responses_function_call() {
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "resp_cli_tool_123_msg",
|
||||
"id": openai_responses_message_item_id("resp_cli_tool_123", 0),
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{
|
||||
@@ -1252,7 +1252,7 @@ fn local_finalize_handles_openai_responses_openai_family_sync_response_even_when
|
||||
"model": "gpt-5",
|
||||
"output": [{
|
||||
"type": "message",
|
||||
"id": "resp_cli_family_123_msg",
|
||||
"id": openai_responses_message_item_id("resp_cli_family_123", 0),
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{
|
||||
|
||||
@@ -2325,16 +2325,11 @@ mod tests {
|
||||
assert_eq!(stored.len(), 1);
|
||||
assert_eq!(stored[0].key_id.as_deref(), Some("normal-key"));
|
||||
assert_eq!(stored[0].candidate_index, 2);
|
||||
assert_eq!(
|
||||
stored[0]
|
||||
.extra_data
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("dispatch_ref"))
|
||||
.and_then(|value| value.get("SingleKey"))
|
||||
.and_then(|value| value.get("key"))
|
||||
.and_then(|value| value.get("key_id")),
|
||||
Some(&json!("normal-key"))
|
||||
);
|
||||
assert!(stored[0]
|
||||
.extra_data
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("dispatch_ref"))
|
||||
.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2482,14 +2477,23 @@ mod tests {
|
||||
|
||||
assert!(should_cache_resolved_candidate_page(&cursor));
|
||||
|
||||
let fixed_order_app = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::disabled().with_system_config_values_for_tests([(
|
||||
"scheduling_mode".to_string(),
|
||||
json!("fixed_order"),
|
||||
)]),
|
||||
);
|
||||
let fixed_order_app = AppState::new().expect("state should build");
|
||||
let fixed_order_policy = ResolvedRoutingPolicy {
|
||||
group_id: Some("routing-group-fixed-order".to_string()),
|
||||
group_version: Some(1),
|
||||
selection_source: "test".to_string(),
|
||||
requested_model: "gpt-5".to_string(),
|
||||
resolved_model: "gpt-5".to_string(),
|
||||
priority_mode: aether_routing_core::RoutingSetPriorityMode::Provider,
|
||||
scheduling_mode: aether_routing_core::RoutingSchedulingMode::FixedOrder,
|
||||
keep_priority_on_conversion: false,
|
||||
sticky_key_attempts: aether_routing_core::DEFAULT_STICKY_KEY_ATTEMPTS,
|
||||
execution_policy: Default::default(),
|
||||
ranking_overlay: Default::default(),
|
||||
mutation_plan: Default::default(),
|
||||
pool_policy_overrides: Default::default(),
|
||||
matched_rules: Vec::new(),
|
||||
};
|
||||
let mut page_cursor = LocalCandidatePreselectionPageCursor::new(
|
||||
PlannerAppState::new(&fixed_order_app),
|
||||
&model_directive_policy,
|
||||
@@ -2499,7 +2503,7 @@ mod tests {
|
||||
true,
|
||||
None,
|
||||
&auth_snapshot,
|
||||
None,
|
||||
Some(&fixed_order_policy),
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
@@ -2517,7 +2521,7 @@ mod tests {
|
||||
auth_snapshot,
|
||||
client_session_affinity: None,
|
||||
required_capabilities: None,
|
||||
routing_policy: None,
|
||||
routing_policy: Some(fixed_order_policy),
|
||||
sticky_session_token: None,
|
||||
request_auth_channel: None,
|
||||
skipped_user_id: "user-1".to_string(),
|
||||
@@ -2615,16 +2619,11 @@ mod tests {
|
||||
);
|
||||
assert_eq!(stored[1].key_id.as_deref(), Some("normal-key"));
|
||||
assert_eq!(stored[1].candidate_index, 1);
|
||||
assert_eq!(
|
||||
stored[1]
|
||||
.extra_data
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("dispatch_ref"))
|
||||
.and_then(|value| value.get("SingleKey"))
|
||||
.and_then(|value| value.get("key"))
|
||||
.and_then(|value| value.get("key_id")),
|
||||
Some(&json!("normal-key"))
|
||||
);
|
||||
assert!(stored[1]
|
||||
.extra_data
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("dispatch_ref"))
|
||||
.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2694,7 +2693,7 @@ mod tests {
|
||||
.as_ref()
|
||||
.and_then(serde_json::Value::as_object)
|
||||
.expect("ranking metadata should persist as object extra data");
|
||||
assert_eq!(extra_data.get("existing"), Some(&json!("value")));
|
||||
assert!(extra_data.get("existing").is_none());
|
||||
assert_eq!(
|
||||
extra_data.get("ranking_mode"),
|
||||
Some(&json!("CacheAffinity"))
|
||||
@@ -2707,14 +2706,7 @@ mod tests {
|
||||
Some(&json!("cached_affinity"))
|
||||
);
|
||||
assert_eq!(extra_data.get("demoted_by"), Some(&json!("cross_format")));
|
||||
assert_eq!(
|
||||
extra_data
|
||||
.get("dispatch_ref")
|
||||
.and_then(|value| value.get("SingleKey"))
|
||||
.and_then(|value| value.get("key"))
|
||||
.and_then(|value| value.get("key_id")),
|
||||
Some(&json!("ranked-key"))
|
||||
);
|
||||
assert!(extra_data.get("dispatch_ref").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -3052,7 +3044,7 @@ mod tests {
|
||||
.as_ref()
|
||||
.and_then(serde_json::Value::as_object)
|
||||
.expect("skipped ranking metadata should persist");
|
||||
assert_eq!(extra_data.get("existing"), Some(&json!("value")));
|
||||
assert!(extra_data.get("existing").is_none());
|
||||
assert_eq!(
|
||||
extra_data.get("ranking_mode"),
|
||||
Some(&json!("CacheAffinity"))
|
||||
|
||||
@@ -278,13 +278,21 @@ mod tests {
|
||||
|
||||
assert_eq!(metadata["transport_diagnostics"]["provider_type"], "codex");
|
||||
assert_eq!(
|
||||
metadata["transport_diagnostics"]["fingerprint"]["transport_profile"]["profile_id"],
|
||||
"chrome_136"
|
||||
metadata["transport_diagnostics"]["key_fingerprint_configured"],
|
||||
Value::Bool(true)
|
||||
);
|
||||
assert_eq!(
|
||||
metadata["transport_diagnostics"]["key_transport_profile_configured"],
|
||||
Value::Bool(true)
|
||||
);
|
||||
assert_eq!(
|
||||
metadata["transport_diagnostics"]["resolved_transport_profile_id"],
|
||||
"chrome_136"
|
||||
);
|
||||
assert_eq!(
|
||||
metadata["transport_diagnostics"]["resolved_transport_profile"]["profile_id"],
|
||||
"chrome_136"
|
||||
);
|
||||
assert_eq!(
|
||||
metadata["transport_diagnostics"]["request_pair"]["conversion_enabled"],
|
||||
Value::Bool(true)
|
||||
|
||||
@@ -251,6 +251,7 @@ mod tests {
|
||||
use aether_ai_serving::{
|
||||
ai_ranking_context, build_ai_rankable_candidate, AiRankableCandidateParts,
|
||||
};
|
||||
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||
use aether_data::repository::{
|
||||
provider_catalog::InMemoryProviderCatalogReadRepository,
|
||||
routing_profiles::InMemoryRoutingGroupRepository,
|
||||
@@ -566,6 +567,15 @@ mod tests {
|
||||
api_formats: Option<serde_json::Value>,
|
||||
allowed_models: Option<serde_json::Value>,
|
||||
) -> StoredProviderCatalogKey {
|
||||
let credential_state = AppState::new()
|
||||
.expect("credential state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::disabled()
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY),
|
||||
);
|
||||
let encrypted_api_key = credential_state
|
||||
.seal_provider_catalog_key_api_key(provider_id, id, "plain-upstream-key")
|
||||
.expect("api key should encrypt");
|
||||
StoredProviderCatalogKey::new(
|
||||
id.to_string(),
|
||||
provider_id.to_string(),
|
||||
@@ -577,7 +587,7 @@ mod tests {
|
||||
.expect("key should build")
|
||||
.with_transport_fields(
|
||||
api_formats,
|
||||
"plain-upstream-key".to_string(),
|
||||
encrypted_api_key,
|
||||
None,
|
||||
None,
|
||||
Some(json!({"openai:chat": 1})),
|
||||
@@ -691,7 +701,7 @@ mod tests {
|
||||
let observed_at_unix_secs = current_unix_secs();
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
)
|
||||
.with_system_config_values_for_tests(vec![
|
||||
("provider_priority_mode".to_string(), json!("provider")),
|
||||
@@ -700,6 +710,7 @@ mod tests {
|
||||
serde_json::to_value(TunnelAttachmentRecord {
|
||||
gateway_instance_id: "gateway-b".to_string(),
|
||||
relay_base_url: "http://gateway-b:8080".to_string(),
|
||||
tunnel_generation: "test-generation-remote".to_string(),
|
||||
conn_count: 1,
|
||||
observed_at_unix_secs,
|
||||
})
|
||||
@@ -710,6 +721,7 @@ mod tests {
|
||||
serde_json::to_value(TunnelAttachmentRecord {
|
||||
gateway_instance_id: "gateway-a".to_string(),
|
||||
relay_base_url: "http://gateway-a:8080".to_string(),
|
||||
tunnel_generation: "test-generation-local".to_string(),
|
||||
conn_count: 1,
|
||||
observed_at_unix_secs,
|
||||
})
|
||||
@@ -768,7 +780,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -821,7 +833,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
)
|
||||
.with_system_config_values_for_tests(vec![(
|
||||
"scheduling_mode".to_string(),
|
||||
@@ -878,7 +890,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -953,7 +965,7 @@ mod tests {
|
||||
.expect("routing strategy should be created");
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
)
|
||||
.with_routing_group_repository_for_tests(routing_repository);
|
||||
let state = AppState::new()
|
||||
@@ -1015,7 +1027,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
)
|
||||
.with_system_config_values_for_tests(vec![(
|
||||
"provider_priority_mode".to_string(),
|
||||
@@ -1081,7 +1093,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1134,7 +1146,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1208,7 +1220,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1288,7 +1300,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1364,7 +1376,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1431,7 +1443,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1514,7 +1526,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1579,7 +1591,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1668,7 +1680,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1754,7 +1766,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1851,7 +1863,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -1956,7 +1968,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -2050,7 +2062,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
|
||||
@@ -1473,6 +1473,7 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::data::GatewayDataState;
|
||||
use crate::AppState;
|
||||
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||
use aether_data::repository::candidate_selection::InMemoryMinimalCandidateSelectionReadRepository;
|
||||
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
||||
use aether_data::DataLayerError;
|
||||
@@ -2174,6 +2175,19 @@ mod tests {
|
||||
None,
|
||||
)
|
||||
.expect("endpoint transport should build");
|
||||
let credential_state = AppState::new()
|
||||
.expect("credential state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::disabled()
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY),
|
||||
);
|
||||
let encrypted_api_key = credential_state
|
||||
.seal_provider_catalog_key_api_key(
|
||||
row.provider_id.as_str(),
|
||||
row.key_id.as_str(),
|
||||
"plain-upstream-key",
|
||||
)
|
||||
.expect("api key should encrypt");
|
||||
let key = StoredProviderCatalogKey::new(
|
||||
row.key_id.clone(),
|
||||
row.provider_id.clone(),
|
||||
@@ -2185,7 +2199,7 @@ mod tests {
|
||||
.expect("key should build")
|
||||
.with_transport_fields(
|
||||
Some(serde_json::json!([row.endpoint_api_format.clone()])),
|
||||
"plain-upstream-key".to_string(),
|
||||
encrypted_api_key,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
@@ -2540,7 +2554,7 @@ mod tests {
|
||||
provider_repository,
|
||||
candidate_repository,
|
||||
)
|
||||
.with_encryption_key_for_tests("development-key");
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY);
|
||||
let app = AppState::new()
|
||||
.expect("gateway state should build")
|
||||
.with_data_state_for_tests(data_state);
|
||||
@@ -2660,7 +2674,7 @@ mod tests {
|
||||
provider_repository,
|
||||
candidate_repository,
|
||||
)
|
||||
.with_encryption_key_for_tests("development-key")
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY)
|
||||
// Legacy keys deliberately disagree with the routing policy: the
|
||||
// resolved policy must be the only source of scheduler ordering.
|
||||
.with_system_config_values_for_tests([
|
||||
|
||||
@@ -11,6 +11,7 @@ use crate::ai_serving::planner::route::{
|
||||
is_matching_stream_request, resolve_execution_runtime_stream_plan_kind,
|
||||
};
|
||||
use crate::ai_serving::{resolve_decision_execution_runtime_auth_context, GatewayControlDecision};
|
||||
use crate::state::VideoTaskRouteAccess;
|
||||
use crate::{AiExecutionDecision, AppState, GatewayError};
|
||||
|
||||
pub(crate) async fn maybe_build_stream_decision_payload(
|
||||
@@ -155,16 +156,37 @@ async fn maybe_build_local_video_task_content_stream_decision_payload(
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let _ = state
|
||||
.hydrate_video_task_for_route(decision.route_family.as_deref(), parts.uri.path())
|
||||
.await?;
|
||||
let Some(user_id) = decision
|
||||
.auth_context
|
||||
.as_ref()
|
||||
.filter(|auth_context| auth_context.access_allowed)
|
||||
.map(|auth_context| auth_context.user_id.trim())
|
||||
.filter(|value| !value.is_empty())
|
||||
else {
|
||||
return Err(crate::video_tasks::not_found_error());
|
||||
};
|
||||
if state
|
||||
.hydrate_video_task_for_route_for_user(
|
||||
decision.route_family.as_deref(),
|
||||
parts.uri.path(),
|
||||
user_id,
|
||||
)
|
||||
.await?
|
||||
!= VideoTaskRouteAccess::Allowed
|
||||
{
|
||||
return Err(crate::video_tasks::not_found_error());
|
||||
}
|
||||
|
||||
let Some(action) = state.video_tasks.prepare_openai_content_stream_action(
|
||||
parts.uri.path(),
|
||||
parts.uri.query(),
|
||||
trace_id,
|
||||
) else {
|
||||
return Ok(None);
|
||||
let Some(action) = state
|
||||
.video_tasks
|
||||
.prepare_openai_content_stream_action_for_user(
|
||||
parts.uri.path(),
|
||||
parts.uri.query(),
|
||||
trace_id,
|
||||
user_id,
|
||||
)
|
||||
else {
|
||||
return Err(crate::video_tasks::not_found_error());
|
||||
};
|
||||
|
||||
let crate::video_tasks::LocalVideoTaskContentAction::StreamPlan(plan) = action else {
|
||||
|
||||
@@ -16,6 +16,7 @@ use crate::ai_serving::{
|
||||
build_execution_runtime_auth_context, resolve_execution_runtime_auth_context,
|
||||
GatewayControlDecision,
|
||||
};
|
||||
use crate::state::VideoTaskRouteAccess;
|
||||
use crate::{AiExecutionDecision, AppState, GatewayError};
|
||||
|
||||
pub(crate) async fn maybe_build_sync_decision_payload(
|
||||
@@ -191,10 +192,6 @@ async fn maybe_build_local_video_task_follow_up_sync_decision_payload(
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let _ = state
|
||||
.hydrate_video_task_for_route(decision.route_family.as_deref(), parts.uri.path())
|
||||
.await?;
|
||||
|
||||
let auth_context = resolve_execution_runtime_auth_context(
|
||||
state,
|
||||
decision,
|
||||
@@ -204,16 +201,30 @@ async fn maybe_build_local_video_task_follow_up_sync_decision_payload(
|
||||
)
|
||||
.await?;
|
||||
let Some(auth_context) = auth_context else {
|
||||
return Ok(None);
|
||||
return Err(crate::video_tasks::not_found_error());
|
||||
};
|
||||
let Some(follow_up) = state.video_tasks.prepare_follow_up_sync_plan(
|
||||
if !auth_context.access_allowed || auth_context.user_id.trim().is_empty() {
|
||||
return Err(crate::video_tasks::not_found_error());
|
||||
}
|
||||
if state
|
||||
.hydrate_video_task_for_route_for_user(
|
||||
decision.route_family.as_deref(),
|
||||
parts.uri.path(),
|
||||
&auth_context.user_id,
|
||||
)
|
||||
.await?
|
||||
!= VideoTaskRouteAccess::Allowed
|
||||
{
|
||||
return Err(crate::video_tasks::not_found_error());
|
||||
}
|
||||
let Some(follow_up) = state.video_tasks.prepare_follow_up_sync_plan_for_user(
|
||||
plan_kind,
|
||||
parts.uri.path(),
|
||||
Some(body_json),
|
||||
Some(&auth_context),
|
||||
trace_id,
|
||||
) else {
|
||||
return Ok(None);
|
||||
return Err(crate::video_tasks::not_found_error());
|
||||
};
|
||||
|
||||
let aether_video_tasks_core::LocalVideoTaskFollowUpPlan {
|
||||
@@ -236,8 +247,7 @@ async fn maybe_build_local_video_task_follow_up_sync_decision_payload(
|
||||
downstream_path = %parts.uri.path(),
|
||||
provider_api_format = %plan.provider_api_format,
|
||||
client_api_format = %plan.client_api_format,
|
||||
upstream_base_url = ?upstream_base_url,
|
||||
upstream_url = %plan.url,
|
||||
upstream_origin = %crate::handlers::shared::security_log_url_origin(&plan.url),
|
||||
"gateway built local video follow-up sync decision payload"
|
||||
);
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ use http::{HeaderMap, HeaderName, HeaderValue};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::ai_serving::planner::common::extract_standard_requested_model;
|
||||
use crate::ai_serving::transport::CodexFingerprintConvergenceContext;
|
||||
use crate::ai_serving::transport::ProviderOutboundRequestContext;
|
||||
use crate::ai_serving::{
|
||||
ClientSurface, ExecutionRuntimeAuthContext, GatewayAuthApiKeySnapshot,
|
||||
GatewayCredentialCarrier, GatewayProviderTransportSnapshot, PlannerAppState,
|
||||
@@ -37,6 +37,10 @@ const ROUTING_GROUP_SELECTION_CACHE_TTL: Duration = Duration::from_secs(30);
|
||||
const ROUTING_GROUP_SELECTION_CACHE_STALE_TTL: Duration = Duration::from_secs(120);
|
||||
const CODEX_ACCOUNT_ID_HEADER: &str = "chatgpt-account-id";
|
||||
const CODEX_FEDRAMP_HEADER: &str = "x-openai-fedramp";
|
||||
const INVALID_ROUTING_PROVIDER_CONTRACT_MESSAGE: &str =
|
||||
"routing provider request violates provider contract";
|
||||
const INVALID_ROUTING_PROVIDER_HEADERS_MESSAGE: &str =
|
||||
"invalid provider request headers in routing mutation";
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct ResolvedLocalDecisionAuthInput {
|
||||
@@ -56,7 +60,7 @@ pub(crate) struct LocalRequestedModelDecisionInput {
|
||||
pub(crate) client_surface: Option<ClientSurface>,
|
||||
pub(crate) gateway_credential_carrier: Option<GatewayCredentialCarrier>,
|
||||
pub(crate) client_session_affinity: Option<ClientSessionAffinity>,
|
||||
pub(crate) codex_fingerprint_context: Option<CodexFingerprintConvergenceContext>,
|
||||
pub(crate) provider_outbound_context: Option<ProviderOutboundRequestContext>,
|
||||
pub(crate) routing_policy: Option<ResolvedRoutingPolicy>,
|
||||
pub(crate) routing_trace_seed: Option<RoutingDecisionTrace>,
|
||||
pub(crate) routing_context: Option<LocalRoutingRequestContext>,
|
||||
@@ -168,7 +172,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
||||
provider_api_format.as_str(),
|
||||
);
|
||||
}
|
||||
apply_codex_fingerprint_convergence_to_decision(
|
||||
apply_provider_outbound_request_policies_to_decision(
|
||||
input,
|
||||
decision,
|
||||
transport,
|
||||
@@ -231,7 +235,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
||||
provider_api_format.as_str(),
|
||||
);
|
||||
}
|
||||
apply_codex_fingerprint_convergence_to_decision(
|
||||
apply_provider_outbound_request_policies_to_decision(
|
||||
input,
|
||||
decision,
|
||||
transport,
|
||||
@@ -312,10 +316,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
||||
)
|
||||
}
|
||||
}
|
||||
.map_err(|violation| GatewayError::Client {
|
||||
status: StatusCode::BAD_REQUEST,
|
||||
message: format!("routing provider_request violates provider contract: {violation:?}"),
|
||||
})?;
|
||||
.map_err(|_| invalid_routing_provider_contract())?;
|
||||
}
|
||||
let provider_model = provider_request_body
|
||||
.get("model")
|
||||
@@ -357,7 +358,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
||||
if original_provider_request_body.is_some() {
|
||||
decision.provider_request_body = Some(provider_request_body);
|
||||
}
|
||||
apply_codex_fingerprint_convergence_to_decision(
|
||||
apply_provider_outbound_request_policies_to_decision(
|
||||
input,
|
||||
decision,
|
||||
transport,
|
||||
@@ -367,7 +368,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn apply_codex_fingerprint_convergence_to_decision(
|
||||
fn apply_provider_outbound_request_policies_to_decision(
|
||||
input: &LocalRequestedModelDecisionInput,
|
||||
decision: &mut AiExecutionDecision,
|
||||
transport: Option<&GatewayProviderTransportSnapshot>,
|
||||
@@ -378,17 +379,17 @@ fn apply_codex_fingerprint_convergence_to_decision(
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let Some(context) = input.codex_fingerprint_context.as_ref() else {
|
||||
let Some(context) = input.provider_outbound_context.as_ref() else {
|
||||
return;
|
||||
};
|
||||
let applied = crate::ai_serving::transport::apply_codex_fingerprint_convergence_with_context(
|
||||
let results = crate::ai_serving::transport::apply_provider_outbound_request_policies(
|
||||
transport,
|
||||
provider_api_format,
|
||||
context,
|
||||
&mut decision.provider_request_headers,
|
||||
provider_request_body,
|
||||
);
|
||||
if applied {
|
||||
if results.iter().any(|result| result.was_applied()) {
|
||||
decision.prompt_cache_key = provider_request_body
|
||||
.get("prompt_cache_key")
|
||||
.and_then(Value::as_str)
|
||||
@@ -396,6 +397,31 @@ fn apply_codex_fingerprint_convergence_to_decision(
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned);
|
||||
}
|
||||
if results.is_empty() {
|
||||
return;
|
||||
}
|
||||
for result in &results {
|
||||
tracing::debug!(
|
||||
event_name = "provider_outbound_policy_evaluated",
|
||||
log_type = "event",
|
||||
policy = ?result.policy,
|
||||
outcome = ?result.outcome,
|
||||
reason = ?result.reason,
|
||||
mutation_scope = ?result.mutation_scope,
|
||||
identity_scope = ?result.identity_scope,
|
||||
"provider outbound request policy evaluated"
|
||||
);
|
||||
}
|
||||
let Some(serde_json::Value::Object(report_context)) = decision.report_context.as_mut() else {
|
||||
return;
|
||||
};
|
||||
report_context.insert(
|
||||
"provider_outbound_policies".to_string(),
|
||||
serde_json::json!({
|
||||
"schema_version": 1,
|
||||
"results": results,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
struct GatewayAuthenticatedDecisionInputPort<'a> {
|
||||
@@ -484,7 +510,7 @@ pub(crate) fn build_local_requested_model_decision_input(
|
||||
client_surface: None,
|
||||
gateway_credential_carrier: None,
|
||||
client_session_affinity: None,
|
||||
codex_fingerprint_context: None,
|
||||
provider_outbound_context: None,
|
||||
routing_policy: None,
|
||||
routing_trace_seed: None,
|
||||
routing_context: None,
|
||||
@@ -499,7 +525,7 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
||||
body_json: &Value,
|
||||
client_api_format: &str,
|
||||
) -> Result<(), GatewayError> {
|
||||
input.codex_fingerprint_context =
|
||||
input.provider_outbound_context =
|
||||
Some(crate::ai_serving::codex_context::resolve_codex_fingerprint_context(parts, body_json));
|
||||
let explicit_group = routing_header_value_str(&parts.headers, ROUTING_GROUP_HEADER);
|
||||
let selected_group = match state.routing_group_read_repository() {
|
||||
@@ -863,10 +889,32 @@ fn routing_selection_error(error: GatewayRoutingSelectionError) -> GatewayError
|
||||
status: StatusCode::SERVICE_UNAVAILABLE,
|
||||
message: "no enabled routing strategy is configured for this request".to_string(),
|
||||
},
|
||||
error => GatewayError::Client {
|
||||
GatewayRoutingSelectionError::NotFound(_) => GatewayError::Client {
|
||||
status: StatusCode::FORBIDDEN,
|
||||
message: error.to_string(),
|
||||
message: "requested routing group was not found".to_string(),
|
||||
},
|
||||
GatewayRoutingSelectionError::Disabled(_) => GatewayError::Client {
|
||||
status: StatusCode::FORBIDDEN,
|
||||
message: "requested routing group is not enabled".to_string(),
|
||||
},
|
||||
GatewayRoutingSelectionError::Forbidden(_) => GatewayError::Client {
|
||||
status: StatusCode::FORBIDDEN,
|
||||
message: "requested routing group is not allowed for this principal".to_string(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn invalid_routing_provider_contract() -> GatewayError {
|
||||
GatewayError::Client {
|
||||
status: StatusCode::BAD_REQUEST,
|
||||
message: INVALID_ROUTING_PROVIDER_CONTRACT_MESSAGE.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn invalid_routing_provider_headers() -> GatewayError {
|
||||
GatewayError::Client {
|
||||
status: StatusCode::BAD_REQUEST,
|
||||
message: INVALID_ROUTING_PROVIDER_HEADERS_MESSAGE.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -921,14 +969,9 @@ fn btree_headers_to_header_map(
|
||||
) -> Result<HeaderMap, GatewayError> {
|
||||
let mut output = HeaderMap::new();
|
||||
for (name, value) in headers {
|
||||
let name = HeaderName::from_bytes(name.as_bytes()).map_err(|err| GatewayError::Client {
|
||||
status: StatusCode::BAD_REQUEST,
|
||||
message: format!("invalid provider request header name in routing mutation: {err}"),
|
||||
})?;
|
||||
let value = HeaderValue::from_str(value).map_err(|err| GatewayError::Client {
|
||||
status: StatusCode::BAD_REQUEST,
|
||||
message: format!("invalid provider request header value in routing mutation: {err}"),
|
||||
})?;
|
||||
let name = HeaderName::from_bytes(name.as_bytes())
|
||||
.map_err(|_| invalid_routing_provider_headers())?;
|
||||
let value = HeaderValue::from_str(value).map_err(|_| invalid_routing_provider_headers())?;
|
||||
output.insert(name, value);
|
||||
}
|
||||
Ok(output)
|
||||
@@ -1151,6 +1194,50 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn routing_selection_errors_do_not_echo_explicit_group() {
|
||||
let secret = "private-group?token=Bearer-secret";
|
||||
|
||||
for error in [
|
||||
GatewayRoutingSelectionError::NotFound(secret.to_string()),
|
||||
GatewayRoutingSelectionError::Disabled(secret.to_string()),
|
||||
GatewayRoutingSelectionError::Forbidden(secret.to_string()),
|
||||
] {
|
||||
let error = routing_selection_error(error);
|
||||
assert!(matches!(
|
||||
error,
|
||||
GatewayError::Client {
|
||||
status: StatusCode::FORBIDDEN,
|
||||
ref message,
|
||||
} if !message.contains(secret)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn routing_provider_errors_do_not_echo_dynamic_details() {
|
||||
let secret = "https://internal.example/?token=Bearer-secret";
|
||||
let contract_error = invalid_routing_provider_contract();
|
||||
let header_error = btree_headers_to_header_map(&BTreeMap::from([(
|
||||
format!("Authorization: {secret}"),
|
||||
secret.to_string(),
|
||||
)]))
|
||||
.expect_err("invalid header should fail");
|
||||
|
||||
for (error, expected_message) in [
|
||||
(contract_error, INVALID_ROUTING_PROVIDER_CONTRACT_MESSAGE),
|
||||
(header_error, INVALID_ROUTING_PROVIDER_HEADERS_MESSAGE),
|
||||
] {
|
||||
assert!(matches!(
|
||||
error,
|
||||
GatewayError::Client {
|
||||
status: StatusCode::BAD_REQUEST,
|
||||
ref message,
|
||||
} if message == expected_message && !message.contains(secret)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn explicit_auth_snapshot_override_does_not_fall_back_to_the_planner_cache() {
|
||||
// AppState::new has no auth snapshot repository. Without the explicit
|
||||
@@ -1328,7 +1415,7 @@ mod tests {
|
||||
client_surface: None,
|
||||
gateway_credential_carrier: None,
|
||||
client_session_affinity: None,
|
||||
codex_fingerprint_context: None,
|
||||
provider_outbound_context: None,
|
||||
routing_policy: None,
|
||||
routing_trace_seed: None,
|
||||
model_directive_policy: Default::default(),
|
||||
@@ -1577,7 +1664,7 @@ mod tests {
|
||||
client_surface: None,
|
||||
gateway_credential_carrier: None,
|
||||
client_session_affinity: None,
|
||||
codex_fingerprint_context: None,
|
||||
provider_outbound_context: None,
|
||||
routing_policy: None,
|
||||
routing_trace_seed: None,
|
||||
model_directive_policy: Default::default(),
|
||||
@@ -1647,7 +1734,7 @@ mod tests {
|
||||
client_surface: None,
|
||||
gateway_credential_carrier: None,
|
||||
client_session_affinity: None,
|
||||
codex_fingerprint_context: None,
|
||||
provider_outbound_context: None,
|
||||
routing_policy: None,
|
||||
routing_trace_seed: None,
|
||||
routing_context: None,
|
||||
@@ -1716,6 +1803,35 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_codex_provider_outbound_policies_are_terminal_noop() {
|
||||
let mut input = sample_decision_input();
|
||||
input.routing_context = None;
|
||||
input.provider_outbound_context = Some(ProviderOutboundRequestContext::new(
|
||||
"logical-turn",
|
||||
1_700_000_000_123,
|
||||
));
|
||||
let mut decision = sample_codex_fingerprint_decision();
|
||||
decision.provider_type = Some("openai".to_string());
|
||||
decision.provider_api_format = Some("openai:responses".to_string());
|
||||
decision.client_api_format = Some("openai:responses".to_string());
|
||||
let mut transport = sample_codex_fingerprint_transport();
|
||||
transport.provider.provider_type = "openai".to_string();
|
||||
|
||||
let original_headers = decision.provider_request_headers.clone();
|
||||
let original_body = decision.provider_request_body.clone();
|
||||
apply_provider_request_routing_policy_to_decision(&input, &mut decision, Some(&transport))
|
||||
.expect("non-Codex terminal finalization should succeed");
|
||||
|
||||
assert_eq!(decision.provider_request_headers, original_headers);
|
||||
assert_eq!(decision.provider_request_body, original_body);
|
||||
assert!(decision
|
||||
.report_context
|
||||
.as_ref()
|
||||
.and_then(|context| context.get("provider_outbound_policies"))
|
||||
.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codex_fingerprint_convergence_runs_at_every_provider_routing_success_exit() {
|
||||
let transport = sample_codex_fingerprint_transport();
|
||||
@@ -1732,8 +1848,8 @@ mod tests {
|
||||
});
|
||||
let mut with_mutation = sample_decision_input();
|
||||
for input in [&mut no_context, &mut empty_mutation, &mut with_mutation] {
|
||||
input.codex_fingerprint_context = Some(
|
||||
CodexFingerprintConvergenceContext::new(
|
||||
input.provider_outbound_context = Some(
|
||||
ProviderOutboundRequestContext::new(
|
||||
uuid::Uuid::new_v4().to_string(),
|
||||
1_756_668_000_000,
|
||||
)
|
||||
@@ -1802,6 +1918,25 @@ mod tests {
|
||||
);
|
||||
assert_eq!(body["client_metadata"]["x-codex-window-id"], window_id);
|
||||
|
||||
let policy_results = decision
|
||||
.report_context
|
||||
.as_ref()
|
||||
.and_then(|context| context.get("provider_outbound_policies"))
|
||||
.and_then(|policies| policies.get("results"))
|
||||
.and_then(Value::as_array)
|
||||
.expect("provider policy results");
|
||||
assert_eq!(
|
||||
policy_results.len(),
|
||||
1,
|
||||
"policy result count at {exit_name}"
|
||||
);
|
||||
assert_eq!(
|
||||
policy_results[0]["policy"],
|
||||
json!("codex_fingerprint_convergence")
|
||||
);
|
||||
assert_eq!(policy_results[0]["outcome"], json!("applied"));
|
||||
assert_eq!(policy_results[0]["reason"], json!("applied"));
|
||||
|
||||
let header_metadata: Value =
|
||||
serde_json::from_str(&decision.provider_request_headers["x-codex-turn-metadata"])
|
||||
.expect("header turn metadata");
|
||||
|
||||
@@ -76,10 +76,12 @@ pub(crate) fn build_local_execution_report_context(
|
||||
let RequestOrigin {
|
||||
client_ip,
|
||||
user_agent,
|
||||
forwarded_headers_trusted,
|
||||
} = parts
|
||||
.request_origin
|
||||
.unwrap_or_else(|| request_origin_from_headers(parts.original_headers));
|
||||
let original_headers = crate::ai_serving::collect_control_headers(parts.original_headers);
|
||||
let original_headers =
|
||||
collect_report_context_original_headers(parts.original_headers, forwarded_headers_trusted);
|
||||
let original_request_body = crate::ai_serving::build_report_context_original_request_echo(
|
||||
parts.original_request_body_json,
|
||||
parts.original_request_body_base64,
|
||||
@@ -106,10 +108,12 @@ pub(crate) fn build_local_execution_report_context(
|
||||
value,
|
||||
);
|
||||
}
|
||||
if let Some(incoming_tls) =
|
||||
crate::ai_serving::tls_fingerprint_from_headers(parts.original_headers)
|
||||
{
|
||||
merge_incoming_tls_fingerprint(&mut extra_fields, incoming_tls);
|
||||
if forwarded_headers_trusted {
|
||||
if let Some(incoming_tls) =
|
||||
crate::ai_serving::tls_fingerprint_from_headers(parts.original_headers)
|
||||
{
|
||||
merge_incoming_tls_fingerprint(&mut extra_fields, incoming_tls);
|
||||
}
|
||||
}
|
||||
insert_pool_key_lease_report_context_fields(&mut extra_fields, parts.pool_key_lease);
|
||||
insert_scheduler_affinity_policy_report_context_field(&mut extra_fields, parts.routing_policy);
|
||||
@@ -189,6 +193,17 @@ pub(crate) fn build_local_execution_report_context(
|
||||
})
|
||||
}
|
||||
|
||||
fn collect_report_context_original_headers(
|
||||
headers: &http::HeaderMap,
|
||||
forwarded_headers_trusted: bool,
|
||||
) -> BTreeMap<String, String> {
|
||||
let mut collected = crate::ai_serving::collect_control_headers(headers);
|
||||
if !forwarded_headers_trusted {
|
||||
collected.retain(|name, _| !name.starts_with("x-aether-tls-"));
|
||||
}
|
||||
collected
|
||||
}
|
||||
|
||||
fn insert_request_path_fields(
|
||||
extra_fields: &mut Map<String, Value>,
|
||||
request_path: Option<&str>,
|
||||
@@ -258,8 +273,8 @@ mod tests {
|
||||
use serde_json::{json, Map, Value};
|
||||
|
||||
use super::{
|
||||
build_local_execution_report_context, provider_stream_event_api_format_for_provider_type,
|
||||
LocalExecutionReportContextParts,
|
||||
build_local_execution_report_context, collect_report_context_original_headers,
|
||||
provider_stream_event_api_format_for_provider_type, LocalExecutionReportContextParts,
|
||||
};
|
||||
use crate::ai_serving::ExecutionRuntimeAuthContext;
|
||||
use crate::ai_serving::RequestOrigin;
|
||||
@@ -289,6 +304,26 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn untrusted_tls_forwarding_headers_are_excluded_from_report_context() {
|
||||
let mut headers = http::HeaderMap::new();
|
||||
headers.insert("x-aether-tls-ja3", "spoofed-ja3".parse().unwrap());
|
||||
headers.insert(http::header::USER_AGENT, "test-client".parse().unwrap());
|
||||
|
||||
let untrusted = collect_report_context_original_headers(&headers, false);
|
||||
assert!(!untrusted.contains_key("x-aether-tls-ja3"));
|
||||
assert_eq!(
|
||||
untrusted.get("user-agent").map(String::as_str),
|
||||
Some("test-client")
|
||||
);
|
||||
|
||||
let trusted = collect_report_context_original_headers(&headers, true);
|
||||
assert_eq!(
|
||||
trusted.get("x-aether-tls-ja3").map(String::as_str),
|
||||
Some("spoofed-ja3")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_execution_report_context_records_request_origin_and_session_affinity() {
|
||||
let auth_context = ExecutionRuntimeAuthContext {
|
||||
@@ -339,6 +374,7 @@ mod tests {
|
||||
request_origin: Some(RequestOrigin {
|
||||
client_ip: Some("203.0.113.8".to_string()),
|
||||
user_agent: Some("Claude-Code/1.0".to_string()),
|
||||
forwarded_headers_trusted: false,
|
||||
}),
|
||||
original_request_body_json: Some(&json!({"model": "gpt-5"})),
|
||||
original_request_body_base64: None,
|
||||
@@ -491,7 +527,11 @@ mod tests {
|
||||
original_headers: &original_headers,
|
||||
request_path: None,
|
||||
request_query_string: None,
|
||||
request_origin: None,
|
||||
request_origin: Some(RequestOrigin {
|
||||
client_ip: None,
|
||||
user_agent: None,
|
||||
forwarded_headers_trusted: true,
|
||||
}),
|
||||
original_request_body_json: Some(&json!({"model": "gpt-5"})),
|
||||
original_request_body_base64: None,
|
||||
client_session_affinity: None,
|
||||
|
||||
@@ -8,7 +8,10 @@ use crate::ai_serving::transport::{
|
||||
GeminiFilesRequestBodyError,
|
||||
};
|
||||
use crate::ai_serving::GEMINI_FILES_UPLOAD_PLAN_KIND;
|
||||
use crate::ai_serving::{CandidateFailureDiagnostic, GatewayProviderTransportSnapshot};
|
||||
use crate::ai_serving::{
|
||||
CandidateFailureDiagnostic, GatewayProviderTransportSnapshot, GEMINI_FILES_DELETE_PLAN_KIND,
|
||||
GEMINI_FILES_DOWNLOAD_PLAN_KIND, GEMINI_FILES_GET_PLAN_KIND,
|
||||
};
|
||||
use crate::AppState;
|
||||
|
||||
use super::support::{
|
||||
@@ -47,6 +50,26 @@ pub(super) async fn resolve_local_gemini_files_candidate_payload_parts(
|
||||
let transport = &attempt.eligible.transport;
|
||||
let effective_headers = input.effective_headers(&parts.headers);
|
||||
|
||||
if matches!(
|
||||
spec_metadata.decision_kind,
|
||||
GEMINI_FILES_GET_PLAN_KIND
|
||||
| GEMINI_FILES_DELETE_PLAN_KIND
|
||||
| GEMINI_FILES_DOWNLOAD_PLAN_KIND
|
||||
) && !candidate_matches_owned_gemini_file_mapping(state, parts, input, attempt).await
|
||||
{
|
||||
mark_skipped_local_gemini_files_candidate(
|
||||
state,
|
||||
input,
|
||||
trace_id,
|
||||
candidate,
|
||||
attempt.candidate_index,
|
||||
&attempt.candidate_id,
|
||||
"gemini_file_mapping_mismatch",
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
}
|
||||
|
||||
if let Some(skip_reason) =
|
||||
gemini_files_transport_unsupported_reason(transport, GEMINI_FILES_CANDIDATE_API_FORMAT)
|
||||
{
|
||||
@@ -191,3 +214,64 @@ pub(super) async fn resolve_local_gemini_files_candidate_payload_parts(
|
||||
file_name,
|
||||
})
|
||||
}
|
||||
|
||||
async fn candidate_matches_owned_gemini_file_mapping(
|
||||
state: &AppState,
|
||||
parts: &http::request::Parts,
|
||||
input: &LocalGeminiFilesDecisionInput,
|
||||
attempt: &LocalGeminiFilesCandidateAttempt,
|
||||
) -> bool {
|
||||
let Some(file_name) = normalize_gemini_file_name_from_path(parts.uri.path()) else {
|
||||
return false;
|
||||
};
|
||||
let user_id = input.auth_context.user_id.trim();
|
||||
if user_id.is_empty() || !state.has_gemini_file_mapping_data_reader() {
|
||||
return false;
|
||||
}
|
||||
let Ok(Some(mapping)) = state
|
||||
.find_active_gemini_file_mapping_for_owner(
|
||||
file_name.as_str(),
|
||||
&attempt.eligible.transport.key.id,
|
||||
user_id,
|
||||
crate::clock::current_unix_secs(),
|
||||
)
|
||||
.await
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
|
||||
mapping.user_id.as_deref().map(str::trim) == Some(user_id)
|
||||
&& mapping.key_id == attempt.eligible.transport.key.id
|
||||
}
|
||||
|
||||
pub(crate) fn normalize_gemini_file_name_from_path(path: &str) -> Option<String> {
|
||||
let suffix = path.strip_prefix("/v1beta/files/")?.trim_matches('/');
|
||||
let suffix = suffix.strip_suffix(":download").unwrap_or(suffix).trim();
|
||||
let suffix = suffix.strip_prefix("files/").unwrap_or(suffix).trim();
|
||||
if suffix.is_empty() || suffix.contains('/') {
|
||||
return None;
|
||||
}
|
||||
Some(format!("files/{suffix}"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::normalize_gemini_file_name_from_path;
|
||||
|
||||
#[test]
|
||||
fn normalizes_supported_gemini_file_object_paths() {
|
||||
assert_eq!(
|
||||
normalize_gemini_file_name_from_path("/v1beta/files/file-123"),
|
||||
Some("files/file-123".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
normalize_gemini_file_name_from_path("/v1beta/files/file-123:download"),
|
||||
Some("files/file-123".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
normalize_gemini_file_name_from_path("/v1beta/files/files/abc-123"),
|
||||
Some("files/abc-123".to_string())
|
||||
);
|
||||
assert_eq!(normalize_gemini_file_name_from_path("/v1beta/files"), None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ use crate::ai_serving::planner::candidate_preparation::{
|
||||
};
|
||||
use crate::ai_serving::planner::spec_metadata::local_openai_image_spec_metadata;
|
||||
use crate::ai_serving::pure::normalize_openai_image_request_with_options;
|
||||
use crate::ai_serving::transport::antigravity::is_antigravity_provider_transport;
|
||||
use crate::ai_serving::transport::{
|
||||
build_grok_browser_headers, build_grok_upstream_url, build_openai_image_headers,
|
||||
build_openai_image_upstream_url, build_standard_provider_request_headers,
|
||||
@@ -338,6 +339,25 @@ async fn resolve_local_openai_image_to_gemini_candidate_payload_parts(
|
||||
let candidate = &attempt.eligible.candidate;
|
||||
let transport = &attempt.eligible.transport;
|
||||
let provider_api_format = "gemini:generate_content";
|
||||
|
||||
// The gemini:generate_content URL hook rewrites an Antigravity endpoint to
|
||||
// /v1internal:, and this image path has no v1internal envelope to match it.
|
||||
// Skip the candidate instead of posting a bare Gemini body that upstream
|
||||
// would only reject.
|
||||
if is_antigravity_provider_transport(transport) {
|
||||
mark_skipped_local_openai_image_candidate(
|
||||
state,
|
||||
input,
|
||||
trace_id,
|
||||
candidate,
|
||||
attempt.candidate_index,
|
||||
&attempt.candidate_id,
|
||||
"transport_unsupported",
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
}
|
||||
|
||||
let effective_headers = input.effective_headers(&parts.headers);
|
||||
|
||||
let prepared_candidate = match prepare_header_authenticated_candidate(
|
||||
|
||||
@@ -504,8 +504,8 @@ fn projects_uuid_prompt_cache_identity_into_missing_session_headers() {
|
||||
);
|
||||
assert_eq!(headers.get("x-client-request-id"), None);
|
||||
assert_eq!(
|
||||
headers.get("user-agent"),
|
||||
Some(&"codex_cli_rs/0.144.1".to_string())
|
||||
headers.get("user-agent").map(String::as_str),
|
||||
Some(aether_ai_formats::CODEX_CLIENT_USER_AGENT)
|
||||
);
|
||||
assert_eq!(headers.get("originator"), Some(&"codex_cli_rs".to_string()));
|
||||
assert!(!headers.contains_key("version"));
|
||||
@@ -614,8 +614,8 @@ fn injects_only_codex_client_headers_for_images_requests() {
|
||||
Some(&"acc-123".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
headers.get("user-agent"),
|
||||
Some(&"codex_cli_rs/0.144.1".to_string())
|
||||
headers.get("user-agent").map(String::as_str),
|
||||
Some(aether_ai_formats::CODEX_CLIENT_USER_AGENT)
|
||||
);
|
||||
assert_eq!(headers.get("originator"), Some(&"codex_cli_rs".to_string()));
|
||||
assert!(!headers.contains_key("version"));
|
||||
@@ -698,8 +698,8 @@ fn preserves_client_context_headers_and_enforces_codex_provider_identity() {
|
||||
Some(&"kept-by-rule-request".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
headers.get("user-agent"),
|
||||
Some(&"codex_cli_rs/0.144.1".to_string())
|
||||
headers.get("user-agent").map(String::as_str),
|
||||
Some(aether_ai_formats::CODEX_CLIENT_USER_AGENT)
|
||||
);
|
||||
assert_eq!(headers.get("originator"), Some(&"codex_cli_rs".to_string()));
|
||||
assert_eq!(
|
||||
@@ -762,8 +762,8 @@ fn compact_projects_uuid_prompt_cache_identity_into_session_headers() {
|
||||
);
|
||||
assert_eq!(headers.get("x-client-request-id"), None);
|
||||
assert_eq!(
|
||||
headers.get("user-agent"),
|
||||
Some(&"codex_cli_rs/0.144.1".to_string())
|
||||
headers.get("user-agent").map(String::as_str),
|
||||
Some(aether_ai_formats::CODEX_CLIENT_USER_AGENT)
|
||||
);
|
||||
assert_eq!(headers.get("originator"), Some(&"codex_cli_rs".to_string()));
|
||||
assert!(!headers.contains_key("version"));
|
||||
|
||||
@@ -378,7 +378,7 @@ mod tests {
|
||||
client_surface: None,
|
||||
gateway_credential_carrier: None,
|
||||
client_session_affinity: None,
|
||||
codex_fingerprint_context: None,
|
||||
provider_outbound_context: None,
|
||||
routing_policy: None,
|
||||
routing_trace_seed: None,
|
||||
routing_context: None,
|
||||
|
||||
@@ -4,6 +4,10 @@ use std::sync::Arc;
|
||||
use aether_contracts::ResolvedTransportProfile;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::ai_serving::planner::antigravity::{
|
||||
build_antigravity_v1internal_provider_request, AntigravityV1InternalRequestError,
|
||||
AntigravityV1InternalRequestInput, ANTIGRAVITY_V1INTERNAL_ENVELOPE_NAME,
|
||||
};
|
||||
use crate::ai_serving::planner::candidate_preparation::{
|
||||
prepare_header_authenticated_candidate, prepare_header_authenticated_candidate_from_auth,
|
||||
OauthPreparationContext,
|
||||
@@ -26,6 +30,7 @@ use crate::ai_serving::planner::standard::{
|
||||
openai_provider_request_contract_failure_extra_data, openai_responses_reasoning_replay_policy,
|
||||
request_body_build_failure_extra_data, request_conversion_failure_extra_data,
|
||||
};
|
||||
use crate::ai_serving::transport::antigravity::is_antigravity_provider_transport;
|
||||
use crate::ai_serving::transport::kiro::{
|
||||
build_kiro_provider_headers, build_kiro_provider_request_body,
|
||||
is_kiro_claude_messages_transport, KiroProviderHeadersInput, KiroRequestAuth,
|
||||
@@ -587,7 +592,7 @@ pub(crate) async fn resolve_local_standard_candidate_payload_parts(
|
||||
}
|
||||
};
|
||||
crate::ai_serving::hydrate_openai_response_history(
|
||||
state.runtime_state(),
|
||||
state,
|
||||
body_json,
|
||||
spec_metadata.api_format,
|
||||
provider_api_format,
|
||||
@@ -837,6 +842,29 @@ pub(crate) async fn resolve_local_standard_candidate_payload_parts(
|
||||
.await);
|
||||
}
|
||||
|
||||
if normalized_provider_api_format == "gemini:generate_content"
|
||||
&& is_antigravity_provider_transport(transport)
|
||||
{
|
||||
return Ok(build_antigravity_cross_format_payload_parts(
|
||||
state,
|
||||
parts,
|
||||
trace_id,
|
||||
body_json,
|
||||
input,
|
||||
attempt,
|
||||
transport,
|
||||
spec_metadata.api_format,
|
||||
provider_api_format,
|
||||
prepared_candidate.mapped_model,
|
||||
prepared_candidate.auth_header,
|
||||
prepared_candidate.auth_value,
|
||||
provider_request_body,
|
||||
upstream_is_stream,
|
||||
redaction.redacted,
|
||||
)
|
||||
.await);
|
||||
}
|
||||
|
||||
if normalized_provider_api_format == "gemini:generate_content"
|
||||
&& is_gemini_cli_provider_transport(transport)
|
||||
{
|
||||
@@ -963,6 +991,145 @@ fn apply_transport_request_body_semantics(
|
||||
)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn build_antigravity_cross_format_payload_parts(
|
||||
state: &AppState,
|
||||
parts: &http::request::Parts,
|
||||
trace_id: &str,
|
||||
original_body_json: &serde_json::Value,
|
||||
input: &LocalStandardDecisionInput,
|
||||
attempt: &LocalStandardCandidateAttempt,
|
||||
transport: &Arc<GatewayProviderTransportSnapshot>,
|
||||
client_api_format: &str,
|
||||
provider_api_format: &str,
|
||||
mapped_model: String,
|
||||
auth_header: String,
|
||||
auth_value: String,
|
||||
gemini_request_body: Value,
|
||||
upstream_is_stream: bool,
|
||||
request_redacted: bool,
|
||||
) -> Option<LocalStandardCandidatePayloadParts> {
|
||||
let candidate = &attempt.eligible.candidate;
|
||||
let effective_headers = input.effective_headers(&parts.headers);
|
||||
let resolved =
|
||||
match build_antigravity_v1internal_provider_request(AntigravityV1InternalRequestInput {
|
||||
state,
|
||||
parts,
|
||||
transport,
|
||||
trace_id,
|
||||
mapped_model: &mapped_model,
|
||||
provider_api_format,
|
||||
auth_header: &auth_header,
|
||||
auth_value: &auth_value,
|
||||
request_headers: effective_headers,
|
||||
original_request_body: original_body_json,
|
||||
gemini_request_body: &gemini_request_body,
|
||||
upstream_is_stream,
|
||||
same_format: false,
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(resolved) => resolved,
|
||||
Err(AntigravityV1InternalRequestError::TransportUnsupported) => {
|
||||
mark_skipped_local_standard_candidate(
|
||||
state,
|
||||
input,
|
||||
trace_id,
|
||||
candidate,
|
||||
attempt.candidate_index,
|
||||
&attempt.candidate_id,
|
||||
"transport_unsupported",
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
}
|
||||
Err(AntigravityV1InternalRequestError::EnvelopeUnsupported) => {
|
||||
mark_skipped_local_standard_candidate_with_extra_data(
|
||||
state,
|
||||
input,
|
||||
trace_id,
|
||||
candidate,
|
||||
attempt.candidate_index,
|
||||
&attempt.candidate_id,
|
||||
"provider_request_body_build_failed",
|
||||
request_body_build_failure_extra_data(
|
||||
original_body_json,
|
||||
client_api_format,
|
||||
provider_api_format,
|
||||
),
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
}
|
||||
Err(AntigravityV1InternalRequestError::UpstreamUrlUnavailable) => {
|
||||
mark_skipped_local_standard_candidate_with_failure_diagnostic(
|
||||
state,
|
||||
input,
|
||||
trace_id,
|
||||
candidate,
|
||||
attempt.candidate_index,
|
||||
&attempt.candidate_id,
|
||||
"upstream_url_missing",
|
||||
CandidateFailureDiagnostic::upstream_url_missing(
|
||||
client_api_format,
|
||||
provider_api_format,
|
||||
"standard_family_antigravity_url",
|
||||
),
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
}
|
||||
Err(AntigravityV1InternalRequestError::HeaderRulesApplyFailed) => {
|
||||
mark_skipped_local_standard_candidate_with_failure_diagnostic(
|
||||
state,
|
||||
input,
|
||||
trace_id,
|
||||
candidate,
|
||||
attempt.candidate_index,
|
||||
&attempt.candidate_id,
|
||||
"transport_header_rules_apply_failed",
|
||||
CandidateFailureDiagnostic::header_rules_apply_failed(
|
||||
client_api_format,
|
||||
provider_api_format,
|
||||
"standard_family_antigravity_headers",
|
||||
),
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
let mut provider_request_headers = resolved.headers.headers;
|
||||
apply_codex_openai_special_headers(
|
||||
&mut provider_request_headers,
|
||||
&resolved.body,
|
||||
effective_headers,
|
||||
resolved.transport.provider.provider_type.as_str(),
|
||||
provider_api_format,
|
||||
Some(trace_id),
|
||||
resolved.transport.key.decrypted_auth_config.as_deref(),
|
||||
);
|
||||
request_identity_response_encoding_when_redacted(
|
||||
&mut provider_request_headers,
|
||||
request_redacted,
|
||||
);
|
||||
|
||||
Some(LocalStandardCandidatePayloadParts {
|
||||
auth_header: resolved.headers.auth_header,
|
||||
auth_value: resolved.headers.auth_value,
|
||||
mapped_model,
|
||||
provider_api_format: provider_api_format.to_string(),
|
||||
provider_request_body: resolved.body,
|
||||
provider_request_headers,
|
||||
upstream_url: resolved.upstream_url,
|
||||
upstream_is_stream,
|
||||
envelope_name: Some(ANTIGRAVITY_V1INTERNAL_ENVELOPE_NAME),
|
||||
transport: resolved.transport,
|
||||
transport_profile: None,
|
||||
request_redacted,
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn build_gemini_cli_cross_format_payload_parts(
|
||||
state: &AppState,
|
||||
|
||||
@@ -2183,7 +2183,7 @@ mod tests {
|
||||
client_surface: None,
|
||||
gateway_credential_carrier: None,
|
||||
client_session_affinity: None,
|
||||
codex_fingerprint_context: None,
|
||||
provider_outbound_context: None,
|
||||
routing_policy: None,
|
||||
routing_trace_seed: None,
|
||||
routing_context: None,
|
||||
@@ -2741,7 +2741,7 @@ mod tests {
|
||||
.provider_request_headers
|
||||
.get("x-client-version")
|
||||
.map(String::as_str),
|
||||
Some("1.2.3")
|
||||
Some("4.3.0")
|
||||
);
|
||||
assert_eq!(
|
||||
payload
|
||||
@@ -2761,7 +2761,7 @@ mod tests {
|
||||
assert_eq!(payload.provider_request_body["model"], "gemini-2.5-pro");
|
||||
assert_eq!(
|
||||
payload.provider_request_body["userAgent"],
|
||||
"antigravity/cli/1.0.16 (aidev_client; os_type=linux; arch=arm64; auth_method=consumer)"
|
||||
"vscode/1.X.X (Antigravity/4.3.0)"
|
||||
);
|
||||
assert_eq!(payload.provider_request_body["requestType"], "agent");
|
||||
assert!(payload.provider_request_body.get("contents").is_none());
|
||||
|
||||
+12
-27
@@ -1,7 +1,6 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use aether_contracts::RequestBody;
|
||||
use tracing::debug;
|
||||
|
||||
use super::super::{
|
||||
augment_sync_report_context, build_ai_execution_plan_from_decision,
|
||||
@@ -10,7 +9,6 @@ use super::super::{
|
||||
AiStreamAttempt,
|
||||
};
|
||||
use crate::ai_serving::planner::common::enforce_provider_body_stream_policy;
|
||||
use crate::ai_serving::planner::redaction::sanitize_upstream_url_for_log;
|
||||
use crate::ai_serving::provider_adaptation_requires_eventstream_accept;
|
||||
use crate::ai_serving::transport::{
|
||||
build_standard_plan_fallback_headers, build_standard_plan_fallback_openai_chat_url,
|
||||
@@ -157,21 +155,16 @@ pub(crate) fn build_openai_responses_stream_plan_from_decision(
|
||||
let Some(auth_pair) = take_ai_upstream_auth_pair(&mut payload) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let (url, url_source) = if let Some(upstream_url) =
|
||||
take_non_empty_string(&mut payload.upstream_url)
|
||||
{
|
||||
(upstream_url, "upstream_url")
|
||||
let url = if let Some(upstream_url) = take_non_empty_string(&mut payload.upstream_url) {
|
||||
upstream_url
|
||||
} else {
|
||||
let Some(upstream_base_url) = take_non_empty_string(&mut payload.upstream_base_url) else {
|
||||
return Ok(None);
|
||||
};
|
||||
(
|
||||
build_standard_plan_fallback_openai_responses_url(
|
||||
&upstream_base_url,
|
||||
parts.uri.query(),
|
||||
compact,
|
||||
),
|
||||
"upstream_base_url",
|
||||
build_standard_plan_fallback_openai_responses_url(
|
||||
&upstream_base_url,
|
||||
parts.uri.query(),
|
||||
compact,
|
||||
)
|
||||
};
|
||||
let Some(provider_request_body_value) = payload.provider_request_body.take() else {
|
||||
@@ -238,16 +231,7 @@ pub(crate) fn build_openai_responses_stream_plan_from_decision(
|
||||
.uri
|
||||
.query()
|
||||
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
||||
let log_decision_upstream_base_url = payload
|
||||
.upstream_base_url
|
||||
.as_deref()
|
||||
.map(sanitize_upstream_url_for_log);
|
||||
let log_decision_upstream_url = payload
|
||||
.upstream_url
|
||||
.as_deref()
|
||||
.map(sanitize_upstream_url_for_log);
|
||||
let log_plan_url = sanitize_upstream_url_for_log(plan.url.as_str());
|
||||
debug!(
|
||||
tracing::debug!(
|
||||
event_name = "local_openai_responses_stream_plan_built",
|
||||
log_type = "debug",
|
||||
request_id = %plan.request_id,
|
||||
@@ -255,12 +239,13 @@ pub(crate) fn build_openai_responses_stream_plan_from_decision(
|
||||
provider_id = %plan.provider_id,
|
||||
endpoint_id = %plan.endpoint_id,
|
||||
key_id = %plan.key_id,
|
||||
downstream_path_and_query = %crate::ai_serving::pure::sanitize_request_path_and_query(
|
||||
parts.uri.path(),
|
||||
parts.uri.query(),
|
||||
).unwrap_or_else(|| "/".to_string()),
|
||||
upstream_origin = %crate::handlers::shared::security_log_url_origin(&plan.url),
|
||||
downstream_path = %parts.uri.path(),
|
||||
downstream_query = ?log_downstream_query,
|
||||
url_source,
|
||||
decision_upstream_base_url = ?log_decision_upstream_base_url,
|
||||
decision_upstream_url = ?log_decision_upstream_url,
|
||||
plan_url = %log_plan_url,
|
||||
client_api_format = %plan.client_api_format,
|
||||
provider_api_format = %plan.provider_api_format,
|
||||
upstream_is_stream = effective_upstream_is_stream,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use aether_contracts::RequestBody;
|
||||
use tracing::debug;
|
||||
|
||||
use super::super::{
|
||||
augment_sync_report_context, build_ai_execution_plan_from_decision,
|
||||
@@ -10,7 +9,6 @@ use super::super::{
|
||||
AiSyncAttempt,
|
||||
};
|
||||
use crate::ai_serving::planner::common::enforce_provider_body_stream_policy;
|
||||
use crate::ai_serving::planner::redaction::sanitize_upstream_url_for_log;
|
||||
use crate::ai_serving::transport::{
|
||||
build_standard_plan_fallback_headers, build_standard_plan_fallback_openai_chat_url,
|
||||
build_standard_plan_fallback_openai_responses_url, StandardPlanFallbackAcceptPolicy,
|
||||
@@ -142,21 +140,16 @@ pub(crate) fn build_openai_responses_sync_plan_from_decision(
|
||||
let Some(auth_pair) = take_ai_upstream_auth_pair(&mut payload) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let (url, url_source) = if let Some(upstream_url) =
|
||||
take_non_empty_string(&mut payload.upstream_url)
|
||||
{
|
||||
(upstream_url, "upstream_url")
|
||||
let url = if let Some(upstream_url) = take_non_empty_string(&mut payload.upstream_url) {
|
||||
upstream_url
|
||||
} else {
|
||||
let Some(upstream_base_url) = take_non_empty_string(&mut payload.upstream_base_url) else {
|
||||
return Ok(None);
|
||||
};
|
||||
(
|
||||
build_standard_plan_fallback_openai_responses_url(
|
||||
&upstream_base_url,
|
||||
parts.uri.query(),
|
||||
compact,
|
||||
),
|
||||
"upstream_base_url",
|
||||
build_standard_plan_fallback_openai_responses_url(
|
||||
&upstream_base_url,
|
||||
parts.uri.query(),
|
||||
compact,
|
||||
)
|
||||
};
|
||||
let Some(provider_request_body_value) = payload.provider_request_body.take() else {
|
||||
@@ -205,16 +198,7 @@ pub(crate) fn build_openai_responses_sync_plan_from_decision(
|
||||
.uri
|
||||
.query()
|
||||
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
||||
let log_decision_upstream_base_url = payload
|
||||
.upstream_base_url
|
||||
.as_deref()
|
||||
.map(sanitize_upstream_url_for_log);
|
||||
let log_decision_upstream_url = payload
|
||||
.upstream_url
|
||||
.as_deref()
|
||||
.map(sanitize_upstream_url_for_log);
|
||||
let log_plan_url = sanitize_upstream_url_for_log(plan.url.as_str());
|
||||
debug!(
|
||||
tracing::debug!(
|
||||
event_name = "local_openai_responses_sync_plan_built",
|
||||
log_type = "debug",
|
||||
request_id = %plan.request_id,
|
||||
@@ -222,12 +206,13 @@ pub(crate) fn build_openai_responses_sync_plan_from_decision(
|
||||
provider_id = %plan.provider_id,
|
||||
endpoint_id = %plan.endpoint_id,
|
||||
key_id = %plan.key_id,
|
||||
downstream_path_and_query = %crate::ai_serving::pure::sanitize_request_path_and_query(
|
||||
parts.uri.path(),
|
||||
parts.uri.query(),
|
||||
).unwrap_or_else(|| "/".to_string()),
|
||||
upstream_origin = %crate::handlers::shared::security_log_url_origin(&plan.url),
|
||||
downstream_path = %parts.uri.path(),
|
||||
downstream_query = ?log_downstream_query,
|
||||
url_source,
|
||||
decision_upstream_base_url = ?log_decision_upstream_base_url,
|
||||
decision_upstream_url = ?log_decision_upstream_url,
|
||||
plan_url = %log_plan_url,
|
||||
client_api_format = %plan.client_api_format,
|
||||
provider_api_format = %plan.provider_api_format,
|
||||
upstream_is_stream = payload.upstream_is_stream,
|
||||
|
||||
+5
-5
@@ -3,7 +3,6 @@ use tracing::debug;
|
||||
|
||||
use crate::ai_serving::build_request_trace_proxy_value;
|
||||
use crate::ai_serving::planner::decision_input::apply_provider_request_routing_policy_to_decision_with_websocket_mode;
|
||||
use crate::ai_serving::planner::redaction::sanitize_upstream_url_for_log;
|
||||
use crate::ai_serving::planner::report_context::{
|
||||
build_local_execution_report_context, insert_native_client_envelope_name,
|
||||
insert_provider_stream_event_api_format, LocalExecutionReportContextParts,
|
||||
@@ -205,12 +204,10 @@ pub(crate) async fn maybe_build_local_openai_responses_decision_payload_for_cand
|
||||
&resolved.transport,
|
||||
);
|
||||
|
||||
let log_base_url = sanitize_upstream_url_for_log(resolved.transport.endpoint.base_url.as_str());
|
||||
let log_request_query = parts
|
||||
.uri
|
||||
.query()
|
||||
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
||||
let log_upstream_url = sanitize_upstream_url_for_log(resolved.upstream_url.as_str());
|
||||
debug!(
|
||||
event_name = "local_openai_responses_decision_payload_built",
|
||||
log_type = "debug",
|
||||
@@ -227,9 +224,12 @@ pub(crate) async fn maybe_build_local_openai_responses_decision_payload_for_cand
|
||||
client_api_format = spec_metadata.api_format,
|
||||
provider_api_format = %resolved.provider_api_format,
|
||||
request_path = %parts.uri.path(),
|
||||
request_path_and_query = %crate::ai_serving::pure::sanitize_request_path_and_query(
|
||||
parts.uri.path(),
|
||||
parts.uri.query(),
|
||||
).unwrap_or_else(|| "/".to_string()),
|
||||
upstream_origin = %crate::handlers::shared::security_log_url_origin(&resolved.upstream_url),
|
||||
request_query = ?log_request_query,
|
||||
upstream_base_url = %log_base_url,
|
||||
upstream_url = %log_upstream_url,
|
||||
upstream_is_stream = resolved.upstream_is_stream,
|
||||
has_envelope = resolved.envelope_name.is_some(),
|
||||
"gateway built local openai responses decision payload"
|
||||
|
||||
+7
-15
@@ -24,7 +24,6 @@ use crate::ai_serving::planner::gemini_cli::{
|
||||
};
|
||||
use crate::ai_serving::planner::redaction::{
|
||||
request_identity_response_encoding_when_redacted, resolve_provider_chat_pii_redaction,
|
||||
sanitize_upstream_url_for_log,
|
||||
};
|
||||
use crate::ai_serving::planner::spec_metadata::local_openai_responses_spec_metadata;
|
||||
use crate::ai_serving::planner::standard::{
|
||||
@@ -428,7 +427,7 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts_with_
|
||||
}
|
||||
};
|
||||
crate::ai_serving::hydrate_openai_response_history(
|
||||
state.runtime_state(),
|
||||
state,
|
||||
body_json,
|
||||
spec_metadata.api_format,
|
||||
provider_api_format,
|
||||
@@ -867,17 +866,10 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts_with_
|
||||
|
||||
let (execution_strategy, conversion_mode) =
|
||||
ai_local_execution_contract_for_formats(spec_metadata.api_format, provider_api_format);
|
||||
let log_base_url = sanitize_upstream_url_for_log(transport.endpoint.base_url.as_str());
|
||||
let log_custom_path = transport
|
||||
.endpoint
|
||||
.custom_path
|
||||
.as_deref()
|
||||
.map(sanitize_upstream_url_for_log);
|
||||
let log_request_query = parts
|
||||
.uri
|
||||
.query()
|
||||
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
||||
let log_upstream_url = sanitize_upstream_url_for_log(upstream_url.as_str());
|
||||
|
||||
debug!(
|
||||
event_name = "local_openai_responses_upstream_url_resolved",
|
||||
@@ -893,12 +885,14 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts_with_
|
||||
provider_api_format = %provider_api_format,
|
||||
execution_strategy = execution_strategy.as_str(),
|
||||
conversion_mode = conversion_mode.as_str(),
|
||||
base_url = %log_base_url,
|
||||
custom_path = ?log_custom_path,
|
||||
request_path_and_query = %crate::ai_serving::pure::sanitize_request_path_and_query(
|
||||
parts.uri.path(),
|
||||
parts.uri.query(),
|
||||
).unwrap_or_else(|| "/".to_string()),
|
||||
upstream_origin = %crate::handlers::shared::security_log_url_origin(&upstream_url),
|
||||
request_path = %parts.uri.path(),
|
||||
request_query = ?log_request_query,
|
||||
mapped_model = %mapped_model,
|
||||
upstream_url = %log_upstream_url,
|
||||
upstream_is_stream,
|
||||
"gateway resolved local openai responses upstream url"
|
||||
);
|
||||
@@ -2011,8 +2005,6 @@ async fn build_kiro_openai_responses_payload_parts(
|
||||
};
|
||||
let (execution_strategy, conversion_mode) =
|
||||
ai_local_execution_contract_for_formats(client_api_format, provider_api_format);
|
||||
let log_upstream_url = sanitize_upstream_url_for_log(upstream_url.as_str());
|
||||
|
||||
debug!(
|
||||
event_name = "local_openai_responses_kiro_upstream_url_resolved",
|
||||
log_type = "debug",
|
||||
@@ -2027,7 +2019,7 @@ async fn build_kiro_openai_responses_payload_parts(
|
||||
provider_api_format = %provider_api_format,
|
||||
execution_strategy = execution_strategy.as_str(),
|
||||
conversion_mode = conversion_mode.as_str(),
|
||||
upstream_url = %log_upstream_url,
|
||||
upstream_origin = %crate::handlers::shared::security_log_url_origin(&upstream_url),
|
||||
upstream_is_stream,
|
||||
"gateway resolved local openai responses kiro upstream url"
|
||||
);
|
||||
|
||||
@@ -177,6 +177,7 @@ pub(crate) use aether_ai_formats::{
|
||||
api_format_defaults_to_client_error_failover, api_format_defaults_to_non_stream,
|
||||
api_format_permission_covers, codex_responses_lite_tool_is_client_executed,
|
||||
intersect_api_format_allowed_lists, is_embedding_api_format, is_rerank_api_format,
|
||||
normalize_openai_responses_message_item_ids, openai_responses_message_item_id,
|
||||
openai_responses_request_operation, openai_responses_synthetic_reasoning_item_id,
|
||||
strip_incompatible_openai_responses_reasoning_items,
|
||||
strip_incompatible_openai_responses_reasoning_items_with_policy, ApiOperation, ClientSurface,
|
||||
|
||||
@@ -2,14 +2,15 @@ use crate::ai_serving::{
|
||||
hydrate_response_history, normalize_api_format_alias, record_converted_response_history,
|
||||
response_history_is_loaded, response_history_storage_key, ResponseHistoryRecord,
|
||||
};
|
||||
use aether_runtime_state::RuntimeState;
|
||||
use serde_json::Value;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::GatewayError;
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
const RESPONSE_HISTORY_SECRET_PURPOSE: &str = "openai-response-history";
|
||||
|
||||
pub(crate) async fn hydrate_openai_response_history(
|
||||
runtime_state: &RuntimeState,
|
||||
state: &AppState,
|
||||
request: &Value,
|
||||
client_api_format: &str,
|
||||
provider_api_format: &str,
|
||||
@@ -33,6 +34,7 @@ pub(crate) async fn hydrate_openai_response_history(
|
||||
}
|
||||
|
||||
let storage_key = response_history_storage_key(previous_response_id, Some(history_scope));
|
||||
let runtime_state = state.runtime_state();
|
||||
let payload = runtime_state.kv_get(&storage_key).await.map_err(|error| {
|
||||
warn!(
|
||||
event_name = "openai_response_history_read_failed",
|
||||
@@ -46,8 +48,24 @@ pub(crate) async fn hydrate_openai_response_history(
|
||||
let Some(payload) = payload else {
|
||||
return Ok(());
|
||||
};
|
||||
let Some(payload) = crate::handlers::shared::open_runtime_secret_payload(
|
||||
state,
|
||||
RESPONSE_HISTORY_SECRET_PURPOSE,
|
||||
&payload,
|
||||
) else {
|
||||
let _ = runtime_state.kv_delete(&storage_key).await;
|
||||
warn!(
|
||||
event_name = "openai_response_history_decryption_failed",
|
||||
log_type = "ops",
|
||||
backend = runtime_state.backend_kind().as_str(),
|
||||
"gateway rejected undecryptable shared OpenAI response history"
|
||||
);
|
||||
return Err(GatewayError::Internal(
|
||||
"OpenAI response history decryption failed".to_string(),
|
||||
));
|
||||
};
|
||||
if let Err(error) =
|
||||
hydrate_response_history(previous_response_id, Some(history_scope), &payload)
|
||||
hydrate_response_history(previous_response_id, Some(history_scope), payload.as_str())
|
||||
{
|
||||
let _ = runtime_state.kv_delete(&storage_key).await;
|
||||
warn!(
|
||||
@@ -65,11 +83,25 @@ pub(crate) async fn hydrate_openai_response_history(
|
||||
}
|
||||
|
||||
pub(crate) async fn persist_response_history_record(
|
||||
runtime_state: &RuntimeState,
|
||||
state: &AppState,
|
||||
record: ResponseHistoryRecord,
|
||||
) {
|
||||
let runtime_state = state.runtime_state();
|
||||
let Some(sealed_payload) = crate::handlers::shared::seal_runtime_secret_payload(
|
||||
state,
|
||||
RESPONSE_HISTORY_SECRET_PURPOSE,
|
||||
&record.payload,
|
||||
) else {
|
||||
warn!(
|
||||
event_name = "openai_response_history_encryption_unavailable",
|
||||
log_type = "ops",
|
||||
backend = runtime_state.backend_kind().as_str(),
|
||||
"gateway refused to persist unencrypted OpenAI response history"
|
||||
);
|
||||
return;
|
||||
};
|
||||
if let Err(error) = runtime_state
|
||||
.kv_set(&record.storage_key, record.payload, Some(record.ttl))
|
||||
.kv_set(&record.storage_key, sealed_payload, Some(record.ttl))
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
@@ -83,7 +115,7 @@ pub(crate) async fn persist_response_history_record(
|
||||
}
|
||||
|
||||
pub(crate) async fn persist_converted_response_history(
|
||||
runtime_state: &RuntimeState,
|
||||
state: &AppState,
|
||||
report_context: &Value,
|
||||
response: Option<&Value>,
|
||||
) {
|
||||
@@ -91,6 +123,126 @@ pub(crate) async fn persist_converted_response_history(
|
||||
return;
|
||||
};
|
||||
if let Some(record) = record_converted_response_history(report_context, response) {
|
||||
persist_response_history_record(runtime_state, record).await;
|
||||
persist_response_history_record(state, record).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||
use aether_runtime_state::{MemoryRuntimeStateConfig, RuntimeState};
|
||||
use serde_json::json;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::{
|
||||
hydrate_openai_response_history, persist_response_history_record, ResponseHistoryRecord,
|
||||
};
|
||||
use crate::{ai_serving::response_history_storage_key, data::GatewayDataState, AppState};
|
||||
|
||||
fn response_history_test_state() -> AppState {
|
||||
AppState::new()
|
||||
.expect("test state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::disabled()
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY),
|
||||
)
|
||||
.with_runtime_state(Arc::new(RuntimeState::memory(
|
||||
MemoryRuntimeStateConfig::default(),
|
||||
)))
|
||||
}
|
||||
|
||||
fn response_history_payload(response_id: &str, scope: &str, marker: &str) -> String {
|
||||
let expires_at_unix_secs = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
.saturating_add(3600);
|
||||
json!({
|
||||
"version": 1,
|
||||
"response_id": response_id,
|
||||
"scope_fingerprint": format!("{:x}", Sha256::digest(scope.trim().as_bytes())),
|
||||
"expires_at_unix_secs": expires_at_unix_secs,
|
||||
"transcript": [{"type": "message", "content": marker}],
|
||||
})
|
||||
.to_string()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn response_history_is_encrypted_at_rest_and_hydrates() {
|
||||
let state = response_history_test_state();
|
||||
let response_id = "resp_gateway_encrypted_history_v1";
|
||||
let scope = "response-history-encrypted-scope";
|
||||
let marker = "private-response-history-marker";
|
||||
let storage_key = response_history_storage_key(response_id, Some(scope));
|
||||
let payload = response_history_payload(response_id, scope, marker);
|
||||
|
||||
persist_response_history_record(
|
||||
&state,
|
||||
ResponseHistoryRecord {
|
||||
storage_key: storage_key.clone(),
|
||||
payload,
|
||||
ttl: Duration::from_secs(6 * 60 * 60),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
let stored = state
|
||||
.runtime_kv_get(&storage_key)
|
||||
.await
|
||||
.expect("history lookup should succeed")
|
||||
.expect("history should be persisted");
|
||||
assert!(crate::handlers::shared::runtime_secret_payload_is_sealed(
|
||||
&stored
|
||||
));
|
||||
assert!(!stored.contains(marker));
|
||||
|
||||
hydrate_openai_response_history(
|
||||
&state,
|
||||
&json!({"previous_response_id": response_id}),
|
||||
"openai:responses",
|
||||
"openai:chat",
|
||||
scope,
|
||||
)
|
||||
.await
|
||||
.expect("encrypted history should hydrate");
|
||||
assert!(crate::ai_serving::response_history_is_loaded(
|
||||
response_id,
|
||||
Some(scope)
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn response_history_reader_rejects_and_deletes_legacy_plaintext() {
|
||||
let state = response_history_test_state();
|
||||
let response_id = "resp_gateway_legacy_history_v1";
|
||||
let scope = "response-history-legacy-scope";
|
||||
let storage_key = response_history_storage_key(response_id, Some(scope));
|
||||
let payload = response_history_payload(response_id, scope, "legacy-private-history");
|
||||
state
|
||||
.runtime_kv_setex(&storage_key, &payload, 6 * 60 * 60)
|
||||
.await
|
||||
.expect("legacy history should store");
|
||||
|
||||
let result = hydrate_openai_response_history(
|
||||
&state,
|
||||
&json!({"previous_response_id": response_id}),
|
||||
"openai:responses",
|
||||
"openai:chat",
|
||||
scope,
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_err());
|
||||
assert!(!crate::ai_serving::response_history_is_loaded(
|
||||
response_id,
|
||||
Some(scope)
|
||||
));
|
||||
assert!(state
|
||||
.runtime_kv_get(&storage_key)
|
||||
.await
|
||||
.expect("history lookup should succeed")
|
||||
.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -62,7 +62,8 @@ pub(crate) use aether_provider_transport::{
|
||||
append_transport_diagnostics_to_value, apply_codex_fingerprint_convergence,
|
||||
apply_codex_fingerprint_convergence_with_context, apply_local_auth_config_header_overrides,
|
||||
apply_local_body_rules, apply_local_body_rules_with_request_headers, apply_local_header_rules,
|
||||
apply_local_header_rules_with_request_headers, apply_standard_provider_request_body_rules,
|
||||
apply_local_header_rules_with_request_headers, apply_provider_outbound_request_policies,
|
||||
apply_standard_provider_request_body_rules,
|
||||
apply_standard_provider_request_body_rules_with_request_headers,
|
||||
apply_transport_request_body_semantics, body_rules_are_locally_supported,
|
||||
body_rules_handle_path, body_rules_have_enabled_rules,
|
||||
@@ -112,7 +113,11 @@ pub(crate) use aether_provider_transport::{
|
||||
GeminiCliRequestAuthSupport, GeminiCliRequestAuthUnsupportedReason,
|
||||
GeminiCliRequestEnvelopeSupport, GeminiFilesHeadersInput, GeminiFilesRequestBodyError,
|
||||
GeminiFilesRequestBodyParts, GrokHeaderInput, LocalResolvedOAuthRequestAuth,
|
||||
ProviderOpenAiImageHeadersInput, ProviderVideoCreateFamily, ProviderVideoCreateHeadersInput,
|
||||
ProviderOpenAiImageHeadersInput, ProviderOutboundRequestContext,
|
||||
ProviderOutboundRequestIdentityScope, ProviderOutboundRequestMutationScope,
|
||||
ProviderOutboundRequestPolicy, ProviderOutboundRequestPolicyOutcome,
|
||||
ProviderOutboundRequestPolicyReason, ProviderOutboundRequestPolicyResult,
|
||||
ProviderVideoCreateFamily, ProviderVideoCreateHeadersInput,
|
||||
SameFormatProviderCompatibilityEdit, SameFormatProviderCompatibilityEditAction,
|
||||
SameFormatProviderFamily, SameFormatProviderHeadersInput, SameFormatProviderRequestBehavior,
|
||||
SameFormatProviderRequestBehaviorParams, SameFormatProviderRequestBodyInput,
|
||||
@@ -121,5 +126,5 @@ pub(crate) use aether_provider_transport::{
|
||||
StandardProviderRequestHeaders, StandardProviderRequestHeadersInput,
|
||||
TransportRequestBodySemanticsError, TransportRequestUrlParams, GEMINI_CLI_USER_AGENT,
|
||||
GEMINI_CLI_V1INTERNAL_ENVELOPE_NAME, GROK_CHAT_PATH, GROK_INTERNAL_HEADER,
|
||||
GROK_RATE_LIMITS_PATH, WINDSURF_ENVELOPE_NAME,
|
||||
GROK_RATE_LIMITS_PATH, PROVIDER_OUTBOUND_CONTEXT_MAX_VALUE_BYTES, WINDSURF_ENVELOPE_NAME,
|
||||
};
|
||||
|
||||
@@ -133,7 +133,7 @@ pub(crate) async fn frontdoor_manifest(State(state): State<AppState>) -> impl In
|
||||
"internal_gateway": {
|
||||
"route_groups": INTERNAL_GATEWAY_ROUTE_GROUPS,
|
||||
"path_prefixes": INTERNAL_GATEWAY_PATH_PREFIXES,
|
||||
"status": "rust_native_control_plane",
|
||||
"status": state.internal_gateway_auth_status(),
|
||||
},
|
||||
},
|
||||
"features": {
|
||||
|
||||
@@ -1,5 +1,14 @@
|
||||
use std::net::SocketAddr;
|
||||
|
||||
use axum::body::Body;
|
||||
use axum::extract::{ConnectInfo, Request, State};
|
||||
use axum::http::{self, HeaderValue, StatusCode};
|
||||
use axum::middleware::{self, Next};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use axum::routing::{get, post};
|
||||
use axum::Router;
|
||||
use axum::{Json, Router};
|
||||
use serde_json::json;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::async_task::{
|
||||
cancel_video_task, get_video_task_detail, get_video_task_stats, get_video_task_video,
|
||||
@@ -10,8 +19,18 @@ use crate::hooks::{get_request_audit_bundle, get_request_usage_audit};
|
||||
use crate::router::metrics;
|
||||
use crate::state::AppState;
|
||||
|
||||
pub(crate) fn mount_operational_routes(router: Router<AppState>) -> Router<AppState> {
|
||||
router
|
||||
#[derive(Clone, Copy)]
|
||||
struct OperationalPermission {
|
||||
required_permissions: &'static [&'static str],
|
||||
write: bool,
|
||||
requires_full_admin_role: bool,
|
||||
}
|
||||
|
||||
pub(crate) fn mount_operational_routes(
|
||||
router: Router<AppState>,
|
||||
state: AppState,
|
||||
) -> Router<AppState> {
|
||||
let operational = Router::<AppState>::new()
|
||||
.route("/_gateway/metrics", get(metrics))
|
||||
.route("/_gateway/async-tasks/video-tasks", get(list_video_tasks))
|
||||
.route(
|
||||
@@ -50,4 +69,236 @@ pub(crate) fn mount_operational_routes(router: Router<AppState>) -> Router<AppSt
|
||||
"/_gateway/audit/request-usage/{request_id}",
|
||||
get(get_request_usage_audit),
|
||||
)
|
||||
.route_layer(middleware::from_fn_with_state(
|
||||
state,
|
||||
authorize_operational_request,
|
||||
));
|
||||
router.merge(operational)
|
||||
}
|
||||
|
||||
async fn authorize_operational_request(
|
||||
State(state): State<AppState>,
|
||||
request: Request,
|
||||
next: Next,
|
||||
) -> Response<Body> {
|
||||
let Some(permission) = operational_permission(request.method(), request.uri().path()) else {
|
||||
return operational_error_response(
|
||||
StatusCode::FORBIDDEN,
|
||||
"operational route permission is not configured",
|
||||
None,
|
||||
);
|
||||
};
|
||||
let Some(remote_addr) = request
|
||||
.extensions()
|
||||
.get::<ConnectInfo<SocketAddr>>()
|
||||
.map(|value| value.0)
|
||||
else {
|
||||
return operational_error_response(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"operational authentication unavailable",
|
||||
None,
|
||||
);
|
||||
};
|
||||
let headers = request.headers().clone();
|
||||
let uri = request.uri().clone();
|
||||
if headers.get_all(http::header::AUTHORIZATION).iter().count() > 1 {
|
||||
return operational_auth_required_response();
|
||||
}
|
||||
|
||||
match crate::control::resolve_local_admin_session_principal(&state, &headers, &uri).await {
|
||||
Ok(Some(principal)) => {
|
||||
if permission.requires_full_admin_role
|
||||
&& !crate::roles::is_full_admin_role(&principal.user_role)
|
||||
{
|
||||
return operational_permission_denied_response(permission.required_permissions[0]);
|
||||
}
|
||||
if permission.write && !crate::roles::can_write_admin_console(&principal.user_role) {
|
||||
return operational_permission_denied_response(permission.required_permissions[0]);
|
||||
}
|
||||
}
|
||||
Ok(None) => {
|
||||
let client_ip = crate::headers::effective_client_ip(&headers, &remote_addr);
|
||||
let authenticated = match crate::management_token_auth::authenticate_management_token(
|
||||
&state, &headers, client_ip,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(authenticated) => authenticated,
|
||||
Err(
|
||||
crate::management_token_auth::ManagementTokenAuthError::Missing
|
||||
| crate::management_token_auth::ManagementTokenAuthError::Invalid,
|
||||
) => return operational_auth_required_response(),
|
||||
Err(crate::management_token_auth::ManagementTokenAuthError::Unavailable) => {
|
||||
return operational_error_response(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"operational authentication unavailable",
|
||||
None,
|
||||
)
|
||||
}
|
||||
};
|
||||
|
||||
if permission.requires_full_admin_role
|
||||
&& !crate::roles::is_full_admin_role(&authenticated.user.role)
|
||||
{
|
||||
return operational_permission_denied_response(permission.required_permissions[0]);
|
||||
}
|
||||
if permission.write && !crate::roles::can_write_admin_console(&authenticated.user.role)
|
||||
{
|
||||
return operational_permission_denied_response(permission.required_permissions[0]);
|
||||
}
|
||||
let missing_permission =
|
||||
permission
|
||||
.required_permissions
|
||||
.iter()
|
||||
.copied()
|
||||
.find(|required| {
|
||||
!management_token_has_operational_permission(
|
||||
&authenticated.permissions,
|
||||
required,
|
||||
)
|
||||
});
|
||||
if let Some(required_permission) = missing_permission {
|
||||
return operational_permission_denied_response(required_permission);
|
||||
}
|
||||
|
||||
let client_ip = client_ip.to_string();
|
||||
if let Err(err) = state
|
||||
.record_management_token_usage(&authenticated.token.id, Some(client_ip.as_str()))
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
token_id = %authenticated.token.id,
|
||||
error = ?err,
|
||||
"gateway failed to record operational management token usage"
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(error = ?err, "operational admin session authentication failed");
|
||||
return operational_error_response(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"operational authentication unavailable",
|
||||
None,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let mut response = next.run(request).await;
|
||||
response.headers_mut().insert(
|
||||
http::header::CACHE_CONTROL,
|
||||
HeaderValue::from_static("no-store"),
|
||||
);
|
||||
response
|
||||
}
|
||||
|
||||
fn operational_permission(method: &http::Method, path: &str) -> Option<OperationalPermission> {
|
||||
if path == "/_gateway/metrics" {
|
||||
return Some(OperationalPermission {
|
||||
required_permissions: &["admin:monitoring:read"],
|
||||
write: false,
|
||||
requires_full_admin_role: false,
|
||||
});
|
||||
}
|
||||
if path.starts_with("/_gateway/async-tasks/video-tasks") {
|
||||
let write = *method == http::Method::POST && path.ends_with("/cancel");
|
||||
return Some(OperationalPermission {
|
||||
required_permissions: if write {
|
||||
&["admin:video_tasks:write"]
|
||||
} else {
|
||||
&["admin:video_tasks:read"]
|
||||
},
|
||||
write,
|
||||
requires_full_admin_role: false,
|
||||
});
|
||||
}
|
||||
if path.starts_with("/_gateway/audit/auth/users/") {
|
||||
return Some(OperationalPermission {
|
||||
required_permissions: &["admin:api_keys:read"],
|
||||
write: false,
|
||||
requires_full_admin_role: false,
|
||||
});
|
||||
}
|
||||
if path.starts_with("/_gateway/audit/request-audit/") {
|
||||
return Some(OperationalPermission {
|
||||
required_permissions: &[
|
||||
"admin:monitoring:admin",
|
||||
"admin:usage:read",
|
||||
"admin:api_keys:read",
|
||||
],
|
||||
write: false,
|
||||
requires_full_admin_role: true,
|
||||
});
|
||||
}
|
||||
if path.starts_with("/_gateway/audit/request-candidates/")
|
||||
|| path.starts_with("/_gateway/audit/decision-trace/")
|
||||
{
|
||||
return Some(OperationalPermission {
|
||||
required_permissions: &["admin:monitoring:admin"],
|
||||
write: false,
|
||||
requires_full_admin_role: true,
|
||||
});
|
||||
}
|
||||
if path.starts_with("/_gateway/audit/") {
|
||||
return Some(OperationalPermission {
|
||||
required_permissions: &["admin:usage:read"],
|
||||
write: false,
|
||||
requires_full_admin_role: false,
|
||||
});
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn management_token_has_operational_permission(
|
||||
permissions: &[String],
|
||||
required_permission: &str,
|
||||
) -> bool {
|
||||
let scope = required_permission
|
||||
.rsplit_once(':')
|
||||
.map(|(scope, _)| scope)
|
||||
.unwrap_or(required_permission);
|
||||
let admin_permission = format!("{scope}:admin");
|
||||
permissions
|
||||
.iter()
|
||||
.any(|permission| permission == required_permission || permission == &admin_permission)
|
||||
}
|
||||
|
||||
fn operational_auth_required_response() -> Response<Body> {
|
||||
let mut response = operational_error_response(
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"admin authentication required",
|
||||
None,
|
||||
);
|
||||
response.headers_mut().insert(
|
||||
http::header::WWW_AUTHENTICATE,
|
||||
HeaderValue::from_static("Bearer"),
|
||||
);
|
||||
response
|
||||
}
|
||||
|
||||
fn operational_permission_denied_response(required_permission: &'static str) -> Response<Body> {
|
||||
operational_error_response(
|
||||
StatusCode::FORBIDDEN,
|
||||
"operational permission denied",
|
||||
Some(required_permission),
|
||||
)
|
||||
}
|
||||
|
||||
fn operational_error_response(
|
||||
status: StatusCode,
|
||||
detail: &'static str,
|
||||
required_permission: Option<&'static str>,
|
||||
) -> Response<Body> {
|
||||
let mut response = (
|
||||
status,
|
||||
Json(json!({
|
||||
"detail": detail,
|
||||
"required_permission": required_permission,
|
||||
})),
|
||||
)
|
||||
.into_response();
|
||||
response.headers_mut().insert(
|
||||
http::header::CACHE_CONTROL,
|
||||
HeaderValue::from_static("no-store"),
|
||||
);
|
||||
response
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ use crate::constants::*;
|
||||
use crate::control::GatewayControlDecision;
|
||||
use crate::control::GatewayLocalAuthRejection;
|
||||
use crate::headers::should_skip_response_header;
|
||||
use crate::plan_usage_policy::PlanUsagePolicyRejection;
|
||||
use crate::rate_limit::FrontdoorUserRpmRejection;
|
||||
use crate::{insert_header_if_missing, GatewayError};
|
||||
|
||||
@@ -52,22 +53,34 @@ pub(crate) fn apply_streaming_response_headers(headers: &mut http::HeaderMap) {
|
||||
);
|
||||
}
|
||||
|
||||
fn apply_gateway_browser_security_headers(headers: &mut http::HeaderMap) {
|
||||
// Provider responses are API data, even when an untrusted provider labels
|
||||
// them as HTML or SVG. Keep a direct navigation to a gateway API route
|
||||
// from becoming same-origin active content, and prevent referrer leakage
|
||||
// if a user follows a link rendered from such a response.
|
||||
headers.insert(
|
||||
http::header::X_CONTENT_TYPE_OPTIONS,
|
||||
HeaderValue::from_static("nosniff"),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("content-security-policy"),
|
||||
HeaderValue::from_static(
|
||||
"default-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'; sandbox",
|
||||
),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("referrer-policy"),
|
||||
HeaderValue::from_static("no-referrer"),
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn build_client_response(
|
||||
upstream_response: reqwest::Response,
|
||||
trace_id: &str,
|
||||
control_decision: Option<&GatewayControlDecision>,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let status = upstream_response.status();
|
||||
let upstream_headers = upstream_response
|
||||
.headers()
|
||||
.iter()
|
||||
.map(|(name, value)| {
|
||||
(
|
||||
name.as_str().to_string(),
|
||||
value.to_str().unwrap_or_default().to_string(),
|
||||
)
|
||||
})
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let upstream_headers = collect_safe_response_headers(upstream_response.headers());
|
||||
let upstream_stream = upstream_response.bytes_stream();
|
||||
build_client_response_from_parts(
|
||||
status.as_u16(),
|
||||
@@ -78,6 +91,40 @@ pub(crate) fn build_client_response(
|
||||
)
|
||||
}
|
||||
|
||||
fn collect_safe_response_headers(headers: &http::HeaderMap) -> BTreeMap<String, String> {
|
||||
let connection_declared = aether_http::connection_declared_header_names(
|
||||
headers
|
||||
.get_all(http::header::CONNECTION)
|
||||
.iter()
|
||||
.filter_map(|value| value.to_str().ok()),
|
||||
);
|
||||
headers
|
||||
.iter()
|
||||
.filter_map(|(name, value)| {
|
||||
let normalized = name.as_str().to_ascii_lowercase();
|
||||
if should_skip_client_response_header(&normalized)
|
||||
|| connection_declared.contains(&normalized)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
value
|
||||
.to_str()
|
||||
.ok()
|
||||
.map(|value| (normalized, value.to_string()))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn should_skip_client_response_header(name: &str) -> bool {
|
||||
should_skip_response_header(name)
|
||||
// A provider Location is relative to the provider, not to the gateway.
|
||||
// Forwarding it lets redirect-following clients bypass the gateway and
|
||||
// can disclose their gateway Authorization header to another origin.
|
||||
// Keep Location available inside execution reports, but never expose
|
||||
// it on the client-facing response boundary.
|
||||
|| name.eq_ignore_ascii_case(http::header::LOCATION.as_str())
|
||||
}
|
||||
|
||||
pub(crate) fn build_client_response_from_parts(
|
||||
status_code: u16,
|
||||
upstream_headers: &BTreeMap<String, String>,
|
||||
@@ -111,8 +158,17 @@ where
|
||||
.body(body)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
|
||||
let connection_declared = aether_http::connection_declared_header_names(
|
||||
upstream_headers
|
||||
.iter()
|
||||
.filter(|(name, _)| name.eq_ignore_ascii_case(http::header::CONNECTION.as_str()))
|
||||
.map(|(_, value)| value.as_str()),
|
||||
);
|
||||
|
||||
for (name, value) in upstream_headers {
|
||||
if should_skip_response_header(name.as_str()) {
|
||||
if should_skip_client_response_header(name.as_str())
|
||||
|| connection_declared.contains(&name.to_ascii_lowercase())
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let header_name = HeaderName::from_bytes(name.as_bytes())
|
||||
@@ -123,6 +179,7 @@ where
|
||||
}
|
||||
mutate_headers(response.headers_mut())?;
|
||||
apply_streaming_response_headers(response.headers_mut());
|
||||
apply_gateway_browser_security_headers(response.headers_mut());
|
||||
insert_header_if_missing(response.headers_mut(), TRACE_ID_HEADER, trace_id)?;
|
||||
insert_header_if_missing(response.headers_mut(), GATEWAY_HEADER, "rust-phase3b")?;
|
||||
if let Some(decision) = control_decision {
|
||||
@@ -258,6 +315,57 @@ pub(crate) fn build_local_user_rpm_limited_response(
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn build_local_plan_usage_limited_response(
|
||||
trace_id: &str,
|
||||
control_decision: Option<&GatewayControlDecision>,
|
||||
rejection: &PlanUsagePolicyRejection,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let message = "套餐使用限制已达到上限,请稍后重试";
|
||||
let fallback_payload = json!({
|
||||
"error": {
|
||||
"type": "plan_usage_limit_exceeded",
|
||||
"message": message,
|
||||
"details": {
|
||||
"metric": rejection.metric,
|
||||
"window": rejection.window,
|
||||
"limit": rejection.limit,
|
||||
"retry_after": rejection.retry_after,
|
||||
}
|
||||
}
|
||||
});
|
||||
let payload = build_local_error_payload(
|
||||
control_decision,
|
||||
None,
|
||||
message,
|
||||
LocalCoreSyncErrorKind::RateLimit,
|
||||
fallback_payload,
|
||||
);
|
||||
let body =
|
||||
serde_json::to_vec(&payload).map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
let headers = BTreeMap::from([
|
||||
("content-type".to_string(), "application/json".to_string()),
|
||||
("Retry-After".to_string(), rejection.retry_after.to_string()),
|
||||
("X-RateLimit-Limit".to_string(), rejection.limit.to_string()),
|
||||
("X-RateLimit-Remaining".to_string(), "0".to_string()),
|
||||
("X-RateLimit-Scope".to_string(), "plan".to_string()),
|
||||
(
|
||||
"X-RateLimit-Metric".to_string(),
|
||||
rejection.metric.to_string(),
|
||||
),
|
||||
(
|
||||
"X-RateLimit-Window".to_string(),
|
||||
rejection.window.to_string(),
|
||||
),
|
||||
]);
|
||||
build_client_response_from_parts(
|
||||
StatusCode::TOO_MANY_REQUESTS.as_u16(),
|
||||
&headers,
|
||||
Body::from(body),
|
||||
trace_id,
|
||||
control_decision,
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn build_local_http_error_response(
|
||||
trace_id: &str,
|
||||
control_decision: Option<&GatewayControlDecision>,
|
||||
@@ -454,11 +562,13 @@ fn local_error_kind_for_status(status: StatusCode) -> LocalCoreSyncErrorKind {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
build_client_response_from_parts, build_local_auth_rejection_response,
|
||||
build_client_response, build_client_response_from_parts,
|
||||
build_client_response_from_parts_with_mutator, build_local_auth_rejection_response,
|
||||
build_local_http_error_response_with_request_path, build_local_overloaded_response,
|
||||
build_local_user_rpm_limited_response,
|
||||
build_local_plan_usage_limited_response, build_local_user_rpm_limited_response,
|
||||
};
|
||||
use crate::control::{GatewayControlDecision, GatewayLocalAuthRejection};
|
||||
use crate::plan_usage_policy::PlanUsagePolicyRejection;
|
||||
use crate::rate_limit::FrontdoorUserRpmRejection;
|
||||
use axum::body::{to_bytes, Body};
|
||||
use std::collections::BTreeMap;
|
||||
@@ -490,6 +600,163 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upstream_security_headers_are_stripped_before_gateway_headers_are_added() {
|
||||
let response = build_client_response_from_parts_with_mutator(
|
||||
200,
|
||||
&BTreeMap::from([
|
||||
("set-cookie".to_string(), "session=attacker".to_string()),
|
||||
(
|
||||
"x-aether-gateway".to_string(),
|
||||
"attacker-gateway".to_string(),
|
||||
),
|
||||
(
|
||||
"x-aether-control-action".to_string(),
|
||||
"attacker-action".to_string(),
|
||||
),
|
||||
(
|
||||
"x-aether-future-control".to_string(),
|
||||
"attacker-future".to_string(),
|
||||
),
|
||||
(
|
||||
"x-accel-redirect".to_string(),
|
||||
"/internal/private-file".to_string(),
|
||||
),
|
||||
("x-sendfile".to_string(), "/etc/passwd".to_string()),
|
||||
(
|
||||
"x-reproxy-url".to_string(),
|
||||
"http://127.0.0.1:9000/private".to_string(),
|
||||
),
|
||||
(
|
||||
"access-control-allow-origin".to_string(),
|
||||
"https://attacker.example".to_string(),
|
||||
),
|
||||
(
|
||||
"access-control-allow-credentials".to_string(),
|
||||
"true".to_string(),
|
||||
),
|
||||
("content-length".to_string(), "999999".to_string()),
|
||||
(
|
||||
"content-security-policy".to_string(),
|
||||
"default-src * 'unsafe-inline' 'unsafe-eval'".to_string(),
|
||||
),
|
||||
(
|
||||
"content-security-policy-report-only".to_string(),
|
||||
"default-src 'none'; report-uri https://attacker.example/csp".to_string(),
|
||||
),
|
||||
(
|
||||
"reporting-endpoints".to_string(),
|
||||
"attacker=\"https://attacker.example/reports\"".to_string(),
|
||||
),
|
||||
("report-to".to_string(), "attacker".to_string()),
|
||||
(
|
||||
"nel".to_string(),
|
||||
"{\"report_to\":\"attacker\"}".to_string(),
|
||||
),
|
||||
(
|
||||
"refresh".to_string(),
|
||||
"0; url=https://attacker.example".to_string(),
|
||||
),
|
||||
("referrer-policy".to_string(), "unsafe-url".to_string()),
|
||||
("x-content-type-options".to_string(), "invalid".to_string()),
|
||||
(
|
||||
"location".to_string(),
|
||||
"https://provider.example/direct".to_string(),
|
||||
),
|
||||
("x-upstream-visible".to_string(), "ok".to_string()),
|
||||
]),
|
||||
Body::empty(),
|
||||
"trace-upstream-header-filter",
|
||||
None,
|
||||
|headers| {
|
||||
headers.insert(
|
||||
http::HeaderName::from_static("x-aether-control-action"),
|
||||
http::HeaderValue::from_static("gateway-action"),
|
||||
);
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.expect("response should build");
|
||||
|
||||
assert!(response.headers().get(http::header::SET_COOKIE).is_none());
|
||||
assert!(response.headers().get("x-aether-future-control").is_none());
|
||||
assert!(response.headers().get("x-accel-redirect").is_none());
|
||||
assert!(response.headers().get("x-sendfile").is_none());
|
||||
assert!(response.headers().get("x-reproxy-url").is_none());
|
||||
assert!(response
|
||||
.headers()
|
||||
.get("access-control-allow-origin")
|
||||
.is_none());
|
||||
assert!(response
|
||||
.headers()
|
||||
.get("access-control-allow-credentials")
|
||||
.is_none());
|
||||
assert!(response
|
||||
.headers()
|
||||
.get(http::header::CONTENT_LENGTH)
|
||||
.is_none());
|
||||
assert!(response
|
||||
.headers()
|
||||
.get("content-security-policy-report-only")
|
||||
.is_none());
|
||||
assert!(response.headers().get("reporting-endpoints").is_none());
|
||||
assert!(response.headers().get("report-to").is_none());
|
||||
assert!(response.headers().get("nel").is_none());
|
||||
assert!(response.headers().get("refresh").is_none());
|
||||
assert!(response.headers().get(http::header::LOCATION).is_none());
|
||||
assert_eq!(
|
||||
response.headers()["content-security-policy"],
|
||||
"default-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'; sandbox"
|
||||
);
|
||||
assert_eq!(response.headers()["referrer-policy"], "no-referrer");
|
||||
assert_eq!(
|
||||
response.headers()[http::header::X_CONTENT_TYPE_OPTIONS],
|
||||
"nosniff"
|
||||
);
|
||||
assert_eq!(response.headers()["x-aether-gateway"], "rust-phase3b");
|
||||
assert_eq!(
|
||||
response.headers()["x-aether-control-action"],
|
||||
"gateway-action"
|
||||
);
|
||||
assert_eq!(response.headers()["x-upstream-visible"], "ok");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raw_response_collector_honors_all_connection_header_lines() {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("listener");
|
||||
let addr = listener.local_addr().expect("address");
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.expect("connection");
|
||||
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
|
||||
let mut request = [0_u8; 1024];
|
||||
let _ = stream.read(&mut request).await.expect("request read");
|
||||
stream
|
||||
.write_all(
|
||||
b"HTTP/1.1 200 OK\r\nConnection: x-first-hop\r\nConnection: x-second-hop\r\nX-First-Hop: first-secret\r\nX-Second-Hop: second-secret\r\nContent-Length: 2\r\n\r\nok",
|
||||
)
|
||||
.await
|
||||
.expect("response write");
|
||||
});
|
||||
let upstream = reqwest::Client::builder()
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.build()
|
||||
.expect("client")
|
||||
.get(format!("http://{addr}/"))
|
||||
.send()
|
||||
.await
|
||||
.expect("upstream response");
|
||||
|
||||
let response = build_client_response(upstream, "trace-connection-lines", None)
|
||||
.expect("client response");
|
||||
server.await.expect("server");
|
||||
|
||||
assert!(response.headers().get("connection").is_none());
|
||||
assert!(response.headers().get("x-first-hop").is_none());
|
||||
assert!(response.headers().get("x-second-hop").is_none());
|
||||
}
|
||||
|
||||
fn claude_decision() -> GatewayControlDecision {
|
||||
GatewayControlDecision::synthetic(
|
||||
"/v1/messages",
|
||||
@@ -581,4 +848,25 @@ mod tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn plan_usage_rejection_exposes_machine_readable_limit_headers() {
|
||||
let response = build_local_plan_usage_limited_response(
|
||||
"trace-plan-limit",
|
||||
None,
|
||||
&PlanUsagePolicyRejection {
|
||||
metric: "request_count",
|
||||
limit: 100.0,
|
||||
retry_after: 42,
|
||||
window: "calendar_week",
|
||||
},
|
||||
)
|
||||
.expect("response");
|
||||
assert_eq!(response.status(), http::StatusCode::TOO_MANY_REQUESTS);
|
||||
assert_eq!(response.headers()["retry-after"], "42");
|
||||
assert_eq!(response.headers()["x-ratelimit-scope"], "plan");
|
||||
assert_eq!(response.headers()["x-ratelimit-window"], "calendar_week");
|
||||
let payload = response_json(response).await;
|
||||
assert_eq!(payload["error"]["type"], "plan_usage_limit_exceeded");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use aether_contracts::ExecutionResult;
|
||||
@@ -21,7 +22,9 @@ use super::{
|
||||
};
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
pub(crate) use self::cancel::{cancel_video_task_record, CancelVideoTaskError};
|
||||
pub(crate) use self::cancel::{
|
||||
cancel_video_task_record, cancel_video_task_record_for_user, CancelVideoTaskError,
|
||||
};
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(crate) struct ListVideoTasksQuery {
|
||||
@@ -146,18 +149,21 @@ pub(crate) async fn get_video_task_video(
|
||||
}
|
||||
|
||||
pub(crate) async fn build_video_task_video_response(
|
||||
state: &AppState,
|
||||
_state: &AppState,
|
||||
task_id: &str,
|
||||
source: VideoTaskVideoSource,
|
||||
) -> Result<axum::response::Response, GatewayError> {
|
||||
match source {
|
||||
VideoTaskVideoSource::Redirect { url } => Ok(Redirect::temporary(&url).into_response()),
|
||||
VideoTaskVideoSource::Redirect { url } => {
|
||||
resolve_public_video_target(&url).await?;
|
||||
Ok(Redirect::temporary(url.as_str()).into_response())
|
||||
}
|
||||
VideoTaskVideoSource::Proxy {
|
||||
url,
|
||||
header_name,
|
||||
header_value,
|
||||
filename,
|
||||
} => proxy_video_stream(state, task_id, &url, &header_name, &header_value, &filename).await,
|
||||
} => proxy_video_stream(task_id, &url, &header_name, &header_value, &filename).await,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -209,25 +215,34 @@ fn video_task_status_name(status: VideoTaskStatus) -> &'static str {
|
||||
}
|
||||
|
||||
async fn proxy_video_stream(
|
||||
state: &AppState,
|
||||
task_id: &str,
|
||||
url: &str,
|
||||
url: &url::Url,
|
||||
header_name: &str,
|
||||
header_value: &str,
|
||||
filename: &str,
|
||||
) -> Result<axum::response::Response, GatewayError> {
|
||||
let response = state
|
||||
.client
|
||||
.get(url)
|
||||
let target = resolve_public_video_target(url).await?;
|
||||
let client = build_pinned_video_client(&target)?;
|
||||
let response = client
|
||||
.get(url.clone())
|
||||
.header(header_name, header_value)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| GatewayError::UpstreamUnavailable {
|
||||
trace_id: task_id.to_string(),
|
||||
message: err.to_string(),
|
||||
message: video_request_failure_message(&err).to_string(),
|
||||
})?;
|
||||
|
||||
if response.status().is_client_error() || response.status().is_server_error() {
|
||||
if response.status().is_redirection() {
|
||||
return Err(GatewayError::UpstreamUnavailable {
|
||||
trace_id: task_id.to_string(),
|
||||
message: format!(
|
||||
"video upstream redirect was rejected with HTTP {}",
|
||||
response.status()
|
||||
),
|
||||
});
|
||||
}
|
||||
if !response.status().is_success() {
|
||||
return Err(GatewayError::UpstreamUnavailable {
|
||||
trace_id: task_id.to_string(),
|
||||
message: format!("video upstream returned HTTP {}", response.status()),
|
||||
@@ -235,47 +250,321 @@ async fn proxy_video_stream(
|
||||
}
|
||||
|
||||
let status = response.status();
|
||||
let content_type = response
|
||||
.headers()
|
||||
.get(axum::http::header::CONTENT_TYPE)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| axum::http::HeaderValue::from_static("video/mp4"));
|
||||
let content_length = response
|
||||
.headers()
|
||||
.get(axum::http::header::CONTENT_LENGTH)
|
||||
.cloned();
|
||||
let cache_control = response
|
||||
.headers()
|
||||
.get(axum::http::header::CACHE_CONTROL)
|
||||
.cloned();
|
||||
// Do not copy the provider's Content-Length onto a newly wrapped stream.
|
||||
// Reqwest may decode transfer/content encodings and the provider controls
|
||||
// the declaration; forwarding a stale value would make the client-facing
|
||||
// HTTP framing disagree with the bytes produced by this Body. Axum/Hyper
|
||||
// will select safe framing for the actual stream.
|
||||
let upstream_headers = response.headers().clone();
|
||||
let body = Body::from_stream(response.bytes_stream());
|
||||
|
||||
let mut outbound = axum::http::Response::builder()
|
||||
.status(status)
|
||||
.body(body)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
outbound
|
||||
.headers_mut()
|
||||
.insert(axum::http::header::CONTENT_TYPE, content_type);
|
||||
outbound.headers_mut().insert(
|
||||
axum::http::header::CONTENT_DISPOSITION,
|
||||
axum::http::HeaderValue::from_str(&format!("inline; filename=\"{filename}\""))
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?,
|
||||
);
|
||||
if let Some(content_length) = content_length {
|
||||
outbound
|
||||
.headers_mut()
|
||||
.insert(axum::http::header::CONTENT_LENGTH, content_length);
|
||||
}
|
||||
if let Some(cache_control) = cache_control {
|
||||
outbound
|
||||
.headers_mut()
|
||||
.insert(axum::http::header::CACHE_CONTROL, cache_control);
|
||||
} else {
|
||||
outbound.headers_mut().insert(
|
||||
axum::http::header::CACHE_CONTROL,
|
||||
axum::http::HeaderValue::from_static("private, max-age=3600"),
|
||||
);
|
||||
}
|
||||
apply_safe_video_response_metadata(outbound.headers_mut(), &upstream_headers, filename)?;
|
||||
Ok(outbound)
|
||||
}
|
||||
|
||||
fn apply_safe_video_response_metadata(
|
||||
outbound: &mut axum::http::HeaderMap,
|
||||
upstream: &axum::http::HeaderMap,
|
||||
filename: &str,
|
||||
) -> Result<(), GatewayError> {
|
||||
let content_type = upstream
|
||||
.get(axum::http::header::CONTENT_TYPE)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(safe_video_content_type)
|
||||
.unwrap_or_else(|| axum::http::HeaderValue::from_static("application/octet-stream"));
|
||||
outbound.insert(axum::http::header::CONTENT_TYPE, content_type);
|
||||
outbound.insert(
|
||||
axum::http::header::CONTENT_DISPOSITION,
|
||||
axum::http::HeaderValue::from_str(&format!(
|
||||
"inline; filename=\"{}\"",
|
||||
safe_video_filename(filename)
|
||||
))
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?,
|
||||
);
|
||||
outbound.remove(axum::http::header::CONTENT_LENGTH);
|
||||
outbound.insert(
|
||||
axum::http::header::CACHE_CONTROL,
|
||||
axum::http::HeaderValue::from_static("private, no-store"),
|
||||
);
|
||||
outbound.insert(
|
||||
axum::http::header::X_CONTENT_TYPE_OPTIONS,
|
||||
axum::http::HeaderValue::from_static("nosniff"),
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn safe_video_content_type(raw_value: &str) -> Option<axum::http::HeaderValue> {
|
||||
let media_type = raw_value.split(';').next()?.trim().to_ascii_lowercase();
|
||||
let subtype = media_type.strip_prefix("video/")?;
|
||||
if subtype.is_empty()
|
||||
|| !subtype.bytes().all(|byte| {
|
||||
byte.is_ascii_alphanumeric()
|
||||
|| matches!(
|
||||
byte,
|
||||
b'!' | b'#' | b'$' | b'&' | b'-' | b'^' | b'_' | b'.' | b'+'
|
||||
)
|
||||
})
|
||||
{
|
||||
return None;
|
||||
}
|
||||
axum::http::HeaderValue::from_str(raw_value).ok()
|
||||
}
|
||||
|
||||
fn safe_video_filename(filename: &str) -> String {
|
||||
let filename = filename
|
||||
.chars()
|
||||
.take(255)
|
||||
.map(|character| {
|
||||
if character.is_ascii_alphanumeric() || matches!(character, '.' | '-' | '_') {
|
||||
character
|
||||
} else {
|
||||
'_'
|
||||
}
|
||||
})
|
||||
.collect::<String>();
|
||||
if filename.is_empty() {
|
||||
"video.mp4".to_string()
|
||||
} else {
|
||||
filename
|
||||
}
|
||||
}
|
||||
|
||||
struct ResolvedVideoTarget {
|
||||
host: String,
|
||||
addrs: Vec<SocketAddr>,
|
||||
}
|
||||
|
||||
async fn resolve_public_video_target(url: &url::Url) -> Result<ResolvedVideoTarget, GatewayError> {
|
||||
if !matches!(url.scheme(), "http" | "https")
|
||||
|| !url.username().is_empty()
|
||||
|| url.password().is_some()
|
||||
{
|
||||
return Err(video_target_rejected(
|
||||
"video URL must be an absolute HTTP(S) URL without credentials",
|
||||
));
|
||||
}
|
||||
let port = url
|
||||
.port_or_known_default()
|
||||
.ok_or_else(|| video_target_rejected("video URL is missing a port"))?;
|
||||
let (host, addrs) = match url.host() {
|
||||
Some(url::Host::Ipv4(ip)) => (ip.to_string(), vec![SocketAddr::new(IpAddr::V4(ip), port)]),
|
||||
Some(url::Host::Ipv6(ip)) => (ip.to_string(), vec![SocketAddr::new(IpAddr::V6(ip), port)]),
|
||||
Some(url::Host::Domain(host)) if !host.is_empty() => {
|
||||
let addrs = aether_http::lookup_host_with_limits(
|
||||
host,
|
||||
port,
|
||||
aether_http::DEFAULT_DNS_LOOKUP_TIMEOUT,
|
||||
)
|
||||
.await
|
||||
.map_err(|_| video_target_rejected("video URL DNS resolution failed"))?;
|
||||
(host.to_string(), addrs)
|
||||
}
|
||||
_ => return Err(video_target_rejected("video URL is missing a host")),
|
||||
};
|
||||
if addrs.is_empty()
|
||||
|| addrs
|
||||
.iter()
|
||||
.any(|addr| aether_http::is_private_or_reserved_ip(addr.ip()))
|
||||
{
|
||||
return Err(video_target_rejected(
|
||||
"video URL resolves to a private or reserved address",
|
||||
));
|
||||
}
|
||||
Ok(ResolvedVideoTarget { host, addrs })
|
||||
}
|
||||
|
||||
fn build_pinned_video_client(
|
||||
target: &ResolvedVideoTarget,
|
||||
) -> Result<reqwest::Client, GatewayError> {
|
||||
let mut builder = aether_http::apply_http_client_config(
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.redirect(reqwest::redirect::Policy::none()),
|
||||
&aether_http::HttpClientConfig {
|
||||
connect_timeout_ms: Some(10_000),
|
||||
request_timeout_ms: Some(300_000),
|
||||
http2_adaptive_window: true,
|
||||
..aether_http::HttpClientConfig::default()
|
||||
},
|
||||
);
|
||||
if target.host.parse::<IpAddr>().is_err() {
|
||||
builder = builder.resolve_to_addrs(&target.host, &target.addrs);
|
||||
}
|
||||
builder
|
||||
.build()
|
||||
.map_err(|_| GatewayError::Internal("video HTTP client initialization failed".to_string()))
|
||||
}
|
||||
|
||||
fn video_target_rejected(message: &str) -> GatewayError {
|
||||
GatewayError::Client {
|
||||
status: axum::http::StatusCode::BAD_GATEWAY,
|
||||
message: message.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn video_request_failure_message(error: &reqwest::Error) -> &'static str {
|
||||
if error.is_timeout() {
|
||||
"video upstream request timed out"
|
||||
} else if error.is_connect() {
|
||||
"video upstream connection failed"
|
||||
} else if error.is_body() || error.is_decode() {
|
||||
"video upstream response failed"
|
||||
} else {
|
||||
"video upstream request failed"
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use axum::response::IntoResponse;
|
||||
|
||||
use super::{
|
||||
apply_safe_video_response_metadata, build_video_task_video_response,
|
||||
resolve_public_video_target, safe_video_content_type, safe_video_filename,
|
||||
VideoTaskVideoSource,
|
||||
};
|
||||
use crate::AppState;
|
||||
|
||||
#[tokio::test]
|
||||
async fn video_redirect_response_accepts_public_target() {
|
||||
let state = AppState::new().expect("gateway state should build");
|
||||
let target = "https://8.8.8.8/video.mp4";
|
||||
|
||||
let response = build_video_task_video_response(
|
||||
&state,
|
||||
"task-public-redirect",
|
||||
VideoTaskVideoSource::Redirect {
|
||||
url: url::Url::parse(target).expect("public target should parse"),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("public redirect should build");
|
||||
|
||||
assert_eq!(
|
||||
response.status(),
|
||||
axum::http::StatusCode::TEMPORARY_REDIRECT
|
||||
);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get(axum::http::header::LOCATION)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some(target)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn video_redirect_response_rejects_private_and_reserved_targets() {
|
||||
let state = AppState::new().expect("gateway state should build");
|
||||
|
||||
for raw_url in [
|
||||
"http://127.0.0.1/video.mp4",
|
||||
"http://169.254.169.254/latest/meta-data",
|
||||
"http://10.0.0.1/video.mp4",
|
||||
"http://[::1]/video.mp4",
|
||||
] {
|
||||
let error = build_video_task_video_response(
|
||||
&state,
|
||||
"task-rejected-redirect",
|
||||
VideoTaskVideoSource::Redirect {
|
||||
url: url::Url::parse(raw_url).expect("target should parse"),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("private or reserved redirect target should be rejected");
|
||||
|
||||
assert_eq!(
|
||||
error.into_response().status(),
|
||||
axum::http::StatusCode::BAD_GATEWAY,
|
||||
"unexpected status for {raw_url}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn video_target_resolution_rejects_private_and_reserved_ip_literals() {
|
||||
for raw_url in [
|
||||
"http://127.0.0.1/video.mp4",
|
||||
"http://169.254.169.254/latest/meta-data",
|
||||
"http://10.0.0.1/video.mp4",
|
||||
"http://[::1]/video.mp4",
|
||||
"http://[::ffff:127.0.0.1]/video.mp4",
|
||||
] {
|
||||
let url = url::Url::parse(raw_url).unwrap();
|
||||
assert!(
|
||||
resolve_public_video_target(&url).await.is_err(),
|
||||
"target should be rejected: {raw_url}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn video_target_resolution_accepts_public_ip_literals() {
|
||||
for raw_url in [
|
||||
"https://8.8.8.8/video.mp4",
|
||||
"https://[2606:4700:4700::1111]/video.mp4",
|
||||
] {
|
||||
let url = url::Url::parse(raw_url).unwrap();
|
||||
assert!(
|
||||
resolve_public_video_target(&url).await.is_ok(),
|
||||
"target should be accepted: {raw_url}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_response_metadata_rejects_active_content_and_sanitizes_filename() {
|
||||
assert!(safe_video_content_type("video/mp4").is_some());
|
||||
assert!(safe_video_content_type("video/webm; charset=binary").is_some());
|
||||
assert!(safe_video_content_type("video/").is_none());
|
||||
assert!(safe_video_content_type("video/; charset=binary").is_none());
|
||||
assert!(safe_video_content_type("text/html").is_none());
|
||||
assert!(safe_video_content_type("video/mp4\r\nx-test: injected").is_none());
|
||||
assert_eq!(
|
||||
safe_video_filename("video_123.mp4\"; filename=\"attack.html"),
|
||||
"video_123.mp4___filename__attack.html"
|
||||
);
|
||||
assert_eq!(safe_video_filename(&"x".repeat(1024)).len(), 255);
|
||||
|
||||
let mut upstream = axum::http::HeaderMap::new();
|
||||
upstream.insert(
|
||||
axum::http::header::CONTENT_TYPE,
|
||||
axum::http::HeaderValue::from_static("text/html"),
|
||||
);
|
||||
upstream.insert(
|
||||
axum::http::header::CONTENT_LENGTH,
|
||||
axum::http::HeaderValue::from_static("999999"),
|
||||
);
|
||||
let mut outbound = upstream.clone();
|
||||
apply_safe_video_response_metadata(
|
||||
&mut outbound,
|
||||
&upstream,
|
||||
"video.mp4\"; filename=\"attack.html",
|
||||
)
|
||||
.expect("video metadata should build");
|
||||
|
||||
assert_eq!(
|
||||
outbound
|
||||
.get(axum::http::header::CONTENT_TYPE)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("application/octet-stream")
|
||||
);
|
||||
assert!(outbound.get(axum::http::header::CONTENT_LENGTH).is_none());
|
||||
assert_eq!(
|
||||
outbound
|
||||
.get(axum::http::header::X_CONTENT_TYPE_OPTIONS)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("nosniff")
|
||||
);
|
||||
assert_eq!(
|
||||
outbound
|
||||
.get(axum::http::header::CONTENT_DISPOSITION)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("inline; filename=\"video.mp4___filename__attack.html\"")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,12 +3,13 @@ use aether_data_contracts::repository::video_tasks::{
|
||||
};
|
||||
use axum::response::IntoResponse;
|
||||
use axum::Json;
|
||||
use serde_json::{json, Map, Value};
|
||||
use serde_json::json;
|
||||
|
||||
use crate::state::VideoTaskRouteAccess;
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
use super::super::finalize_video_task_if_terminal;
|
||||
use super::super::read_video_task_detail;
|
||||
use super::super::{read_video_task_detail, read_video_task_detail_for_user};
|
||||
use super::current_unix_secs;
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -29,7 +30,31 @@ pub(crate) async fn cancel_video_task_record(
|
||||
state: &AppState,
|
||||
task_id: &str,
|
||||
) -> Result<StoredVideoTask, CancelVideoTaskError> {
|
||||
let Some(task) = read_video_task_detail(state, task_id).await? else {
|
||||
cancel_video_task_record_inner(state, task_id, None).await
|
||||
}
|
||||
|
||||
pub(crate) async fn cancel_video_task_record_for_user(
|
||||
state: &AppState,
|
||||
task_id: &str,
|
||||
user_id: &str,
|
||||
) -> Result<StoredVideoTask, CancelVideoTaskError> {
|
||||
let user_id = user_id.trim();
|
||||
if user_id.is_empty() {
|
||||
return Err(CancelVideoTaskError::NotFound);
|
||||
}
|
||||
cancel_video_task_record_inner(state, task_id, Some(user_id)).await
|
||||
}
|
||||
|
||||
async fn cancel_video_task_record_inner(
|
||||
state: &AppState,
|
||||
task_id: &str,
|
||||
expected_user_id: Option<&str>,
|
||||
) -> Result<StoredVideoTask, CancelVideoTaskError> {
|
||||
let task = match expected_user_id {
|
||||
Some(user_id) => read_video_task_detail_for_user(state, task_id, user_id).await?,
|
||||
None => read_video_task_detail(state, task_id).await?,
|
||||
};
|
||||
let Some(task) = task else {
|
||||
return Err(CancelVideoTaskError::NotFound);
|
||||
};
|
||||
|
||||
@@ -45,39 +70,84 @@ pub(crate) async fn cancel_video_task_record(
|
||||
}
|
||||
|
||||
let trace_id = format!("async-task-admin-cancel-{task_id}");
|
||||
let mut finalize_mutation = None;
|
||||
if let Some(cancel_plan) = build_video_task_cancel_plan(&task) {
|
||||
state
|
||||
.hydrate_video_task_for_route(Some(cancel_plan.route_family), &cancel_plan.request_path)
|
||||
.await?;
|
||||
|
||||
let body_json = json!({});
|
||||
let follow_up = state.video_tasks.prepare_follow_up_sync_plan(
|
||||
cancel_plan.plan_kind,
|
||||
&cancel_plan.request_path,
|
||||
Some(&body_json),
|
||||
None,
|
||||
&trace_id,
|
||||
);
|
||||
let follow_up = if let Some(user_id) = expected_user_id {
|
||||
if state
|
||||
.hydrate_video_task_for_route_for_user(
|
||||
Some(cancel_plan.route_family),
|
||||
&cancel_plan.request_path,
|
||||
user_id,
|
||||
)
|
||||
.await?
|
||||
!= VideoTaskRouteAccess::Allowed
|
||||
{
|
||||
return Err(CancelVideoTaskError::NotFound);
|
||||
}
|
||||
state.video_tasks.prepare_follow_up_sync_plan_for_user_id(
|
||||
cancel_plan.plan_kind,
|
||||
&cancel_plan.request_path,
|
||||
Some(&body_json),
|
||||
user_id,
|
||||
task.api_key_id.as_deref(),
|
||||
&trace_id,
|
||||
)
|
||||
} else {
|
||||
state
|
||||
.hydrate_video_task_for_route(
|
||||
Some(cancel_plan.route_family),
|
||||
&cancel_plan.request_path,
|
||||
)
|
||||
.await?;
|
||||
state.video_tasks.prepare_follow_up_sync_plan(
|
||||
cancel_plan.plan_kind,
|
||||
&cancel_plan.request_path,
|
||||
Some(&body_json),
|
||||
None,
|
||||
&trace_id,
|
||||
)
|
||||
};
|
||||
|
||||
if let Some(follow_up) = follow_up {
|
||||
execute_video_task_cancel_plan(state, &trace_id, follow_up.plan)
|
||||
.await
|
||||
.map_err(CancelVideoTaskError::Response)?;
|
||||
finalize_mutation = Some((
|
||||
cancel_plan.request_path,
|
||||
cancel_plan.report_kind.to_string(),
|
||||
));
|
||||
} else if expected_user_id.is_none() {
|
||||
finalize_mutation = Some((
|
||||
cancel_plan.request_path,
|
||||
cancel_plan.report_kind.to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
state
|
||||
.video_tasks
|
||||
.apply_finalize_mutation(&cancel_plan.request_path, cancel_plan.report_kind);
|
||||
}
|
||||
|
||||
let request_metadata = build_cancelled_request_metadata(state, &task).await?;
|
||||
let stored = persist_cancelled_video_task(state, &task, request_metadata)
|
||||
.await?
|
||||
.ok_or_else(|| {
|
||||
CancelVideoTaskError::Gateway(GatewayError::Internal(
|
||||
let stored = match persist_cancelled_video_task(state, &task).await? {
|
||||
Some(stored) => stored,
|
||||
None => {
|
||||
let current = match expected_user_id {
|
||||
Some(user_id) => read_video_task_detail_for_user(state, task_id, user_id).await?,
|
||||
None => read_video_task_detail(state, task_id).await?,
|
||||
};
|
||||
let Some(current) = current else {
|
||||
return Err(CancelVideoTaskError::NotFound);
|
||||
};
|
||||
if !current.status.is_active() {
|
||||
return Err(CancelVideoTaskError::InvalidStatus(current.status));
|
||||
}
|
||||
return Err(CancelVideoTaskError::Gateway(GatewayError::Internal(
|
||||
"video task repository is unavailable".to_string(),
|
||||
))
|
||||
})?;
|
||||
)));
|
||||
}
|
||||
};
|
||||
if let Some((request_path, report_kind)) = finalize_mutation {
|
||||
state
|
||||
.video_tasks
|
||||
.apply_finalize_mutation(&request_path, &report_kind);
|
||||
}
|
||||
finalize_video_task_if_terminal(state, &stored).await;
|
||||
Ok(stored)
|
||||
}
|
||||
@@ -91,12 +161,7 @@ struct VideoTaskCancelPlan<'a> {
|
||||
}
|
||||
|
||||
fn build_video_task_cancel_plan(task: &StoredVideoTask) -> Option<VideoTaskCancelPlan<'_>> {
|
||||
let provider_api_format = task
|
||||
.provider_api_format
|
||||
.as_deref()
|
||||
.or(task.client_api_format.as_deref())
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())?;
|
||||
let provider_api_format = task.effective_api_format()?;
|
||||
|
||||
match provider_api_format {
|
||||
"openai:video" => Some(VideoTaskCancelPlan {
|
||||
@@ -131,99 +196,52 @@ async fn execute_video_task_cancel_plan(
|
||||
let result =
|
||||
crate::execution_runtime::execute_execution_runtime_sync_plan(state, Some(trace_id), &plan)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
.map_err(|_| {
|
||||
GatewayError::UpstreamUnavailable {
|
||||
trace_id: trace_id.to_string(),
|
||||
message: format!("{err:?}"),
|
||||
message: "video cancellation request failed".to_string(),
|
||||
}
|
||||
.into_response()
|
||||
})?;
|
||||
|
||||
if result.status_code >= 400 {
|
||||
let status = axum::http::StatusCode::from_u16(result.status_code)
|
||||
.unwrap_or(axum::http::StatusCode::BAD_GATEWAY);
|
||||
let body_json = result
|
||||
.body
|
||||
.and_then(|body| body.json_body)
|
||||
.unwrap_or_else(|| {
|
||||
json!({
|
||||
"error": {
|
||||
"message": result
|
||||
.error
|
||||
.as_ref()
|
||||
.map(|error| error.message.clone())
|
||||
.unwrap_or_else(|| {
|
||||
format!("execution runtime returned {}", result.status_code)
|
||||
}),
|
||||
}
|
||||
})
|
||||
});
|
||||
return Err((status, Json(body_json)).into_response());
|
||||
return Err(build_video_task_cancel_upstream_error_response(&result));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn build_cancelled_request_metadata(
|
||||
state: &AppState,
|
||||
task: &StoredVideoTask,
|
||||
) -> Result<Option<Value>, GatewayError> {
|
||||
let mut metadata = match task.request_metadata.clone() {
|
||||
Some(Value::Object(object)) => object,
|
||||
_ => Map::new(),
|
||||
};
|
||||
let mut snapshot_value = metadata.get("rust_local_snapshot").cloned();
|
||||
if snapshot_value.is_none() {
|
||||
snapshot_value = state
|
||||
.reconstruct_video_task_snapshot(task)
|
||||
.await?
|
||||
.map(|snapshot| {
|
||||
serde_json::to_value(snapshot)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))
|
||||
})
|
||||
.transpose()?;
|
||||
}
|
||||
if let Some(snapshot_value_ref) = snapshot_value.as_mut() {
|
||||
mark_snapshot_value_cancelled(snapshot_value_ref);
|
||||
metadata.insert(
|
||||
"rust_owner".to_string(),
|
||||
Value::String("async_task".to_string()),
|
||||
);
|
||||
metadata.insert(
|
||||
"rust_local_snapshot".to_string(),
|
||||
snapshot_value_ref.clone(),
|
||||
);
|
||||
return Ok(Some(Value::Object(metadata)));
|
||||
}
|
||||
|
||||
Ok(task.request_metadata.clone())
|
||||
}
|
||||
|
||||
fn mark_snapshot_value_cancelled(snapshot_value: &mut Value) {
|
||||
if let Some(object) = snapshot_value
|
||||
.get_mut("OpenAi")
|
||||
.and_then(Value::as_object_mut)
|
||||
{
|
||||
object.insert("status".to_string(), Value::String("Cancelled".to_string()));
|
||||
return;
|
||||
}
|
||||
if let Some(object) = snapshot_value
|
||||
.get_mut("Gemini")
|
||||
.and_then(Value::as_object_mut)
|
||||
{
|
||||
object.insert("status".to_string(), Value::String("Cancelled".to_string()));
|
||||
}
|
||||
fn build_video_task_cancel_upstream_error_response(
|
||||
result: &aether_contracts::ExecutionResult,
|
||||
) -> axum::response::Response {
|
||||
let status = axum::http::StatusCode::from_u16(result.status_code)
|
||||
.unwrap_or(axum::http::StatusCode::BAD_GATEWAY);
|
||||
tracing::warn!(
|
||||
event_name = "video_task_cancel_upstream_error",
|
||||
upstream_status = result.status_code,
|
||||
"video cancellation upstream response body discarded"
|
||||
);
|
||||
(
|
||||
status,
|
||||
Json(json!({
|
||||
"error": {
|
||||
"message": format!(
|
||||
"video cancellation upstream returned HTTP {}",
|
||||
result.status_code
|
||||
),
|
||||
}
|
||||
})),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
async fn persist_cancelled_video_task(
|
||||
state: &AppState,
|
||||
task: &StoredVideoTask,
|
||||
request_metadata: Option<Value>,
|
||||
) -> Result<Option<StoredVideoTask>, GatewayError> {
|
||||
let now_unix_secs = current_unix_secs();
|
||||
state
|
||||
.data
|
||||
.upsert_video_task(UpsertVideoTask {
|
||||
.update_active_video_task(UpsertVideoTask {
|
||||
id: task.id.clone(),
|
||||
short_id: task.short_id.clone(),
|
||||
request_id: task.request_id.clone(),
|
||||
@@ -240,14 +258,14 @@ async fn persist_cancelled_video_task(
|
||||
format_converted: task.format_converted,
|
||||
model: task.model.clone(),
|
||||
prompt: task.prompt.clone(),
|
||||
original_request_body: task.original_request_body.clone(),
|
||||
original_request_body: None,
|
||||
duration_seconds: task.duration_seconds,
|
||||
resolution: task.resolution.clone(),
|
||||
aspect_ratio: task.aspect_ratio.clone(),
|
||||
size: task.size.clone(),
|
||||
status: VideoTaskStatus::Cancelled,
|
||||
progress_percent: task.progress_percent,
|
||||
progress_message: task.progress_message.clone(),
|
||||
progress_message: None,
|
||||
retry_count: task.retry_count,
|
||||
poll_interval_seconds: task.poll_interval_seconds,
|
||||
next_poll_at_unix_secs: None,
|
||||
@@ -258,10 +276,73 @@ async fn persist_cancelled_video_task(
|
||||
completed_at_unix_secs: Some(now_unix_secs),
|
||||
updated_at_unix_secs: now_unix_secs,
|
||||
error_code: task.error_code.clone(),
|
||||
error_message: task.error_message.clone(),
|
||||
error_message: None,
|
||||
video_url: task.video_url.clone(),
|
||||
request_metadata,
|
||||
request_metadata: None,
|
||||
})
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use aether_contracts::{
|
||||
ExecutionError, ExecutionErrorKind, ExecutionPhase, ExecutionResult, ResponseBody,
|
||||
};
|
||||
use axum::body::to_bytes;
|
||||
use serde_json::json;
|
||||
|
||||
use super::build_video_task_cancel_upstream_error_response;
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancellation_upstream_errors_do_not_expose_runtime_payloads() {
|
||||
let result = ExecutionResult {
|
||||
request_id: "cancel-secret-request-id".to_string(),
|
||||
candidate_id: Some("cancel-secret-candidate-id".to_string()),
|
||||
status_code: 502,
|
||||
headers: BTreeMap::from([(
|
||||
"x-internal-secret".to_string(),
|
||||
"cancel-secret-header".to_string(),
|
||||
)]),
|
||||
response_observation: None,
|
||||
body: Some(ResponseBody {
|
||||
json_body: Some(json!({
|
||||
"error": {
|
||||
"message": "cancel-secret-upstream-body",
|
||||
}
|
||||
})),
|
||||
body_bytes_b64: None,
|
||||
}),
|
||||
telemetry: None,
|
||||
error: Some(ExecutionError {
|
||||
kind: ExecutionErrorKind::Upstream5xx,
|
||||
phase: ExecutionPhase::FirstByte,
|
||||
message: "cancel-secret-runtime-error".to_string(),
|
||||
upstream_status: Some(502),
|
||||
retryable: true,
|
||||
failover_recommended: false,
|
||||
}),
|
||||
};
|
||||
|
||||
let response = build_video_task_cancel_upstream_error_response(&result);
|
||||
assert_eq!(response.status(), axum::http::StatusCode::BAD_GATEWAY);
|
||||
assert!(response.headers().get("x-internal-secret").is_none());
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("response body should read");
|
||||
let payload: serde_json::Value =
|
||||
serde_json::from_slice(&body).expect("response body should parse");
|
||||
|
||||
assert_eq!(
|
||||
payload,
|
||||
json!({
|
||||
"error": {
|
||||
"message": "video cancellation upstream returned HTTP 502",
|
||||
}
|
||||
})
|
||||
);
|
||||
let body = String::from_utf8(body.to_vec()).expect("response body should be utf-8");
|
||||
assert!(!body.contains("cancel-secret"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,13 +6,14 @@ pub(crate) use crate::video_tasks::VideoTaskService;
|
||||
pub use crate::video_tasks::VideoTaskTruthSourceMode;
|
||||
pub(crate) use http::{
|
||||
build_video_task_video_response, cancel_video_task, cancel_video_task_record,
|
||||
get_video_task_detail, get_video_task_stats, get_video_task_video, list_video_tasks,
|
||||
CancelVideoTaskError,
|
||||
cancel_video_task_record_for_user, get_video_task_detail, get_video_task_stats,
|
||||
get_video_task_video, list_video_tasks, CancelVideoTaskError,
|
||||
};
|
||||
pub(crate) use query::{
|
||||
read_video_task_detail, read_video_task_page, read_video_task_page_summary,
|
||||
read_video_task_stats, read_video_task_video_source, VideoTaskPageResponse,
|
||||
VideoTaskStatsResponse, VideoTaskVideoSource,
|
||||
read_video_task_detail, read_video_task_detail_for_user, read_video_task_page,
|
||||
read_video_task_page_summary, read_video_task_stats, read_video_task_video_source,
|
||||
video_task_video_source_from_task, VideoTaskPageResponse, VideoTaskStatsResponse,
|
||||
VideoTaskVideoSource,
|
||||
};
|
||||
pub(crate) use runtime::{
|
||||
execute_video_task_refresh_plan, finalize_video_task_if_terminal, spawn_video_task_poller,
|
||||
|
||||
@@ -26,13 +26,12 @@ pub(crate) struct VideoTaskStatsResponse {
|
||||
pub(crate) processing_count: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) enum VideoTaskVideoSource {
|
||||
Redirect {
|
||||
url: String,
|
||||
url: url::Url,
|
||||
},
|
||||
Proxy {
|
||||
url: String,
|
||||
url: url::Url,
|
||||
header_name: String,
|
||||
header_value: String,
|
||||
filename: String,
|
||||
@@ -102,6 +101,14 @@ pub(crate) async fn read_video_task_detail(
|
||||
state.find_video_task_by_id(task_id).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_video_task_detail_for_user(
|
||||
state: &AppState,
|
||||
task_id: &str,
|
||||
user_id: &str,
|
||||
) -> Result<Option<StoredVideoTask>, GatewayError> {
|
||||
state.find_video_task_by_id_for_user(task_id, user_id).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_video_task_video_source(
|
||||
state: &AppState,
|
||||
task_id: &str,
|
||||
@@ -109,6 +116,13 @@ pub(crate) async fn read_video_task_video_source(
|
||||
let Some(task) = read_video_task_detail(state, task_id).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
video_task_video_source_from_task(state, &task).await
|
||||
}
|
||||
|
||||
pub(crate) async fn video_task_video_source_from_task(
|
||||
state: &AppState,
|
||||
task: &StoredVideoTask,
|
||||
) -> Result<Option<VideoTaskVideoSource>, GatewayError> {
|
||||
let Some(video_url) = task
|
||||
.video_url
|
||||
.as_deref()
|
||||
@@ -119,7 +133,9 @@ pub(crate) async fn read_video_task_video_source(
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
if !video_url.contains("generativelanguage.googleapis.com") {
|
||||
let video_url = parse_video_url(&video_url)?;
|
||||
|
||||
if task.effective_api_format() != Some("gemini:video") {
|
||||
return Ok(Some(VideoTaskVideoSource::Redirect { url: video_url }));
|
||||
}
|
||||
|
||||
@@ -148,6 +164,15 @@ pub(crate) async fn read_video_task_video_source(
|
||||
));
|
||||
};
|
||||
|
||||
let endpoint_url = parse_video_url(transport.endpoint.base_url.trim()).map_err(|_| {
|
||||
GatewayError::Internal("provider endpoint URL is invalid for proxied video".to_string())
|
||||
})?;
|
||||
if !video_urls_share_origin(&endpoint_url, &video_url) {
|
||||
return Err(GatewayError::Client {
|
||||
status: axum::http::StatusCode::BAD_GATEWAY,
|
||||
message: "video URL origin does not match its provider endpoint".to_string(),
|
||||
});
|
||||
}
|
||||
let api_key = transport.key.decrypted_api_key.trim();
|
||||
if api_key.is_empty() {
|
||||
return Err(GatewayError::Internal(
|
||||
@@ -159,10 +184,34 @@ pub(crate) async fn read_video_task_video_source(
|
||||
url: video_url,
|
||||
header_name: "x-goog-api-key".to_string(),
|
||||
header_value: api_key.to_string(),
|
||||
filename: format!("video_{task_id}.mp4"),
|
||||
filename: format!("video_{}.mp4", task.id),
|
||||
}))
|
||||
}
|
||||
|
||||
fn parse_video_url(raw_url: &str) -> Result<url::Url, GatewayError> {
|
||||
let url = url::Url::parse(raw_url.trim()).map_err(|_| GatewayError::Client {
|
||||
status: axum::http::StatusCode::BAD_GATEWAY,
|
||||
message: "video URL is invalid".to_string(),
|
||||
})?;
|
||||
if !matches!(url.scheme(), "http" | "https")
|
||||
|| url.host_str().is_none()
|
||||
|| !url.username().is_empty()
|
||||
|| url.password().is_some()
|
||||
{
|
||||
return Err(GatewayError::Client {
|
||||
status: axum::http::StatusCode::BAD_GATEWAY,
|
||||
message: "video URL must be an absolute HTTP(S) URL without credentials".to_string(),
|
||||
});
|
||||
}
|
||||
Ok(url)
|
||||
}
|
||||
|
||||
fn video_urls_share_origin(left: &url::Url, right: &url::Url) -> bool {
|
||||
left.scheme() == right.scheme()
|
||||
&& left.host() == right.host()
|
||||
&& left.port_or_known_default() == right.port_or_known_default()
|
||||
}
|
||||
|
||||
pub(crate) async fn read_video_task_stats(
|
||||
state: &AppState,
|
||||
filter: &VideoTaskQueryFilter,
|
||||
@@ -226,3 +275,212 @@ fn status_key(status: VideoTaskStatus) -> String {
|
||||
fn start_of_utc_day(now_unix_secs: u64) -> u64 {
|
||||
now_unix_secs - (now_unix_secs % 86_400)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
||||
use aether_data::repository::video_tasks::InMemoryVideoTaskRepository;
|
||||
use aether_data_contracts::repository::provider_catalog::{
|
||||
ProviderCatalogReadRepository, StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
|
||||
StoredProviderCatalogProvider,
|
||||
};
|
||||
use aether_data_contracts::repository::video_tasks::{UpsertVideoTask, VideoTaskStatus};
|
||||
use serde_json::json;
|
||||
|
||||
use super::{
|
||||
parse_video_url, video_task_video_source_from_task, video_urls_share_origin,
|
||||
VideoTaskVideoSource,
|
||||
};
|
||||
use crate::{data::GatewayDataState, AppState};
|
||||
|
||||
fn legacy_gemini_video_task() -> aether_data_contracts::repository::video_tasks::StoredVideoTask
|
||||
{
|
||||
UpsertVideoTask {
|
||||
id: "legacy-gemini-task".to_string(),
|
||||
short_id: Some("legacy-short".to_string()),
|
||||
request_id: "legacy-request".to_string(),
|
||||
user_id: Some("user-1".to_string()),
|
||||
api_key_id: Some("client-key-1".to_string()),
|
||||
username: None,
|
||||
api_key_name: None,
|
||||
external_task_id: Some("operations/upstream-1".to_string()),
|
||||
provider_id: Some("provider-1".to_string()),
|
||||
endpoint_id: Some("endpoint-1".to_string()),
|
||||
key_id: Some("provider-key-1".to_string()),
|
||||
client_api_format: Some("gemini:video".to_string()),
|
||||
provider_api_format: None,
|
||||
format_converted: false,
|
||||
model: Some("veo-3".to_string()),
|
||||
prompt: None,
|
||||
original_request_body: None,
|
||||
duration_seconds: Some(8),
|
||||
resolution: Some("720p".to_string()),
|
||||
aspect_ratio: Some("16:9".to_string()),
|
||||
size: Some("1280x720".to_string()),
|
||||
status: VideoTaskStatus::Completed,
|
||||
progress_percent: 100,
|
||||
progress_message: None,
|
||||
retry_count: 0,
|
||||
poll_interval_seconds: 10,
|
||||
next_poll_at_unix_secs: None,
|
||||
poll_count: 1,
|
||||
max_poll_count: 360,
|
||||
created_at_unix_ms: 1,
|
||||
submitted_at_unix_secs: Some(1),
|
||||
completed_at_unix_secs: Some(2),
|
||||
updated_at_unix_secs: 2,
|
||||
error_code: None,
|
||||
error_message: None,
|
||||
video_url: Some(
|
||||
"https://generativelanguage.googleapis.com/v1beta/files/video-1:download?alt=media"
|
||||
.to_string(),
|
||||
),
|
||||
request_metadata: None,
|
||||
}
|
||||
.into_stored()
|
||||
}
|
||||
|
||||
fn state_with_gemini_transport() -> AppState {
|
||||
let state = AppState::new().expect("gateway state should build");
|
||||
let provider = StoredProviderCatalogProvider::new(
|
||||
"provider-1".to_string(),
|
||||
"Gemini".to_string(),
|
||||
Some("https://ai.google.dev".to_string()),
|
||||
"gemini".to_string(),
|
||||
)
|
||||
.expect("provider should build");
|
||||
let endpoint = StoredProviderCatalogEndpoint::new(
|
||||
"endpoint-1".to_string(),
|
||||
"provider-1".to_string(),
|
||||
"gemini:video".to_string(),
|
||||
None,
|
||||
None,
|
||||
true,
|
||||
)
|
||||
.expect("endpoint should build")
|
||||
.with_transport_fields(
|
||||
"https://generativelanguage.googleapis.com".to_string(),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.expect("endpoint transport should build");
|
||||
let encrypted_api_key = state
|
||||
.seal_provider_catalog_key_api_key(
|
||||
"provider-1",
|
||||
"provider-key-1",
|
||||
"gemini-provider-secret",
|
||||
)
|
||||
.expect("provider key should encrypt");
|
||||
let key = StoredProviderCatalogKey::new(
|
||||
"provider-key-1".to_string(),
|
||||
"provider-1".to_string(),
|
||||
"default".to_string(),
|
||||
"api_key".to_string(),
|
||||
None,
|
||||
true,
|
||||
)
|
||||
.expect("provider key should build")
|
||||
.with_transport_fields(
|
||||
Some(json!(["gemini:video"])),
|
||||
encrypted_api_key,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.expect("provider key transport should build");
|
||||
let provider_catalog: Arc<dyn ProviderCatalogReadRepository> = Arc::new(
|
||||
InMemoryProviderCatalogReadRepository::seed(vec![provider], vec![endpoint], vec![key]),
|
||||
);
|
||||
let video_tasks = Arc::new(InMemoryVideoTaskRepository::default());
|
||||
let data = GatewayDataState::with_video_task_repository_and_provider_transport_for_tests(
|
||||
video_tasks,
|
||||
provider_catalog,
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
state.with_data_state_for_tests(data)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_url_parser_rejects_non_http_and_embedded_credentials() {
|
||||
for raw_url in [
|
||||
"file:///etc/passwd",
|
||||
"data:video/mp4;base64,AAAA",
|
||||
"https://[email protected]/video.mp4",
|
||||
"https://user:[email protected]/video.mp4",
|
||||
"/relative/video.mp4",
|
||||
] {
|
||||
assert!(
|
||||
parse_video_url(raw_url).is_err(),
|
||||
"URL should be rejected: {raw_url}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_origin_comparison_uses_scheme_host_and_effective_port() {
|
||||
let base = parse_video_url("https://generativelanguage.googleapis.com/v1beta").unwrap();
|
||||
for same_origin in [
|
||||
"https://generativelanguage.googleapis.com/file",
|
||||
"https://generativelanguage.googleapis.com:443/file",
|
||||
] {
|
||||
assert!(video_urls_share_origin(
|
||||
&base,
|
||||
&parse_video_url(same_origin).unwrap()
|
||||
));
|
||||
}
|
||||
for different_origin in [
|
||||
"http://generativelanguage.googleapis.com/file",
|
||||
"https://generativelanguage.googleapis.com:444/file",
|
||||
"https://generativelanguage.googleapis.com.evil.test/file",
|
||||
"https://evil.test/generativelanguage.googleapis.com/file",
|
||||
] {
|
||||
assert!(!video_urls_share_origin(
|
||||
&base,
|
||||
&parse_video_url(different_origin).unwrap()
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_gemini_client_format_uses_authenticated_proxy_source() {
|
||||
let source = video_task_video_source_from_task(
|
||||
&state_with_gemini_transport(),
|
||||
&legacy_gemini_video_task(),
|
||||
)
|
||||
.await
|
||||
.expect("video source should resolve")
|
||||
.expect("video source should exist");
|
||||
|
||||
match source {
|
||||
VideoTaskVideoSource::Proxy {
|
||||
url,
|
||||
header_name,
|
||||
header_value,
|
||||
filename,
|
||||
} => {
|
||||
assert_eq!(
|
||||
url.as_str(),
|
||||
"https://generativelanguage.googleapis.com/v1beta/files/video-1:download?alt=media"
|
||||
);
|
||||
assert_eq!(header_name, "x-goog-api-key");
|
||||
assert_eq!(header_value, "gemini-provider-secret");
|
||||
assert_eq!(filename, "video_legacy-gemini-task.mp4");
|
||||
}
|
||||
VideoTaskVideoSource::Redirect { .. } => {
|
||||
panic!("legacy Gemini video must not bypass the authenticated proxy")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@ const VIDEO_TASK_POLL_CLAIM_SECONDS: u64 = 30;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct VideoTaskRefreshError {
|
||||
message: String,
|
||||
category: &'static str,
|
||||
permanent: bool,
|
||||
}
|
||||
|
||||
@@ -55,7 +55,7 @@ pub(crate) async fn execute_video_task_refresh_plan(
|
||||
warn!(
|
||||
event_name = "video_task_refresh_failed",
|
||||
log_type = "event",
|
||||
error = %err.message,
|
||||
error_category = err.category,
|
||||
permanent = err.permanent,
|
||||
"gateway video task refresh failed"
|
||||
);
|
||||
@@ -79,23 +79,32 @@ async fn poll_video_tasks_once(state: &AppState, batch_size: usize) -> Result<us
|
||||
let mut refreshed = 0usize;
|
||||
for (index, task) in tasks.into_iter().enumerate() {
|
||||
let trace_id = format!("video-task-poller-{index}");
|
||||
let Some(snapshot) = state.reconstruct_video_task_snapshot(&task).await? else {
|
||||
continue;
|
||||
};
|
||||
let Some(refresh_plan) = state
|
||||
.video_tasks
|
||||
.prepare_poll_refresh_plan_for_stored_task(&task, &trace_id)
|
||||
.prepare_poll_refresh_plan_for_snapshot(snapshot.clone(), &trace_id)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
|
||||
match fetch_video_task_refresh_attempt(state, &refresh_plan).await? {
|
||||
VideoTaskRefreshAttempt::Success { provider_body } => {
|
||||
let Some(updated) =
|
||||
build_successful_poll_update(&task, &provider_body, now_unix_secs)?
|
||||
let Some(updated) = build_successful_poll_update(
|
||||
&task,
|
||||
snapshot.clone(),
|
||||
&provider_body,
|
||||
now_unix_secs,
|
||||
)?
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
match state.update_active_video_task(updated).await? {
|
||||
Some(stored) => {
|
||||
if let Some(snapshot) = LocalVideoTaskSnapshot::from_stored_task(&stored) {
|
||||
if let Some(snapshot) =
|
||||
state.reconstruct_video_task_snapshot(&stored).await?
|
||||
{
|
||||
state.video_tasks.record_snapshot(snapshot);
|
||||
}
|
||||
info!(
|
||||
@@ -116,7 +125,9 @@ async fn poll_video_tasks_once(state: &AppState, batch_size: usize) -> Result<us
|
||||
let updated = build_failed_poll_update(&task, &err, now_unix_secs);
|
||||
match state.update_active_video_task(updated).await? {
|
||||
Some(stored) => {
|
||||
if let Some(snapshot) = LocalVideoTaskSnapshot::from_stored_task(&stored) {
|
||||
if let Some(snapshot) =
|
||||
state.reconstruct_video_task_snapshot(&stored).await?
|
||||
{
|
||||
state.video_tasks.record_snapshot(snapshot);
|
||||
}
|
||||
info!(
|
||||
@@ -190,9 +201,9 @@ async fn fetch_video_task_refresh_attempt(
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(err) => {
|
||||
Err(_) => {
|
||||
return Ok(VideoTaskRefreshAttempt::Error(VideoTaskRefreshError {
|
||||
message: format!("{err:?}"),
|
||||
category: "transport_error",
|
||||
permanent: false,
|
||||
}));
|
||||
}
|
||||
@@ -209,7 +220,7 @@ async fn fetch_video_task_refresh_attempt(
|
||||
.and_then(|body| body.as_object().cloned())
|
||||
else {
|
||||
return Ok(VideoTaskRefreshAttempt::Error(VideoTaskRefreshError {
|
||||
message: "video task refresh missing json provider body".to_string(),
|
||||
category: "invalid_provider_response",
|
||||
permanent: false,
|
||||
}));
|
||||
};
|
||||
@@ -223,20 +234,19 @@ fn classify_refresh_result_error(result: &ExecutionResult) -> VideoTaskRefreshEr
|
||||
.as_ref()
|
||||
.and_then(|error| error.upstream_status)
|
||||
.unwrap_or(result.status_code);
|
||||
let message = result
|
||||
.error
|
||||
.as_ref()
|
||||
.map(|error| error.message.clone())
|
||||
.or_else(|| {
|
||||
result
|
||||
.body
|
||||
.as_ref()
|
||||
.and_then(|body| body.json_body.as_ref())
|
||||
.and_then(|value| value.get("error"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_string)
|
||||
})
|
||||
.unwrap_or_else(|| format!("upstream returned {status_code}"));
|
||||
let category = if status_code == 401 {
|
||||
"authentication_error"
|
||||
} else if status_code == 403 {
|
||||
"permission_denied"
|
||||
} else if status_code == 404 {
|
||||
"not_found"
|
||||
} else if status_code == 429 {
|
||||
"rate_limit"
|
||||
} else if status_code >= 500 {
|
||||
"server_error"
|
||||
} else {
|
||||
"provider_error"
|
||||
};
|
||||
let permanent = result.error.as_ref().map_or(
|
||||
matches!(status_code, 400 | 401 | 403 | 404 | 422),
|
||||
|error| match error.kind {
|
||||
@@ -253,17 +263,18 @@ fn classify_refresh_result_error(result: &ExecutionResult) -> VideoTaskRefreshEr
|
||||
},
|
||||
);
|
||||
|
||||
VideoTaskRefreshError { message, permanent }
|
||||
VideoTaskRefreshError {
|
||||
category,
|
||||
permanent,
|
||||
}
|
||||
}
|
||||
|
||||
fn build_successful_poll_update(
|
||||
task: &StoredVideoTask,
|
||||
mut snapshot: LocalVideoTaskSnapshot,
|
||||
provider_body: &Map<String, Value>,
|
||||
now_unix_secs: u64,
|
||||
) -> Result<Option<UpsertVideoTask>, GatewayError> {
|
||||
let Some(mut snapshot) = LocalVideoTaskSnapshot::from_stored_task(task) else {
|
||||
return Ok(None);
|
||||
};
|
||||
snapshot.apply_provider_body(provider_body);
|
||||
|
||||
let mut record = snapshot.to_upsert_record();
|
||||
@@ -283,10 +294,7 @@ fn build_successful_poll_update(
|
||||
record.format_converted = task.format_converted;
|
||||
record.model = task.model.clone().or(record.model);
|
||||
record.prompt = task.prompt.clone().or(record.prompt);
|
||||
record.original_request_body = task
|
||||
.original_request_body
|
||||
.clone()
|
||||
.or(record.original_request_body);
|
||||
record.original_request_body = None;
|
||||
record.duration_seconds = task.duration_seconds.or(record.duration_seconds);
|
||||
record.resolution = task.resolution.clone().or(record.resolution);
|
||||
record.aspect_ratio = task.aspect_ratio.clone().or(record.aspect_ratio);
|
||||
@@ -309,17 +317,11 @@ fn build_successful_poll_update(
|
||||
if record.status.is_active() && record.poll_count >= record.max_poll_count {
|
||||
record.status = VideoTaskStatus::Failed;
|
||||
record.error_code = Some("poll_timeout".to_string());
|
||||
record.error_message = Some(format!("Task timed out after {} polls", record.poll_count));
|
||||
record.error_message = None;
|
||||
record.completed_at_unix_secs = Some(now_unix_secs);
|
||||
record.next_poll_at_unix_secs = None;
|
||||
}
|
||||
record.request_metadata = merge_video_task_request_metadata(
|
||||
task.request_metadata.clone(),
|
||||
&snapshot,
|
||||
Some(provider_body),
|
||||
None,
|
||||
)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
record.request_metadata = None;
|
||||
|
||||
Ok(Some(record))
|
||||
}
|
||||
@@ -332,11 +334,11 @@ fn build_failed_poll_update(
|
||||
let mut record = stored_task_to_upsert(task);
|
||||
record.updated_at_unix_secs = now_unix_secs;
|
||||
record.poll_count = task.poll_count.saturating_add(1);
|
||||
record.progress_message = Some(format!("Poll error: {}", err.message));
|
||||
record.progress_message = None;
|
||||
if err.permanent {
|
||||
record.status = VideoTaskStatus::Failed;
|
||||
record.error_code = Some("poll_permanent_error".to_string());
|
||||
record.error_message = Some(err.message.clone());
|
||||
record.error_message = None;
|
||||
record.completed_at_unix_secs = Some(now_unix_secs);
|
||||
record.next_poll_at_unix_secs = None;
|
||||
} else {
|
||||
@@ -348,28 +350,15 @@ fn build_failed_poll_update(
|
||||
if record.status.is_active() && record.poll_count >= record.max_poll_count {
|
||||
record.status = VideoTaskStatus::Failed;
|
||||
record.error_code = Some("poll_timeout".to_string());
|
||||
record.error_message = Some(format!("Task timed out after {} polls", record.poll_count));
|
||||
record.error_message = None;
|
||||
record.completed_at_unix_secs = Some(now_unix_secs);
|
||||
record.next_poll_at_unix_secs = None;
|
||||
}
|
||||
record.request_metadata = LocalVideoTaskSnapshot::from_stored_task(task)
|
||||
.and_then(|snapshot| {
|
||||
merge_video_task_request_metadata(
|
||||
task.request_metadata.clone(),
|
||||
&snapshot,
|
||||
None,
|
||||
Some(err),
|
||||
)
|
||||
.ok()
|
||||
.flatten()
|
||||
})
|
||||
.or(task.request_metadata.clone());
|
||||
record.request_metadata = None;
|
||||
record
|
||||
}
|
||||
|
||||
fn stored_task_to_upsert(task: &StoredVideoTask) -> UpsertVideoTask {
|
||||
let snapshot_record =
|
||||
LocalVideoTaskSnapshot::from_stored_task(task).map(|snapshot| snapshot.to_upsert_record());
|
||||
UpsertVideoTask {
|
||||
id: task.id.clone(),
|
||||
short_id: task.short_id.clone(),
|
||||
@@ -386,39 +375,15 @@ fn stored_task_to_upsert(task: &StoredVideoTask) -> UpsertVideoTask {
|
||||
provider_api_format: task.provider_api_format.clone(),
|
||||
format_converted: task.format_converted,
|
||||
model: task.model.clone(),
|
||||
prompt: task.prompt.clone().or_else(|| {
|
||||
snapshot_record
|
||||
.as_ref()
|
||||
.and_then(|record| record.prompt.clone())
|
||||
}),
|
||||
original_request_body: task.original_request_body.clone().or_else(|| {
|
||||
snapshot_record
|
||||
.as_ref()
|
||||
.and_then(|record| record.original_request_body.clone())
|
||||
}),
|
||||
duration_seconds: task.duration_seconds.or_else(|| {
|
||||
snapshot_record
|
||||
.as_ref()
|
||||
.and_then(|record| record.duration_seconds)
|
||||
}),
|
||||
resolution: task.resolution.clone().or_else(|| {
|
||||
snapshot_record
|
||||
.as_ref()
|
||||
.and_then(|record| record.resolution.clone())
|
||||
}),
|
||||
aspect_ratio: task.aspect_ratio.clone().or_else(|| {
|
||||
snapshot_record
|
||||
.as_ref()
|
||||
.and_then(|record| record.aspect_ratio.clone())
|
||||
}),
|
||||
size: task.size.clone().or_else(|| {
|
||||
snapshot_record
|
||||
.as_ref()
|
||||
.and_then(|record| record.size.clone())
|
||||
}),
|
||||
prompt: task.prompt.clone(),
|
||||
original_request_body: None,
|
||||
duration_seconds: task.duration_seconds,
|
||||
resolution: task.resolution.clone(),
|
||||
aspect_ratio: task.aspect_ratio.clone(),
|
||||
size: task.size.clone(),
|
||||
status: task.status,
|
||||
progress_percent: task.progress_percent,
|
||||
progress_message: task.progress_message.clone(),
|
||||
progress_message: None,
|
||||
retry_count: task.retry_count,
|
||||
poll_interval_seconds: task.poll_interval_seconds.max(1),
|
||||
next_poll_at_unix_secs: task.next_poll_at_unix_secs,
|
||||
@@ -429,9 +394,9 @@ fn stored_task_to_upsert(task: &StoredVideoTask) -> UpsertVideoTask {
|
||||
completed_at_unix_secs: task.completed_at_unix_secs,
|
||||
updated_at_unix_secs: task.updated_at_unix_secs,
|
||||
error_code: task.error_code.clone(),
|
||||
error_message: task.error_message.clone(),
|
||||
error_message: None,
|
||||
video_url: task.video_url.clone(),
|
||||
request_metadata: task.request_metadata.clone(),
|
||||
request_metadata: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -443,44 +408,6 @@ fn compute_poll_backoff_seconds(poll_interval_seconds: u32, retry_count: u32) ->
|
||||
.min(MAX_VIDEO_TASK_POLL_BACKOFF_SECONDS)
|
||||
}
|
||||
|
||||
fn merge_video_task_request_metadata(
|
||||
existing: Option<Value>,
|
||||
snapshot: &LocalVideoTaskSnapshot,
|
||||
provider_body: Option<&Map<String, Value>>,
|
||||
poll_error: Option<&VideoTaskRefreshError>,
|
||||
) -> Result<Option<Value>, serde_json::Error> {
|
||||
let mut metadata = match existing {
|
||||
Some(Value::Object(object)) => object,
|
||||
_ => Map::new(),
|
||||
};
|
||||
metadata.insert(
|
||||
"rust_owner".to_string(),
|
||||
Value::String("async_task".to_string()),
|
||||
);
|
||||
metadata.insert(
|
||||
"rust_local_snapshot".to_string(),
|
||||
serde_json::to_value(snapshot)?,
|
||||
);
|
||||
if let Some(provider_body) = provider_body {
|
||||
metadata.insert(
|
||||
"poll_raw_response".to_string(),
|
||||
Value::Object(provider_body.clone()),
|
||||
);
|
||||
metadata.remove("poll_error");
|
||||
}
|
||||
if let Some(poll_error) = poll_error {
|
||||
metadata.insert(
|
||||
"poll_error".to_string(),
|
||||
serde_json::json!({
|
||||
"message": poll_error.message,
|
||||
"permanent": poll_error.permanent,
|
||||
"observed_at_unix_secs": now_unix_secs(),
|
||||
}),
|
||||
);
|
||||
}
|
||||
Ok(Some(Value::Object(metadata)))
|
||||
}
|
||||
|
||||
pub(crate) async fn finalize_video_task_if_terminal(state: &AppState, task: &StoredVideoTask) {
|
||||
let Some(event) = build_video_task_terminal_usage_event(task) else {
|
||||
return;
|
||||
@@ -543,9 +470,9 @@ fn build_video_task_terminal_usage_event(task: &StoredVideoTask) -> Option<Usage
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let provider_name = LocalVideoTaskSnapshot::from_stored_task(task)
|
||||
.and_then(|snapshot| snapshot.provider_name().map(str::to_string))
|
||||
.or_else(|| task.provider_id.clone())
|
||||
let provider_name = task
|
||||
.provider_id
|
||||
.clone()
|
||||
.unwrap_or_else(|| "unknown".to_string());
|
||||
let response_time_ms = task
|
||||
.submitted_at_unix_secs
|
||||
@@ -580,10 +507,10 @@ fn build_video_task_terminal_usage_event(task: &StoredVideoTask) -> Option<Usage
|
||||
has_format_conversion: Some(task.format_converted),
|
||||
is_stream: Some(false),
|
||||
status_code,
|
||||
error_message: task.error_message.clone().or(task.error_code.clone()),
|
||||
error_message: task.error_code.clone(),
|
||||
response_time_ms,
|
||||
request_body: task.original_request_body.clone(),
|
||||
request_metadata: task.request_metadata.clone(),
|
||||
request_body: None,
|
||||
request_metadata: None,
|
||||
..UsageEventData::default()
|
||||
},
|
||||
))
|
||||
@@ -701,48 +628,36 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stored_task_to_upsert_restores_sparse_fields_from_snapshot() {
|
||||
fn stored_task_to_upsert_does_not_restore_sensitive_legacy_snapshot_fields() {
|
||||
let record = stored_task_to_upsert(&sample_sparse_stored_task());
|
||||
|
||||
assert_eq!(record.prompt.as_deref(), Some("hello"));
|
||||
assert_eq!(
|
||||
record.original_request_body,
|
||||
Some(json!({
|
||||
"prompt": "hello",
|
||||
"seconds": "4",
|
||||
"resolution": "720p",
|
||||
"aspect_ratio": "16:9",
|
||||
"size": "1280x720"
|
||||
}))
|
||||
);
|
||||
assert_eq!(record.duration_seconds, Some(4));
|
||||
assert_eq!(record.resolution.as_deref(), Some("720p"));
|
||||
assert_eq!(record.aspect_ratio.as_deref(), Some("16:9"));
|
||||
assert_eq!(record.size.as_deref(), Some("1280x720"));
|
||||
assert!(record.prompt.is_none());
|
||||
assert!(record.original_request_body.is_none());
|
||||
assert!(record.duration_seconds.is_none());
|
||||
assert!(record.resolution.is_none());
|
||||
assert!(record.aspect_ratio.is_none());
|
||||
assert!(record.size.is_none());
|
||||
assert!(record.progress_message.is_none());
|
||||
assert!(record.error_message.is_none());
|
||||
assert!(record.request_metadata.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failed_poll_update_keeps_snapshot_backed_request_body() {
|
||||
fn failed_poll_update_drops_snapshot_backed_sensitive_fields() {
|
||||
let record = build_failed_poll_update(
|
||||
&sample_sparse_stored_task(),
|
||||
&VideoTaskRefreshError {
|
||||
message: "temporary failure".to_string(),
|
||||
category: "transport_error",
|
||||
permanent: false,
|
||||
},
|
||||
100,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
record.original_request_body,
|
||||
Some(json!({
|
||||
"prompt": "hello",
|
||||
"seconds": "4",
|
||||
"resolution": "720p",
|
||||
"aspect_ratio": "16:9",
|
||||
"size": "1280x720"
|
||||
}))
|
||||
);
|
||||
assert_eq!(record.prompt.as_deref(), Some("hello"));
|
||||
assert_eq!(record.resolution.as_deref(), Some("720p"));
|
||||
assert!(record.original_request_body.is_none());
|
||||
assert!(record.prompt.is_none());
|
||||
assert!(record.resolution.is_none());
|
||||
assert!(record.progress_message.is_none());
|
||||
assert!(record.error_message.is_none());
|
||||
assert!(record.request_metadata.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,6 +34,7 @@ pub(crate) fn emit_admin_audit(
|
||||
path_and_query: &str,
|
||||
control_decision: Option<&GatewayControlDecision>,
|
||||
) {
|
||||
let sanitized_path_and_query = sanitize_admin_audit_path(path_and_query);
|
||||
let Some(decision) = control_decision else {
|
||||
return;
|
||||
};
|
||||
@@ -64,9 +65,10 @@ pub(crate) fn emit_admin_audit(
|
||||
},
|
||||
route_kind,
|
||||
default_target_type(route_family),
|
||||
path_and_query.to_string(),
|
||||
sanitized_path_and_query.clone(),
|
||||
)
|
||||
};
|
||||
let target_id = sanitize_admin_audit_target_id(target_id);
|
||||
|
||||
let (audit_status, log_level) = classify_admin_audit_response(method, response.status());
|
||||
if log_level == AdminAuditLogLevel::Info {
|
||||
@@ -83,7 +85,7 @@ pub(crate) fn emit_admin_audit(
|
||||
route_family,
|
||||
route_kind,
|
||||
method = %method,
|
||||
path = %path_and_query,
|
||||
path = %sanitized_path_and_query,
|
||||
action,
|
||||
target_type,
|
||||
target_id = %target_id,
|
||||
@@ -103,7 +105,7 @@ pub(crate) fn emit_admin_audit(
|
||||
route_family,
|
||||
route_kind,
|
||||
method = %method,
|
||||
path = %path_and_query,
|
||||
path = %sanitized_path_and_query,
|
||||
action,
|
||||
target_type,
|
||||
target_id = %target_id,
|
||||
@@ -112,6 +114,17 @@ pub(crate) fn emit_admin_audit(
|
||||
}
|
||||
}
|
||||
|
||||
fn sanitize_admin_audit_path(path_and_query: &str) -> String {
|
||||
crate::middleware::sanitize_access_log_path(path_and_query)
|
||||
}
|
||||
|
||||
fn sanitize_admin_audit_target_id(target_id: String) -> String {
|
||||
if target_id.trim_start().starts_with('/') {
|
||||
return sanitize_admin_audit_path(&target_id);
|
||||
}
|
||||
target_id
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum AdminAuditLogLevel {
|
||||
Info,
|
||||
@@ -151,7 +164,10 @@ fn is_admin_read_method(method: &http::Method) -> bool {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{classify_admin_audit_response, AdminAuditLogLevel};
|
||||
use super::{
|
||||
classify_admin_audit_response, sanitize_admin_audit_path, sanitize_admin_audit_target_id,
|
||||
AdminAuditLogLevel,
|
||||
};
|
||||
use axum::http::{Method, StatusCode};
|
||||
|
||||
#[test]
|
||||
@@ -169,4 +185,32 @@ mod tests {
|
||||
("failed", AdminAuditLogLevel::Warn)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audit_paths_drop_sensitive_query_values() {
|
||||
assert_eq!(
|
||||
sanitize_admin_audit_path(
|
||||
"/api/admin/providers?token=secret&api_key=live-key&limit=25"
|
||||
),
|
||||
"/api/admin/providers?limit=25"
|
||||
);
|
||||
assert_eq!(
|
||||
sanitize_admin_audit_path("/install/one-time-secret?view=raw"),
|
||||
"/install/[redacted]?view=raw"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn path_shaped_audit_targets_drop_sensitive_query_values() {
|
||||
assert_eq!(
|
||||
sanitize_admin_audit_target_id(
|
||||
"/api/admin/monitoring/trace/request-1?token=secret&limit=25".to_string(),
|
||||
),
|
||||
"/api/admin/monitoring/trace/request-1?limit=25"
|
||||
);
|
||||
assert_eq!(
|
||||
sanitize_admin_audit_target_id("resource-id?literal".to_string()),
|
||||
"resource-id?literal"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,7 +26,10 @@ pub(crate) async fn get_request_candidate_trace(
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()).into_response())?;
|
||||
|
||||
match trace {
|
||||
Some(trace) => Ok(Json(trace)),
|
||||
Some(mut trace) => {
|
||||
trace.sanitize_sensitive_diagnostics();
|
||||
Ok(Json(trace))
|
||||
}
|
||||
None => Err((
|
||||
axum::http::StatusCode::NOT_FOUND,
|
||||
Json(json!({
|
||||
@@ -52,7 +55,10 @@ pub(crate) async fn get_decision_trace(
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()).into_response())?;
|
||||
|
||||
match trace {
|
||||
Some(trace) => Ok(Json(trace)),
|
||||
Some(mut trace) => {
|
||||
trace.sanitize_sensitive_diagnostics();
|
||||
Ok(Json(trace))
|
||||
}
|
||||
None => Err((
|
||||
axum::http::StatusCode::NOT_FOUND,
|
||||
Json(json!({
|
||||
|
||||
@@ -5,7 +5,7 @@ use serde_json::{Map, Value};
|
||||
use super::schedule::{BackupSchedule, BackupScheduleUnit};
|
||||
use super::scopes::BackupScope;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub(crate) struct S3BackupConfig {
|
||||
pub(crate) enabled: bool,
|
||||
pub(crate) scope: BackupScope,
|
||||
@@ -22,6 +22,28 @@ pub(crate) struct S3BackupConfig {
|
||||
pub(crate) retention_count: u32,
|
||||
}
|
||||
|
||||
impl fmt::Debug for S3BackupConfig {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
let endpoint_origin = sanitized_endpoint_origin(&self.endpoint);
|
||||
formatter
|
||||
.debug_struct("S3BackupConfig")
|
||||
.field("enabled", &self.enabled)
|
||||
.field("scope", &self.scope)
|
||||
.field("endpoint_origin", &endpoint_origin)
|
||||
.field("region", &self.region)
|
||||
.field("user_agent", &self.user_agent)
|
||||
.field("bucket", &self.bucket)
|
||||
.field("prefix", &self.prefix)
|
||||
.field("has_access_key_id", &!self.access_key_id.is_empty())
|
||||
.field("has_secret_access_key", &!self.secret_access_key.is_empty())
|
||||
.field("path_style", &self.path_style)
|
||||
.field("compression", &self.compression)
|
||||
.field("schedule", &self.schedule)
|
||||
.field("retention_count", &self.retention_count)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct BackupConfigError {
|
||||
message: String,
|
||||
@@ -84,6 +106,9 @@ impl S3BackupConfig {
|
||||
"Endpoint(S3 地址)",
|
||||
enabled,
|
||||
)?;
|
||||
if enabled {
|
||||
validate_s3_endpoint(&endpoint)?;
|
||||
}
|
||||
let bucket =
|
||||
required_or_disabled_string(entries, "backup_s3_bucket", "Bucket(存储桶)", enabled)?;
|
||||
let access_key_id = required_or_disabled_string(
|
||||
@@ -99,6 +124,11 @@ impl S3BackupConfig {
|
||||
enabled,
|
||||
)?;
|
||||
|
||||
let prefix = normalize_s3_prefix(
|
||||
&optional_string(entries, "backup_s3_prefix")?
|
||||
.unwrap_or_else(|| "aether/backups/".to_string()),
|
||||
)?;
|
||||
|
||||
Ok(Self {
|
||||
enabled,
|
||||
scope,
|
||||
@@ -108,8 +138,7 @@ impl S3BackupConfig {
|
||||
user_agent: optional_string(entries, "backup_s3_user_agent")?
|
||||
.unwrap_or_else(|| "rclone/v1.68.0".to_string()),
|
||||
bucket,
|
||||
prefix: optional_string(entries, "backup_s3_prefix")?
|
||||
.unwrap_or_else(|| "aether/backups/".to_string()),
|
||||
prefix,
|
||||
access_key_id,
|
||||
secret_access_key,
|
||||
path_style: optional_bool(entries, "backup_s3_path_style")?.unwrap_or(true),
|
||||
@@ -121,6 +150,48 @@ impl S3BackupConfig {
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_s3_prefix(prefix: &str) -> Result<String, BackupConfigError> {
|
||||
let prefix = prefix.trim().trim_matches('/');
|
||||
if prefix.is_empty() {
|
||||
return Ok(String::new());
|
||||
}
|
||||
if prefix
|
||||
.split('/')
|
||||
.any(|segment| segment.is_empty() || segment == "." || segment == "..")
|
||||
|| prefix.contains('\\')
|
||||
{
|
||||
return Err(BackupConfigError::new(
|
||||
"Prefix(备份前缀)不能包含空路径段、相对路径段或反斜杠",
|
||||
));
|
||||
}
|
||||
|
||||
Ok(format!("{prefix}/"))
|
||||
}
|
||||
|
||||
fn validate_s3_endpoint(endpoint: &str) -> Result<(), BackupConfigError> {
|
||||
let parsed = url::Url::parse(endpoint)
|
||||
.map_err(|_| BackupConfigError::new("Endpoint(S3 地址)必须是有效的 HTTPS URL"))?;
|
||||
if parsed.scheme() != "https"
|
||||
|| parsed.host_str().is_none()
|
||||
|| !parsed.username().is_empty()
|
||||
|| parsed.password().is_some()
|
||||
|| parsed.query().is_some()
|
||||
|| parsed.fragment().is_some()
|
||||
{
|
||||
return Err(BackupConfigError::new(
|
||||
"Endpoint(S3 地址)必须使用 HTTPS,且不能包含用户凭据、查询参数或片段",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn sanitized_endpoint_origin(endpoint: &str) -> String {
|
||||
url::Url::parse(endpoint)
|
||||
.ok()
|
||||
.map(|parsed| parsed.origin().ascii_serialization())
|
||||
.unwrap_or_else(|| "<invalid>".to_string())
|
||||
}
|
||||
|
||||
fn validate_range(label: &str, value: u32, min: u32, max: u32) -> Result<(), BackupConfigError> {
|
||||
if (min..=max).contains(&value) {
|
||||
Ok(())
|
||||
@@ -374,6 +445,69 @@ mod tests {
|
||||
assert!(err.to_string().contains("Endpoint"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_insecure_or_credential_bearing_endpoints() {
|
||||
for endpoint in [
|
||||
"http://s3.example.com",
|
||||
"https://user:[email protected]",
|
||||
"https://s3.example.com?token=secret",
|
||||
"https://s3.example.com/#fragment",
|
||||
] {
|
||||
let entries = serde_json::json!({
|
||||
"backup_s3_enabled": true,
|
||||
"backup_s3_endpoint": endpoint,
|
||||
"backup_s3_bucket": "aether-backups",
|
||||
"backup_s3_access_key_id": "access",
|
||||
"backup_s3_secret_access_key": "secret"
|
||||
});
|
||||
|
||||
let error = S3BackupConfig::from_json_map(entries.as_object().unwrap())
|
||||
.expect_err("unsafe endpoint should fail closed");
|
||||
assert!(error.to_string().contains("Endpoint"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn debug_output_does_not_expose_s3_credentials() {
|
||||
let entries = serde_json::json!({
|
||||
"backup_s3_enabled": true,
|
||||
"backup_s3_endpoint": "https://s3.example.com/path",
|
||||
"backup_s3_bucket": "aether-backups",
|
||||
"backup_s3_access_key_id": "access-key-value",
|
||||
"backup_s3_secret_access_key": "secret-key-value"
|
||||
});
|
||||
let config = S3BackupConfig::from_json_map(entries.as_object().unwrap())
|
||||
.expect("config should parse");
|
||||
|
||||
let debug = format!("{config:?}");
|
||||
assert!(debug.contains("https://s3.example.com"));
|
||||
assert!(!debug.contains("/path"));
|
||||
assert!(!debug.contains("access-key-value"));
|
||||
assert!(!debug.contains("secret-key-value"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonicalizes_s3_backup_prefix_once() {
|
||||
let entries = serde_json::json!({
|
||||
"backup_s3_enabled": true,
|
||||
"backup_s3_endpoint": "https://s3.example.com",
|
||||
"backup_s3_bucket": "aether-backups",
|
||||
"backup_s3_prefix": "/prod/backups//",
|
||||
"backup_s3_access_key_id": "access",
|
||||
"backup_s3_secret_access_key": "secret"
|
||||
});
|
||||
let config = S3BackupConfig::from_json_map(entries.as_object().unwrap())
|
||||
.expect("prefix should be canonicalized");
|
||||
|
||||
assert_eq!(config.prefix, "prod/backups/");
|
||||
|
||||
for invalid_prefix in ["prod//backups", "prod/../backups", "prod\\backups"] {
|
||||
let mut entries = entries.clone();
|
||||
entries["backup_s3_prefix"] = serde_json::json!(invalid_prefix);
|
||||
assert!(S3BackupConfig::from_json_map(entries.as_object().unwrap()).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn applies_default_values_from_system_config_contract() {
|
||||
let entries = serde_json::json!({
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -6,5 +6,167 @@ pub(crate) mod store;
|
||||
pub(crate) mod task;
|
||||
pub(crate) mod worker;
|
||||
|
||||
pub use executor::{
|
||||
restore_backup_json, BackupDecryptionKey, BackupRestoreError, BackupRestoreLimits,
|
||||
RestoredBackupJson, DEFAULT_BACKUP_MAX_ENCRYPTED_BYTES, DEFAULT_BACKUP_MAX_JSON_BYTES,
|
||||
};
|
||||
|
||||
use axum::body::Bytes;
|
||||
use serde_json::Value;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum BackupRestoreScope {
|
||||
Config,
|
||||
Users,
|
||||
Data,
|
||||
}
|
||||
|
||||
impl BackupRestoreScope {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Config => "config",
|
||||
Self::Users => "users",
|
||||
Self::Data => "data",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("backup database apply failed: {0}")]
|
||||
pub struct BackupApplyError(String);
|
||||
|
||||
pub async fn apply_restored_backup(
|
||||
app: &crate::AppState,
|
||||
restored: RestoredBackupJson,
|
||||
scope: BackupRestoreScope,
|
||||
operator_id: Option<&str>,
|
||||
) -> Result<Result<Value, (http::StatusCode, Value)>, BackupApplyError> {
|
||||
let (json_bytes, authority) = restored.into_authenticated_parts();
|
||||
if authority.scope() != scope {
|
||||
return Err(BackupApplyError(format!(
|
||||
"authenticated {} backup cannot be applied to {} scope",
|
||||
authority.scope().as_str(),
|
||||
scope.as_str(),
|
||||
)));
|
||||
}
|
||||
let request_body = Bytes::from(json_bytes);
|
||||
let state = crate::admin_api::AdminAppState::new(app);
|
||||
let result = crate::admin_api::execute_admin_system_import_exclusively(app, async {
|
||||
match scope {
|
||||
BackupRestoreScope::Config => {
|
||||
state
|
||||
.restore_admin_system_config_backup(&request_body, authority)
|
||||
.await
|
||||
}
|
||||
BackupRestoreScope::Users => {
|
||||
state
|
||||
.restore_admin_system_users_backup(&request_body, operator_id, authority)
|
||||
.await
|
||||
}
|
||||
BackupRestoreScope::Data => {
|
||||
state
|
||||
.restore_admin_system_data_backup(&request_body, operator_id, authority)
|
||||
.await
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|error| {
|
||||
let message = match error {
|
||||
crate::admin_api::AdminSystemImportLockError::Conflict => {
|
||||
"another system import or restore is already running"
|
||||
}
|
||||
crate::admin_api::AdminSystemImportLockError::Unavailable => {
|
||||
"system import coordination is unavailable"
|
||||
}
|
||||
crate::admin_api::AdminSystemImportLockError::Lost => {
|
||||
"system import coordination lease was lost; restore was cancelled and may have partially applied changes"
|
||||
}
|
||||
};
|
||||
BackupApplyError(message.to_string())
|
||||
})?;
|
||||
result.map_err(|error| BackupApplyError(error.into_message()))
|
||||
}
|
||||
|
||||
pub(crate) const S3_BACKUP_ENABLED_KEY: &str = "backup_s3_enabled";
|
||||
pub(crate) const S3_BACKUP_LAST_SLOT_KEY: &str = "backup_s3_last_slot";
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
apply_restored_backup, BackupDecryptionKey, BackupRestoreLimits, BackupRestoreScope,
|
||||
RestoredBackupJson,
|
||||
};
|
||||
use crate::backup::executor::encrypt_backup_bytes;
|
||||
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||
use serde_json::json;
|
||||
|
||||
fn authenticated_users_backup() -> RestoredBackupJson {
|
||||
let object_key = "prod/aether-users-backup-20260830-120000.json.zst.aes256gcm";
|
||||
let compressed = zstd::stream::encode_all(
|
||||
serde_json::to_vec(&json!({
|
||||
"version": "1.5",
|
||||
"exported_at": "2026-08-30T12:00:00Z",
|
||||
"users": [],
|
||||
"standalone_keys": [],
|
||||
}))
|
||||
.expect("test backup should serialize")
|
||||
.as_slice(),
|
||||
0,
|
||||
)
|
||||
.expect("test backup should compress");
|
||||
let (envelope, _) =
|
||||
encrypt_backup_bytes(DEVELOPMENT_ENCRYPTION_KEY, object_key, &compressed)
|
||||
.expect("test backup should encrypt");
|
||||
super::restore_backup_json(
|
||||
object_key,
|
||||
&envelope,
|
||||
&[BackupDecryptionKey::current(DEVELOPMENT_ENCRYPTION_KEY)
|
||||
.expect("test restore key should build")],
|
||||
BackupRestoreLimits::default(),
|
||||
)
|
||||
.expect("test backup should authenticate")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authenticated_backup_cannot_be_applied_to_a_different_scope() {
|
||||
let restored = authenticated_users_backup();
|
||||
|
||||
let error = apply_restored_backup(
|
||||
&crate::AppState::new().expect("test state should build"),
|
||||
restored,
|
||||
BackupRestoreScope::Config,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect_err("scope mismatch must fail before database access");
|
||||
|
||||
assert_eq!(
|
||||
error.to_string(),
|
||||
"backup database apply failed: authenticated users backup cannot be applied to config scope"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authenticated_backup_apply_uses_the_shared_system_import_lock() {
|
||||
let app = crate::AppState::new().expect("test state should build");
|
||||
let lock = crate::admin_api::try_acquire_admin_system_import_lease(&app)
|
||||
.await
|
||||
.expect("test should acquire the shared import lease");
|
||||
|
||||
let error = apply_restored_backup(
|
||||
&app,
|
||||
authenticated_users_backup(),
|
||||
BackupRestoreScope::Users,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect_err("restore must not interleave with another system import");
|
||||
|
||||
assert_eq!(
|
||||
error.to_string(),
|
||||
"backup database apply failed: another system import or restore is already running"
|
||||
);
|
||||
crate::admin_api::release_admin_system_import_lease(&app, &lock).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
use std::fmt;
|
||||
|
||||
const ENCRYPTED_BACKUP_FILE_SUFFIX: &str = ".json.zst.aes256gcm";
|
||||
const LEGACY_PLAINTEXT_BACKUP_FILE_SUFFIX: &str = ".json.zst";
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum BackupScope {
|
||||
Config,
|
||||
@@ -52,10 +55,81 @@ impl BackupScope {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn from_encrypted_object_key(object_key: &str) -> Option<Self> {
|
||||
if object_key.is_empty()
|
||||
|| object_key.starts_with('/')
|
||||
|| object_key.contains('\0')
|
||||
|| object_key.contains('\\')
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let mut segments = object_key.split('/').peekable();
|
||||
let mut file_name = None;
|
||||
while let Some(segment) = segments.next() {
|
||||
if segment.is_empty()
|
||||
|| segment == "."
|
||||
|| segment == ".."
|
||||
|| segment.chars().any(char::is_control)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
if segments.peek().is_none() {
|
||||
file_name = Some(segment);
|
||||
}
|
||||
}
|
||||
let file_name = file_name?;
|
||||
|
||||
[Self::Config, Self::Users, Self::Data]
|
||||
.into_iter()
|
||||
.find(|scope| {
|
||||
file_name
|
||||
.strip_prefix(&format!("{}-", scope.file_stem()))
|
||||
.and_then(|rest| rest.strip_suffix(ENCRYPTED_BACKUP_FILE_SUFFIX))
|
||||
.is_some_and(is_aether_backup_object_id)
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn matching_backup_keys(
|
||||
self,
|
||||
prefix: &str,
|
||||
keys: impl IntoIterator<Item = String>,
|
||||
) -> Vec<String> {
|
||||
self.matching_backup_keys_with_suffixes(
|
||||
prefix,
|
||||
keys,
|
||||
&[
|
||||
ENCRYPTED_BACKUP_FILE_SUFFIX,
|
||||
LEGACY_PLAINTEXT_BACKUP_FILE_SUFFIX,
|
||||
],
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn matching_encrypted_backup_keys(
|
||||
self,
|
||||
prefix: &str,
|
||||
keys: impl IntoIterator<Item = String>,
|
||||
) -> Vec<String> {
|
||||
self.matching_backup_keys_with_suffixes(prefix, keys, &[ENCRYPTED_BACKUP_FILE_SUFFIX])
|
||||
}
|
||||
|
||||
pub(crate) fn matching_legacy_plaintext_backup_keys(
|
||||
self,
|
||||
prefix: &str,
|
||||
keys: impl IntoIterator<Item = String>,
|
||||
) -> Vec<String> {
|
||||
self.matching_backup_keys_with_suffixes(
|
||||
prefix,
|
||||
keys,
|
||||
&[LEGACY_PLAINTEXT_BACKUP_FILE_SUFFIX],
|
||||
)
|
||||
}
|
||||
|
||||
fn matching_backup_keys_with_suffixes(
|
||||
self,
|
||||
prefix: &str,
|
||||
keys: impl IntoIterator<Item = String>,
|
||||
file_suffixes: &[&str],
|
||||
) -> Vec<String> {
|
||||
let normalized_prefix = normalized_prefix(prefix);
|
||||
let expected_prefix = if normalized_prefix.is_empty() {
|
||||
@@ -64,7 +138,6 @@ impl BackupScope {
|
||||
format!("{normalized_prefix}/")
|
||||
};
|
||||
let file_prefix = format!("{}-", self.file_stem());
|
||||
let file_suffix = ".json.zst";
|
||||
|
||||
keys.into_iter()
|
||||
.filter(|key| {
|
||||
@@ -74,20 +147,24 @@ impl BackupScope {
|
||||
if file_name.contains('/') {
|
||||
return false;
|
||||
}
|
||||
let Some(timestamp) = file_name
|
||||
.strip_prefix(&file_prefix)
|
||||
.and_then(|rest| rest.strip_suffix(file_suffix))
|
||||
else {
|
||||
let Some(timestamp) = file_name.strip_prefix(&file_prefix).and_then(|rest| {
|
||||
file_suffixes
|
||||
.iter()
|
||||
.find_map(|suffix| rest.strip_suffix(suffix))
|
||||
}) else {
|
||||
return false;
|
||||
};
|
||||
|
||||
is_aether_backup_timestamp(timestamp)
|
||||
is_aether_backup_object_id(timestamp)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn file_name(self, timestamp: &str) -> String {
|
||||
format!("{}-{timestamp}.json.zst", self.file_stem())
|
||||
format!(
|
||||
"{}-{timestamp}{ENCRYPTED_BACKUP_FILE_SUFFIX}",
|
||||
self.file_stem()
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -110,6 +187,25 @@ fn is_aether_backup_timestamp(timestamp: &str) -> bool {
|
||||
&& bytes[9..].iter().all(|byte| byte.is_ascii_digit())
|
||||
}
|
||||
|
||||
fn is_aether_backup_object_id(value: &str) -> bool {
|
||||
if is_aether_backup_timestamp(value) {
|
||||
return true;
|
||||
}
|
||||
|
||||
let Some((timestamp, collision_digest)) = value.split_once('-').and_then(|(date, rest)| {
|
||||
let (time, digest) = rest.split_once('-')?;
|
||||
Some((format!("{date}-{time}"), digest))
|
||||
}) else {
|
||||
return false;
|
||||
};
|
||||
|
||||
is_aether_backup_timestamp(×tamp)
|
||||
&& collision_digest.len() == 64
|
||||
&& collision_digest
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::BackupScope;
|
||||
@@ -130,15 +226,15 @@ mod tests {
|
||||
|
||||
assert_eq!(
|
||||
BackupScope::Config.object_key("prod/", "20260524-031500"),
|
||||
"prod/aether-config-backup-20260524-031500.json.zst"
|
||||
"prod/aether-config-backup-20260524-031500.json.zst.aes256gcm"
|
||||
);
|
||||
assert_eq!(
|
||||
BackupScope::Users.object_key("prod/", "20260524-031500"),
|
||||
"prod/aether-users-backup-20260524-031500.json.zst"
|
||||
"prod/aether-users-backup-20260524-031500.json.zst.aes256gcm"
|
||||
);
|
||||
assert_eq!(
|
||||
BackupScope::Data.object_key("prod/", "20260524-031500"),
|
||||
"prod/aether-data-backup-20260524-031500.json.zst"
|
||||
"prod/aether-data-backup-20260524-031500.json.zst.aes256gcm"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -146,7 +242,7 @@ mod tests {
|
||||
fn retention_filter_only_matches_same_scope() {
|
||||
let keys = vec![
|
||||
"prod/aether-config-backup-20260524-010000.json.zst".to_string(),
|
||||
"prod/aether-users-backup-20260524-010000.json.zst".to_string(),
|
||||
"prod/aether-users-backup-20260524-010000.json.zst.aes256gcm".to_string(),
|
||||
"prod/aether-data-backup-20260524-010000.json.zst".to_string(),
|
||||
"prod/random.json.zst".to_string(),
|
||||
];
|
||||
@@ -155,14 +251,18 @@ mod tests {
|
||||
|
||||
assert_eq!(
|
||||
matched,
|
||||
vec!["prod/aether-users-backup-20260524-010000.json.zst"]
|
||||
vec!["prod/aether-users-backup-20260524-010000.json.zst.aes256gcm"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retention_filter_requires_aether_timestamp_format() {
|
||||
let collision_digest = "a".repeat(64);
|
||||
let keys = vec![
|
||||
"prod/aether-users-backup-20260524-010000.json.zst".to_string(),
|
||||
format!(
|
||||
"prod/aether-users-backup-20260524-010000-{collision_digest}.json.zst.aes256gcm"
|
||||
),
|
||||
"prod/aether-users-backup-foo.json.zst".to_string(),
|
||||
"prod/aether-users-backup-2026052-010000.json.zst".to_string(),
|
||||
"prod/aether-users-backup-202605240-010000.json.zst".to_string(),
|
||||
@@ -171,13 +271,19 @@ mod tests {
|
||||
"prod/aether-users-backup-20260524010000.json.zst".to_string(),
|
||||
"prod/aether-users-backup-2026052a-010000.json.zst".to_string(),
|
||||
"prod/aether-users-backup-20260524-01000x.json.zst".to_string(),
|
||||
"prod/aether-users-backup-20260524-010000-short.json.zst.aes256gcm".to_string(),
|
||||
];
|
||||
|
||||
let matched = BackupScope::Users.matching_backup_keys("prod/", keys);
|
||||
|
||||
assert_eq!(
|
||||
matched,
|
||||
vec!["prod/aether-users-backup-20260524-010000.json.zst"]
|
||||
vec![
|
||||
"prod/aether-users-backup-20260524-010000.json.zst".to_string(),
|
||||
format!(
|
||||
"prod/aether-users-backup-20260524-010000-{collision_digest}.json.zst.aes256gcm"
|
||||
),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -185,11 +291,11 @@ mod tests {
|
||||
fn backup_key_prefix_boundaries_are_exact() {
|
||||
assert_eq!(
|
||||
BackupScope::Config.object_key("", "20260524-031500"),
|
||||
"aether-config-backup-20260524-031500.json.zst"
|
||||
"aether-config-backup-20260524-031500.json.zst.aes256gcm"
|
||||
);
|
||||
assert_eq!(
|
||||
BackupScope::Config.object_key("prod", "20260524-031500"),
|
||||
"prod/aether-config-backup-20260524-031500.json.zst"
|
||||
"prod/aether-config-backup-20260524-031500.json.zst.aes256gcm"
|
||||
);
|
||||
|
||||
let keys = vec![
|
||||
@@ -208,4 +314,36 @@ mod tests {
|
||||
vec!["prod/aether-config-backup-20260524-010000.json.zst"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encrypted_object_key_parser_binds_scope_and_rejects_path_traversal() {
|
||||
let collision_digest = "a".repeat(64);
|
||||
assert_eq!(
|
||||
BackupScope::from_encrypted_object_key(
|
||||
"prod/aether-config-backup-20260524-010000.json.zst.aes256gcm"
|
||||
),
|
||||
Some(BackupScope::Config)
|
||||
);
|
||||
assert_eq!(
|
||||
BackupScope::from_encrypted_object_key(&format!(
|
||||
"prod/aether-users-backup-20260524-010000-{collision_digest}.json.zst.aes256gcm"
|
||||
)),
|
||||
Some(BackupScope::Users)
|
||||
);
|
||||
for key in [
|
||||
"../aether-data-backup-20260524-010000.json.zst.aes256gcm",
|
||||
"/aether-data-backup-20260524-010000.json.zst.aes256gcm",
|
||||
"prod//aether-data-backup-20260524-010000.json.zst.aes256gcm",
|
||||
"prod/./aether-data-backup-20260524-010000.json.zst.aes256gcm",
|
||||
"prod\\aether-data-backup-20260524-010000.json.zst.aes256gcm",
|
||||
"prod/aether-data-backup-invalid.json.zst.aes256gcm",
|
||||
"prod/unrelated-20260524-010000.json.zst.aes256gcm",
|
||||
] {
|
||||
assert_eq!(
|
||||
BackupScope::from_encrypted_object_key(key),
|
||||
None,
|
||||
"unsafe or unrelated key: {key}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,23 +2,45 @@ use std::collections::BTreeMap;
|
||||
use std::fmt;
|
||||
use std::sync::Arc;
|
||||
|
||||
use bytes::Bytes;
|
||||
use bytes::{Bytes, BytesMut};
|
||||
use futures_util::TryStreamExt;
|
||||
use object_store::aws::AmazonS3Builder;
|
||||
use object_store::path::Path;
|
||||
use object_store::{ClientOptions, ObjectStore};
|
||||
use object_store::{ClientOptions, ObjectStore, ObjectStoreExt, PutMode, PutOptions};
|
||||
use reqwest::header::HeaderValue;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use super::config::S3BackupConfig;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum BackupObjectCreateResult {
|
||||
Created,
|
||||
AlreadyExists,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub(crate) trait BackupObjectStore: Send + Sync {
|
||||
async fn put_object(&self, key: &str, bytes: Bytes) -> Result<(), BackupStoreError>;
|
||||
|
||||
async fn list_keys(&self, prefix: &str) -> Result<Vec<String>, BackupStoreError>;
|
||||
async fn put_object_if_absent(
|
||||
&self,
|
||||
key: &str,
|
||||
bytes: Bytes,
|
||||
) -> Result<BackupObjectCreateResult, BackupStoreError>;
|
||||
|
||||
async fn get_object_limited(
|
||||
&self,
|
||||
key: &str,
|
||||
max_bytes: usize,
|
||||
) -> Result<Bytes, BackupStoreError>;
|
||||
|
||||
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError>;
|
||||
|
||||
async fn list_keys_limited(
|
||||
&self,
|
||||
prefix: &str,
|
||||
max_objects: usize,
|
||||
) -> Result<Vec<String>, BackupStoreError>;
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -60,21 +82,72 @@ impl BackupObjectStore for FakeBackupObjectStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn list_keys(&self, prefix: &str) -> Result<Vec<String>, BackupStoreError> {
|
||||
async fn put_object_if_absent(
|
||||
&self,
|
||||
key: &str,
|
||||
bytes: Bytes,
|
||||
) -> Result<BackupObjectCreateResult, BackupStoreError> {
|
||||
let mut objects = self.objects.write().await;
|
||||
if objects.contains_key(key) {
|
||||
Ok(BackupObjectCreateResult::AlreadyExists)
|
||||
} else {
|
||||
objects.insert(key.to_string(), bytes);
|
||||
Ok(BackupObjectCreateResult::Created)
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_object_limited(
|
||||
&self,
|
||||
key: &str,
|
||||
max_bytes: usize,
|
||||
) -> Result<Bytes, BackupStoreError> {
|
||||
let bytes = self
|
||||
.objects
|
||||
.read()
|
||||
.await
|
||||
.get(key)
|
||||
.cloned()
|
||||
.ok_or_else(|| BackupStoreError::new(format!("backup object `{key}` not found")))?;
|
||||
if bytes.len() > max_bytes {
|
||||
return Err(BackupStoreError::new(format!(
|
||||
"backup object `{key}` exceeds the configured {max_bytes} byte read limit"
|
||||
)));
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError> {
|
||||
self.objects.write().await.remove(key);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn list_keys_limited(
|
||||
&self,
|
||||
prefix: &str,
|
||||
max_objects: usize,
|
||||
) -> Result<Vec<String>, BackupStoreError> {
|
||||
let prefix = directory_list_prefix(prefix);
|
||||
Ok(self
|
||||
let keys: Vec<_> = self
|
||||
.objects
|
||||
.read()
|
||||
.await
|
||||
.keys()
|
||||
.filter(|key| key.starts_with(&prefix))
|
||||
.cloned()
|
||||
.collect())
|
||||
.collect();
|
||||
if keys.len() > max_objects {
|
||||
return Err(BackupStoreError::new(format!(
|
||||
"backup object listing exceeds the configured {max_objects} object limit"
|
||||
)));
|
||||
}
|
||||
Ok(keys)
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError> {
|
||||
self.objects.write().await.remove(key);
|
||||
Ok(())
|
||||
#[cfg(test)]
|
||||
impl FakeBackupObjectStore {
|
||||
pub(crate) async fn object_bytes(&self, key: &str) -> Option<Bytes> {
|
||||
self.objects.read().await.get(key).cloned()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -125,17 +198,68 @@ impl BackupObjectStore for ObjectStoreS3BackupStore {
|
||||
.map_err(|error| BackupStoreError::object_store("put", key, error))
|
||||
}
|
||||
|
||||
async fn list_keys(&self, prefix: &str) -> Result<Vec<String>, BackupStoreError> {
|
||||
let prefix_path = list_prefix_path(prefix);
|
||||
let mut keys = self
|
||||
async fn put_object_if_absent(
|
||||
&self,
|
||||
key: &str,
|
||||
bytes: Bytes,
|
||||
) -> Result<BackupObjectCreateResult, BackupStoreError> {
|
||||
let options = PutOptions {
|
||||
mode: PutMode::Create,
|
||||
..PutOptions::default()
|
||||
};
|
||||
match self
|
||||
.store
|
||||
.list(prefix_path.as_ref())
|
||||
.map_ok(|meta| meta.location.to_string())
|
||||
.try_collect::<Vec<_>>()
|
||||
.put_opts(&Path::from(key), bytes.into(), options)
|
||||
.await
|
||||
.map_err(|error| BackupStoreError::object_store("list", prefix, error))?;
|
||||
keys.sort();
|
||||
Ok(keys)
|
||||
{
|
||||
Ok(_) => Ok(BackupObjectCreateResult::Created),
|
||||
Err(object_store::Error::AlreadyExists { .. }) => {
|
||||
Ok(BackupObjectCreateResult::AlreadyExists)
|
||||
}
|
||||
Err(error) => Err(BackupStoreError::object_store(
|
||||
"conditional put",
|
||||
key,
|
||||
error,
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_object_limited(
|
||||
&self,
|
||||
key: &str,
|
||||
max_bytes: usize,
|
||||
) -> Result<Bytes, BackupStoreError> {
|
||||
let result = self
|
||||
.store
|
||||
.get(&Path::from(key))
|
||||
.await
|
||||
.map_err(|error| BackupStoreError::object_store("get", key, error))?;
|
||||
if result.meta.size > u64::try_from(max_bytes).unwrap_or(u64::MAX) {
|
||||
return Err(BackupStoreError::new(format!(
|
||||
"backup object `{key}` exceeds the configured {max_bytes} byte read limit"
|
||||
)));
|
||||
}
|
||||
let object_size = result.meta.size;
|
||||
let mut stream = result.into_stream();
|
||||
let mut bytes = BytesMut::with_capacity(
|
||||
usize::try_from(object_size)
|
||||
.unwrap_or(max_bytes)
|
||||
.min(max_bytes)
|
||||
.min(8 * 1024 * 1024),
|
||||
);
|
||||
while let Some(chunk) = stream
|
||||
.try_next()
|
||||
.await
|
||||
.map_err(|error| BackupStoreError::object_store("read", key, error))?
|
||||
{
|
||||
if bytes.len().saturating_add(chunk.len()) > max_bytes {
|
||||
return Err(BackupStoreError::new(format!(
|
||||
"backup object `{key}` exceeds the configured {max_bytes} byte read limit"
|
||||
)));
|
||||
}
|
||||
bytes.extend_from_slice(&chunk);
|
||||
}
|
||||
Ok(bytes.freeze())
|
||||
}
|
||||
|
||||
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError> {
|
||||
@@ -144,6 +268,30 @@ impl BackupObjectStore for ObjectStoreS3BackupStore {
|
||||
.await
|
||||
.map_err(|error| BackupStoreError::object_store("delete", key, error))
|
||||
}
|
||||
|
||||
async fn list_keys_limited(
|
||||
&self,
|
||||
prefix: &str,
|
||||
max_objects: usize,
|
||||
) -> Result<Vec<String>, BackupStoreError> {
|
||||
let prefix_path = list_prefix_path(prefix);
|
||||
let mut objects = self.store.list(prefix_path.as_ref());
|
||||
let mut keys = Vec::new();
|
||||
while let Some(meta) = objects
|
||||
.try_next()
|
||||
.await
|
||||
.map_err(|error| BackupStoreError::object_store("list", prefix, error))?
|
||||
{
|
||||
if keys.len() >= max_objects {
|
||||
return Err(BackupStoreError::new(format!(
|
||||
"backup object listing exceeds the configured {max_objects} object limit"
|
||||
)));
|
||||
}
|
||||
keys.push(meta.location.to_string());
|
||||
}
|
||||
keys.sort();
|
||||
Ok(keys)
|
||||
}
|
||||
}
|
||||
|
||||
fn directory_list_prefix(prefix: &str) -> String {
|
||||
@@ -166,10 +314,12 @@ fn list_prefix_path(prefix: &str) -> Option<Path> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{list_prefix_path, BackupObjectStore, FakeBackupObjectStore};
|
||||
use super::{
|
||||
list_prefix_path, BackupObjectCreateResult, BackupObjectStore, FakeBackupObjectStore,
|
||||
};
|
||||
|
||||
#[tokio::test]
|
||||
async fn fake_backup_object_store_puts_lists_and_deletes() {
|
||||
async fn fake_backup_object_store_puts_and_lists() {
|
||||
let store = FakeBackupObjectStore::default();
|
||||
store
|
||||
.put_object(
|
||||
@@ -186,17 +336,59 @@ mod tests {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let keys = store.list_keys("prod/").await.unwrap();
|
||||
assert_eq!(keys.len(), 2);
|
||||
|
||||
store
|
||||
.delete_object("prod/aether-data-backup-20260524-010000.json.zst")
|
||||
.await
|
||||
.unwrap();
|
||||
let keys = store.list_keys("prod/").await.unwrap();
|
||||
let keys = store.list_keys_limited("prod/", 2).await.unwrap();
|
||||
assert_eq!(
|
||||
keys,
|
||||
vec!["prod/aether-data-backup-20260524-020000.json.zst"]
|
||||
vec![
|
||||
"prod/aether-data-backup-20260524-010000.json.zst",
|
||||
"prod/aether-data-backup-20260524-020000.json.zst",
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fake_backup_object_store_enforces_read_and_listing_limits() {
|
||||
let store = FakeBackupObjectStore::default();
|
||||
store
|
||||
.put_object("prod/one", bytes::Bytes::from_static(b"1234"))
|
||||
.await
|
||||
.unwrap();
|
||||
store
|
||||
.put_object("prod/two", bytes::Bytes::from_static(b"5678"))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(store.get_object_limited("prod/one", 3).await.is_err());
|
||||
assert_eq!(
|
||||
store.get_object_limited("prod/one", 4).await.unwrap(),
|
||||
bytes::Bytes::from_static(b"1234")
|
||||
);
|
||||
assert!(store.list_keys_limited("prod/", 1).await.is_err());
|
||||
assert_eq!(store.list_keys_limited("prod/", 2).await.unwrap().len(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fake_backup_object_store_conditional_put_never_overwrites() {
|
||||
let store = FakeBackupObjectStore::default();
|
||||
let key = "prod/aether-data-backup-20260524-010000.json.zst.aes256gcm";
|
||||
|
||||
assert_eq!(
|
||||
store
|
||||
.put_object_if_absent(key, bytes::Bytes::from_static(b"first"))
|
||||
.await
|
||||
.unwrap(),
|
||||
BackupObjectCreateResult::Created
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.put_object_if_absent(key, bytes::Bytes::from_static(b"second"))
|
||||
.await
|
||||
.unwrap(),
|
||||
BackupObjectCreateResult::AlreadyExists
|
||||
);
|
||||
assert_eq!(
|
||||
store.object_bytes(key).await.as_deref(),
|
||||
Some(b"first".as_slice())
|
||||
);
|
||||
}
|
||||
|
||||
@@ -218,7 +410,7 @@ mod tests {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let keys = store.list_keys("prod").await.unwrap();
|
||||
let keys = store.list_keys_limited("prod", 10).await.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
keys,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use std::fmt;
|
||||
use std::future::Future;
|
||||
use std::time::Duration;
|
||||
|
||||
use aether_admin::system::admin_system_config_default_value;
|
||||
@@ -12,14 +13,15 @@ use chrono::Utc;
|
||||
use futures_util::FutureExt;
|
||||
use serde::Serialize;
|
||||
use serde_json::{json, Map, Value};
|
||||
use tokio::task::{JoinError, JoinHandle};
|
||||
use tracing::warn;
|
||||
|
||||
use super::config::S3BackupConfig;
|
||||
use super::executor::{run_backup_with_store, BackupRunResult};
|
||||
use super::scopes::BackupScope;
|
||||
use super::store::ObjectStoreS3BackupStore;
|
||||
use crate::admin_api::AdminAppState;
|
||||
use crate::handlers::shared::decrypt_catalog_secret_with_fallbacks;
|
||||
use crate::admin_api::{AdminAppState, SystemExportMode};
|
||||
use crate::handlers::shared::decrypt_or_migrate_system_config_secret;
|
||||
use crate::task_runtime::{
|
||||
append_event_with_logging, build_task_run_id, now_unix_secs, spawn_fire_and_forget,
|
||||
task_definition, update_run_status, upsert_run_with_logging, TASK_KEY_SYSTEM_S3_BACKUP,
|
||||
@@ -48,6 +50,9 @@ const S3_BACKUP_CONFIG_KEYS: &[&str] = &[
|
||||
];
|
||||
|
||||
const S3_BACKUP_QUEUED_MESSAGE: &str = "S3 备份任务已提交";
|
||||
const S3_BACKUP_INTERNAL_ERROR_DETAIL: &str = "S3 备份服务暂时不可用";
|
||||
const S3_BACKUP_TASK_FAILURE_CODE: &str = "s3_backup_failed";
|
||||
const S3_BACKUP_SLOT_RECORD_FAILURE_CODE: &str = "s3_backup_slot_record_failed";
|
||||
const S3_BACKUP_TASK_LOCK_KEY: &str = "task_runtime:lock:system.s3.backup";
|
||||
const S3_BACKUP_TASK_LOCK_TTL: Duration = Duration::from_secs(60 * 60 * 6);
|
||||
const S3_BACKUP_TASK_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(60 * 5);
|
||||
@@ -67,6 +72,16 @@ pub(crate) struct S3BackupTaskError {
|
||||
detail: String,
|
||||
}
|
||||
|
||||
enum BackupLockRenewalFailure<E> {
|
||||
Lost,
|
||||
Backend(E),
|
||||
}
|
||||
|
||||
enum BackupLockRaceOutcome<T> {
|
||||
BackupCompleted(T),
|
||||
LeaseLost(Result<(), JoinError>),
|
||||
}
|
||||
|
||||
impl S3BackupTaskError {
|
||||
fn bad_request(detail: impl Into<String>) -> Self {
|
||||
Self {
|
||||
@@ -114,8 +129,12 @@ impl fmt::Display for S3BackupTaskError {
|
||||
impl std::error::Error for S3BackupTaskError {}
|
||||
|
||||
impl From<GatewayError> for S3BackupTaskError {
|
||||
fn from(error: GatewayError) -> Self {
|
||||
Self::internal(format!("{error:?}"))
|
||||
fn from(_error: GatewayError) -> Self {
|
||||
warn!(
|
||||
error_category = "dependency_failed",
|
||||
"S3 backup dependency failed"
|
||||
);
|
||||
Self::internal(S3_BACKUP_INTERNAL_ERROR_DETAIL)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -220,8 +239,6 @@ fn s3_backup_task_payload_json(
|
||||
) -> Value {
|
||||
let mut payload = json!({
|
||||
"scope": config.scope.as_config_value(),
|
||||
"bucket": config.bucket.clone(),
|
||||
"prefix": config.prefix.clone(),
|
||||
"compression": config.compression.clone(),
|
||||
"trigger": trigger,
|
||||
});
|
||||
@@ -259,7 +276,7 @@ fn spawn_s3_backup_worker(
|
||||
Some(100),
|
||||
Some("S3 备份任务异常退出".to_string()),
|
||||
None,
|
||||
Some("S3 backup task panicked".to_string()),
|
||||
Some("background_task_panicked".to_string()),
|
||||
None,
|
||||
Some(now_unix_secs()),
|
||||
)
|
||||
@@ -294,16 +311,67 @@ async fn run_s3_backup_worker_inner(
|
||||
.await;
|
||||
append_event_with_logging(&app, &run_id, "running", "S3 backup task started", None).await;
|
||||
|
||||
let heartbeat = spawn_s3_backup_task_heartbeat(app.clone(), run_id.clone(), lock);
|
||||
let result = run_s3_backup_once(&app, &config).await;
|
||||
heartbeat.abort();
|
||||
let _ = heartbeat.await;
|
||||
let heartbeat = spawn_s3_backup_task_heartbeat(app.clone(), run_id.clone(), lock.clone());
|
||||
let result = match race_backup_with_lock_heartbeat(run_s3_backup_once(&app, &config), heartbeat)
|
||||
.await
|
||||
{
|
||||
BackupLockRaceOutcome::BackupCompleted(result) => {
|
||||
match require_successful_backup_lock_renewal(
|
||||
app.runtime_state
|
||||
.lock_renew(&lock, S3_BACKUP_TASK_LOCK_TTL)
|
||||
.await,
|
||||
) {
|
||||
Ok(()) => result,
|
||||
Err(BackupLockRenewalFailure::Lost) => {
|
||||
warn!(
|
||||
run_id = %run_id,
|
||||
lock_key = %lock.key,
|
||||
"S3 backup task lost its distributed lock before publishing completion"
|
||||
);
|
||||
Err(S3BackupTaskError::service_unavailable(
|
||||
"S3 备份任务锁已失效,任务完成状态未发布",
|
||||
))
|
||||
}
|
||||
Err(BackupLockRenewalFailure::Backend(error)) => {
|
||||
warn!(
|
||||
run_id = %run_id,
|
||||
lock_key = %lock.key,
|
||||
error = %error,
|
||||
"S3 backup task could not verify its distributed lock before publishing completion"
|
||||
);
|
||||
Err(S3BackupTaskError::service_unavailable(
|
||||
"无法确认 S3 备份任务锁所有权,任务完成状态未发布",
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
BackupLockRaceOutcome::LeaseLost(heartbeat_result) => {
|
||||
match heartbeat_result {
|
||||
Ok(()) => warn!(
|
||||
run_id = %run_id,
|
||||
"S3 backup task stopped after losing its distributed lock"
|
||||
),
|
||||
Err(error) => warn!(
|
||||
run_id = %run_id,
|
||||
error = %error,
|
||||
"S3 backup lock heartbeat task failed"
|
||||
),
|
||||
}
|
||||
Err(S3BackupTaskError::service_unavailable(
|
||||
"S3 备份任务锁已失效,任务已停止",
|
||||
))
|
||||
}
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(result) => {
|
||||
if let Some(slot) = scheduled_backup_slot_to_record(scheduled_slot.as_deref(), true) {
|
||||
if let Err(error) = record_scheduled_backup_slot(&app, &slot).await {
|
||||
warn!(error = ?error, run_id = %run_id, "S3 backup slot record failed");
|
||||
if record_scheduled_backup_slot(&app, &slot).await.is_err() {
|
||||
warn!(
|
||||
error_category = "slot_record_failed",
|
||||
run_id = %run_id,
|
||||
"S3 backup slot record failed"
|
||||
);
|
||||
let _ = update_run_status(
|
||||
&app,
|
||||
&run_id,
|
||||
@@ -311,7 +379,7 @@ async fn run_s3_backup_worker_inner(
|
||||
Some(100),
|
||||
Some("S3 备份任务完成,但记录调度时间失败".to_string()),
|
||||
None,
|
||||
Some(format!("S3 backup slot record failed: {error:?}")),
|
||||
Some(S3_BACKUP_SLOT_RECORD_FAILURE_CODE.to_string()),
|
||||
None,
|
||||
Some(now_unix_secs()),
|
||||
)
|
||||
@@ -321,7 +389,7 @@ async fn run_s3_backup_worker_inner(
|
||||
&run_id,
|
||||
"failed",
|
||||
"S3 backup slot record failed",
|
||||
Some(json!({ "error": format!("{error:?}") })),
|
||||
Some(json!({ "error_code": S3_BACKUP_SLOT_RECORD_FAILURE_CODE })),
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
@@ -349,8 +417,12 @@ async fn run_s3_backup_worker_inner(
|
||||
)
|
||||
.await;
|
||||
}
|
||||
Err(error) => {
|
||||
warn!(error = %error, run_id = %run_id, "S3 backup task failed");
|
||||
Err(_) => {
|
||||
warn!(
|
||||
error_category = "backup_execution_failed",
|
||||
run_id = %run_id,
|
||||
"S3 backup task failed"
|
||||
);
|
||||
let _ = update_run_status(
|
||||
&app,
|
||||
&run_id,
|
||||
@@ -358,7 +430,7 @@ async fn run_s3_backup_worker_inner(
|
||||
Some(100),
|
||||
Some("S3 备份任务失败".to_string()),
|
||||
None,
|
||||
Some(error.to_string()),
|
||||
Some(S3_BACKUP_TASK_FAILURE_CODE.to_string()),
|
||||
None,
|
||||
Some(now_unix_secs()),
|
||||
)
|
||||
@@ -368,13 +440,44 @@ async fn run_s3_backup_worker_inner(
|
||||
&run_id,
|
||||
"failed",
|
||||
"S3 backup task failed",
|
||||
Some(json!({ "error": error.to_string() })),
|
||||
Some(json!({ "error_code": S3_BACKUP_TASK_FAILURE_CODE })),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn race_backup_with_lock_heartbeat<F, T>(
|
||||
backup: F,
|
||||
mut heartbeat: JoinHandle<()>,
|
||||
) -> BackupLockRaceOutcome<T>
|
||||
where
|
||||
F: Future<Output = T>,
|
||||
{
|
||||
tokio::pin!(backup);
|
||||
tokio::select! {
|
||||
biased;
|
||||
heartbeat_result = &mut heartbeat => {
|
||||
BackupLockRaceOutcome::LeaseLost(heartbeat_result)
|
||||
}
|
||||
result = &mut backup => {
|
||||
heartbeat.abort();
|
||||
let _ = heartbeat.await;
|
||||
BackupLockRaceOutcome::BackupCompleted(result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn require_successful_backup_lock_renewal<E>(
|
||||
result: Result<bool, E>,
|
||||
) -> Result<(), BackupLockRenewalFailure<E>> {
|
||||
match result {
|
||||
Ok(true) => Ok(()),
|
||||
Ok(false) => Err(BackupLockRenewalFailure::Lost),
|
||||
Err(error) => Err(BackupLockRenewalFailure::Backend(error)),
|
||||
}
|
||||
}
|
||||
|
||||
fn spawn_s3_backup_task_heartbeat(
|
||||
app: AppState,
|
||||
run_id: String,
|
||||
@@ -386,10 +489,30 @@ fn spawn_s3_backup_task_heartbeat(
|
||||
interval.tick().await;
|
||||
loop {
|
||||
interval.tick().await;
|
||||
let _ = app
|
||||
.runtime_state
|
||||
.lock_renew(&lock, S3_BACKUP_TASK_LOCK_TTL)
|
||||
.await;
|
||||
match require_successful_backup_lock_renewal(
|
||||
app.runtime_state
|
||||
.lock_renew(&lock, S3_BACKUP_TASK_LOCK_TTL)
|
||||
.await,
|
||||
) {
|
||||
Ok(()) => {}
|
||||
Err(BackupLockRenewalFailure::Lost) => {
|
||||
warn!(
|
||||
run_id = %run_id,
|
||||
lock_key = %lock.key,
|
||||
"S3 backup task distributed lock is no longer owned"
|
||||
);
|
||||
return;
|
||||
}
|
||||
Err(BackupLockRenewalFailure::Backend(error)) => {
|
||||
warn!(
|
||||
run_id = %run_id,
|
||||
lock_key = %lock.key,
|
||||
error = %error,
|
||||
"S3 backup task distributed lock renewal failed"
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
let _ = update_run_status(
|
||||
&app,
|
||||
&run_id,
|
||||
@@ -458,9 +581,15 @@ async fn acquire_s3_backup_task_lock(
|
||||
Ok(None) => Err(S3BackupTaskError::conflict(
|
||||
"已有 S3 备份任务正在执行,请等待当前任务完成后再试",
|
||||
)),
|
||||
Err(error) => Err(S3BackupTaskError::service_unavailable(format!(
|
||||
"无法获取 S3 备份任务锁:{error}"
|
||||
))),
|
||||
Err(_) => {
|
||||
warn!(
|
||||
error_category = "lock_acquisition_failed",
|
||||
"S3 backup task lock acquisition failed"
|
||||
);
|
||||
Err(S3BackupTaskError::service_unavailable(
|
||||
"无法获取 S3 备份任务锁,请稍后重试",
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -509,25 +638,77 @@ async fn run_s3_backup_once(
|
||||
app: &AppState,
|
||||
config: &S3BackupConfig,
|
||||
) -> Result<BackupRunResult, S3BackupTaskError> {
|
||||
let admin_state = AdminAppState::new(app);
|
||||
let payload = match config.scope {
|
||||
BackupScope::Config => {
|
||||
admin_state
|
||||
.build_admin_system_config_export_payload()
|
||||
.await?
|
||||
}
|
||||
BackupScope::Users => {
|
||||
admin_state
|
||||
.build_admin_system_users_export_payload()
|
||||
.await?
|
||||
}
|
||||
BackupScope::Data => admin_state.build_admin_system_data_export_payload().await?,
|
||||
let Some(encryption_secret) = effective_backup_encryption_secret(app) else {
|
||||
return Err(S3BackupTaskError::service_unavailable(
|
||||
"S3 备份需要 AETHER_BACKUP_ENCRYPTION_KEY 或可用的数据加密密钥",
|
||||
));
|
||||
};
|
||||
let store = ObjectStoreS3BackupStore::from_config(config)
|
||||
.map_err(|error| S3BackupTaskError::internal(error.to_string()))?;
|
||||
run_backup_with_store(config, &store, payload, Utc::now())
|
||||
let payload = build_s3_backup_payload_exclusively(app, config.scope).await?;
|
||||
let store = ObjectStoreS3BackupStore::from_config(config).map_err(|_| {
|
||||
warn!(
|
||||
error_category = "object_store_initialization_failed",
|
||||
"S3 backup object store initialization failed"
|
||||
);
|
||||
S3BackupTaskError::internal(S3_BACKUP_INTERNAL_ERROR_DETAIL)
|
||||
})?;
|
||||
run_backup_with_store(config, &store, payload, Utc::now(), &encryption_secret)
|
||||
.await
|
||||
.map_err(|error| S3BackupTaskError::internal(error.to_string()))
|
||||
.map_err(|_| {
|
||||
warn!(
|
||||
error_category = "backup_execution_failed",
|
||||
"S3 backup execution failed"
|
||||
);
|
||||
S3BackupTaskError::internal(S3_BACKUP_INTERNAL_ERROR_DETAIL)
|
||||
})
|
||||
}
|
||||
|
||||
async fn build_s3_backup_payload_exclusively(
|
||||
app: &AppState,
|
||||
scope: BackupScope,
|
||||
) -> Result<Value, S3BackupTaskError> {
|
||||
let admin_state = AdminAppState::new(app);
|
||||
crate::admin_api::execute_admin_system_import_exclusively(app, async {
|
||||
match scope {
|
||||
BackupScope::Config => {
|
||||
admin_state
|
||||
.build_admin_system_config_export_payload(SystemExportMode::RecoveryBackup)
|
||||
.await
|
||||
}
|
||||
BackupScope::Users => {
|
||||
admin_state
|
||||
.build_admin_system_users_export_payload(SystemExportMode::RecoveryBackup)
|
||||
.await
|
||||
}
|
||||
BackupScope::Data => {
|
||||
admin_state
|
||||
.build_admin_system_data_export_payload(SystemExportMode::RecoveryBackup)
|
||||
.await
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|error| {
|
||||
warn!(
|
||||
error_category = "system_import_coordination_failed",
|
||||
lock_error = ?error,
|
||||
"S3 backup snapshot could not acquire or retain the system import lock"
|
||||
);
|
||||
S3BackupTaskError::service_unavailable(S3_BACKUP_INTERNAL_ERROR_DETAIL)
|
||||
})?
|
||||
.map_err(S3BackupTaskError::from)
|
||||
}
|
||||
|
||||
fn effective_backup_encryption_secret(app: &AppState) -> Option<String> {
|
||||
std::env::var("AETHER_BACKUP_ENCRYPTION_KEY")
|
||||
.ok()
|
||||
.map(|value| value.trim().to_string())
|
||||
.filter(|value| !value.is_empty())
|
||||
.or_else(|| {
|
||||
app.encryption_key()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
})
|
||||
}
|
||||
|
||||
async fn load_s3_backup_config_for_run(
|
||||
@@ -551,7 +732,7 @@ pub(crate) async fn load_s3_backup_config_values(
|
||||
.or_else(|| admin_system_config_default_value(key));
|
||||
if let Some(value) = value {
|
||||
let value = if *key == "backup_s3_secret_access_key" {
|
||||
decrypt_s3_secret_access_key(app, value)?
|
||||
decrypt_s3_secret_access_key(app, value).await?
|
||||
} else {
|
||||
value
|
||||
};
|
||||
@@ -561,39 +742,56 @@ pub(crate) async fn load_s3_backup_config_values(
|
||||
Ok(values)
|
||||
}
|
||||
|
||||
fn decrypt_s3_secret_access_key(app: &AppState, value: Value) -> Result<Value, S3BackupTaskError> {
|
||||
let Some(ciphertext) = value
|
||||
async fn decrypt_s3_secret_access_key(
|
||||
app: &AppState,
|
||||
value: Value,
|
||||
) -> Result<Value, S3BackupTaskError> {
|
||||
let Some(stored_value) = value
|
||||
.as_str()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
else {
|
||||
return Ok(value);
|
||||
};
|
||||
let Some(plaintext) = decrypt_catalog_secret_with_fallbacks(app.encryption_key(), ciphertext)
|
||||
else {
|
||||
return Err(S3BackupTaskError::bad_request(
|
||||
let plaintext = decrypt_or_migrate_system_config_secret(
|
||||
app,
|
||||
"backup_s3_secret_access_key",
|
||||
stored_value.to_string(),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
S3BackupTaskError::bad_request(
|
||||
"S3 备份配置无效:Secret Access Key(访问密钥)无法解密,请重新填写",
|
||||
));
|
||||
};
|
||||
)
|
||||
})?;
|
||||
Ok(Value::String(plaintext))
|
||||
}
|
||||
|
||||
fn backup_run_result_json(result: &BackupRunResult) -> Value {
|
||||
json!({
|
||||
"scope": result.scope.as_config_value(),
|
||||
"bucket": result.bucket,
|
||||
"object_key": result.object_key,
|
||||
"bytes": result.bytes,
|
||||
"sha256": result.sha256,
|
||||
"export_version": result.export_version,
|
||||
"exported_at": result.exported_at,
|
||||
"compression": result.compression,
|
||||
"deleted_old_objects": result.deleted_old_objects,
|
||||
"encryption": result.encryption,
|
||||
"legacy_encrypted_copies_created": result.legacy_encrypted_copies_created,
|
||||
"legacy_encrypted_copies_verified": result.legacy_encrypted_copies_verified,
|
||||
"legacy_plaintext_objects_deleted": result.legacy_plaintext_objects_deleted,
|
||||
"legacy_plaintext_objects_retained": result.legacy_plaintext_objects_retained,
|
||||
"retention_cleanup_candidates": result.retention_cleanup_candidates,
|
||||
"automatic_deletions": result.legacy_plaintext_objects_deleted,
|
||||
"object_cleanup_mode": "legacy_plaintext_deleted_after_verified_encryption",
|
||||
"versioned_storage_cleanup_required": result.versioned_storage_cleanup_required,
|
||||
"versioned_storage_cleanup_notice": "legacy_plaintext_versions_require_external_cleanup",
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::convert::Infallible;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
|
||||
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
|
||||
@@ -603,9 +801,78 @@ mod tests {
|
||||
};
|
||||
|
||||
use crate::data::GatewayDataState;
|
||||
use crate::handlers::shared::decrypt_system_config_secret;
|
||||
use crate::state::AppState;
|
||||
use crate::task_runtime::{now_unix_secs, TASK_KEY_SYSTEM_S3_BACKUP};
|
||||
|
||||
#[test]
|
||||
fn backup_lock_renewal_requires_ownership_and_preserves_backend_errors() {
|
||||
assert!(matches!(
|
||||
super::require_successful_backup_lock_renewal::<Infallible>(Ok(true)),
|
||||
Ok(())
|
||||
));
|
||||
assert!(matches!(
|
||||
super::require_successful_backup_lock_renewal::<Infallible>(Ok(false)),
|
||||
Err(super::BackupLockRenewalFailure::Lost)
|
||||
));
|
||||
assert!(matches!(
|
||||
super::require_successful_backup_lock_renewal(Err("redis unavailable")),
|
||||
Err(super::BackupLockRenewalFailure::Backend(
|
||||
"redis unavailable"
|
||||
))
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lost_backup_lock_stops_race_without_publishing_backup_result() {
|
||||
let destructive_stage_reached = Arc::new(AtomicBool::new(false));
|
||||
let destructive_stage_for_backup = Arc::clone(&destructive_stage_reached);
|
||||
let backup = async move {
|
||||
std::future::pending::<()>().await;
|
||||
destructive_stage_for_backup.store(true, Ordering::Release);
|
||||
Ok::<(), super::S3BackupTaskError>(())
|
||||
};
|
||||
let heartbeat = tokio::spawn(async {});
|
||||
let outcome = super::race_backup_with_lock_heartbeat(backup, heartbeat).await;
|
||||
assert!(matches!(
|
||||
outcome,
|
||||
super::BackupLockRaceOutcome::LeaseLost(Ok(()))
|
||||
));
|
||||
assert!(!destructive_stage_reached.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completed_heartbeat_wins_when_backup_completion_is_also_ready() {
|
||||
let heartbeat = tokio::spawn(async {});
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
let outcome = super::race_backup_with_lock_heartbeat(async { 42_u8 }, heartbeat).await;
|
||||
|
||||
assert!(matches!(
|
||||
outcome,
|
||||
super::BackupLockRaceOutcome::LeaseLost(Ok(()))
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn s3_backup_snapshot_refuses_to_overlap_system_import() {
|
||||
let app = AppState::new().expect("app state should build");
|
||||
let lease = crate::admin_api::try_acquire_admin_system_import_lease(&app)
|
||||
.await
|
||||
.expect("test should acquire the system import lease");
|
||||
|
||||
let error = super::build_s3_backup_payload_exclusively(
|
||||
&app,
|
||||
crate::backup::scopes::BackupScope::Config,
|
||||
)
|
||||
.await
|
||||
.expect_err("backup snapshot must not overlap a system import");
|
||||
|
||||
crate::admin_api::release_admin_system_import_lease(&app, &lease).await;
|
||||
assert_eq!(error.status(), axum::http::StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(error.detail(), super::S3_BACKUP_INTERNAL_ERROR_DETAIL);
|
||||
}
|
||||
|
||||
fn valid_s3_backup_config_values() -> Vec<(String, serde_json::Value)> {
|
||||
vec![
|
||||
(
|
||||
@@ -632,6 +899,92 @@ mod tests {
|
||||
]
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_plaintext_s3_secret_is_migrated_when_config_loads() {
|
||||
let plaintext = "legacy-s3-secret-access-key";
|
||||
let mut entries = valid_s3_backup_config_values();
|
||||
entries
|
||||
.iter_mut()
|
||||
.find(|(key, _)| key == "backup_s3_secret_access_key")
|
||||
.expect("secret config fixture should exist")
|
||||
.1 = serde_json::json!(plaintext);
|
||||
let app = AppState::new()
|
||||
.expect("app state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::disabled()
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY)
|
||||
.with_system_config_values_for_tests(entries),
|
||||
);
|
||||
|
||||
let values = super::load_s3_backup_config_values(&app)
|
||||
.await
|
||||
.expect("legacy S3 config should load");
|
||||
assert_eq!(
|
||||
values.get("backup_s3_secret_access_key"),
|
||||
Some(&serde_json::json!(plaintext))
|
||||
);
|
||||
|
||||
let stored = app
|
||||
.read_system_config_json_value_strong("backup_s3_secret_access_key")
|
||||
.await
|
||||
.expect("stored S3 secret should read")
|
||||
.and_then(|value| value.as_str().map(ToOwned::to_owned))
|
||||
.expect("stored S3 secret should remain a string");
|
||||
assert_ne!(stored, plaintext);
|
||||
assert_eq!(
|
||||
decrypt_system_config_secret(&app, "backup_s3_secret_access_key", &stored)
|
||||
.expect("migrated S3 secret should decrypt"),
|
||||
plaintext
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn undecryptable_s3_fernet_secret_fails_closed() {
|
||||
let plaintext = "s3-secret-from-unavailable-key";
|
||||
let ciphertext = encrypt_python_fernet_plaintext("unavailable-s3-key", plaintext)
|
||||
.expect("unknown-key fixture should encrypt");
|
||||
let mut entries = valid_s3_backup_config_values();
|
||||
entries
|
||||
.iter_mut()
|
||||
.find(|(key, _)| key == "backup_s3_secret_access_key")
|
||||
.expect("secret config fixture should exist")
|
||||
.1 = serde_json::json!(ciphertext.clone());
|
||||
let app = AppState::new()
|
||||
.expect("app state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::disabled()
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY)
|
||||
.with_system_config_values_for_tests(entries),
|
||||
);
|
||||
|
||||
let error = super::load_s3_backup_config_values(&app)
|
||||
.await
|
||||
.expect_err("unknown-key S3 ciphertext must fail closed");
|
||||
let error_text = error.to_string();
|
||||
assert!(!error_text.contains(plaintext));
|
||||
assert!(!error_text.contains(&ciphertext));
|
||||
assert_eq!(
|
||||
app.read_system_config_json_value_strong("backup_s3_secret_access_key")
|
||||
.await
|
||||
.expect("stored S3 secret should read"),
|
||||
Some(serde_json::json!(ciphertext))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gateway_dependency_errors_are_not_exposed_to_backup_clients() {
|
||||
let error = super::S3BackupTaskError::from(crate::GatewayError::Internal(
|
||||
"postgresql://admin:[email protected]/aether".to_string(),
|
||||
));
|
||||
|
||||
assert_eq!(
|
||||
error.status(),
|
||||
axum::http::StatusCode::INTERNAL_SERVER_ERROR
|
||||
);
|
||||
assert_eq!(error.detail(), super::S3_BACKUP_INTERNAL_ERROR_DETAIL);
|
||||
assert!(!error.detail().contains("database-secret"));
|
||||
}
|
||||
|
||||
fn stored_s3_backup_run(status: BackgroundTaskStatus) -> StoredBackgroundTaskRun {
|
||||
let now = now_unix_secs();
|
||||
StoredBackgroundTaskRun {
|
||||
@@ -774,7 +1127,9 @@ mod tests {
|
||||
|
||||
let payload = super::s3_backup_task_payload_json(&config, "manual", None);
|
||||
|
||||
assert!(payload["bucket"].is_string());
|
||||
assert!(payload.get("bucket").is_none());
|
||||
assert!(payload.get("prefix").is_none());
|
||||
assert_eq!(payload["scope"], serde_json::json!("data"));
|
||||
assert_eq!(payload["trigger"], serde_json::json!("manual"));
|
||||
assert!(!payload.to_string().contains("secret"));
|
||||
}
|
||||
|
||||
@@ -29,8 +29,11 @@ pub(crate) fn spawn_s3_backup_worker(app: AppState) -> Option<JoinHandle<()>> {
|
||||
interval.tick().await;
|
||||
loop {
|
||||
interval.tick().await;
|
||||
if let Err(error) = run_s3_backup_schedule_tick(&app, Utc::now()).await {
|
||||
warn!(error = ?error, "S3 backup schedule tick failed");
|
||||
if run_s3_backup_schedule_tick(&app, Utc::now()).await.is_err() {
|
||||
warn!(
|
||||
error_category = "schedule_tick_failed",
|
||||
"S3 backup schedule tick failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -43,15 +46,21 @@ async fn run_s3_backup_schedule_tick(
|
||||
) -> Result<(), GatewayError> {
|
||||
let values = match super::task::load_s3_backup_config_values(app).await {
|
||||
Ok(values) => values,
|
||||
Err(error) => {
|
||||
warn!(error = %error, "S3 backup schedule config load failed");
|
||||
Err(_) => {
|
||||
warn!(
|
||||
error_category = "config_load_failed",
|
||||
"S3 backup schedule config load failed"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
let config = match S3BackupConfig::from_json_map(&values) {
|
||||
Ok(config) => config,
|
||||
Err(error) => {
|
||||
warn!(error = %error, "S3 backup schedule config is invalid");
|
||||
Err(_) => {
|
||||
warn!(
|
||||
error_category = "config_invalid",
|
||||
"S3 backup schedule config is invalid"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
@@ -68,8 +77,11 @@ async fn run_s3_backup_schedule_tick(
|
||||
|
||||
match super::task::start_s3_backup_task_for_schedule(app.clone(), slot).await {
|
||||
Ok(_) => {}
|
||||
Err(error) => {
|
||||
warn!(error = %error, "S3 backup scheduled task submission failed");
|
||||
Err(_) => {
|
||||
warn!(
|
||||
error_category = "task_submission_failed",
|
||||
"S3 backup scheduled task submission failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
use crate::handlers::shared::{
|
||||
decrypt_catalog_secret_with_fallbacks, system_config_bool, system_config_string,
|
||||
bark_device_key_binding, canonical_bark_server_url, decrypt_or_migrate_bark_device_key,
|
||||
system_config_bool, system_config_string,
|
||||
};
|
||||
use crate::{AppState, GatewayError};
|
||||
use serde_json::{json, Value};
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
|
||||
pub(crate) const BARK_PUSH_ENABLED_KEY: &str = "module.bark_push.enabled";
|
||||
pub(crate) const BARK_PUSH_DEVICE_KEY_KEY: &str = "module.bark_push.device_key";
|
||||
@@ -10,8 +12,20 @@ pub(crate) const BARK_PUSH_SERVER_URL_KEY: &str = "module.bark_push.server_url";
|
||||
pub(crate) const BARK_PUSH_TEMPLATE_KEY: &str = "module.bark_push.template";
|
||||
|
||||
const DEFAULT_BARK_API_BASE: &str = "https://api.day.app";
|
||||
const BARK_ALLOW_HTTP_ENV: &str = "AETHER_BARK_ALLOW_HTTP";
|
||||
const BARK_ALLOW_PRIVATE_TARGETS_ENV: &str = "AETHER_BARK_ALLOW_PRIVATE_TARGETS";
|
||||
const MAX_BARK_RESPONSE_BYTES: usize = 64 * 1024;
|
||||
const BARK_CONNECT_TIMEOUT_MS: u64 = 10_000;
|
||||
const BARK_REQUEST_TIMEOUT_MS: u64 = 300_000;
|
||||
const MAX_BARK_SERVER_URL_BYTES: usize = 2 * 1024;
|
||||
const MAX_BARK_DEVICE_KEY_BYTES: usize = 512;
|
||||
const MAX_BARK_TEMPLATE_BYTES: usize = 256 * 1024;
|
||||
const MAX_BARK_TITLE_BYTES: usize = 512;
|
||||
const MAX_BARK_BODY_BYTES: usize = 2 * 1024 * 1024;
|
||||
const MAX_BARK_RENDERED_BODY_BYTES: usize = 2 * 1024 * 1024;
|
||||
const MAX_BARK_RESOLVED_ADDRESSES: usize = 32;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct BarkPushConfig {
|
||||
pub(crate) enabled: bool,
|
||||
pub(crate) device_key: Option<String>,
|
||||
@@ -19,6 +33,21 @@ pub(crate) struct BarkPushConfig {
|
||||
pub(crate) template: Option<String>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for BarkPushConfig {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("BarkPushConfig")
|
||||
.field("enabled", &self.enabled)
|
||||
.field(
|
||||
"device_key",
|
||||
&self.device_key.as_ref().map(|_| "[REDACTED]"),
|
||||
)
|
||||
.field("server_url", &self.server_url)
|
||||
.field("template", &self.template.as_ref().map(|_| "[REDACTED]"))
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn bark_push_module_enabled(state: &AppState) -> Result<bool, GatewayError> {
|
||||
let value = state
|
||||
.read_system_config_json_value(BARK_PUSH_ENABLED_KEY)
|
||||
@@ -35,23 +64,34 @@ pub(crate) async fn read_bark_push_config(
|
||||
state: &AppState,
|
||||
) -> Result<BarkPushConfig, GatewayError> {
|
||||
let enabled = bark_push_module_enabled(state).await?;
|
||||
let device_key = state
|
||||
.read_system_config_json_value(BARK_PUSH_DEVICE_KEY_KEY)
|
||||
.await?
|
||||
.and_then(|value| system_config_string(Some(&value)))
|
||||
.map(|value| {
|
||||
decrypt_catalog_secret_with_fallbacks(state.encryption_key(), &value).unwrap_or(value)
|
||||
});
|
||||
let server_url = state
|
||||
.read_system_config_json_value(BARK_PUSH_SERVER_URL_KEY)
|
||||
.await?
|
||||
.and_then(|value| system_config_string(Some(&value)))
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
.unwrap_or_else(|| DEFAULT_BARK_API_BASE.to_string());
|
||||
validate_bark_config_field("server_url", &server_url, MAX_BARK_SERVER_URL_BYTES)?;
|
||||
let server_url = normalized_bark_server_url(&server_url)?;
|
||||
let binding = bark_device_key_binding(&server_url)
|
||||
.ok_or_else(|| GatewayError::Internal("Bark 服务器地址不是合法 URL".to_string()))?;
|
||||
let device_key = state
|
||||
.read_system_config_json_value(BARK_PUSH_DEVICE_KEY_KEY)
|
||||
.await?
|
||||
.and_then(|value| system_config_string(Some(&value)));
|
||||
let device_key = match device_key {
|
||||
Some(value) => Some(decrypt_or_migrate_bark_device_key(state, &binding, value).await?),
|
||||
None => None,
|
||||
};
|
||||
if let Some(device_key) = device_key.as_deref() {
|
||||
validate_bark_config_field("device_key", device_key, MAX_BARK_DEVICE_KEY_BYTES)?;
|
||||
}
|
||||
let template = state
|
||||
.read_system_config_json_value(BARK_PUSH_TEMPLATE_KEY)
|
||||
.await?
|
||||
.and_then(|value| system_config_string(Some(&value)));
|
||||
if let Some(template) = template.as_deref() {
|
||||
validate_bark_config_field("template", template, MAX_BARK_TEMPLATE_BYTES)?;
|
||||
}
|
||||
|
||||
Ok(BarkPushConfig {
|
||||
enabled,
|
||||
@@ -62,7 +102,7 @@ pub(crate) async fn read_bark_push_config(
|
||||
}
|
||||
|
||||
pub(crate) async fn send_bark_push(
|
||||
state: &AppState,
|
||||
_state: &AppState,
|
||||
config: &BarkPushConfig,
|
||||
title: &str,
|
||||
markdown_body: &str,
|
||||
@@ -76,11 +116,13 @@ pub(crate) async fn send_bark_push(
|
||||
"Bark Device Key 不能为空".to_string(),
|
||||
));
|
||||
}
|
||||
let server_url = normalized_bark_server_url(&config.server_url)?;
|
||||
let body = render_bark_body(config.template.as_deref(), title, markdown_body);
|
||||
let response = state
|
||||
.client
|
||||
.post(format!("{server_url}/push"))
|
||||
validate_bark_config_field("device_key", device_key, MAX_BARK_DEVICE_KEY_BYTES)?;
|
||||
validate_bark_content_field("title", title, MAX_BARK_TITLE_BYTES)?;
|
||||
validate_bark_content_field("body", markdown_body, MAX_BARK_BODY_BYTES)?;
|
||||
let (client, push_url) = build_bark_push_client_and_url(&config.server_url).await?;
|
||||
let body = render_bark_body(config.template.as_deref(), title, markdown_body)?;
|
||||
let response = client
|
||||
.post(push_url)
|
||||
.json(&json!({
|
||||
"device_key": device_key,
|
||||
"title": title,
|
||||
@@ -88,16 +130,14 @@ pub(crate) async fn send_bark_push(
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
.map_err(|err| GatewayError::Internal(bark_request_error_message(&err)))?;
|
||||
let status = response.status();
|
||||
let text = response
|
||||
.text()
|
||||
let body = aether_http::read_response_bytes_with_limit(response, MAX_BARK_RESPONSE_BYTES)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
.map_err(|err| GatewayError::Internal(bark_response_body_error_message(&err)))?;
|
||||
let text = String::from_utf8_lossy(&body);
|
||||
if !status.is_success() {
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"Bark 返回 HTTP {status}: {text}"
|
||||
)));
|
||||
return Err(GatewayError::Internal(format!("Bark 返回 HTTP {status}")));
|
||||
}
|
||||
if let Ok(payload) = serde_json::from_str::<Value>(&text) {
|
||||
let code_is_ok = payload
|
||||
@@ -114,53 +154,259 @@ pub(crate) async fn send_bark_push(
|
||||
})
|
||||
.unwrap_or(true);
|
||||
if !code_is_ok {
|
||||
return Err(GatewayError::Internal(format!("Bark 返回失败: {payload}")));
|
||||
return Err(GatewayError::Internal("Bark 返回失败".to_string()));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn normalized_bark_server_url(server_url: &str) -> Result<String, GatewayError> {
|
||||
let server_url = server_url.trim().trim_end_matches('/');
|
||||
if server_url.is_empty() {
|
||||
return Err(GatewayError::Internal(
|
||||
"Bark 服务器地址不能为空".to_string(),
|
||||
));
|
||||
}
|
||||
if !server_url.starts_with("https://") && !server_url.starts_with("http://") {
|
||||
return Err(GatewayError::Internal(
|
||||
"Bark 服务器地址必须以 http:// 或 https:// 开头".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(server_url.to_string())
|
||||
fn bark_request_error_message(error: &reqwest::Error) -> String {
|
||||
format!("Bark 请求失败 ({})", bark_reqwest_error_kind(error))
|
||||
}
|
||||
|
||||
fn render_bark_body(template: Option<&str>, title: &str, markdown_body: &str) -> String {
|
||||
match template {
|
||||
Some(template) if !template.trim().is_empty() => template
|
||||
.replace("{title}", title)
|
||||
.replace("{body}", markdown_body),
|
||||
_ => markdown_body.to_string(),
|
||||
fn bark_response_body_error_message(error: &aether_http::ResponseBodyReadError) -> String {
|
||||
match error {
|
||||
aether_http::ResponseBodyReadError::TooLarge { max_bytes } => {
|
||||
format!("Bark 响应超过 {max_bytes} 字节")
|
||||
}
|
||||
aether_http::ResponseBodyReadError::Read(error) => {
|
||||
format!("Bark 响应读取失败 ({})", bark_reqwest_error_kind(error))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn bark_reqwest_error_kind(error: &reqwest::Error) -> &'static str {
|
||||
if error.is_timeout() {
|
||||
"timeout"
|
||||
} else if error.is_connect() {
|
||||
"connect"
|
||||
} else if error.is_request() {
|
||||
"request"
|
||||
} else {
|
||||
"transport"
|
||||
}
|
||||
}
|
||||
|
||||
fn normalized_bark_server_url(server_url: &str) -> Result<String, GatewayError> {
|
||||
canonical_bark_server_url(server_url)
|
||||
.ok_or_else(|| GatewayError::Internal("Bark 服务器地址不是合法 URL".to_string()))
|
||||
}
|
||||
|
||||
async fn build_bark_push_client_and_url(
|
||||
server_url: &str,
|
||||
) -> Result<(reqwest::Client, url::Url), GatewayError> {
|
||||
validate_bark_config_field("server_url", server_url, MAX_BARK_SERVER_URL_BYTES)?;
|
||||
let normalized = normalized_bark_server_url(server_url)?;
|
||||
let mut push_url = url::Url::parse(&normalized)
|
||||
.map_err(|_| GatewayError::Internal("Bark 服务器地址不是合法 URL".to_string()))?;
|
||||
validate_bark_transport_policy(&push_url, env_flag_enabled(BARK_ALLOW_HTTP_ENV))?;
|
||||
|
||||
let host = push_url
|
||||
.host_str()
|
||||
.ok_or_else(|| GatewayError::Internal("Bark 服务器地址缺少主机名".to_string()))?
|
||||
.to_string();
|
||||
let port = push_url
|
||||
.port_or_known_default()
|
||||
.ok_or_else(|| GatewayError::Internal("Bark 服务器地址缺少端口".to_string()))?;
|
||||
let addresses = if let Ok(ip) = host.parse::<IpAddr>() {
|
||||
vec![SocketAddr::new(ip, port)]
|
||||
} else {
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_millis(BARK_CONNECT_TIMEOUT_MS),
|
||||
tokio::net::lookup_host((host.as_str(), port)),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| GatewayError::Internal("Bark 服务器 DNS 解析超时".to_string()))?
|
||||
.map_err(|_| GatewayError::Internal("Bark 服务器 DNS 解析失败".to_string()))?
|
||||
.take(MAX_BARK_RESOLVED_ADDRESSES)
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
let allow_benchmarking_ip = push_url.scheme() == "https"
|
||||
&& push_url.port_or_known_default() == Some(443)
|
||||
&& host.eq_ignore_ascii_case("api.day.app");
|
||||
validate_bark_resolved_addresses(
|
||||
&addresses,
|
||||
env_flag_enabled(BARK_ALLOW_PRIVATE_TARGETS_ENV),
|
||||
allow_benchmarking_ip,
|
||||
)?;
|
||||
|
||||
push_url
|
||||
.path_segments_mut()
|
||||
.map_err(|_| GatewayError::Internal("Bark 服务器地址不能作为基础 URL".to_string()))?
|
||||
.pop_if_empty()
|
||||
.push("push");
|
||||
|
||||
let mut builder = aether_http::apply_http_client_config(
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.redirect(reqwest::redirect::Policy::none()),
|
||||
&aether_http::HttpClientConfig {
|
||||
connect_timeout_ms: Some(BARK_CONNECT_TIMEOUT_MS),
|
||||
request_timeout_ms: Some(BARK_REQUEST_TIMEOUT_MS),
|
||||
http2_adaptive_window: true,
|
||||
..aether_http::HttpClientConfig::default()
|
||||
},
|
||||
);
|
||||
if host.parse::<IpAddr>().is_err() {
|
||||
builder = builder.resolve_to_addrs(&host, &addresses);
|
||||
}
|
||||
let client = builder
|
||||
.build()
|
||||
.map_err(|_| GatewayError::Internal("Bark HTTP 客户端初始化失败".to_string()))?;
|
||||
Ok((client, push_url))
|
||||
}
|
||||
|
||||
fn validate_bark_transport_policy(url: &url::Url, allow_http: bool) -> Result<(), GatewayError> {
|
||||
if url.scheme() == "http" && !allow_http {
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"Bark 服务器必须使用 HTTPS;如确需明文 HTTP,请显式设置 {BARK_ALLOW_HTTP_ENV}=true"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_bark_resolved_addresses(
|
||||
addresses: &[SocketAddr],
|
||||
allow_private: bool,
|
||||
allow_benchmarking_ip: bool,
|
||||
) -> Result<(), GatewayError> {
|
||||
if addresses.is_empty() {
|
||||
return Err(GatewayError::Internal(
|
||||
"Bark 服务器 DNS 解析未返回地址".to_string(),
|
||||
));
|
||||
}
|
||||
if !allow_private
|
||||
&& addresses.iter().any(|address| {
|
||||
aether_http::is_private_or_reserved_ip(address.ip())
|
||||
&& !(allow_benchmarking_ip
|
||||
&& aether_http::is_ipv4_benchmarking_fake_ip(address.ip()))
|
||||
})
|
||||
{
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"Bark 服务器解析到私有或保留地址;如确需内网自建服务,请显式设置 {BARK_ALLOW_PRIVATE_TARGETS_ENV}=true"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn env_flag_enabled(key: &str) -> bool {
|
||||
std::env::var(key).ok().is_some_and(|value| {
|
||||
matches!(
|
||||
value.trim().to_ascii_lowercase().as_str(),
|
||||
"1" | "true" | "yes" | "on"
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_bark_config_field(
|
||||
field: &str,
|
||||
value: &str,
|
||||
max_bytes: usize,
|
||||
) -> Result<(), GatewayError> {
|
||||
if value.len() > max_bytes || value.bytes().any(|byte| byte == 0) {
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"Bark {field} exceeds the allowed size or contains a NUL byte"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_bark_content_field(
|
||||
field: &str,
|
||||
value: &str,
|
||||
max_bytes: usize,
|
||||
) -> Result<(), GatewayError> {
|
||||
validate_bark_config_field(field, value, max_bytes)
|
||||
}
|
||||
|
||||
fn render_bark_body(
|
||||
template: Option<&str>,
|
||||
title: &str,
|
||||
markdown_body: &str,
|
||||
) -> Result<String, GatewayError> {
|
||||
validate_bark_content_field("title", title, MAX_BARK_TITLE_BYTES)?;
|
||||
validate_bark_content_field("body", markdown_body, MAX_BARK_BODY_BYTES)?;
|
||||
let template = template
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
.unwrap_or("{body}");
|
||||
validate_bark_config_field("template", template, MAX_BARK_TEMPLATE_BYTES)?;
|
||||
|
||||
let mut rendered = String::with_capacity(template.len().min(MAX_BARK_RENDERED_BODY_BYTES));
|
||||
let mut cursor = 0usize;
|
||||
while cursor < template.len() {
|
||||
let remaining = &template[cursor..];
|
||||
let title_match = remaining.find("{title}");
|
||||
let body_match = remaining.find("{body}");
|
||||
let next = match (title_match, body_match) {
|
||||
(None, None) => {
|
||||
append_bark_rendered_part(&mut rendered, remaining)?;
|
||||
cursor = template.len();
|
||||
continue;
|
||||
}
|
||||
(Some(index), None) => (index, "{title}", title),
|
||||
(None, Some(index)) => (index, "{body}", markdown_body),
|
||||
(Some(title_index), Some(body_index)) if title_index <= body_index => {
|
||||
(title_index, "{title}", title)
|
||||
}
|
||||
(Some(_), Some(body_index)) => (body_index, "{body}", markdown_body),
|
||||
};
|
||||
append_bark_rendered_part(&mut rendered, &remaining[..next.0])?;
|
||||
append_bark_rendered_part(&mut rendered, next.2)?;
|
||||
cursor += next.0 + next.1.len();
|
||||
}
|
||||
if rendered.is_empty() && template.is_empty() {
|
||||
return Ok(String::new());
|
||||
}
|
||||
Ok(rendered)
|
||||
}
|
||||
|
||||
fn append_bark_rendered_part(output: &mut String, part: &str) -> Result<(), GatewayError> {
|
||||
let next_len = output
|
||||
.len()
|
||||
.checked_add(part.len())
|
||||
.ok_or_else(|| GatewayError::Internal("Bark rendered body is too large".to_string()))?;
|
||||
if next_len > MAX_BARK_RENDERED_BODY_BYTES {
|
||||
return Err(GatewayError::Internal(
|
||||
"Bark rendered body exceeds the allowed size".to_string(),
|
||||
));
|
||||
}
|
||||
output.push_str(part);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{normalized_bark_server_url, render_bark_body};
|
||||
use super::{
|
||||
bark_request_error_message, bark_response_body_error_message, normalized_bark_server_url,
|
||||
render_bark_body, validate_bark_resolved_addresses, validate_bark_transport_policy,
|
||||
};
|
||||
use std::net::SocketAddr;
|
||||
|
||||
#[test]
|
||||
fn bark_body_uses_template_when_provided() {
|
||||
let rendered = render_bark_body(Some("{title}\n\n{body}"), "告警", "原始正文");
|
||||
let rendered = render_bark_body(Some("{title}\n\n{body}"), "告警", "原始正文")
|
||||
.expect("template should render");
|
||||
assert_eq!(rendered, "告警\n\n原始正文");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bark_body_falls_back_to_markdown_body_for_empty_template() {
|
||||
assert_eq!(render_bark_body(None, "告警", "原始正文"), "原始正文");
|
||||
assert_eq!(
|
||||
render_bark_body(Some(" "), "告警", "原始正文"),
|
||||
render_bark_body(None, "告警", "原始正文").expect("fallback should render"),
|
||||
"原始正文"
|
||||
);
|
||||
assert_eq!(
|
||||
render_bark_body(Some(" "), "告警", "原始正文").expect("fallback should render"),
|
||||
"原始正文"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bark_body_rejects_template_expansion_bombs_and_oversized_content() {
|
||||
let template = "x".repeat(super::MAX_BARK_TEMPLATE_BYTES + 1);
|
||||
assert!(render_bark_body(Some(&template), "告警", "正文").is_err());
|
||||
let body = "x".repeat(super::MAX_BARK_BODY_BYTES + 1);
|
||||
assert!(render_bark_body(None, "告警", &body).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -170,4 +416,61 @@ mod tests {
|
||||
"https://api.day.app"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bark_server_url_rejects_credentials_query_and_fragments() {
|
||||
for invalid in [
|
||||
"https://[email protected]",
|
||||
"https://example.com?target=internal",
|
||||
"https://example.com/#fragment",
|
||||
] {
|
||||
assert!(normalized_bark_server_url(invalid).is_err(), "{invalid}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bark_http_transport_requires_explicit_opt_in() {
|
||||
let url = url::Url::parse("http://bark.example.com").unwrap();
|
||||
assert!(validate_bark_transport_policy(&url, false).is_err());
|
||||
assert!(validate_bark_transport_policy(&url, true).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bark_private_targets_require_explicit_opt_in() {
|
||||
let private = [SocketAddr::from(([127, 0, 0, 1], 443))];
|
||||
assert!(validate_bark_resolved_addresses(&private, false, false).is_err());
|
||||
assert!(validate_bark_resolved_addresses(&private, true, false).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bark_builtin_server_allows_benchmarking_ip_only_with_https_default_port() {
|
||||
let fake = [SocketAddr::from(([198, 18, 75, 234], 443))];
|
||||
assert!(validate_bark_resolved_addresses(&fake, false, true).is_ok());
|
||||
assert!(validate_bark_resolved_addresses(
|
||||
&[fake[0], SocketAddr::from(([127, 0, 0, 1], 443))],
|
||||
false,
|
||||
true,
|
||||
)
|
||||
.is_err());
|
||||
assert!(validate_bark_resolved_addresses(&fake, false, false).is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bark_transport_errors_do_not_expose_server_url_or_response_body() {
|
||||
let secret = "bark-secret-query";
|
||||
let error = reqwest::Client::new()
|
||||
.post(format!("ftp://bark.example.test/push?token={secret}"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("unsupported URL scheme should fail before network I/O");
|
||||
|
||||
let message = bark_request_error_message(&error);
|
||||
assert!(!message.contains(secret));
|
||||
assert!(!message.contains("bark.example.test"));
|
||||
|
||||
let body_error = aether_http::ResponseBodyReadError::Read(error);
|
||||
let message = bark_response_body_error_message(&body_error);
|
||||
assert!(!message.contains(secret));
|
||||
assert!(!message.contains("bark.example.test"));
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -193,6 +193,7 @@ pub(crate) fn parse_probe_url(raw: &str) -> Result<Url, ProbeFailure> {
|
||||
|| url.password().is_some()
|
||||
|| url.query().is_some()
|
||||
|| url.fragment().is_some()
|
||||
|| (url.scheme() == "ws" && !aether_http::url_has_literal_loopback_host(&url))
|
||||
{
|
||||
return Err(ProbeFailure::InvalidEndpoint);
|
||||
}
|
||||
@@ -210,6 +211,7 @@ pub(crate) const fn turn_timeout(args: &ProbeArgs) -> Duration {
|
||||
|
||||
async fn run_probe(config: &ProbeConfig, started_at: Instant) -> Result<ProbeReport, ProbeFailure> {
|
||||
let client = wreq::Client::builder()
|
||||
.no_proxy()
|
||||
.connect_timeout(config.turn_timeout)
|
||||
.timeout(config.turn_timeout)
|
||||
.build()
|
||||
@@ -434,7 +436,14 @@ mod tests {
|
||||
#[test]
|
||||
fn probe_url_rejects_credentials_and_query_strings() {
|
||||
assert!(parse_probe_url("wss://example.test/v1/responses").is_ok());
|
||||
assert!(parse_probe_url("ws://localhost:8080/v1/responses").is_ok());
|
||||
assert!(parse_probe_url("ws://127.42.0.1:8080/v1/responses").is_ok());
|
||||
assert!(parse_probe_url("ws://[::1]:8080/v1/responses").is_ok());
|
||||
assert!(parse_probe_url("https://example.test/v1/responses").is_err());
|
||||
assert!(parse_probe_url("ws://example.test/v1/responses").is_err());
|
||||
assert!(parse_probe_url("ws://10.0.0.1/v1/responses").is_err());
|
||||
assert!(parse_probe_url("ws://0.0.0.0:8080/v1/responses").is_err());
|
||||
assert!(parse_probe_url("ws://[::ffff:127.0.0.1]:8080/v1/responses").is_err());
|
||||
assert!(parse_probe_url("wss://[email protected]/v1/responses").is_err());
|
||||
assert!(parse_probe_url("wss://example.test/v1/responses?token=secret").is_err());
|
||||
}
|
||||
|
||||
@@ -422,6 +422,7 @@ mod tests {
|
||||
local_rejection: None,
|
||||
allowed_models: None,
|
||||
ip_rules: None,
|
||||
verified_api_key_hash: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -173,6 +173,15 @@ impl SystemConfigCache {
|
||||
self.detach_all_loads();
|
||||
}
|
||||
|
||||
pub(crate) fn invalidate(&self, key: &str) {
|
||||
let Ok(_mutation) = self.mutation.lock() else {
|
||||
return;
|
||||
};
|
||||
self.generation.fetch_add(1, Ordering::AcqRel);
|
||||
self.entries.remove(&key.to_string());
|
||||
self.detach_all_loads();
|
||||
}
|
||||
|
||||
pub(crate) fn insert_if_generation(
|
||||
&self,
|
||||
key: String,
|
||||
|
||||
@@ -18,6 +18,7 @@ pub(crate) const TUNNEL_AFFINITY_FORWARDED_BY_HEADER: &str =
|
||||
"x-aether-tunnel-affinity-forwarded-by";
|
||||
pub(crate) const TUNNEL_AFFINITY_OWNER_INSTANCE_HEADER: &str =
|
||||
"x-aether-tunnel-affinity-owner-instance-id";
|
||||
pub(crate) const TUNNEL_AFFINITY_NODE_ID_HEADER: &str = "x-aether-tunnel-affinity-node-id";
|
||||
pub(crate) const EXECUTION_PATH_PUBLIC_PROXY_PASSTHROUGH: &str = "public_proxy_passthrough";
|
||||
pub(crate) const EXECUTION_PATH_LOCAL_PROXY_PASSTHROUGH_REMOVED: &str =
|
||||
"local_proxy_passthrough_removed";
|
||||
|
||||
@@ -23,7 +23,7 @@ pub(crate) fn extract_requested_model(
|
||||
body: &Bytes,
|
||||
) -> Option<String> {
|
||||
if decision.route_family.as_deref() == Some("gemini") {
|
||||
if let Some(model) = extract_gemini_model_from_path(uri.path()) {
|
||||
if let Some(model) = extract_gemini_requested_model_from_path(uri.path()) {
|
||||
return Some(model);
|
||||
}
|
||||
}
|
||||
@@ -43,23 +43,46 @@ pub(crate) fn extract_requested_model(
|
||||
.filter(|value| !value.is_empty())
|
||||
}
|
||||
|
||||
fn extract_gemini_requested_model_from_path(path: &str) -> Option<String> {
|
||||
let model = extract_gemini_model_from_path(path)?;
|
||||
Some(
|
||||
model
|
||||
.split_once("/operations/")
|
||||
.map(|(model, _)| model)
|
||||
.unwrap_or(model.as_str())
|
||||
.to_string(),
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn extract_request_credentials(
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
auth_endpoint_signature: &str,
|
||||
) -> GatewayExtractedCredentials {
|
||||
extract_request_credentials_with_trusted_auth(headers, uri, auth_endpoint_signature, cfg!(test))
|
||||
}
|
||||
|
||||
pub(super) fn extract_request_credentials_with_trusted_auth(
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
auth_endpoint_signature: &str,
|
||||
trusted_auth_verified: bool,
|
||||
) -> GatewayExtractedCredentials {
|
||||
let bundle = GatewayCredentialBundle {
|
||||
authorization_bearer: header_value_str(headers, http::header::AUTHORIZATION.as_str())
|
||||
.as_deref()
|
||||
.and_then(extract_bearer_token)
|
||||
.map(ToOwned::to_owned),
|
||||
authorization_bearer: unique_header_value_str(
|
||||
headers,
|
||||
http::header::AUTHORIZATION.as_str(),
|
||||
)
|
||||
.as_deref()
|
||||
.and_then(extract_bearer_token)
|
||||
.map(ToOwned::to_owned),
|
||||
x_api_key: header_value_str(headers, "x-api-key"),
|
||||
api_key: header_value_str(headers, "api-key"),
|
||||
x_goog_api_key: header_value_str(headers, "x-goog-api-key"),
|
||||
query_key: extract_query_api_key(uri),
|
||||
cookie_header: header_value_str(headers, http::header::COOKIE.as_str()),
|
||||
};
|
||||
let trusted_headers = extract_trusted_auth_headers(headers);
|
||||
let trusted_headers = extract_trusted_auth_headers(headers, trusted_auth_verified);
|
||||
let trusted_admin_headers = extract_trusted_admin_headers(headers);
|
||||
let primary = select_primary_credential(auth_endpoint_signature, &bundle);
|
||||
|
||||
@@ -71,6 +94,20 @@ pub(super) fn extract_request_credentials(
|
||||
}
|
||||
}
|
||||
|
||||
fn unique_header_value_str(headers: &http::HeaderMap, key: &str) -> Option<String> {
|
||||
let mut values = headers.get_all(key).iter();
|
||||
let value = values.next()?;
|
||||
if values.next().is_some() {
|
||||
return None;
|
||||
}
|
||||
value
|
||||
.to_str()
|
||||
.ok()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
}
|
||||
|
||||
pub(in crate::control) fn resolve_gateway_credential_carrier(
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
@@ -85,6 +122,7 @@ pub(in crate::control) fn resolve_gateway_credential_carrier(
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn has_trusted_gateway_marker(headers: &http::HeaderMap) -> bool {
|
||||
header_value_str(headers, crate::constants::GATEWAY_HEADER)
|
||||
.unwrap_or_default()
|
||||
@@ -97,13 +135,32 @@ pub(super) fn build_auth_context_cache_key(
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
auth_endpoint_signature: &str,
|
||||
) -> Option<String> {
|
||||
build_auth_context_cache_key_with_trusted_auth(
|
||||
headers,
|
||||
uri,
|
||||
auth_endpoint_signature,
|
||||
cfg!(test),
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn build_auth_context_cache_key_with_trusted_auth(
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
auth_endpoint_signature: &str,
|
||||
trusted_auth_verified: bool,
|
||||
) -> Option<String> {
|
||||
let signature = auth_endpoint_signature.trim();
|
||||
if signature.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let extracted = extract_request_credentials(headers, uri, signature);
|
||||
let extracted = extract_request_credentials_with_trusted_auth(
|
||||
headers,
|
||||
uri,
|
||||
signature,
|
||||
trusted_auth_verified,
|
||||
);
|
||||
let trusted_headers = extracted.trusted_headers;
|
||||
let bundle = extracted.bundle;
|
||||
if bundle.authorization_bearer.is_none()
|
||||
@@ -135,7 +192,7 @@ pub(super) fn build_auth_context_cache_key(
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
Some(format!(
|
||||
let raw_cache_identity = format!(
|
||||
"{signature}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}",
|
||||
bundle.authorization_bearer.unwrap_or_default(),
|
||||
bundle.x_api_key.unwrap_or_default(),
|
||||
@@ -147,11 +204,26 @@ pub(super) fn build_auth_context_cache_key(
|
||||
trusted_api_key_id,
|
||||
trusted_balance_remaining,
|
||||
trusted_access_allowed,
|
||||
))
|
||||
);
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(raw_cache_identity.as_bytes());
|
||||
Some(format!("auth-context:sha256:{:x}", hasher.finalize()))
|
||||
}
|
||||
|
||||
fn extract_trusted_auth_headers(headers: &http::HeaderMap) -> Option<GatewayTrustedAuthHeaders> {
|
||||
if !has_trusted_gateway_marker(headers) {
|
||||
fn extract_trusted_auth_headers(
|
||||
headers: &http::HeaderMap,
|
||||
trusted_auth_verified: bool,
|
||||
) -> Option<GatewayTrustedAuthHeaders> {
|
||||
if !trusted_auth_verified {
|
||||
return None;
|
||||
}
|
||||
#[cfg(test)]
|
||||
if !header_value_str(headers, crate::constants::GATEWAY_HEADER)
|
||||
.unwrap_or_default()
|
||||
.trim()
|
||||
.to_ascii_lowercase()
|
||||
.starts_with("rust-phase3")
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let user_id = header_value_str(headers, crate::constants::TRUSTED_AUTH_USER_ID_HEADER)
|
||||
@@ -387,7 +459,7 @@ fn extract_bearer_token(value: &str) -> Option<&str> {
|
||||
return None;
|
||||
}
|
||||
let token = token.trim();
|
||||
if token.is_empty() {
|
||||
if token.is_empty() || token.chars().any(char::is_whitespace) {
|
||||
None
|
||||
} else {
|
||||
Some(token)
|
||||
@@ -472,6 +544,46 @@ mod tests {
|
||||
assert_eq!(requested_model.as_deref(), Some("gpt-5.4"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_requested_model_handles_gemini_generation_and_operation_paths() {
|
||||
let generation_decision = GatewayControlDecision::synthetic(
|
||||
"/v1beta/models/gemini-2.5-pro:generateContent",
|
||||
Some("ai_public".to_string()),
|
||||
Some("gemini".to_string()),
|
||||
Some("generate_content".to_string()),
|
||||
Some("gemini:generate_content".to_string()),
|
||||
);
|
||||
let operation_decision = GatewayControlDecision::synthetic(
|
||||
"/v1beta/models/veo-3/operations/task-123:cancel",
|
||||
Some("ai_public".to_string()),
|
||||
Some("gemini".to_string()),
|
||||
Some("video".to_string()),
|
||||
Some("gemini:video".to_string()),
|
||||
);
|
||||
let headers = http::HeaderMap::new();
|
||||
|
||||
assert_eq!(
|
||||
extract_requested_model(
|
||||
&generation_decision,
|
||||
&uri("/v1beta/models/gemini-2.5-pro:generateContent"),
|
||||
&headers,
|
||||
&Bytes::new(),
|
||||
)
|
||||
.as_deref(),
|
||||
Some("gemini-2.5-pro")
|
||||
);
|
||||
assert_eq!(
|
||||
extract_requested_model(
|
||||
&operation_decision,
|
||||
&uri("/v1beta/models/veo-3/operations/task-123:cancel"),
|
||||
&headers,
|
||||
&Bytes::new(),
|
||||
)
|
||||
.as_deref(),
|
||||
Some("veo-3")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selects_openai_bearer_as_provider_api_key() {
|
||||
let mut headers = http::HeaderMap::new();
|
||||
@@ -491,6 +603,33 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_duplicate_or_combined_authorization_credentials() {
|
||||
let mut duplicate = http::HeaderMap::new();
|
||||
duplicate.append(
|
||||
http::header::AUTHORIZATION,
|
||||
"Bearer first-token".parse().unwrap(),
|
||||
);
|
||||
duplicate.append(
|
||||
http::header::AUTHORIZATION,
|
||||
"Bearer second-token".parse().unwrap(),
|
||||
);
|
||||
let extracted =
|
||||
extract_request_credentials(&duplicate, &uri("/api/admin/system"), "admin:operational");
|
||||
assert!(extracted.bundle.authorization_bearer.is_none());
|
||||
assert!(extracted.primary.is_none());
|
||||
|
||||
let mut combined = http::HeaderMap::new();
|
||||
combined.insert(
|
||||
http::header::AUTHORIZATION,
|
||||
"Bearer first-token, Bearer second-token".parse().unwrap(),
|
||||
);
|
||||
let extracted =
|
||||
extract_request_credentials(&combined, &uri("/api/admin/system"), "admin:operational");
|
||||
assert!(extracted.bundle.authorization_bearer.is_none());
|
||||
assert!(extracted.primary.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selects_codex_live_bearer_as_provider_api_key() {
|
||||
let mut headers = http::HeaderMap::new();
|
||||
@@ -608,7 +747,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cache_key_includes_cookie_header() {
|
||||
fn cache_key_hashes_cookie_header_instead_of_retaining_session_secret() {
|
||||
let mut headers = http::HeaderMap::new();
|
||||
headers.insert(http::header::COOKIE, "session=abc123".parse().unwrap());
|
||||
|
||||
@@ -618,7 +757,8 @@ mod tests {
|
||||
"internal:session",
|
||||
)
|
||||
.expect("cache key should exist");
|
||||
assert!(cache_key.contains("session=abc123"));
|
||||
assert!(cache_key.starts_with("auth-context:sha256:"));
|
||||
assert!(!cache_key.contains("session=abc123"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -669,12 +809,10 @@ mod tests {
|
||||
.expect("trusted cache key should exist");
|
||||
|
||||
assert_ne!(first, second);
|
||||
assert!(first.contains("user-1"));
|
||||
assert!(first.contains("key-1"));
|
||||
assert!(first.contains("1.5"));
|
||||
assert!(first.contains("true"));
|
||||
assert!(second.contains("user-2"));
|
||||
assert!(second.contains("false"));
|
||||
for raw_identity in ["user-1", "key-1", "1.5", "user-2"] {
|
||||
assert!(!first.contains(raw_identity));
|
||||
assert!(!second.contains(raw_identity));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -224,7 +224,7 @@ fn wallet_finite_available_usd(
|
||||
Some(wallet.balance.max(0.0) + wallet.gift_balance.max(0.0))
|
||||
}
|
||||
|
||||
async fn estimate_execution_plan_cost_upper_bound_usd(
|
||||
pub(crate) async fn estimate_execution_plan_cost_upper_bound_usd(
|
||||
state: &AppState,
|
||||
plan: &aether_contracts::ExecutionPlan,
|
||||
report_context: Option<&serde_json::Value>,
|
||||
@@ -925,6 +925,7 @@ mod tests {
|
||||
local_rejection: None,
|
||||
allowed_models: Some(allowed_models),
|
||||
ip_rules: None,
|
||||
verified_api_key_hash: None,
|
||||
});
|
||||
decision
|
||||
}
|
||||
|
||||
@@ -7,13 +7,16 @@ mod types;
|
||||
pub(crate) use credentials::extract_requested_model;
|
||||
pub(super) use credentials::resolve_gateway_credential_carrier;
|
||||
pub(crate) use gate::{
|
||||
execution_plan_balance_capacity_rejection, request_model_local_rejection,
|
||||
should_buffer_request_for_local_auth, trusted_auth_local_rejection, GatewayLocalAuthRejection,
|
||||
estimate_execution_plan_cost_upper_bound_usd, execution_plan_balance_capacity_rejection,
|
||||
request_model_local_rejection, should_buffer_request_for_local_auth,
|
||||
trusted_auth_local_rejection, GatewayLocalAuthRejection,
|
||||
};
|
||||
pub(crate) use resolution::{
|
||||
refresh_execution_runtime_auth_context, refresh_execution_runtime_auth_context_with_snapshot,
|
||||
resolve_execution_runtime_auth_context, GatewayAdminPrincipalContext,
|
||||
GatewayControlAuthContext,
|
||||
resolve_execution_runtime_auth_context, resolve_local_admin_session_principal,
|
||||
GatewayAdminPrincipalContext, GatewayControlAuthContext,
|
||||
};
|
||||
pub(super) use resolution::{
|
||||
resolve_control_decision_auth_with_trusted_auth, ControlDecisionAuthResolution,
|
||||
};
|
||||
pub(super) use resolution::{resolve_control_decision_auth, ControlDecisionAuthResolution};
|
||||
pub(crate) use types::GatewayCredentialCarrier;
|
||||
|
||||
@@ -4,11 +4,9 @@ use aether_data_contracts::repository::provider_catalog::{
|
||||
StoredProviderCatalogEndpoint, StoredProviderCatalogProvider,
|
||||
};
|
||||
use axum::http::Uri;
|
||||
use base64::Engine as _;
|
||||
use hmac::Mac;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use tracing::{debug, info};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::wallet_runtime::{
|
||||
local_rejection_from_wallet_access, resolve_wallet_auth_gate_uncached,
|
||||
@@ -17,7 +15,8 @@ use crate::{AppState, GatewayError};
|
||||
|
||||
use super::super::GatewayControlDecision;
|
||||
use super::credentials::{
|
||||
build_auth_context_cache_key, current_unix_secs, extract_request_credentials,
|
||||
build_auth_context_cache_key, build_auth_context_cache_key_with_trusted_auth,
|
||||
current_unix_secs, extract_request_credentials, extract_request_credentials_with_trusted_auth,
|
||||
extract_trusted_admin_headers, hash_api_key,
|
||||
};
|
||||
use super::gate::GatewayLocalAuthRejection;
|
||||
@@ -27,6 +26,9 @@ use super::types::{
|
||||
};
|
||||
use crate::cache::{AuthContextCacheGeneration, AuthContextInflightRegistration};
|
||||
use crate::headers::header_value_str;
|
||||
use crate::local_auth_token::{
|
||||
decode_local_auth_token, local_auth_token_identity_matches_user, LocalAuthTokenType,
|
||||
};
|
||||
|
||||
const AUTH_CONTEXT_CACHE_TTL: Duration = Duration::from_secs(60);
|
||||
const AUTH_CONTEXT_CACHE_REFRESH_INTERVAL: Duration = Duration::from_secs(10);
|
||||
@@ -93,6 +95,30 @@ pub(crate) struct GatewayControlAuthContext {
|
||||
pub(crate) allowed_models: Option<Vec<String>>,
|
||||
#[serde(skip)]
|
||||
pub(crate) ip_rules: Option<Vec<String>>,
|
||||
/// Credential verifier that established this API-key identity. Long-lived
|
||||
/// executions use it to prove that a later row with the same IDs is still
|
||||
/// the record authenticated by the original request.
|
||||
#[serde(skip)]
|
||||
pub(crate) verified_api_key_hash: Option<VerifiedApiKeyHash>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct VerifiedApiKeyHash(String);
|
||||
|
||||
impl VerifiedApiKeyHash {
|
||||
fn new(value: String) -> Self {
|
||||
Self(value)
|
||||
}
|
||||
|
||||
fn as_str(&self) -> &str {
|
||||
self.0.as_str()
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for VerifiedApiKeyHash {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter.write_str("VerifiedApiKeyHash([REDACTED])")
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -109,11 +135,30 @@ pub(in super::super) enum ControlDecisionAuthResolution {
|
||||
}
|
||||
|
||||
pub(in super::super) async fn resolve_control_decision_auth(
|
||||
state: &AppState,
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
trace_id: &str,
|
||||
decision: GatewayControlDecision,
|
||||
) -> Result<ControlDecisionAuthResolution, GatewayError> {
|
||||
resolve_control_decision_auth_with_trusted_auth(
|
||||
state,
|
||||
headers,
|
||||
uri,
|
||||
trace_id,
|
||||
decision,
|
||||
cfg!(test),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in super::super) async fn resolve_control_decision_auth_with_trusted_auth(
|
||||
state: &AppState,
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
trace_id: &str,
|
||||
mut decision: GatewayControlDecision,
|
||||
trusted_auth_verified: bool,
|
||||
) -> Result<ControlDecisionAuthResolution, GatewayError> {
|
||||
if let Some(admin_principal) =
|
||||
resolve_trusted_admin_principal(headers, decision.auth_endpoint_signature.as_deref())
|
||||
@@ -132,10 +177,18 @@ pub(in super::super) async fn resolve_control_decision_auth(
|
||||
decision.admin_principal = Some(admin_principal);
|
||||
}
|
||||
|
||||
let auth_context_cache_key = decision
|
||||
.auth_endpoint_signature
|
||||
.as_deref()
|
||||
.and_then(|signature| build_auth_context_cache_key(headers, uri, signature));
|
||||
let auth_context_cache_key =
|
||||
decision
|
||||
.auth_endpoint_signature
|
||||
.as_deref()
|
||||
.and_then(|signature| {
|
||||
build_auth_context_cache_key_with_trusted_auth(
|
||||
headers,
|
||||
uri,
|
||||
signature,
|
||||
trusted_auth_verified,
|
||||
)
|
||||
});
|
||||
|
||||
let mut resolved_auth_context = None;
|
||||
if let Some(cache_key) = auth_context_cache_key.as_deref() {
|
||||
@@ -149,6 +202,7 @@ pub(in super::super) async fn resolve_control_decision_auth(
|
||||
decision.auth_endpoint_signature.as_deref(),
|
||||
headers,
|
||||
uri,
|
||||
trusted_auth_verified,
|
||||
)
|
||||
.await?,
|
||||
);
|
||||
@@ -168,6 +222,7 @@ pub(in super::super) async fn resolve_control_decision_auth(
|
||||
uri,
|
||||
decision.auth_endpoint_signature.as_deref(),
|
||||
true,
|
||||
trusted_auth_verified,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -336,7 +391,7 @@ async fn resolve_local_admin_principal(
|
||||
let Some(access_token) = extracted.bundle.authorization_bearer.as_deref() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let claims = match decode_local_auth_token(access_token, "access") {
|
||||
let claims = match decode_local_auth_token(access_token, LocalAuthTokenType::Access) {
|
||||
Ok(claims) => claims,
|
||||
Err(_) => return Ok(None),
|
||||
};
|
||||
@@ -351,6 +406,14 @@ async fn resolve_local_admin_principal(
|
||||
resolve_local_admin_principal_from_claims(state, headers, uri, &claims).await
|
||||
}
|
||||
|
||||
pub(crate) async fn resolve_local_admin_session_principal(
|
||||
state: &AppState,
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
) -> Result<Option<GatewayAdminPrincipalContext>, GatewayError> {
|
||||
resolve_local_admin_principal(state, headers, uri, Some("admin:operational")).await
|
||||
}
|
||||
|
||||
async fn resolve_local_admin_principal_from_claims(
|
||||
state: &AppState,
|
||||
headers: &http::HeaderMap,
|
||||
@@ -373,6 +436,9 @@ async fn resolve_local_admin_principal_from_claims(
|
||||
if !user.is_active || user.is_deleted || !crate::roles::can_access_admin_console(&user.role) {
|
||||
return Ok(None);
|
||||
}
|
||||
if !local_auth_token_identity_matches_user(claims, &user) {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let now = chrono::Utc::now();
|
||||
let Some(session) = state.find_user_session(user_id, session_id).await? else {
|
||||
@@ -380,6 +446,7 @@ async fn resolve_local_admin_principal_from_claims(
|
||||
};
|
||||
if session.is_revoked()
|
||||
|| session.is_expired(now)
|
||||
|| session.security_version != user.security_version
|
||||
|| session.client_device_id != client_device_id
|
||||
{
|
||||
return Ok(None);
|
||||
@@ -431,68 +498,6 @@ fn local_admin_user_agent(headers: &http::HeaderMap) -> Option<String> {
|
||||
.map(|value| value.chars().take(1000).collect())
|
||||
}
|
||||
|
||||
fn local_auth_secret() -> String {
|
||||
std::env::var("JWT_SECRET_KEY")
|
||||
.ok()
|
||||
.map(|value| value.trim().to_string())
|
||||
.filter(|value| !value.is_empty())
|
||||
.unwrap_or_else(|| "aether-rust-dev-jwt-secret".to_string())
|
||||
}
|
||||
|
||||
fn decode_local_auth_token(
|
||||
token: &str,
|
||||
expected_type: &str,
|
||||
) -> Result<serde_json::Map<String, Value>, String> {
|
||||
let mut parts = token.split('.');
|
||||
let Some(header_segment) = parts.next() else {
|
||||
return Err("invalid token".to_string());
|
||||
};
|
||||
let Some(payload_segment) = parts.next() else {
|
||||
return Err("invalid token".to_string());
|
||||
};
|
||||
let Some(signature_segment) = parts.next() else {
|
||||
return Err("invalid token".to_string());
|
||||
};
|
||||
if parts.next().is_some() {
|
||||
return Err("invalid token".to_string());
|
||||
}
|
||||
|
||||
let signing_input = format!("{header_segment}.{payload_segment}");
|
||||
let signature = base64::engine::general_purpose::URL_SAFE_NO_PAD
|
||||
.decode(signature_segment)
|
||||
.map_err(|_| "invalid token".to_string())?;
|
||||
let mut mac = hmac::Hmac::<sha2::Sha256>::new_from_slice(local_auth_secret().as_bytes())
|
||||
.map_err(|_| "invalid token".to_string())?;
|
||||
mac.update(signing_input.as_bytes());
|
||||
mac.verify_slice(&signature)
|
||||
.map_err(|_| "invalid token".to_string())?;
|
||||
|
||||
let payload_bytes = base64::engine::general_purpose::URL_SAFE_NO_PAD
|
||||
.decode(payload_segment)
|
||||
.map_err(|_| "invalid token".to_string())?;
|
||||
let payload =
|
||||
serde_json::from_slice::<Value>(&payload_bytes).map_err(|_| "invalid token".to_string())?;
|
||||
let payload = payload
|
||||
.as_object()
|
||||
.cloned()
|
||||
.ok_or_else(|| "invalid token".to_string())?;
|
||||
let actual_type = payload
|
||||
.get("type")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or_default();
|
||||
if actual_type != expected_type {
|
||||
return Err("invalid token".to_string());
|
||||
}
|
||||
let exp = payload
|
||||
.get("exp")
|
||||
.and_then(Value::as_i64)
|
||||
.ok_or_else(|| "invalid token".to_string())?;
|
||||
if exp <= chrono::Utc::now().timestamp() {
|
||||
return Err("expired token".to_string());
|
||||
}
|
||||
Ok(payload)
|
||||
}
|
||||
|
||||
pub(crate) async fn resolve_execution_runtime_auth_context(
|
||||
state: &AppState,
|
||||
decision: &GatewayControlDecision,
|
||||
@@ -525,6 +530,7 @@ pub(crate) async fn resolve_execution_runtime_auth_context(
|
||||
Some(auth_endpoint_signature),
|
||||
headers,
|
||||
uri,
|
||||
cfg!(test),
|
||||
)
|
||||
.await
|
||||
.map(Some);
|
||||
@@ -539,6 +545,7 @@ pub(crate) async fn resolve_execution_runtime_auth_context(
|
||||
uri,
|
||||
Some(auth_endpoint_signature),
|
||||
true,
|
||||
cfg!(test),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
@@ -558,6 +565,7 @@ async fn revalidate_cached_auth_context(
|
||||
auth_endpoint_signature: Option<&str>,
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
trusted_auth_verified: bool,
|
||||
) -> Result<GatewayControlAuthContext, GatewayError> {
|
||||
if is_negative_auth_context(&auth_context)
|
||||
|| !auth_context.access_allowed
|
||||
@@ -581,6 +589,7 @@ async fn revalidate_cached_auth_context(
|
||||
uri,
|
||||
auth_context.clone(),
|
||||
auth_endpoint_signature,
|
||||
trusted_auth_verified,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -622,6 +631,7 @@ async fn revalidate_cached_auth_context(
|
||||
uri,
|
||||
auth_context,
|
||||
auth_endpoint_signature,
|
||||
trusted_auth_verified,
|
||||
)
|
||||
.await;
|
||||
if refreshed.is_err() {
|
||||
@@ -639,9 +649,16 @@ async fn resolve_security_fresh_auth_context(
|
||||
uri: &Uri,
|
||||
stale: GatewayControlAuthContext,
|
||||
auth_endpoint_signature: Option<&str>,
|
||||
trusted_auth_verified: bool,
|
||||
) -> Result<GatewayControlAuthContext, GatewayError> {
|
||||
if let Some(refreshed) =
|
||||
resolve_data_backed_auth_context(state, headers, uri, auth_endpoint_signature).await?
|
||||
if let Some(refreshed) = resolve_data_backed_auth_context_with_trusted_auth(
|
||||
state,
|
||||
headers,
|
||||
uri,
|
||||
auth_endpoint_signature,
|
||||
trusted_auth_verified,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
return Ok(refreshed);
|
||||
}
|
||||
@@ -660,19 +677,27 @@ async fn resolve_data_backed_auth_context_cached(
|
||||
uri: &Uri,
|
||||
auth_endpoint_signature: Option<&str>,
|
||||
cache_negative: bool,
|
||||
trusted_auth_verified: bool,
|
||||
) -> Result<Option<GatewayControlAuthContext>, GatewayError> {
|
||||
let Some(cache_key) = cache_key else {
|
||||
return resolve_data_backed_auth_context(state, headers, uri, auth_endpoint_signature)
|
||||
.await;
|
||||
return resolve_data_backed_auth_context_with_trusted_auth(
|
||||
state,
|
||||
headers,
|
||||
uri,
|
||||
auth_endpoint_signature,
|
||||
trusted_auth_verified,
|
||||
)
|
||||
.await;
|
||||
};
|
||||
loop {
|
||||
match state.auth_context_cache.register_inflight(cache_key) {
|
||||
AuthContextInflightRegistration::Leader(guard) => {
|
||||
let resolved = match resolve_data_backed_auth_context(
|
||||
let resolved = match resolve_data_backed_auth_context_with_trusted_auth(
|
||||
state,
|
||||
headers,
|
||||
uri,
|
||||
auth_endpoint_signature,
|
||||
trusted_auth_verified,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -708,11 +733,12 @@ async fn resolve_data_backed_auth_context_cached(
|
||||
}
|
||||
}
|
||||
AuthContextInflightRegistration::Bypass => {
|
||||
return resolve_data_backed_auth_context(
|
||||
return resolve_data_backed_auth_context_with_trusted_auth(
|
||||
state,
|
||||
headers,
|
||||
uri,
|
||||
auth_endpoint_signature,
|
||||
trusted_auth_verified,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -768,28 +794,39 @@ pub(crate) async fn refresh_execution_runtime_auth_context_with_snapshot(
|
||||
return Ok((auth_context, None));
|
||||
}
|
||||
|
||||
let verified_api_key_hash = auth_context.verified_api_key_hash.clone();
|
||||
let snapshot = {
|
||||
let _permit = state.acquire_auth_snapshot_load_gate().await?;
|
||||
state
|
||||
.data
|
||||
.read_auth_api_key_snapshot_strong(
|
||||
&auth_context.user_id,
|
||||
&auth_context.api_key_id,
|
||||
current_unix_secs(),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||
if let Some(key_hash) = verified_api_key_hash.as_ref() {
|
||||
state
|
||||
.data
|
||||
.read_auth_api_key_snapshot_by_key_hash_strong(
|
||||
key_hash.as_str(),
|
||||
current_unix_secs(),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||
} else {
|
||||
state
|
||||
.data
|
||||
.read_auth_api_key_snapshot_strong(
|
||||
&auth_context.user_id,
|
||||
&auth_context.api_key_id,
|
||||
current_unix_secs(),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||
}
|
||||
};
|
||||
let Some(snapshot) = snapshot else {
|
||||
let mut denied = auth_context;
|
||||
denied.access_allowed = false;
|
||||
denied.local_rejection = Some(GatewayLocalAuthRejection::InvalidApiKey);
|
||||
denied.balance_remaining = None;
|
||||
return Ok((denied, None));
|
||||
return Ok((deny_refreshed_auth_context(auth_context), None));
|
||||
};
|
||||
if snapshot.user_id != auth_context.user_id || snapshot.api_key_id != auth_context.api_key_id {
|
||||
return Ok((deny_refreshed_auth_context(auth_context), None));
|
||||
};
|
||||
|
||||
let wallet_access = resolve_wallet_auth_gate_uncached(state, &snapshot).await?;
|
||||
let refreshed = build_data_backed_auth_context(
|
||||
let mut refreshed = build_data_backed_auth_context(
|
||||
state,
|
||||
snapshot.clone(),
|
||||
auth_endpoint_signature,
|
||||
@@ -798,9 +835,19 @@ pub(crate) async fn refresh_execution_runtime_auth_context_with_snapshot(
|
||||
wallet_access,
|
||||
)
|
||||
.await;
|
||||
refreshed.verified_api_key_hash = verified_api_key_hash;
|
||||
Ok((refreshed, Some(snapshot)))
|
||||
}
|
||||
|
||||
fn deny_refreshed_auth_context(
|
||||
mut auth_context: GatewayControlAuthContext,
|
||||
) -> GatewayControlAuthContext {
|
||||
auth_context.access_allowed = false;
|
||||
auth_context.local_rejection = Some(GatewayLocalAuthRejection::InvalidApiKey);
|
||||
auth_context.balance_remaining = None;
|
||||
auth_context
|
||||
}
|
||||
|
||||
fn put_cached_auth_context(
|
||||
state: &AppState,
|
||||
cache_key: String,
|
||||
@@ -913,6 +960,23 @@ pub(super) async fn resolve_data_backed_auth_context(
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
auth_endpoint_signature: Option<&str>,
|
||||
) -> Result<Option<GatewayControlAuthContext>, GatewayError> {
|
||||
resolve_data_backed_auth_context_with_trusted_auth(
|
||||
state,
|
||||
headers,
|
||||
uri,
|
||||
auth_endpoint_signature,
|
||||
cfg!(test),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn resolve_data_backed_auth_context_with_trusted_auth(
|
||||
state: &AppState,
|
||||
headers: &http::HeaderMap,
|
||||
uri: &Uri,
|
||||
auth_endpoint_signature: Option<&str>,
|
||||
trusted_auth_verified: bool,
|
||||
) -> Result<Option<GatewayControlAuthContext>, GatewayError> {
|
||||
let Some(signature) = auth_endpoint_signature
|
||||
.map(str::trim)
|
||||
@@ -923,7 +987,12 @@ pub(super) async fn resolve_data_backed_auth_context(
|
||||
if !state.has_auth_api_key_reader() {
|
||||
return Ok(None);
|
||||
}
|
||||
let extracted = extract_request_credentials(headers, uri, signature);
|
||||
let extracted = extract_request_credentials_with_trusted_auth(
|
||||
headers,
|
||||
uri,
|
||||
signature,
|
||||
trusted_auth_verified,
|
||||
);
|
||||
let principal = derive_principal_candidate(&extracted);
|
||||
let now_unix_secs = current_unix_secs();
|
||||
|
||||
@@ -955,6 +1024,7 @@ pub(super) async fn resolve_data_backed_auth_context(
|
||||
local_rejection: Some(GatewayLocalAuthRejection::InvalidApiKey),
|
||||
allowed_models: None,
|
||||
ip_rules: None,
|
||||
verified_api_key_hash: None,
|
||||
}));
|
||||
};
|
||||
|
||||
@@ -963,17 +1033,17 @@ pub(super) async fn resolve_data_backed_auth_context(
|
||||
.await;
|
||||
|
||||
let wallet_access = resolve_wallet_auth_gate_uncached(state, &snapshot).await?;
|
||||
Ok(Some(
|
||||
build_data_backed_auth_context(
|
||||
state,
|
||||
snapshot,
|
||||
signature,
|
||||
None,
|
||||
None,
|
||||
wallet_access,
|
||||
)
|
||||
.await,
|
||||
))
|
||||
let mut auth_context = build_data_backed_auth_context(
|
||||
state,
|
||||
snapshot,
|
||||
signature,
|
||||
None,
|
||||
None,
|
||||
wallet_access,
|
||||
)
|
||||
.await;
|
||||
auth_context.verified_api_key_hash = Some(VerifiedApiKeyHash::new(key_hash));
|
||||
Ok(Some(auth_context))
|
||||
}
|
||||
Some(GatewayPrincipalCandidate::DeferredBearerToken { raw, carrier }) => {
|
||||
if let Some(auth_context) = resolve_antigravity_bearer_bridge_auth_context(
|
||||
@@ -1068,6 +1138,7 @@ async fn resolve_antigravity_bearer_bridge_auth_context(
|
||||
local_rejection: Some(GatewayLocalAuthRejection::InvalidApiKey),
|
||||
allowed_models: None,
|
||||
ip_rules: None,
|
||||
verified_api_key_hash: None,
|
||||
}));
|
||||
};
|
||||
|
||||
@@ -1127,6 +1198,7 @@ async fn resolve_trusted_auth_context(
|
||||
local_rejection: Some(GatewayLocalAuthRejection::InvalidApiKey),
|
||||
allowed_models: None,
|
||||
ip_rules: None,
|
||||
verified_api_key_hash: None,
|
||||
}));
|
||||
};
|
||||
|
||||
@@ -1158,9 +1230,7 @@ async fn build_data_backed_auth_context(
|
||||
let invalid_api_key = !snapshot.user_is_active
|
||||
|| snapshot.user_is_deleted
|
||||
|| !snapshot.api_key_is_active
|
||||
|| snapshot
|
||||
.api_key_expires_at_unix_secs
|
||||
.is_some_and(|expires_at| expires_at < current_unix_secs());
|
||||
|| api_key_is_expired(snapshot.api_key_expires_at_unix_secs, current_unix_secs());
|
||||
let locked_api_key = snapshot.api_key_is_locked && !snapshot.api_key_is_standalone;
|
||||
let key_access_allowed = header_access_allowed
|
||||
.map(|value| value && snapshot.currently_usable)
|
||||
@@ -1225,9 +1295,14 @@ async fn build_data_backed_auth_context(
|
||||
local_rejection,
|
||||
allowed_models,
|
||||
ip_rules: snapshot.api_key_ip_rules,
|
||||
verified_api_key_hash: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn api_key_is_expired(expires_at_unix_secs: Option<u64>, now_unix_secs: u64) -> bool {
|
||||
expires_at_unix_secs.is_some_and(|expires_at| expires_at <= now_unix_secs)
|
||||
}
|
||||
|
||||
fn contains_api_format_or_alias(items: &[String], target: &str) -> bool {
|
||||
items.iter().any(|item| api_format_matches(item, target))
|
||||
}
|
||||
@@ -1282,18 +1357,21 @@ async fn auth_snapshot_allows_requested_provider(
|
||||
return true;
|
||||
}
|
||||
if !state.has_provider_catalog_data_reader() {
|
||||
return true;
|
||||
debug!(
|
||||
"deny requested provider {}: provider catalog is unavailable for allowlist resolution",
|
||||
requested_provider
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
let providers = match state.list_provider_catalog_providers(true).await {
|
||||
Ok(value) => value,
|
||||
Err(err) => {
|
||||
debug!(
|
||||
"skip local provider auth gate for requested provider {}: provider catalog lookup failed: {:?}",
|
||||
requested_provider,
|
||||
err
|
||||
warn!(
|
||||
"deny requested provider {}: provider catalog lookup failed: {:?}",
|
||||
requested_provider, err
|
||||
);
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1331,11 +1409,11 @@ async fn auth_snapshot_allows_requested_provider(
|
||||
{
|
||||
Ok(value) => value,
|
||||
Err(err) => {
|
||||
debug!(
|
||||
"skip local provider auth gate for requested provider {}: provider endpoint lookup failed: {:?}",
|
||||
warn!(
|
||||
"deny requested provider {}: provider endpoint lookup failed: {:?}",
|
||||
requested_provider, err
|
||||
);
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1421,12 +1499,12 @@ fn get_cached_auth_context_with_age(
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use aether_data::repository::auth::{
|
||||
AuthApiKeyWriteRepository, InMemoryAuthApiKeySnapshotRepository, StoredAuthApiKeySnapshot,
|
||||
AuthApiKeyWriteRepository, CreateUserApiKeyRecord, InMemoryAuthApiKeySnapshotRepository,
|
||||
StoredAuthApiKeySnapshot,
|
||||
};
|
||||
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
||||
use aether_data::repository::wallet::{
|
||||
@@ -1441,9 +1519,10 @@ mod tests {
|
||||
use futures_util::future::join_all;
|
||||
|
||||
use super::{
|
||||
get_cached_auth_context, resolve_control_decision_auth, resolve_data_backed_auth_context,
|
||||
resolve_execution_runtime_auth_context, ControlDecisionAuthResolution,
|
||||
GatewayLocalAuthRejection,
|
||||
api_key_is_expired, get_cached_auth_context,
|
||||
refresh_execution_runtime_auth_context_with_snapshot, resolve_control_decision_auth,
|
||||
resolve_data_backed_auth_context, resolve_execution_runtime_auth_context,
|
||||
ControlDecisionAuthResolution, GatewayLocalAuthRejection,
|
||||
};
|
||||
use crate::control::auth::credentials::{build_auth_context_cache_key, hash_api_key};
|
||||
use crate::control::GatewayControlDecision;
|
||||
@@ -1481,6 +1560,14 @@ mod tests {
|
||||
path.parse().expect("uri should parse")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn api_key_expiry_is_inclusive_at_the_declared_second() {
|
||||
assert!(!api_key_is_expired(None, 100));
|
||||
assert!(!api_key_is_expired(Some(101), 100));
|
||||
assert!(api_key_is_expired(Some(100), 100));
|
||||
assert!(api_key_is_expired(Some(99), 100));
|
||||
}
|
||||
|
||||
fn sample_provider(id: &str, name: &str, provider_type: &str) -> StoredProviderCatalogProvider {
|
||||
StoredProviderCatalogProvider::new(
|
||||
id.to_string(),
|
||||
@@ -1507,40 +1594,24 @@ mod tests {
|
||||
.expect("endpoint should build")
|
||||
}
|
||||
|
||||
struct SqliteAuthConfigNodes {
|
||||
struct PostgresAuthConfigNodes {
|
||||
first: AppState,
|
||||
second: AppState,
|
||||
_database: TemporarySqliteDatabase,
|
||||
_database: aether_testkit::ManagedPostgresServer,
|
||||
}
|
||||
|
||||
struct TemporarySqliteDatabase(PathBuf);
|
||||
|
||||
impl Drop for TemporarySqliteDatabase {
|
||||
fn drop(&mut self) {
|
||||
for suffix in ["", "-shm", "-wal"] {
|
||||
let mut path = self.0.as_os_str().to_os_string();
|
||||
path.push(suffix);
|
||||
let _ = std::fs::remove_file(PathBuf::from(path));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn sqlite_auth_config_nodes(
|
||||
async fn postgres_auth_config_nodes(
|
||||
auth_repository: Arc<InMemoryAuthApiKeySnapshotRepository>,
|
||||
) -> SqliteAuthConfigNodes {
|
||||
let database_path = std::env::temp_dir().join(format!(
|
||||
"aether-auth-config-{}.sqlite",
|
||||
uuid::Uuid::new_v4()
|
||||
));
|
||||
) -> PostgresAuthConfigNodes {
|
||||
let server = aether_testkit::ManagedPostgresServer::start()
|
||||
.await
|
||||
.expect("temporary PostgreSQL should start");
|
||||
let mut pool = SqlPoolConfig::default();
|
||||
pool.min_connections = 0;
|
||||
pool.max_connections = 4;
|
||||
let database = SqlDatabaseConfig::new(
|
||||
DatabaseDriver::Sqlite,
|
||||
format!("sqlite://{}", database_path.display()),
|
||||
pool,
|
||||
)
|
||||
.expect("sqlite config should build");
|
||||
let database =
|
||||
SqlDatabaseConfig::new(DatabaseDriver::Postgres, server.database_url(), pool)
|
||||
.expect("postgres config should build");
|
||||
let config = GatewayDataConfig::from_database_config(database);
|
||||
let first_data = GatewayDataState::from_config(config.clone())
|
||||
.expect("first data state should build")
|
||||
@@ -1549,27 +1620,27 @@ mod tests {
|
||||
assert!(first_data
|
||||
.run_database_migrations()
|
||||
.await
|
||||
.expect("sqlite migrations should run"));
|
||||
.expect("postgres migrations should run"));
|
||||
let second_data = GatewayDataState::from_config(config)
|
||||
.expect("second data state should build")
|
||||
.with_auth_api_key_reader(auth_repository)
|
||||
.without_wallet_reader_for_tests();
|
||||
|
||||
SqliteAuthConfigNodes {
|
||||
PostgresAuthConfigNodes {
|
||||
first: AppState::new()
|
||||
.expect("first app state should build")
|
||||
.with_data_state_for_tests(first_data),
|
||||
second: AppState::new()
|
||||
.expect("second app state should build")
|
||||
.with_data_state_for_tests(second_data),
|
||||
_database: TemporarySqliteDatabase(database_path),
|
||||
_database: server,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn strong_system_config_read_bypasses_app_and_data_caches() {
|
||||
let nodes =
|
||||
sqlite_auth_config_nodes(Arc::new(InMemoryAuthApiKeySnapshotRepository::seed([])))
|
||||
postgres_auth_config_nodes(Arc::new(InMemoryAuthApiKeySnapshotRepository::seed([])))
|
||||
.await;
|
||||
let key = format!("test.auth.strong-read.{}", uuid::Uuid::new_v4());
|
||||
let old_value = serde_json::json!({"version": "old"});
|
||||
@@ -1769,6 +1840,97 @@ mod tests {
|
||||
assert_eq!(repository.touch_count("key-1"), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn long_lived_refresh_rejects_same_ids_recreated_with_a_different_credential() {
|
||||
let old_api_key = "sk-old-websocket-credential";
|
||||
let new_api_key = "sk-new-websocket-credential";
|
||||
let old_key_hash = hash_api_key(old_api_key);
|
||||
let new_key_hash = hash_api_key(new_api_key);
|
||||
let mut old_snapshot = sample_snapshot("key-stable-id", "user-stable-id");
|
||||
old_snapshot.user_allowed_api_formats = Some(vec!["openai:responses".to_string()]);
|
||||
old_snapshot.api_key_allowed_api_formats = Some(vec!["openai:responses".to_string()]);
|
||||
let repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
|
||||
Some(old_key_hash.clone()),
|
||||
old_snapshot,
|
||||
)]));
|
||||
let data = GatewayDataState::with_auth_api_key_repository_for_tests(repository.clone());
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(data);
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
http::header::AUTHORIZATION,
|
||||
format!("Bearer {old_api_key}").parse().unwrap(),
|
||||
);
|
||||
|
||||
let original = resolve_data_backed_auth_context(
|
||||
&state,
|
||||
&headers,
|
||||
&uri("/v1/responses"),
|
||||
Some("openai:responses"),
|
||||
)
|
||||
.await
|
||||
.expect("initial auth resolution should succeed")
|
||||
.expect("the old API key should authenticate");
|
||||
assert!(original.access_allowed);
|
||||
assert!(original.verified_api_key_hash.is_some());
|
||||
assert!(
|
||||
!format!("{original:?}").contains(&old_key_hash),
|
||||
"the credential verifier must stay redacted from Debug output"
|
||||
);
|
||||
|
||||
assert!(repository
|
||||
.delete_user_api_key("user-stable-id", "key-stable-id")
|
||||
.await
|
||||
.expect("old API key deletion should succeed"));
|
||||
repository
|
||||
.create_user_api_key(CreateUserApiKeyRecord {
|
||||
user_id: "user-stable-id".to_string(),
|
||||
api_key_id: "key-stable-id".to_string(),
|
||||
key_hash: new_key_hash,
|
||||
key_encrypted: None,
|
||||
name: Some("restored-with-new-secret".to_string()),
|
||||
allowed_providers: Some(vec!["openai".to_string()]),
|
||||
allowed_api_formats: Some(vec!["openai:responses".to_string()]),
|
||||
allowed_models: Some(vec!["gpt-4.1".to_string()]),
|
||||
ip_rules: None,
|
||||
rate_limit: 60,
|
||||
concurrent_limit: Some(5),
|
||||
force_capabilities: None,
|
||||
feature_settings: None,
|
||||
is_active: true,
|
||||
expires_at_unix_secs: Some(4_102_444_800),
|
||||
auto_delete_on_expiry: false,
|
||||
total_requests: 0,
|
||||
total_tokens: 0,
|
||||
total_cost_usd: 0.0,
|
||||
})
|
||||
.await
|
||||
.expect("same-ID API key recreation should resolve")
|
||||
.expect("same-ID API key recreation should persist");
|
||||
|
||||
let (refreshed, snapshot) = refresh_execution_runtime_auth_context_with_snapshot(
|
||||
&state,
|
||||
original,
|
||||
Some("openai:responses"),
|
||||
)
|
||||
.await
|
||||
.expect("long-lived auth refresh should resolve");
|
||||
|
||||
assert!(!refreshed.access_allowed);
|
||||
assert_eq!(
|
||||
refreshed.local_rejection,
|
||||
Some(GatewayLocalAuthRejection::InvalidApiKey)
|
||||
);
|
||||
assert!(snapshot.is_none());
|
||||
assert_eq!(repository.key_hash_lookup_count(&old_key_hash), 1);
|
||||
assert_eq!(
|
||||
repository.snapshot_lookup_count("key-stable-id"),
|
||||
0,
|
||||
"a bound long-lived credential must not fall back to identity-only lookup"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn control_auth_context_singleflights_concurrent_cache_misses() {
|
||||
let api_key = "sk-test-concurrent-auth-miss";
|
||||
@@ -2396,6 +2558,44 @@ mod tests {
|
||||
assert_eq!(auth_context.local_rejection, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn data_backed_auth_context_denies_unresolved_provider_id_without_catalog_reader() {
|
||||
let api_key = "sk-test-provider-no-catalog";
|
||||
let mut snapshot = sample_snapshot("key-no-catalog", "user-no-catalog");
|
||||
snapshot.user_allowed_providers = Some(vec!["provider-custom-claude".to_string()]);
|
||||
snapshot.api_key_allowed_providers = Some(vec!["provider-custom-claude".to_string()]);
|
||||
snapshot.user_allowed_api_formats = None;
|
||||
snapshot.api_key_allowed_api_formats = None;
|
||||
let repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
|
||||
Some(hash_api_key(api_key)),
|
||||
snapshot,
|
||||
)]));
|
||||
let data = GatewayDataState::with_auth_api_key_reader_for_tests(repository);
|
||||
let state = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(data);
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("x-api-key", api_key.parse().unwrap());
|
||||
|
||||
let auth_context = resolve_data_backed_auth_context(
|
||||
&state,
|
||||
&headers,
|
||||
&uri("/v1/messages"),
|
||||
Some("claude:messages"),
|
||||
)
|
||||
.await
|
||||
.expect("resolution should succeed")
|
||||
.expect("auth context should exist");
|
||||
|
||||
assert!(!auth_context.access_allowed);
|
||||
assert_eq!(
|
||||
auth_context.local_rejection,
|
||||
Some(GatewayLocalAuthRejection::ProviderNotAllowed {
|
||||
provider: "claude".to_string(),
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn due_antigravity_bearer_refresh_observes_cross_node_allowlist_revocation() {
|
||||
let raw_bearer = "google-oauth-access-token-revoked-cross-node";
|
||||
@@ -2410,7 +2610,7 @@ mod tests {
|
||||
let auth_repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
|
||||
None, snapshot,
|
||||
)]));
|
||||
let nodes = sqlite_auth_config_nodes(auth_repository.clone()).await;
|
||||
let nodes = postgres_auth_config_nodes(auth_repository.clone()).await;
|
||||
nodes
|
||||
.first
|
||||
.auth_context_cache
|
||||
|
||||
@@ -44,7 +44,7 @@ pub(super) struct GatewayTrustedAdminHeaders {
|
||||
pub(super) management_token_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
#[derive(Clone, Default, PartialEq, Eq)]
|
||||
pub(super) struct GatewayCredentialBundle {
|
||||
pub(super) authorization_bearer: Option<String>,
|
||||
pub(super) x_api_key: Option<String>,
|
||||
@@ -54,7 +54,25 @@ pub(super) struct GatewayCredentialBundle {
|
||||
pub(super) cookie_header: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
impl std::fmt::Debug for GatewayCredentialBundle {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let redacted = |value: &Option<String>| value.as_ref().map(|_| "[REDACTED]");
|
||||
formatter
|
||||
.debug_struct("GatewayCredentialBundle")
|
||||
.field(
|
||||
"authorization_bearer",
|
||||
&redacted(&self.authorization_bearer),
|
||||
)
|
||||
.field("x_api_key", &redacted(&self.x_api_key))
|
||||
.field("api_key", &redacted(&self.api_key))
|
||||
.field("x_goog_api_key", &redacted(&self.x_goog_api_key))
|
||||
.field("query_key", &redacted(&self.query_key))
|
||||
.field("cookie_header", &redacted(&self.cookie_header))
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub(super) enum GatewayPrimaryCredential {
|
||||
ProviderApiKey {
|
||||
raw: String,
|
||||
@@ -70,6 +88,21 @@ pub(super) enum GatewayPrimaryCredential {
|
||||
},
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for GatewayPrimaryCredential {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let (variant, carrier) = match self {
|
||||
Self::ProviderApiKey { carrier, .. } => ("ProviderApiKey", carrier),
|
||||
Self::BearerToken { carrier, .. } => ("BearerToken", carrier),
|
||||
Self::CookieHeader { carrier, .. } => ("CookieHeader", carrier),
|
||||
};
|
||||
formatter
|
||||
.debug_struct(variant)
|
||||
.field("raw", &"[REDACTED]")
|
||||
.field("carrier", carrier)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(super) struct GatewayExtractedCredentials {
|
||||
pub(super) trusted_headers: Option<GatewayTrustedAuthHeaders>,
|
||||
@@ -78,7 +111,7 @@ pub(super) struct GatewayExtractedCredentials {
|
||||
pub(super) primary: Option<GatewayPrimaryCredential>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
#[derive(Clone, PartialEq)]
|
||||
pub(super) enum GatewayPrincipalCandidate {
|
||||
TrustedHeaders(GatewayTrustedAuthHeaders),
|
||||
ApiKeyHash {
|
||||
@@ -94,3 +127,58 @@ pub(super) enum GatewayPrincipalCandidate {
|
||||
carrier: GatewayCredentialCarrier,
|
||||
},
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for GatewayPrincipalCandidate {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::TrustedHeaders(headers) => formatter
|
||||
.debug_tuple("TrustedHeaders")
|
||||
.field(headers)
|
||||
.finish(),
|
||||
Self::ApiKeyHash { carrier, .. } => formatter
|
||||
.debug_struct("ApiKeyHash")
|
||||
.field("key_hash", &"[REDACTED]")
|
||||
.field("carrier", carrier)
|
||||
.finish(),
|
||||
Self::DeferredBearerToken { carrier, .. } => formatter
|
||||
.debug_struct("DeferredBearerToken")
|
||||
.field("raw", &"[REDACTED]")
|
||||
.field("carrier", carrier)
|
||||
.finish(),
|
||||
Self::DeferredCookieHeader { carrier, .. } => formatter
|
||||
.debug_struct("DeferredCookieHeader")
|
||||
.field("raw", &"[REDACTED]")
|
||||
.field("carrier", carrier)
|
||||
.finish(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod debug_redaction_tests {
|
||||
use super::{GatewayCredentialBundle, GatewayCredentialCarrier, GatewayPrimaryCredential};
|
||||
|
||||
#[test]
|
||||
fn gateway_credential_debug_output_redacts_raw_authorization_values() {
|
||||
let bundle = GatewayCredentialBundle {
|
||||
authorization_bearer: Some("bundle-bearer-canary".to_string()),
|
||||
api_key: Some("bundle-api-key-canary".to_string()),
|
||||
cookie_header: Some("bundle-cookie-canary".to_string()),
|
||||
..GatewayCredentialBundle::default()
|
||||
};
|
||||
let primary = GatewayPrimaryCredential::ProviderApiKey {
|
||||
raw: "primary-api-key-canary".to_string(),
|
||||
carrier: GatewayCredentialCarrier::ApiKey,
|
||||
};
|
||||
let debug = format!("{bundle:?} {primary:?}");
|
||||
assert!(debug.contains("[REDACTED]"));
|
||||
for secret in [
|
||||
"bundle-bearer-canary",
|
||||
"bundle-api-key-canary",
|
||||
"bundle-cookie-canary",
|
||||
"primary-api-key-canary",
|
||||
] {
|
||||
assert!(!debug.contains(secret), "debug output leaked {secret}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -8,9 +8,10 @@ mod public;
|
||||
mod route;
|
||||
|
||||
pub(crate) use auth::{
|
||||
execution_plan_balance_capacity_rejection, extract_requested_model,
|
||||
refresh_execution_runtime_auth_context, refresh_execution_runtime_auth_context_with_snapshot,
|
||||
request_model_local_rejection, resolve_execution_runtime_auth_context,
|
||||
estimate_execution_plan_cost_upper_bound_usd, execution_plan_balance_capacity_rejection,
|
||||
extract_requested_model, refresh_execution_runtime_auth_context,
|
||||
refresh_execution_runtime_auth_context_with_snapshot, request_model_local_rejection,
|
||||
resolve_execution_runtime_auth_context, resolve_local_admin_session_principal,
|
||||
should_buffer_request_for_local_auth, trusted_auth_local_rejection,
|
||||
GatewayAdminPrincipalContext, GatewayControlAuthContext, GatewayCredentialCarrier,
|
||||
GatewayLocalAuthRejection,
|
||||
@@ -18,14 +19,16 @@ pub(crate) use auth::{
|
||||
pub(crate) use execute::{allows_control_execute_emergency, maybe_execute_via_control};
|
||||
pub(crate) use management_token_permissions::{
|
||||
all_assignable_management_token_permissions,
|
||||
audit_admin_read_only_management_token_permissions,
|
||||
audit_admin_read_only_management_token_permissions, legacy_full_management_token_permissions,
|
||||
management_token_permission_catalog_payload, management_token_permission_keys_from_value,
|
||||
management_token_permission_mode_and_summary,
|
||||
management_token_permissions_cover_all_assignable_permissions,
|
||||
management_token_permission_mode_and_summary, management_token_principal_has_permission,
|
||||
management_token_required_permission, normalize_assignable_management_token_permissions,
|
||||
read_only_management_token_permissions, validate_management_token_admin_route_permission,
|
||||
};
|
||||
pub(crate) use public::{resolve_public_request_context, GatewayPublicRequestContext};
|
||||
pub(crate) use public::{
|
||||
resolve_public_request_context, resolve_public_request_context_with_trusted_auth,
|
||||
resolve_public_request_context_without_trusted_auth, GatewayPublicRequestContext,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use route::classify_control_route;
|
||||
pub(crate) use route::{resolve_control_route, GatewayControlDecision};
|
||||
|
||||
@@ -2,7 +2,9 @@ use axum::http::Uri;
|
||||
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
use super::{resolve_control_route, GatewayControlDecision};
|
||||
use super::{
|
||||
resolve_control_route, route::resolve_control_route_with_trusted_auth, GatewayControlDecision,
|
||||
};
|
||||
|
||||
pub(crate) type GatewayPublicRequestContext =
|
||||
aether_gateway_control::PublicRequestContext<GatewayControlDecision>;
|
||||
@@ -23,3 +25,41 @@ pub(crate) async fn resolve_public_request_context(
|
||||
control_decision,
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn resolve_public_request_context_with_trusted_auth(
|
||||
state: &AppState,
|
||||
method: &http::Method,
|
||||
uri: &Uri,
|
||||
headers: &http::HeaderMap,
|
||||
trace_id: &str,
|
||||
) -> Result<GatewayPublicRequestContext, GatewayError> {
|
||||
let control_decision =
|
||||
resolve_control_route_with_trusted_auth(state, method, uri, headers, trace_id, true)
|
||||
.await?;
|
||||
Ok(GatewayPublicRequestContext::from_request_parts(
|
||||
trace_id,
|
||||
method,
|
||||
uri,
|
||||
headers,
|
||||
control_decision,
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn resolve_public_request_context_without_trusted_auth(
|
||||
state: &AppState,
|
||||
method: &http::Method,
|
||||
uri: &Uri,
|
||||
headers: &http::HeaderMap,
|
||||
trace_id: &str,
|
||||
) -> Result<GatewayPublicRequestContext, GatewayError> {
|
||||
let control_decision =
|
||||
resolve_control_route_with_trusted_auth(state, method, uri, headers, trace_id, false)
|
||||
.await?;
|
||||
Ok(GatewayPublicRequestContext::from_request_parts(
|
||||
trace_id,
|
||||
method,
|
||||
uri,
|
||||
headers,
|
||||
control_decision,
|
||||
))
|
||||
}
|
||||
|
||||
@@ -5,19 +5,21 @@ pub(super) fn classify_internal_route(
|
||||
method: &http::Method,
|
||||
normalized_path: &str,
|
||||
) -> Option<ClassifiedRoute> {
|
||||
if method == http::Method::POST && normalized_path.starts_with("/api/internal/gateway/") {
|
||||
let route_kind = match normalized_path {
|
||||
"/api/internal/gateway/resolve" => "resolve",
|
||||
"/api/internal/gateway/auth-context" => "auth_context",
|
||||
"/api/internal/gateway/decision-sync" => "decision_sync",
|
||||
"/api/internal/gateway/decision-stream" => "decision_stream",
|
||||
"/api/internal/gateway/plan-sync" => "plan_sync",
|
||||
"/api/internal/gateway/plan-stream" => "plan_stream",
|
||||
"/api/internal/gateway/report-sync" => "report_sync",
|
||||
"/api/internal/gateway/report-stream" => "report_stream",
|
||||
"/api/internal/gateway/finalize-sync" => "finalize_sync",
|
||||
"/api/internal/gateway/execute-sync" => "execute_sync",
|
||||
"/api/internal/gateway/execute-stream" => "execute_stream",
|
||||
if normalized_path == "/api/internal/gateway"
|
||||
|| normalized_path.starts_with("/api/internal/gateway/")
|
||||
{
|
||||
let route_kind = match (method, normalized_path) {
|
||||
(&http::Method::POST, "/api/internal/gateway/resolve") => "resolve",
|
||||
(&http::Method::POST, "/api/internal/gateway/auth-context") => "auth_context",
|
||||
(&http::Method::POST, "/api/internal/gateway/decision-sync") => "decision_sync",
|
||||
(&http::Method::POST, "/api/internal/gateway/decision-stream") => "decision_stream",
|
||||
(&http::Method::POST, "/api/internal/gateway/plan-sync") => "plan_sync",
|
||||
(&http::Method::POST, "/api/internal/gateway/plan-stream") => "plan_stream",
|
||||
(&http::Method::POST, "/api/internal/gateway/report-sync") => "report_sync",
|
||||
(&http::Method::POST, "/api/internal/gateway/report-stream") => "report_stream",
|
||||
(&http::Method::POST, "/api/internal/gateway/finalize-sync") => "finalize_sync",
|
||||
(&http::Method::POST, "/api/internal/gateway/execute-sync") => "execute_sync",
|
||||
(&http::Method::POST, "/api/internal/gateway/execute-stream") => "execute_stream",
|
||||
_ => "unhandled",
|
||||
};
|
||||
Some(classified(
|
||||
|
||||
@@ -11,7 +11,7 @@ mod oauth;
|
||||
mod public_support;
|
||||
|
||||
use super::auth::{
|
||||
resolve_control_decision_auth, resolve_gateway_credential_carrier,
|
||||
resolve_control_decision_auth_with_trusted_auth, resolve_gateway_credential_carrier,
|
||||
ControlDecisionAuthResolution, GatewayCredentialCarrier,
|
||||
};
|
||||
use super::{GatewayAdminPrincipalContext, GatewayControlAuthContext, GatewayLocalAuthRejection};
|
||||
@@ -175,6 +175,17 @@ pub(crate) async fn resolve_control_route(
|
||||
uri: &Uri,
|
||||
headers: &http::HeaderMap,
|
||||
trace_id: &str,
|
||||
) -> Result<Option<GatewayControlDecision>, GatewayError> {
|
||||
resolve_control_route_with_trusted_auth(state, method, uri, headers, trace_id, cfg!(test)).await
|
||||
}
|
||||
|
||||
pub(crate) async fn resolve_control_route_with_trusted_auth(
|
||||
state: &AppState,
|
||||
method: &http::Method,
|
||||
uri: &Uri,
|
||||
headers: &http::HeaderMap,
|
||||
trace_id: &str,
|
||||
trusted_auth_verified: bool,
|
||||
) -> Result<Option<GatewayControlDecision>, GatewayError> {
|
||||
let Some(mut decision) = classify_control_route(method, uri, headers) else {
|
||||
return Ok(None);
|
||||
@@ -185,7 +196,16 @@ pub(crate) async fn resolve_control_route(
|
||||
crate::system_features::ModelDirectivePolicySnapshot::load(state).await;
|
||||
}
|
||||
|
||||
match resolve_control_decision_auth(state, headers, uri, trace_id, decision).await? {
|
||||
match resolve_control_decision_auth_with_trusted_auth(
|
||||
state,
|
||||
headers,
|
||||
uri,
|
||||
trace_id,
|
||||
decision,
|
||||
trusted_auth_verified,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
ControlDecisionAuthResolution::Resolved(decision) => Ok(Some(decision)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -285,7 +285,7 @@ pub(super) fn classify_oauth_route(
|
||||
"admin_proxy",
|
||||
"provider_oauth_manage",
|
||||
"batch_import_oauth",
|
||||
"admin:pool",
|
||||
"admin:provider_oauth",
|
||||
false,
|
||||
))
|
||||
} else if method == http::Method::POST
|
||||
@@ -296,7 +296,7 @@ pub(super) fn classify_oauth_route(
|
||||
"admin_proxy",
|
||||
"provider_oauth_manage",
|
||||
"start_batch_import_oauth_task",
|
||||
"admin:pool",
|
||||
"admin:provider_oauth",
|
||||
false,
|
||||
))
|
||||
} else if method == http::Method::GET
|
||||
@@ -307,7 +307,7 @@ pub(super) fn classify_oauth_route(
|
||||
"admin_proxy",
|
||||
"provider_oauth_manage",
|
||||
"get_batch_import_task_status",
|
||||
"admin:pool",
|
||||
"admin:provider_oauth",
|
||||
false,
|
||||
))
|
||||
} else if method == http::Method::POST
|
||||
|
||||
@@ -197,18 +197,22 @@ pub(super) fn classify_public_support_route(
|
||||
"public:auth",
|
||||
false,
|
||||
))
|
||||
} else if matches!(method, &http::Method::GET | &http::Method::POST)
|
||||
// Authentication state-changing endpoints must never be dispatched for GET.
|
||||
// Besides violating HTTP method semantics, accepting GET here would allow
|
||||
// browser prefetches/cross-site requests to trigger login, refresh, logout,
|
||||
// registration, or verification side effects. `/me` is the sole read route.
|
||||
} else if (method == http::Method::POST
|
||||
&& matches!(
|
||||
normalized_path,
|
||||
"/api/auth/login"
|
||||
| "/api/auth/refresh"
|
||||
| "/api/auth/register"
|
||||
| "/api/auth/me"
|
||||
| "/api/auth/logout"
|
||||
| "/api/auth/send-verification-code"
|
||||
| "/api/auth/verify-email"
|
||||
| "/api/auth/verification-status"
|
||||
)
|
||||
))
|
||||
|| (method == http::Method::GET && normalized_path == "/api/auth/me")
|
||||
{
|
||||
let route_kind = match normalized_path {
|
||||
"/api/auth/login" => "login",
|
||||
|
||||
@@ -628,7 +628,7 @@ fn classifies_admin_management_token_write_routes_and_permission_catalog() {
|
||||
http::Method::POST,
|
||||
"/api/admin/management-tokens",
|
||||
"create_token",
|
||||
"admin:management_tokens:write",
|
||||
"admin:management_tokens:admin",
|
||||
),
|
||||
(
|
||||
http::Method::PUT,
|
||||
@@ -640,7 +640,7 @@ fn classifies_admin_management_token_write_routes_and_permission_catalog() {
|
||||
http::Method::POST,
|
||||
"/api/admin/management-tokens/token-123/regenerate",
|
||||
"regenerate_token",
|
||||
"admin:management_tokens:write",
|
||||
"admin:management_tokens:admin",
|
||||
),
|
||||
];
|
||||
|
||||
|
||||
@@ -68,11 +68,11 @@ fn classifies_admin_provider_oauth_batch_import_task_status_as_admin_proxy_route
|
||||
);
|
||||
assert_eq!(
|
||||
decision.auth_endpoint_signature.as_deref(),
|
||||
Some("admin:pool")
|
||||
Some("admin:provider_oauth")
|
||||
);
|
||||
assert_eq!(
|
||||
management_token_required_permission(&http::Method::GET, &decision).as_deref(),
|
||||
Some("admin:pool:read")
|
||||
Some("admin:provider_oauth:read")
|
||||
);
|
||||
assert!(!decision.is_execution_runtime_candidate());
|
||||
}
|
||||
@@ -86,42 +86,42 @@ fn classifies_admin_provider_oauth_maintenance_routes_as_admin_proxy_route() {
|
||||
"/api/admin/provider-oauth/keys/key-123/complete",
|
||||
"complete_key_oauth",
|
||||
"admin:provider_oauth",
|
||||
"admin:provider_oauth:write",
|
||||
"admin:provider_oauth:admin",
|
||||
),
|
||||
(
|
||||
http::Method::POST,
|
||||
"/api/admin/provider-oauth/keys/key-123/refresh",
|
||||
"refresh_key_oauth",
|
||||
"admin:provider_oauth",
|
||||
"admin:provider_oauth:write",
|
||||
"admin:provider_oauth:admin",
|
||||
),
|
||||
(
|
||||
http::Method::POST,
|
||||
"/api/admin/provider-oauth/providers/provider-123/complete",
|
||||
"complete_provider_oauth",
|
||||
"admin:provider_oauth",
|
||||
"admin:provider_oauth:write",
|
||||
"admin:provider_oauth:admin",
|
||||
),
|
||||
(
|
||||
http::Method::POST,
|
||||
"/api/admin/provider-oauth/providers/provider-123/import-refresh-token",
|
||||
"import_refresh_token",
|
||||
"admin:provider_oauth",
|
||||
"admin:provider_oauth:write",
|
||||
"admin:provider_oauth:admin",
|
||||
),
|
||||
(
|
||||
http::Method::POST,
|
||||
"/api/admin/provider-oauth/providers/provider-123/cookie-authorize",
|
||||
"cookie_authorize",
|
||||
"admin:provider_oauth",
|
||||
"admin:provider_oauth:write",
|
||||
"admin:provider_oauth:admin",
|
||||
),
|
||||
(
|
||||
http::Method::POST,
|
||||
"/api/admin/provider-oauth/providers/provider-123/cookie-authorize/tasks",
|
||||
"start_cookie_authorize_task",
|
||||
"admin:provider_oauth",
|
||||
"admin:provider_oauth:write",
|
||||
"admin:provider_oauth:admin",
|
||||
),
|
||||
(
|
||||
http::Method::GET,
|
||||
@@ -135,7 +135,7 @@ fn classifies_admin_provider_oauth_maintenance_routes_as_admin_proxy_route() {
|
||||
"/api/admin/provider-oauth/providers/provider-123/agent-identity-import/tasks",
|
||||
"start_agent_identity_import_task",
|
||||
"admin:provider_oauth",
|
||||
"admin:provider_oauth:write",
|
||||
"admin:provider_oauth:admin",
|
||||
),
|
||||
(
|
||||
http::Method::GET,
|
||||
@@ -148,22 +148,22 @@ fn classifies_admin_provider_oauth_maintenance_routes_as_admin_proxy_route() {
|
||||
http::Method::POST,
|
||||
"/api/admin/provider-oauth/providers/provider-123/batch-import",
|
||||
"batch_import_oauth",
|
||||
"admin:pool",
|
||||
"admin:pool:write",
|
||||
"admin:provider_oauth",
|
||||
"admin:provider_oauth:admin",
|
||||
),
|
||||
(
|
||||
http::Method::POST,
|
||||
"/api/admin/provider-oauth/providers/provider-123/batch-import/tasks",
|
||||
"start_batch_import_oauth_task",
|
||||
"admin:pool",
|
||||
"admin:pool:write",
|
||||
"admin:provider_oauth",
|
||||
"admin:provider_oauth:admin",
|
||||
),
|
||||
(
|
||||
http::Method::GET,
|
||||
"/api/admin/provider-oauth/providers/provider-123/batch-import/tasks/task-123",
|
||||
"get_batch_import_task_status",
|
||||
"admin:pool",
|
||||
"admin:pool:read",
|
||||
"admin:provider_oauth",
|
||||
"admin:provider_oauth:read",
|
||||
),
|
||||
(
|
||||
http::Method::POST,
|
||||
@@ -177,7 +177,7 @@ fn classifies_admin_provider_oauth_maintenance_routes_as_admin_proxy_route() {
|
||||
"/api/admin/provider-oauth/providers/provider-123/device-poll",
|
||||
"device_poll",
|
||||
"admin:provider_oauth",
|
||||
"admin:provider_oauth:write",
|
||||
"admin:provider_oauth:admin",
|
||||
),
|
||||
] {
|
||||
let uri: Uri = path.parse().expect("uri should parse");
|
||||
|
||||
@@ -947,6 +947,34 @@ fn classifies_auth_routes_as_public_support_route() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn does_not_classify_state_changing_auth_routes_for_get() {
|
||||
for path in [
|
||||
"/api/auth/login",
|
||||
"/api/auth/refresh",
|
||||
"/api/auth/register",
|
||||
"/api/auth/logout",
|
||||
"/api/auth/send-verification-code",
|
||||
"/api/auth/verify-email",
|
||||
"/api/auth/verification-status",
|
||||
] {
|
||||
let headers = headers(&[]);
|
||||
let uri: Uri = path.parse().expect("uri should parse");
|
||||
assert!(
|
||||
classify_control_route(&http::Method::GET, &uri, &headers).is_none(),
|
||||
"state-changing auth route {path} must not accept GET"
|
||||
);
|
||||
}
|
||||
|
||||
let headers = headers(&[]);
|
||||
let uri: Uri = "/api/auth/me".parse().expect("uri should parse");
|
||||
assert_eq!(
|
||||
classify_control_route(&http::Method::GET, &uri, &headers)
|
||||
.and_then(|decision| decision.route_kind),
|
||||
Some("me".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classifies_oauth_public_providers_route() {
|
||||
let headers = headers(&[]);
|
||||
|
||||
@@ -113,10 +113,7 @@ impl GatewayDataConfig {
|
||||
return (self.clone(), None);
|
||||
};
|
||||
let total_max = database.pool.max_connections;
|
||||
if total_max < 2
|
||||
|| configured_background_max == Some(0)
|
||||
|| is_private_sqlite_memory_database(database)
|
||||
{
|
||||
if total_max < 2 || configured_background_max == Some(0) {
|
||||
return (self.clone(), None);
|
||||
}
|
||||
|
||||
@@ -152,14 +149,9 @@ impl GatewayDataConfig {
|
||||
}
|
||||
}
|
||||
|
||||
fn is_private_sqlite_memory_database(database: &aether_data::SqlDatabaseConfig) -> bool {
|
||||
database.driver == aether_data::DatabaseDriver::Sqlite
|
||||
&& matches!(database.url.trim(), "sqlite::memory:" | "sqlite://:memory:")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::GatewayDataConfig;
|
||||
use super::{GatewayDataConfig, PostgresPoolConfig};
|
||||
use aether_data::{DatabaseDriver, SqlDatabaseConfig, SqlPoolConfig};
|
||||
|
||||
#[test]
|
||||
@@ -226,7 +218,10 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn runtime_pool_split_can_be_disabled_or_degrade_for_single_connection() {
|
||||
let mut database = SqlDatabaseConfig::sqlite_default();
|
||||
let mut database = SqlDatabaseConfig::from_postgres_config(PostgresPoolConfig {
|
||||
database_url: "postgres://localhost/aether".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
database.pool.max_connections = 1;
|
||||
let config = GatewayDataConfig::from_database_config(database);
|
||||
assert!(config
|
||||
@@ -234,7 +229,10 @@ mod tests {
|
||||
.1
|
||||
.is_none());
|
||||
|
||||
let mut database = SqlDatabaseConfig::sqlite_default();
|
||||
let mut database = SqlDatabaseConfig::from_postgres_config(PostgresPoolConfig {
|
||||
database_url: "postgres://localhost/aether".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
database.pool.max_connections = 8;
|
||||
let config = GatewayDataConfig::from_database_config(database);
|
||||
assert!(config
|
||||
@@ -242,34 +240,4 @@ mod tests {
|
||||
.1
|
||||
.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_pool_split_keeps_private_sqlite_memory_database_in_one_pool() {
|
||||
for url in ["sqlite::memory:", "sqlite://:memory:"] {
|
||||
let config = GatewayDataConfig::from_database_config(
|
||||
SqlDatabaseConfig::new(
|
||||
DatabaseDriver::Sqlite,
|
||||
url,
|
||||
SqlPoolConfig {
|
||||
min_connections: 1,
|
||||
max_connections: 8,
|
||||
..SqlPoolConfig::default()
|
||||
},
|
||||
)
|
||||
.expect("sqlite memory database config should be valid"),
|
||||
);
|
||||
|
||||
let (foreground, background) = config.split_runtime_pools_with_background_max(Some(2));
|
||||
|
||||
assert!(background.is_none(), "private SQLite URL {url} was split");
|
||||
assert_eq!(
|
||||
foreground
|
||||
.database()
|
||||
.expect("foreground database")
|
||||
.pool
|
||||
.max_connections,
|
||||
8
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -172,7 +172,7 @@ mod tests {
|
||||
candidates: vec![DecisionTraceCandidate {
|
||||
candidate: sample_candidate("req-1"),
|
||||
provider_name: Some("OpenAI".to_string()),
|
||||
provider_website: Some("https://openai.com".to_string()),
|
||||
provider_website: Some("https://openai.com/".to_string()),
|
||||
provider_type: Some("custom".to_string()),
|
||||
provider_priority: Some(0),
|
||||
provider_keep_priority_on_conversion: Some(false),
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,23 +1,42 @@
|
||||
use super::{
|
||||
ApiKeyLastUsedDelta, DataLayerError, GatewayDataState, GeminiFileMappingListQuery,
|
||||
GeminiFileMappingStats, ProviderCatalogKeyAdaptiveStateUpdate,
|
||||
ProviderCatalogKeyAdminCasUpdate, ProviderCatalogKeyHealthStateUpdate,
|
||||
ProviderCatalogKeyListQuery, ProviderCatalogKeyOAuthCredentialCasDelete,
|
||||
ProviderCatalogKeyOAuthRuntimeStateCasUpdate, ProviderCatalogKeyRuntimeMetadataUpdate,
|
||||
ProviderCatalogKeyStatusSnapshotUpdate, PublicHealthStatusCount, PublicHealthTimelineBucket,
|
||||
StoredGeminiFileMapping, StoredGeminiFileMappingListPage, StoredProviderCatalogEndpoint,
|
||||
StoredProviderCatalogKey, StoredProviderCatalogKeyMaintenanceSummary,
|
||||
StoredProviderCatalogKeyPage, StoredProviderCatalogKeyStats, StoredProviderCatalogProvider,
|
||||
StoredRequestCandidate, UpsertGeminiFileMappingRecord, UpsertRequestCandidateRecord,
|
||||
ProviderCatalogKeyAdminCasUpdate, ProviderCatalogKeyCredentialsCasUpdate,
|
||||
ProviderCatalogKeyHealthStateUpdate, ProviderCatalogKeyListQuery,
|
||||
ProviderCatalogKeyOAuthCredentialCasDelete, ProviderCatalogKeyOAuthRuntimeStateCasUpdate,
|
||||
ProviderCatalogKeyRuntimeMetadataUpdate, ProviderCatalogKeyStatusSnapshotUpdate,
|
||||
ProviderCatalogProviderConfigCasUpdate, ProviderCatalogProxyCasUpdate, PublicHealthStatusCount,
|
||||
PublicHealthTimelineBucket, StoredGeminiFileMapping, StoredGeminiFileMappingListPage,
|
||||
StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
|
||||
StoredProviderCatalogKeyMaintenanceSummary, StoredProviderCatalogKeyPage,
|
||||
StoredProviderCatalogKeyStats, StoredProviderCatalogProvider, StoredRequestCandidate,
|
||||
UpsertGeminiFileMappingRecord, UpsertRequestCandidateRecord,
|
||||
};
|
||||
|
||||
fn sanitize_request_candidate_rows(
|
||||
mut candidates: Vec<StoredRequestCandidate>,
|
||||
) -> Vec<StoredRequestCandidate> {
|
||||
for candidate in &mut candidates {
|
||||
candidate.sanitize_sensitive_diagnostics();
|
||||
}
|
||||
candidates
|
||||
}
|
||||
|
||||
fn sanitize_request_candidate_row(mut candidate: StoredRequestCandidate) -> StoredRequestCandidate {
|
||||
candidate.sanitize_sensitive_diagnostics();
|
||||
candidate
|
||||
}
|
||||
|
||||
impl GatewayDataState {
|
||||
pub(crate) async fn list_request_candidates_by_request_id(
|
||||
&self,
|
||||
request_id: &str,
|
||||
) -> Result<Vec<StoredRequestCandidate>, DataLayerError> {
|
||||
match &self.request_candidate_reader {
|
||||
Some(repository) => repository.list_by_request_id(request_id).await,
|
||||
Some(repository) => repository
|
||||
.list_by_request_id(request_id)
|
||||
.await
|
||||
.map(sanitize_request_candidate_rows),
|
||||
None => Ok(Vec::new()),
|
||||
}
|
||||
}
|
||||
@@ -27,7 +46,10 @@ impl GatewayDataState {
|
||||
request_id: &str,
|
||||
) -> Result<Vec<StoredRequestCandidate>, DataLayerError> {
|
||||
match &self.request_candidate_reader {
|
||||
Some(repository) => repository.list_attempted_by_request_id(request_id).await,
|
||||
Some(repository) => repository
|
||||
.list_attempted_by_request_id(request_id)
|
||||
.await
|
||||
.map(sanitize_request_candidate_rows),
|
||||
None => Ok(Vec::new()),
|
||||
}
|
||||
}
|
||||
@@ -38,7 +60,10 @@ impl GatewayDataState {
|
||||
limit: usize,
|
||||
) -> Result<Vec<StoredRequestCandidate>, DataLayerError> {
|
||||
match &self.request_candidate_reader {
|
||||
Some(repository) => repository.list_by_provider_id(provider_id, limit).await,
|
||||
Some(repository) => repository
|
||||
.list_by_provider_id(provider_id, limit)
|
||||
.await
|
||||
.map(sanitize_request_candidate_rows),
|
||||
None => Ok(Vec::new()),
|
||||
}
|
||||
}
|
||||
@@ -48,7 +73,10 @@ impl GatewayDataState {
|
||||
limit: usize,
|
||||
) -> Result<Vec<StoredRequestCandidate>, DataLayerError> {
|
||||
match &self.request_candidate_reader {
|
||||
Some(repository) => repository.list_recent(limit).await,
|
||||
Some(repository) => repository
|
||||
.list_recent(limit)
|
||||
.await
|
||||
.map(sanitize_request_candidate_rows),
|
||||
None => Ok(Vec::new()),
|
||||
}
|
||||
}
|
||||
@@ -60,11 +88,10 @@ impl GatewayDataState {
|
||||
limit: usize,
|
||||
) -> Result<Vec<StoredRequestCandidate>, DataLayerError> {
|
||||
match &self.request_candidate_reader {
|
||||
Some(repository) => {
|
||||
repository
|
||||
.list_finalized_by_endpoint_ids_since(endpoint_ids, since_unix_secs, limit)
|
||||
.await
|
||||
}
|
||||
Some(repository) => repository
|
||||
.list_finalized_by_endpoint_ids_since(endpoint_ids, since_unix_secs, limit)
|
||||
.await
|
||||
.map(sanitize_request_candidate_rows),
|
||||
None => Ok(Vec::new()),
|
||||
}
|
||||
}
|
||||
@@ -108,14 +135,19 @@ impl GatewayDataState {
|
||||
|
||||
pub(crate) async fn upsert_request_candidate(
|
||||
&self,
|
||||
candidate: UpsertRequestCandidateRecord,
|
||||
mut candidate: UpsertRequestCandidateRecord,
|
||||
) -> Result<Option<StoredRequestCandidate>, DataLayerError> {
|
||||
candidate.sanitize_for_persistence();
|
||||
crate::request_diagnostics::observe_db_operation(
|
||||
"request_candidate_upsert",
|
||||
self.database_pool_summary(),
|
||||
async {
|
||||
match &self.request_candidate_writer {
|
||||
Some(repository) => repository.upsert(candidate).await.map(Some),
|
||||
Some(repository) => repository
|
||||
.upsert(candidate)
|
||||
.await
|
||||
.map(sanitize_request_candidate_row)
|
||||
.map(Some),
|
||||
None => Ok(None),
|
||||
}
|
||||
},
|
||||
@@ -170,6 +202,16 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn upsert_gemini_file_mapping_if_owner_matches(
|
||||
&self,
|
||||
record: UpsertGeminiFileMappingRecord,
|
||||
) -> Result<Option<StoredGeminiFileMapping>, DataLayerError> {
|
||||
match &self.gemini_file_mapping_writer {
|
||||
Some(repository) => repository.upsert_if_owner_matches(record).await,
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn list_gemini_file_mappings(
|
||||
&self,
|
||||
query: &GeminiFileMappingListQuery,
|
||||
@@ -183,6 +225,49 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn find_gemini_file_mapping_by_file_name(
|
||||
&self,
|
||||
file_name: &str,
|
||||
) -> Result<Option<StoredGeminiFileMapping>, DataLayerError> {
|
||||
match &self.gemini_file_mapping_reader {
|
||||
Some(repository) => repository.find_by_file_name(file_name).await,
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn find_active_gemini_file_mapping_for_user(
|
||||
&self,
|
||||
file_name: &str,
|
||||
user_id: &str,
|
||||
now_unix_secs: u64,
|
||||
) -> Result<Option<StoredGeminiFileMapping>, DataLayerError> {
|
||||
match &self.gemini_file_mapping_reader {
|
||||
Some(repository) => {
|
||||
repository
|
||||
.find_active_by_file_name_for_user(file_name, user_id, now_unix_secs)
|
||||
.await
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn find_active_gemini_file_mapping_for_owner(
|
||||
&self,
|
||||
file_name: &str,
|
||||
key_id: &str,
|
||||
user_id: &str,
|
||||
now_unix_secs: u64,
|
||||
) -> Result<Option<StoredGeminiFileMapping>, DataLayerError> {
|
||||
match &self.gemini_file_mapping_reader {
|
||||
Some(repository) => {
|
||||
repository
|
||||
.find_active_by_file_name_for_owner(file_name, key_id, user_id, now_unix_secs)
|
||||
.await
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn summarize_gemini_file_mappings(
|
||||
&self,
|
||||
now_unix_secs: u64,
|
||||
@@ -208,6 +293,37 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_gemini_file_mapping_by_file_name_for_user(
|
||||
&self,
|
||||
file_name: &str,
|
||||
user_id: &str,
|
||||
) -> Result<bool, DataLayerError> {
|
||||
match &self.gemini_file_mapping_writer {
|
||||
Some(repository) => {
|
||||
repository
|
||||
.delete_by_file_name_for_user(file_name, user_id)
|
||||
.await
|
||||
}
|
||||
None => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_gemini_file_mapping_by_file_name_for_owner(
|
||||
&self,
|
||||
file_name: &str,
|
||||
key_id: &str,
|
||||
user_id: &str,
|
||||
) -> Result<bool, DataLayerError> {
|
||||
match &self.gemini_file_mapping_writer {
|
||||
Some(repository) => {
|
||||
repository
|
||||
.delete_by_file_name_for_owner(file_name, key_id, user_id)
|
||||
.await
|
||||
}
|
||||
None => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_gemini_file_mapping_by_id(
|
||||
&self,
|
||||
mapping_id: &str,
|
||||
@@ -357,38 +473,11 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn update_provider_catalog_key_oauth_credentials(
|
||||
&self,
|
||||
key_id: &str,
|
||||
encrypted_api_key: &str,
|
||||
encrypted_auth_config: Option<&str>,
|
||||
expires_at_unix_secs: Option<u64>,
|
||||
) -> Result<bool, DataLayerError> {
|
||||
let updated = match &self.provider_catalog_writer {
|
||||
Some(repository) => {
|
||||
repository
|
||||
.update_key_oauth_credentials(
|
||||
key_id,
|
||||
encrypted_api_key,
|
||||
encrypted_auth_config,
|
||||
expires_at_unix_secs,
|
||||
)
|
||||
.await
|
||||
}
|
||||
None => Ok(false),
|
||||
}?;
|
||||
if updated {
|
||||
self.clear_provider_catalog_cache();
|
||||
}
|
||||
Ok(updated)
|
||||
}
|
||||
|
||||
pub(crate) async fn update_provider_catalog_key_oauth_runtime_state(
|
||||
&self,
|
||||
key_id: &str,
|
||||
oauth_invalid_at_unix_secs: Option<u64>,
|
||||
oauth_invalid_reason: Option<&str>,
|
||||
encrypted_auth_config_update: Option<&str>,
|
||||
updated_at_unix_secs: Option<u64>,
|
||||
) -> Result<bool, DataLayerError> {
|
||||
let updated = match &self.provider_catalog_writer {
|
||||
@@ -398,7 +487,6 @@ impl GatewayDataState {
|
||||
key_id,
|
||||
oauth_invalid_at_unix_secs,
|
||||
oauth_invalid_reason,
|
||||
encrypted_auth_config_update,
|
||||
updated_at_unix_secs,
|
||||
)
|
||||
.await
|
||||
@@ -475,6 +563,30 @@ impl GatewayDataState {
|
||||
Ok(updated)
|
||||
}
|
||||
|
||||
pub(crate) async fn compare_and_swap_provider_catalog_provider_config(
|
||||
&self,
|
||||
update: &ProviderCatalogProviderConfigCasUpdate,
|
||||
) -> Result<bool, DataLayerError> {
|
||||
let updated = match &self.provider_catalog_writer {
|
||||
Some(repository) => repository.compare_and_swap_provider_config(update).await,
|
||||
None => Ok(false),
|
||||
}?;
|
||||
self.clear_provider_catalog_cache();
|
||||
Ok(updated)
|
||||
}
|
||||
|
||||
pub(crate) async fn compare_and_swap_provider_catalog_provider_proxy(
|
||||
&self,
|
||||
update: &ProviderCatalogProxyCasUpdate,
|
||||
) -> Result<bool, DataLayerError> {
|
||||
let updated = match &self.provider_catalog_writer {
|
||||
Some(repository) => repository.compare_and_swap_provider_proxy(update).await,
|
||||
None => Ok(false),
|
||||
}?;
|
||||
self.clear_provider_catalog_cache();
|
||||
Ok(updated)
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_provider_catalog_provider(
|
||||
&self,
|
||||
provider_id: &str,
|
||||
@@ -543,6 +655,18 @@ impl GatewayDataState {
|
||||
Ok(updated)
|
||||
}
|
||||
|
||||
pub(crate) async fn compare_and_swap_provider_catalog_endpoint_proxy(
|
||||
&self,
|
||||
update: &ProviderCatalogProxyCasUpdate,
|
||||
) -> Result<bool, DataLayerError> {
|
||||
let updated = match &self.provider_catalog_writer {
|
||||
Some(repository) => repository.compare_and_swap_endpoint_proxy(update).await,
|
||||
None => Ok(false),
|
||||
}?;
|
||||
self.clear_provider_catalog_cache();
|
||||
Ok(updated)
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_provider_catalog_endpoint(
|
||||
&self,
|
||||
endpoint_id: &str,
|
||||
@@ -571,6 +695,32 @@ impl GatewayDataState {
|
||||
Ok(updated)
|
||||
}
|
||||
|
||||
pub(crate) async fn compare_and_swap_provider_catalog_key_proxy(
|
||||
&self,
|
||||
update: &ProviderCatalogProxyCasUpdate,
|
||||
) -> Result<bool, DataLayerError> {
|
||||
let updated = match &self.provider_catalog_writer {
|
||||
Some(repository) => repository.compare_and_swap_key_proxy(update).await,
|
||||
None => Ok(false),
|
||||
}?;
|
||||
self.clear_provider_catalog_cache();
|
||||
Ok(updated)
|
||||
}
|
||||
|
||||
pub(crate) async fn compare_and_swap_provider_catalog_key_credentials(
|
||||
&self,
|
||||
update: &ProviderCatalogKeyCredentialsCasUpdate,
|
||||
) -> Result<bool, DataLayerError> {
|
||||
let updated = match &self.provider_catalog_writer {
|
||||
Some(repository) => repository.compare_and_swap_key_credentials(update).await,
|
||||
None => Ok(false),
|
||||
}?;
|
||||
// Clear on both outcomes: a CAS miss proves the cached credential
|
||||
// generation was stale and the retry must observe the winning record.
|
||||
self.clear_provider_catalog_cache();
|
||||
Ok(updated)
|
||||
}
|
||||
|
||||
pub(crate) async fn compare_and_update_provider_catalog_key_admin_state(
|
||||
&self,
|
||||
update: &ProviderCatalogKeyAdminCasUpdate,
|
||||
@@ -843,3 +993,81 @@ impl GatewayDataState {
|
||||
Ok(updated)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod request_candidate_security_tests {
|
||||
use serde_json::json;
|
||||
|
||||
use super::{
|
||||
sanitize_request_candidate_row, sanitize_request_candidate_rows, StoredRequestCandidate,
|
||||
};
|
||||
use aether_data_contracts::repository::candidates::RequestCandidateStatus;
|
||||
|
||||
fn untrusted_candidate() -> StoredRequestCandidate {
|
||||
let mut candidate = StoredRequestCandidate::new(
|
||||
"candidate-untrusted".to_string(),
|
||||
"request-1".to_string(),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
0,
|
||||
0,
|
||||
Some("provider-1".to_string()),
|
||||
Some("endpoint-1".to_string()),
|
||||
Some("key-1".to_string()),
|
||||
RequestCandidateStatus::Failed,
|
||||
None,
|
||||
false,
|
||||
Some(500),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
1,
|
||||
None,
|
||||
Some(2),
|
||||
)
|
||||
.expect("candidate should build");
|
||||
candidate.skip_reason = Some("Bearer candidate-secret".to_string());
|
||||
candidate.error_type = Some("candidate-secret".to_string());
|
||||
candidate.error_message = Some("Bearer candidate-secret".to_string());
|
||||
candidate.extra_data = Some(json!({
|
||||
"gateway_execution_runtime": true,
|
||||
"request_body": {"token": "candidate-secret"}
|
||||
}));
|
||||
candidate.required_capabilities = Some(json!({
|
||||
"vision": 1,
|
||||
"tenant_secret": "candidate-secret"
|
||||
}));
|
||||
candidate
|
||||
}
|
||||
|
||||
fn assert_candidate_is_sanitized(candidate: &StoredRequestCandidate) {
|
||||
assert_eq!(candidate.skip_reason.as_deref(), Some("unclassified_skip"));
|
||||
assert_eq!(candidate.error_type.as_deref(), Some("unclassified_error"));
|
||||
assert!(candidate.error_message.is_none());
|
||||
assert_eq!(
|
||||
candidate.extra_data,
|
||||
Some(json!({"gateway_execution_runtime": true}))
|
||||
);
|
||||
assert_eq!(
|
||||
candidate.required_capabilities,
|
||||
Some(json!({"vision": true}))
|
||||
);
|
||||
assert!(!serde_json::to_string(candidate)
|
||||
.expect("candidate should serialize")
|
||||
.contains("candidate-secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gateway_candidate_boundary_sanitizes_repository_rows_and_write_results() {
|
||||
let candidate = sanitize_request_candidate_row(untrusted_candidate());
|
||||
assert_candidate_is_sanitized(&candidate);
|
||||
|
||||
let candidates = sanitize_request_candidate_rows(vec![untrusted_candidate()]);
|
||||
assert_candidate_is_sanitized(&candidates[0]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -895,6 +895,44 @@ impl GatewayDataState {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn compare_and_set_system_config_string_value(
|
||||
&self,
|
||||
key: &str,
|
||||
expected: &str,
|
||||
replacement: &str,
|
||||
) -> Result<bool, DataLayerError> {
|
||||
if let Some(values) = &self.system_config_values {
|
||||
let updated = {
|
||||
let mut values = values.write().expect("system config values lock");
|
||||
match values.get_mut(key) {
|
||||
Some(entry) if entry.value.as_str() == Some(expected) => {
|
||||
entry.value = serde_json::Value::String(replacement.to_string());
|
||||
entry.updated_at_unix_secs =
|
||||
Some(current_system_config_updated_at_unix_secs());
|
||||
true
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
};
|
||||
self.clear_cached_system_config_value(key);
|
||||
return Ok(updated);
|
||||
}
|
||||
|
||||
let result = match self.backends.as_ref() {
|
||||
Some(backends) => {
|
||||
crate::request_diagnostics::observe_db_operation(
|
||||
"system_config_compare_and_set",
|
||||
self.database_pool_summary(),
|
||||
backends.compare_and_set_system_config_string_value(key, expected, replacement),
|
||||
)
|
||||
.await
|
||||
}
|
||||
None => Ok(false),
|
||||
};
|
||||
self.clear_cached_system_config_value(key);
|
||||
result
|
||||
}
|
||||
|
||||
pub(crate) async fn upsert_system_config_value(
|
||||
&self,
|
||||
key: &str,
|
||||
@@ -1078,10 +1116,7 @@ impl GatewayDataState {
|
||||
}
|
||||
|
||||
fn database_driver_supports_usage_counter_flush(driver: Option<DatabaseDriver>) -> bool {
|
||||
matches!(
|
||||
driver,
|
||||
Some(DatabaseDriver::Postgres | DatabaseDriver::Mysql | DatabaseDriver::Sqlite)
|
||||
)
|
||||
matches!(driver, Some(DatabaseDriver::Postgres))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -1094,12 +1129,6 @@ mod usage_counter_flush_backend_tests {
|
||||
assert!(database_driver_supports_usage_counter_flush(Some(
|
||||
DatabaseDriver::Postgres
|
||||
)));
|
||||
assert!(database_driver_supports_usage_counter_flush(Some(
|
||||
DatabaseDriver::Mysql
|
||||
)));
|
||||
assert!(database_driver_supports_usage_counter_flush(Some(
|
||||
DatabaseDriver::Sqlite
|
||||
)));
|
||||
assert!(!database_driver_supports_usage_counter_flush(None));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,7 +11,11 @@ use aether_data_contracts::repository::candidates::DecisionTrace;
|
||||
use aether_data_contracts::repository::provider_catalog::{
|
||||
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
|
||||
};
|
||||
use aether_data_contracts::repository::settlement::{StoredUsageSettlement, UsageSettlementInput};
|
||||
use aether_data_contracts::repository::proxy_nodes::ProxyNodeTrafficMutation;
|
||||
use aether_data_contracts::repository::settlement::{
|
||||
ReconcileUsagePolicyCostInput, StoredUsagePolicyCostReservation, StoredUsageSettlement,
|
||||
UsageSettlementInput,
|
||||
};
|
||||
use aether_data_contracts::repository::usage::{
|
||||
ProxyNodeCounterDelta, StoredRequestUsageAudit, UpsertUsageRecord, UsageWriteRepository,
|
||||
};
|
||||
@@ -34,7 +38,7 @@ const LEGACY_REQUEST_LOG_LEVEL_KEY: &str = "request_log_level";
|
||||
|
||||
fn usage_request_record_level_from_value(value: Option<&Value>) -> UsageRequestRecordLevel {
|
||||
let Some(value) = value.and_then(Value::as_str).map(str::trim) else {
|
||||
return UsageRequestRecordLevel::Full;
|
||||
return UsageRequestRecordLevel::Basic;
|
||||
};
|
||||
|
||||
if value.eq_ignore_ascii_case("basic")
|
||||
@@ -45,7 +49,9 @@ fn usage_request_record_level_from_value(value: Option<&Value>) -> UsageRequestR
|
||||
{
|
||||
UsageRequestRecordLevel::Basic
|
||||
} else {
|
||||
UsageRequestRecordLevel::Full
|
||||
// Raw HTTP payload capture is disabled at the runtime boundary. The setting remains
|
||||
// accepted for compatibility, but no longer authorizes collecting request/response data.
|
||||
UsageRequestRecordLevel::Basic
|
||||
}
|
||||
}
|
||||
|
||||
@@ -95,6 +101,14 @@ impl StoredVideoTaskReadSide for GatewayDataState {
|
||||
) -> Result<Option<StoredVideoTask>, DataLayerError> {
|
||||
GatewayDataState::find_video_task(self, key).await
|
||||
}
|
||||
|
||||
async fn find_stored_video_task_for_user(
|
||||
&self,
|
||||
key: VideoTaskLookupKey<'_>,
|
||||
user_id: &str,
|
||||
) -> Result<Option<StoredVideoTask>, DataLayerError> {
|
||||
GatewayDataState::find_video_task_for_user(self, key, user_id).await
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -219,6 +233,13 @@ impl UsageSettlementWriter for GatewayDataState {
|
||||
GatewayDataState::has_settlement_writer(self)
|
||||
}
|
||||
|
||||
async fn reconcile_usage_policy_cost(
|
||||
&self,
|
||||
input: ReconcileUsagePolicyCostInput,
|
||||
) -> Result<Option<StoredUsagePolicyCostReservation>, DataLayerError> {
|
||||
GatewayDataState::reconcile_usage_policy_cost(self, input).await
|
||||
}
|
||||
|
||||
async fn settle_usage(
|
||||
&self,
|
||||
input: UsageSettlementInput,
|
||||
@@ -285,10 +306,26 @@ impl aether_usage_runtime::ManualProxyNodeCounter for GatewayDataState {
|
||||
failed_delta: i64,
|
||||
latency_ms: Option<i64>,
|
||||
) -> Result<(), DataLayerError> {
|
||||
// This API predates incarnation fences and only receives a node id. Read
|
||||
// the selected node first so every durable path can bind the delta to
|
||||
// the observed generation. If the node disappeared, fail closed rather
|
||||
// than allowing a bare id to target a replacement node.
|
||||
let Some(node) = self.find_proxy_node(node_id).await? else {
|
||||
return Ok(());
|
||||
};
|
||||
if !node.is_manual {
|
||||
return Ok(());
|
||||
}
|
||||
let expected_tunnel_generation = node.tunnel_generation.trim().to_string();
|
||||
if expected_tunnel_generation.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if let Some(repository) = &self.usage_writer {
|
||||
let enqueued = repository
|
||||
.enqueue_proxy_node_counter_delta(ProxyNodeCounterDelta {
|
||||
node_id: node_id.to_string(),
|
||||
expected_tunnel_generation: Some(expected_tunnel_generation.clone()),
|
||||
total_requests_delta: total_delta,
|
||||
failed_requests_delta: failed_delta,
|
||||
dns_failures_delta: 0,
|
||||
@@ -300,14 +337,25 @@ impl aether_usage_runtime::ManualProxyNodeCounter for GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
match &self.proxy_node_writer {
|
||||
Some(repository) => {
|
||||
repository
|
||||
.increment_manual_node_requests(node_id, total_delta, failed_delta, latency_ms)
|
||||
.await
|
||||
}
|
||||
None => Ok(()),
|
||||
// The legacy increment method has no generation argument and would
|
||||
// re-read the current row, which is vulnerable to an id reuse between
|
||||
// the read above and the write. Use the fenced traffic mutation as the
|
||||
// only fallback. It intentionally omits the legacy latency-only field;
|
||||
// preserving counters safely is more important than an unfenced write.
|
||||
if let Some(repository) = &self.proxy_node_writer {
|
||||
let _ = repository
|
||||
.record_traffic(&ProxyNodeTrafficMutation {
|
||||
node_id: node_id.to_string(),
|
||||
expected_tunnel_generation: Some(expected_tunnel_generation),
|
||||
total_requests_delta: total_delta,
|
||||
failed_requests_delta: failed_delta,
|
||||
dns_failures_delta: 0,
|
||||
stream_errors_delta: 0,
|
||||
})
|
||||
.await?;
|
||||
}
|
||||
let _ = latency_ms;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -452,6 +500,20 @@ mod tests {
|
||||
assert_eq!(level, UsageRequestRecordLevel::Basic);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_runtime_access_disables_full_http_capture() {
|
||||
let state = GatewayDataState::disabled().with_system_config_values_for_tests([(
|
||||
"request_record_level".to_string(),
|
||||
json!("full"),
|
||||
)]);
|
||||
|
||||
let level = UsageRuntimeAccess::request_record_level(&state)
|
||||
.await
|
||||
.expect("request record level should read");
|
||||
|
||||
assert_eq!(level, UsageRequestRecordLevel::Basic);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_runtime_access_falls_back_to_legacy_request_log_level_alias() {
|
||||
let state = GatewayDataState::disabled().with_system_config_values_for_tests([(
|
||||
@@ -467,14 +529,14 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_runtime_access_defaults_missing_request_record_level_to_full() {
|
||||
async fn usage_runtime_access_defaults_missing_request_record_level_to_basic() {
|
||||
let state = GatewayDataState::disabled();
|
||||
|
||||
let level = UsageRuntimeAccess::request_record_level(&state)
|
||||
.await
|
||||
.expect("missing request record level should fall back");
|
||||
|
||||
assert_eq!(level, UsageRequestRecordLevel::Full);
|
||||
assert_eq!(level, UsageRequestRecordLevel::Basic);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -488,6 +550,20 @@ mod tests {
|
||||
.await
|
||||
.expect("body capture policy should read");
|
||||
|
||||
assert_eq!(policy.record_level, UsageRequestRecordLevel::Full);
|
||||
assert_eq!(policy.record_level, UsageRequestRecordLevel::Basic);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_runtime_access_fails_closed_for_unknown_record_level() {
|
||||
let state = GatewayDataState::disabled().with_system_config_values_for_tests([(
|
||||
"request_record_level".to_string(),
|
||||
json!("everything"),
|
||||
)]);
|
||||
|
||||
let level = UsageRuntimeAccess::request_record_level(&state)
|
||||
.await
|
||||
.expect("request record level should read");
|
||||
|
||||
assert_eq!(level, UsageRequestRecordLevel::Basic);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,8 +27,8 @@ use aether_data::repository::auth::{
|
||||
StoredAuthApiKeyExportRecord, StoredAuthApiKeySnapshot,
|
||||
};
|
||||
use aether_data::repository::auth_modules::{
|
||||
AuthModuleReadRepository, AuthModuleWriteRepository, StoredLdapModuleConfig,
|
||||
StoredOAuthProviderModuleConfig,
|
||||
AuthModuleReadRepository, AuthModuleWriteRepository, CompareAndSwapLdapConfigResult,
|
||||
LdapBindPasswordUpdate, StoredLdapModuleConfig, StoredOAuthProviderModuleConfig,
|
||||
};
|
||||
use aether_data::repository::gemini_file_mappings::{
|
||||
GeminiFileMappingListQuery, GeminiFileMappingReadRepository, GeminiFileMappingStats,
|
||||
@@ -36,9 +36,10 @@ use aether_data::repository::gemini_file_mappings::{
|
||||
UpsertGeminiFileMappingRecord,
|
||||
};
|
||||
use aether_data::repository::management_tokens::{
|
||||
CreateManagementTokenRecord, ManagementTokenListQuery, ManagementTokenReadRepository,
|
||||
ManagementTokenWriteRepository, RegenerateManagementTokenSecret, StoredManagementToken,
|
||||
StoredManagementTokenListPage, StoredManagementTokenWithUser, UpdateManagementTokenRecord,
|
||||
ActivateManagementTokenIfMatches, CreateManagementTokenRecord, ManagementTokenListQuery,
|
||||
ManagementTokenReadRepository, ManagementTokenWriteRepository, RegenerateManagementTokenSecret,
|
||||
StoredManagementToken, StoredManagementTokenListPage, StoredManagementTokenWithUser,
|
||||
UpdateManagementTokenRecord,
|
||||
};
|
||||
use aether_data::repository::oauth_providers::{
|
||||
OAuthProviderReadRepository, OAuthProviderWriteRepository, StoredOAuthProviderConfig,
|
||||
@@ -63,21 +64,24 @@ pub(crate) use aether_data::repository::users::{
|
||||
use aether_data::repository::wallet::{
|
||||
AdjustWalletBalanceInput, AdminPaymentOrderListQuery, AdminRedeemCodeBatchListQuery,
|
||||
AdminRedeemCodeListQuery, AdminWalletLedgerQuery, AdminWalletListQuery,
|
||||
AdminWalletRefundRequestListQuery, CompleteAdminWalletRefundInput,
|
||||
CreateAdminRedeemCodeBatchInput, CreateAdminRedeemCodeBatchResult,
|
||||
CreateManualWalletRechargeInput, CreatePlanPurchaseOrderInput, CreatePlanPurchaseOrderOutcome,
|
||||
CreateWalletRechargeOrderInput, CreateWalletRechargeOrderOutcome,
|
||||
CreateWalletRefundRequestInput, CreateWalletRefundRequestOutcome, CreditAdminPaymentOrderInput,
|
||||
AdminWalletRefundRequestListQuery, CompareAndSwapPaymentOrderStripeClientSecretInput,
|
||||
CompleteAdminWalletRefundInput, CreateAdminRedeemCodeBatchInput,
|
||||
CreateAdminRedeemCodeBatchResult, CreateManualWalletRechargeInput,
|
||||
CreatePlanPurchaseOrderInput, CreatePlanPurchaseOrderOutcome, CreateWalletRechargeOrderInput,
|
||||
CreateWalletRechargeOrderOutcome, CreateWalletRefundRequestInput,
|
||||
CreateWalletRefundRequestOutcome, CreditAdminPaymentOrderInput,
|
||||
DeleteAdminRedeemCodeBatchInput, DisableAdminRedeemCodeBatchInput, DisableAdminRedeemCodeInput,
|
||||
FailAdminWalletRefundInput, ProcessAdminWalletRefundInput, ProcessPaymentCallbackInput,
|
||||
ProcessPaymentCallbackOutcome, RedeemWalletCodeInput, RedeemWalletCodeOutcome,
|
||||
FailAdminWalletRefundInput, FailWalletRechargeCheckoutInput, InitializeAuthWalletOutcome,
|
||||
ProcessAdminWalletRefundInput, ProcessPaymentCallbackInput, ProcessPaymentCallbackOutcome,
|
||||
ReclaimWalletRechargeCheckoutInput, RedeemWalletCodeInput, RedeemWalletCodeOutcome,
|
||||
StoredAdminPaymentCallback, StoredAdminPaymentCallbackPage, StoredAdminPaymentOrder,
|
||||
StoredAdminPaymentOrderPage, StoredAdminRedeemCode, StoredAdminRedeemCodeBatch,
|
||||
StoredAdminRedeemCodeBatchPage, StoredAdminRedeemCodePage, StoredAdminWalletLedgerPage,
|
||||
StoredAdminWalletListPage, StoredAdminWalletRefund, StoredAdminWalletRefundPage,
|
||||
StoredAdminWalletRefundRequestPage, StoredAdminWalletTransaction,
|
||||
StoredAdminWalletTransactionPage, StoredWalletDailyUsageLedger,
|
||||
StoredWalletDailyUsageLedgerPage, StoredWalletSnapshot, WalletLookupKey, WalletMutationOutcome,
|
||||
StoredWalletDailyUsageLedgerPage, StoredWalletSnapshot, UpdateAdminWalletRefundGatewayInput,
|
||||
UpdateWalletRechargeCheckoutInput, WalletLookupKey, WalletMutationOutcome,
|
||||
WalletReadRepository, WalletWriteRepository,
|
||||
};
|
||||
use aether_data::{
|
||||
@@ -92,8 +96,9 @@ use aether_data_contracts::repository::background_tasks::{
|
||||
use aether_data_contracts::repository::billing::{
|
||||
AdminBillingCollectorRecord, AdminBillingCollectorWriteInput, AdminBillingMutationOutcome,
|
||||
AdminBillingPresetApplyResult, AdminBillingRuleRecord, AdminBillingRuleWriteInput,
|
||||
BillingPlanRecord, BillingPlanWriteInput, BillingReadRepository, PaymentGatewayConfigRecord,
|
||||
PaymentGatewayConfigWriteInput, StoredBillingModelContext, UserDailyQuotaAvailabilityRecord,
|
||||
BillingPlanRecord, BillingPlanWriteInput, BillingReadRepository,
|
||||
PaymentGatewayConfigCasWriteInput, PaymentGatewayConfigRecord, PaymentGatewayConfigWriteInput,
|
||||
PaymentGatewaySecretCasUpdate, StoredBillingModelContext, UserDailyQuotaAvailabilityRecord,
|
||||
UserPlanEntitlementRecord,
|
||||
};
|
||||
use aether_data_contracts::repository::candidate_selection::{
|
||||
@@ -122,12 +127,14 @@ use aether_data_contracts::repository::pool_scores::{
|
||||
};
|
||||
use aether_data_contracts::repository::provider_catalog::{
|
||||
ProviderCatalogKeyAdaptiveStateUpdate, ProviderCatalogKeyAdminCasUpdate,
|
||||
ProviderCatalogKeyHealthStateUpdate, ProviderCatalogKeyListQuery,
|
||||
ProviderCatalogKeyOAuthCredentialCasDelete, ProviderCatalogKeyOAuthRuntimeStateCasUpdate,
|
||||
ProviderCatalogKeyRuntimeMetadataUpdate, ProviderCatalogKeyStatusSnapshotUpdate,
|
||||
ProviderCatalogReadRepository, ProviderCatalogWriteRepository, StoredProviderCatalogEndpoint,
|
||||
StoredProviderCatalogKey, StoredProviderCatalogKeyMaintenanceSummary,
|
||||
StoredProviderCatalogKeyPage, StoredProviderCatalogKeyStats, StoredProviderCatalogProvider,
|
||||
ProviderCatalogKeyCredentialsCasUpdate, ProviderCatalogKeyHealthStateUpdate,
|
||||
ProviderCatalogKeyListQuery, ProviderCatalogKeyOAuthCredentialCasDelete,
|
||||
ProviderCatalogKeyOAuthRuntimeStateCasUpdate, ProviderCatalogKeyRuntimeMetadataUpdate,
|
||||
ProviderCatalogKeyStatusSnapshotUpdate, ProviderCatalogProviderConfigCasUpdate,
|
||||
ProviderCatalogProxyCasUpdate, ProviderCatalogReadRepository, ProviderCatalogWriteRepository,
|
||||
StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
|
||||
StoredProviderCatalogKeyMaintenanceSummary, StoredProviderCatalogKeyPage,
|
||||
StoredProviderCatalogKeyStats, StoredProviderCatalogProvider,
|
||||
};
|
||||
use aether_data_contracts::repository::quota::{
|
||||
ProviderQuotaReadRepository, ProviderQuotaWriteRepository, StoredProviderQuotaSnapshot,
|
||||
@@ -136,7 +143,10 @@ use aether_data_contracts::repository::routing_profiles::{
|
||||
RoutingGroupReadRepository, RoutingGroupWriteRepository,
|
||||
};
|
||||
use aether_data_contracts::repository::settlement::{
|
||||
SettlementWriteRepository, StoredUsageSettlement, UsageSettlementInput,
|
||||
ReconcileUsagePolicyCostInput, ReleaseUsagePolicyRequestAdmissionInput,
|
||||
ReserveUsagePolicyCostInput, ReserveUsagePolicyCostOutcome, ReserveUsagePolicyRequestInput,
|
||||
ReserveUsagePolicyRequestOutcome, SettlementWriteRepository, StoredUsagePolicyCostReservation,
|
||||
StoredUsagePolicyRequestAdmission, StoredUsageSettlement, UsageSettlementInput,
|
||||
};
|
||||
use aether_data_contracts::repository::usage::{
|
||||
ApiKeyLastUsedDelta, ManagementTokenCounterDelta, PendingUsageCleanupSummary,
|
||||
@@ -150,9 +160,9 @@ use aether_data_contracts::repository::video_tasks::{
|
||||
use aether_runtime_state::RuntimeQueueStore;
|
||||
|
||||
pub(crate) use self::referrals::{
|
||||
ReferralAdminStats, ReferralMutationStatus, ReferralRelationshipListQuery,
|
||||
ReferralRelationshipRecord, ReferralRewardConfig, ReferralRewardListQuery,
|
||||
ReferralRewardRecord, ReferralUserDashboard,
|
||||
ReferralAdminStats, ReferralMutationStatus, ReferralReconciliationSummary,
|
||||
ReferralRelationshipListQuery, ReferralRelationshipRecord, ReferralRewardConfig,
|
||||
ReferralRewardListQuery, ReferralRewardRecord, ReferralUserDashboard,
|
||||
};
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
|
||||
@@ -4,9 +4,9 @@ use aether_data::DataLayerError;
|
||||
use super::GatewayDataState;
|
||||
|
||||
pub(crate) use aether_data::backend::{
|
||||
ReferralAdminStats, ReferralMutationStatus, ReferralRelationshipListQuery,
|
||||
ReferralRelationshipRecord, ReferralRewardConfig, ReferralRewardListQuery,
|
||||
ReferralRewardRecord, ReferralUserDashboard,
|
||||
ReferralAdminStats, ReferralMutationStatus, ReferralReconciliationSummary,
|
||||
ReferralRelationshipListQuery, ReferralRelationshipRecord, ReferralRewardConfig,
|
||||
ReferralRewardListQuery, ReferralRewardRecord, ReferralUserDashboard,
|
||||
};
|
||||
|
||||
impl GatewayDataState {
|
||||
@@ -115,4 +115,13 @@ impl GatewayDataState {
|
||||
.reverse_referral_rewards_for_order(order_id, amount_usd)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn reconcile_referral_rewards_once(
|
||||
&self,
|
||||
reward_config: Option<ReferralRewardConfig>,
|
||||
) -> Result<ReferralReconciliationSummary, DataLayerError> {
|
||||
self.referrals()
|
||||
.reconcile_referral_rewards_once(reward_config)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -684,7 +684,7 @@ mod tests {
|
||||
.unwrap_or_default();
|
||||
// One Arc is retained by the map and every active request
|
||||
// owns one through its leader guard or follower state.
|
||||
if participant_count >= participants + 1 {
|
||||
if participant_count > participants {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
@@ -7,33 +7,39 @@ use super::{
|
||||
AdminWalletRefundRequestListQuery, AnnouncementListQuery, AuditLogListQuery,
|
||||
BackgroundTaskListQuery, BackgroundTaskSummary, BillingModelContextCacheKey,
|
||||
BillingModelContextCacheState, BillingModelContextInflightState, BillingPlanRecord,
|
||||
BillingPlanWriteInput, CompleteAdminWalletRefundInput, CreateAdminRedeemCodeBatchInput,
|
||||
BillingPlanWriteInput, CompareAndSwapPaymentOrderStripeClientSecretInput,
|
||||
CompleteAdminWalletRefundInput, CreateAdminRedeemCodeBatchInput,
|
||||
CreateAdminRedeemCodeBatchResult, CreateAnnouncementRecord, CreateManualWalletRechargeInput,
|
||||
CreatePlanPurchaseOrderInput, CreatePlanPurchaseOrderOutcome, CreateWalletRechargeOrderInput,
|
||||
CreateWalletRechargeOrderOutcome, CreateWalletRefundRequestInput,
|
||||
CreateWalletRefundRequestOutcome, CreditAdminPaymentOrderInput, DataLayerError,
|
||||
DatabaseMaintenanceSummary, DecisionTrace, DeleteAdminRedeemCodeBatchInput,
|
||||
DisableAdminRedeemCodeBatchInput, DisableAdminRedeemCodeInput, FailAdminWalletRefundInput,
|
||||
GatewayDataState, GatewayProviderTransportSnapshot, LocalVideoTaskReadResponse,
|
||||
PaymentGatewayConfigRecord, PaymentGatewayConfigWriteInput, ProcessAdminWalletRefundInput,
|
||||
ProcessPaymentCallbackInput, ProcessPaymentCallbackOutcome, RedeemWalletCodeInput,
|
||||
RedeemWalletCodeOutcome, RequestAuditBundle, RequestCandidateTrace, StoredAdminAuditLogPage,
|
||||
StoredAdminPaymentCallbackPage, StoredAdminPaymentOrder, StoredAdminPaymentOrderPage,
|
||||
StoredAdminRedeemCodeBatch, StoredAdminRedeemCodeBatchPage, StoredAdminRedeemCodePage,
|
||||
StoredAdminWalletLedgerPage, StoredAdminWalletListPage, StoredAdminWalletRefund,
|
||||
StoredAdminWalletRefundPage, StoredAdminWalletRefundRequestPage, StoredAdminWalletTransaction,
|
||||
FailWalletRechargeCheckoutInput, GatewayDataState, GatewayProviderTransportSnapshot,
|
||||
LocalVideoTaskReadResponse, PaymentGatewayConfigCasWriteInput, PaymentGatewayConfigRecord,
|
||||
PaymentGatewayConfigWriteInput, PaymentGatewaySecretCasUpdate, ProcessAdminWalletRefundInput,
|
||||
ProcessPaymentCallbackInput, ProcessPaymentCallbackOutcome, ReclaimWalletRechargeCheckoutInput,
|
||||
ReconcileUsagePolicyCostInput, RedeemWalletCodeInput, RedeemWalletCodeOutcome,
|
||||
ReleaseUsagePolicyRequestAdmissionInput, RequestAuditBundle, RequestCandidateTrace,
|
||||
ReserveUsagePolicyCostInput, ReserveUsagePolicyCostOutcome, ReserveUsagePolicyRequestInput,
|
||||
ReserveUsagePolicyRequestOutcome, StoredAdminAuditLogPage, StoredAdminPaymentCallbackPage,
|
||||
StoredAdminPaymentOrder, StoredAdminPaymentOrderPage, StoredAdminRedeemCodeBatch,
|
||||
StoredAdminRedeemCodeBatchPage, StoredAdminRedeemCodePage, StoredAdminWalletLedgerPage,
|
||||
StoredAdminWalletListPage, StoredAdminWalletRefund, StoredAdminWalletRefundPage,
|
||||
StoredAdminWalletRefundRequestPage, StoredAdminWalletTransaction,
|
||||
StoredAdminWalletTransactionPage, StoredAnnouncement, StoredAnnouncementPage,
|
||||
StoredBackgroundTaskEvent, StoredBackgroundTaskRun, StoredBackgroundTaskRunPage,
|
||||
StoredBillingModelContext, StoredProviderQuotaSnapshot, StoredProviderUsageSummary,
|
||||
StoredRequestUsageAudit, StoredSuspiciousActivity, StoredUsageSettlement,
|
||||
StoredUserAuditLogPage, StoredUserAuthRecord, StoredUserExportRow, StoredUserSummary,
|
||||
StoredVideoTask, StoredWalletDailyUsageLedger, StoredWalletDailyUsageLedgerPage,
|
||||
StoredWalletSnapshot, UpdateAnnouncementRecord, UpsertBackgroundTaskEvent,
|
||||
UpsertBackgroundTaskRun, UpsertUsageRecord, UpsertVideoTask, UsageSettlementInput,
|
||||
UserDailyQuotaAvailabilityRecord, UserPlanEntitlementRecord, VideoTaskLookupKey,
|
||||
VideoTaskModelCount, VideoTaskQueryFilter, VideoTaskStatusCount,
|
||||
WalletDailyUsageAggregationInput, WalletDailyUsageAggregationResult, WalletLookupKey,
|
||||
WalletMutationOutcome,
|
||||
StoredRequestUsageAudit, StoredSuspiciousActivity, StoredUsagePolicyCostReservation,
|
||||
StoredUsagePolicyRequestAdmission, StoredUsageSettlement, StoredUserAuditLogPage,
|
||||
StoredUserAuthRecord, StoredUserExportRow, StoredUserSummary, StoredVideoTask,
|
||||
StoredWalletDailyUsageLedger, StoredWalletDailyUsageLedgerPage, StoredWalletSnapshot,
|
||||
UpdateAdminWalletRefundGatewayInput, UpdateAnnouncementRecord,
|
||||
UpdateWalletRechargeCheckoutInput, UpsertBackgroundTaskEvent, UpsertBackgroundTaskRun,
|
||||
UpsertUsageRecord, UpsertVideoTask, UsageSettlementInput, UserDailyQuotaAvailabilityRecord,
|
||||
UserPlanEntitlementRecord, VideoTaskLookupKey, VideoTaskModelCount, VideoTaskQueryFilter,
|
||||
VideoTaskStatusCount, WalletDailyUsageAggregationInput, WalletDailyUsageAggregationResult,
|
||||
WalletLookupKey, WalletMutationOutcome,
|
||||
};
|
||||
use aether_data_contracts::repository::usage::{
|
||||
PendingUsageCleanupSummary, ProviderApiKeyWindowUsageRequest,
|
||||
@@ -43,7 +49,9 @@ use aether_data_contracts::repository::usage::{
|
||||
UsageDailyHeatmapQuery,
|
||||
};
|
||||
use aether_runtime_state::RuntimeQueueStore;
|
||||
use aether_video_tasks_core::read_data_backed_video_task_response;
|
||||
use aether_video_tasks_core::{
|
||||
read_data_backed_video_task_response, read_data_backed_video_task_response_for_user,
|
||||
};
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::time::timeout;
|
||||
|
||||
@@ -558,6 +566,17 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn find_video_task_for_user(
|
||||
&self,
|
||||
key: VideoTaskLookupKey<'_>,
|
||||
user_id: &str,
|
||||
) -> Result<Option<StoredVideoTask>, DataLayerError> {
|
||||
match &self.video_task_reader {
|
||||
Some(repository) => repository.find_for_user(key, user_id).await,
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn list_video_task_page(
|
||||
&self,
|
||||
filter: &VideoTaskQueryFilter,
|
||||
@@ -894,6 +913,21 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn find_wallet_recharge_order_by_order_no(
|
||||
&self,
|
||||
user_id: &str,
|
||||
order_no: &str,
|
||||
) -> Result<Option<StoredAdminPaymentOrder>, DataLayerError> {
|
||||
match &self.wallet_reader {
|
||||
Some(repository) => {
|
||||
repository
|
||||
.find_wallet_recharge_order_by_order_no(user_id, order_no)
|
||||
.await
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn find_pending_plan_purchase_order_by_user_id(
|
||||
&self,
|
||||
user_id: &str,
|
||||
@@ -909,6 +943,16 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn find_payment_order_by_order_no(
|
||||
&self,
|
||||
order_no: &str,
|
||||
) -> Result<Option<StoredAdminPaymentOrder>, DataLayerError> {
|
||||
match &self.wallet_reader {
|
||||
Some(repository) => repository.find_payment_order_by_order_no(order_no).await,
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn find_wallet_refund(
|
||||
&self,
|
||||
wallet_id: &str,
|
||||
@@ -934,6 +978,58 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn update_wallet_recharge_checkout(
|
||||
&self,
|
||||
input: UpdateWalletRechargeCheckoutInput,
|
||||
) -> Result<Option<WalletMutationOutcome<StoredAdminPaymentOrder>>, DataLayerError> {
|
||||
match &self.wallet_writer {
|
||||
Some(repository) => repository
|
||||
.update_wallet_recharge_checkout(input)
|
||||
.await
|
||||
.map(Some),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn compare_and_swap_payment_order_stripe_client_secret(
|
||||
&self,
|
||||
input: CompareAndSwapPaymentOrderStripeClientSecretInput,
|
||||
) -> Result<Option<bool>, DataLayerError> {
|
||||
match &self.wallet_writer {
|
||||
Some(repository) => repository
|
||||
.compare_and_swap_payment_order_stripe_client_secret(input)
|
||||
.await
|
||||
.map(Some),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn fail_wallet_recharge_checkout(
|
||||
&self,
|
||||
input: FailWalletRechargeCheckoutInput,
|
||||
) -> Result<Option<WalletMutationOutcome<StoredAdminPaymentOrder>>, DataLayerError> {
|
||||
match &self.wallet_writer {
|
||||
Some(repository) => repository
|
||||
.fail_wallet_recharge_checkout(input)
|
||||
.await
|
||||
.map(Some),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn reclaim_wallet_recharge_checkout(
|
||||
&self,
|
||||
input: ReclaimWalletRechargeCheckoutInput,
|
||||
) -> Result<Option<WalletMutationOutcome<StoredAdminPaymentOrder>>, DataLayerError> {
|
||||
match &self.wallet_writer {
|
||||
Some(repository) => repository
|
||||
.reclaim_wallet_recharge_checkout(input)
|
||||
.await
|
||||
.map(Some),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn create_plan_purchase_order(
|
||||
&self,
|
||||
input: CreatePlanPurchaseOrderInput,
|
||||
@@ -1009,6 +1105,19 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn update_admin_wallet_refund_gateway(
|
||||
&self,
|
||||
input: UpdateAdminWalletRefundGatewayInput,
|
||||
) -> Result<Option<WalletMutationOutcome<StoredAdminWalletRefund>>, DataLayerError> {
|
||||
match &self.wallet_writer {
|
||||
Some(repository) => repository
|
||||
.update_admin_wallet_refund_gateway(input)
|
||||
.await
|
||||
.map(Some),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn complete_admin_wallet_refund(
|
||||
&self,
|
||||
input: CompleteAdminWalletRefundInput,
|
||||
@@ -1151,6 +1260,83 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn reserve_usage_policy_cost(
|
||||
&self,
|
||||
input: ReserveUsagePolicyCostInput,
|
||||
) -> Result<Option<ReserveUsagePolicyCostOutcome>, DataLayerError> {
|
||||
match &self.settlement_writer {
|
||||
Some(repository) => repository.reserve_usage_policy_cost(input).await.map(Some),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn reserve_usage_policy_request(
|
||||
&self,
|
||||
input: ReserveUsagePolicyRequestInput,
|
||||
) -> Result<Option<ReserveUsagePolicyRequestOutcome>, DataLayerError> {
|
||||
match &self.settlement_writer {
|
||||
Some(repository) => repository
|
||||
.reserve_usage_policy_request(input)
|
||||
.await
|
||||
.map(Some),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn release_usage_policy_request_admission(
|
||||
&self,
|
||||
input: ReleaseUsagePolicyRequestAdmissionInput,
|
||||
) -> Result<Option<StoredUsagePolicyRequestAdmission>, DataLayerError> {
|
||||
match &self.settlement_writer {
|
||||
Some(repository) => {
|
||||
repository
|
||||
.release_usage_policy_request_admission(input)
|
||||
.await
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn cleanup_usage_policy_request_admissions(
|
||||
&self,
|
||||
now_unix_secs: u64,
|
||||
batch_size: usize,
|
||||
) -> Result<usize, DataLayerError> {
|
||||
match &self.settlement_writer {
|
||||
Some(repository) => {
|
||||
repository
|
||||
.cleanup_usage_policy_request_admissions(now_unix_secs, batch_size)
|
||||
.await
|
||||
}
|
||||
None => Ok(0),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn reconcile_usage_policy_cost(
|
||||
&self,
|
||||
input: ReconcileUsagePolicyCostInput,
|
||||
) -> Result<Option<StoredUsagePolicyCostReservation>, DataLayerError> {
|
||||
match &self.settlement_writer {
|
||||
Some(repository) => repository.reconcile_usage_policy_cost(input).await,
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn cleanup_usage_policy_cost_reservations(
|
||||
&self,
|
||||
now_unix_secs: u64,
|
||||
batch_size: usize,
|
||||
) -> Result<usize, DataLayerError> {
|
||||
match &self.settlement_writer {
|
||||
Some(repository) => {
|
||||
repository
|
||||
.cleanup_usage_policy_cost_reservations(now_unix_secs, batch_size)
|
||||
.await
|
||||
}
|
||||
None => Ok(0),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn reset_due_provider_quotas(
|
||||
&self,
|
||||
now_unix_secs: u64,
|
||||
@@ -2496,6 +2682,48 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn find_payment_gateway_config_strong(
|
||||
&self,
|
||||
provider: &str,
|
||||
) -> Result<Option<PaymentGatewayConfigRecord>, DataLayerError> {
|
||||
match &self.billing_reader {
|
||||
Some(repository) => {
|
||||
repository
|
||||
.find_payment_gateway_config_strong(provider)
|
||||
.await
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn compare_and_swap_payment_gateway_secret(
|
||||
&self,
|
||||
update: &PaymentGatewaySecretCasUpdate,
|
||||
) -> Result<bool, DataLayerError> {
|
||||
match &self.billing_reader {
|
||||
Some(repository) => {
|
||||
repository
|
||||
.compare_and_swap_payment_gateway_secret(update)
|
||||
.await
|
||||
}
|
||||
None => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn compare_and_swap_payment_gateway_config(
|
||||
&self,
|
||||
input: &PaymentGatewayConfigCasWriteInput,
|
||||
) -> Result<AdminBillingMutationOutcome<PaymentGatewayConfigRecord>, DataLayerError> {
|
||||
match &self.billing_reader {
|
||||
Some(repository) => {
|
||||
repository
|
||||
.compare_and_swap_payment_gateway_config(input)
|
||||
.await
|
||||
}
|
||||
None => Ok(AdminBillingMutationOutcome::Unavailable),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn upsert_payment_gateway_config(
|
||||
&self,
|
||||
input: &PaymentGatewayConfigWriteInput,
|
||||
@@ -2667,6 +2895,16 @@ impl GatewayDataState {
|
||||
read_data_backed_video_task_response(self, route_family, request_path).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_video_task_response_for_user(
|
||||
&self,
|
||||
route_family: Option<&str>,
|
||||
request_path: &str,
|
||||
user_id: &str,
|
||||
) -> Result<Option<LocalVideoTaskReadResponse>, DataLayerError> {
|
||||
read_data_backed_video_task_response_for_user(self, route_family, request_path, user_id)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn find_background_task_run(
|
||||
&self,
|
||||
run_id: &str,
|
||||
|
||||
@@ -1,10 +1,15 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::{Arc, RwLock};
|
||||
|
||||
use aether_data::repository::routing_profiles::InMemoryRoutingGroupRepository;
|
||||
use aether_data_contracts::repository::candidates::RequestCandidateRepository;
|
||||
use aether_data_contracts::repository::pool_scores::PoolMemberScoreRepository;
|
||||
use aether_data_contracts::repository::quota::ProviderQuotaRepository;
|
||||
use aether_data_contracts::repository::routing_profiles::{
|
||||
StoredRoutingGroup, StoredRoutingGroupBinding, StoredRoutingGroupVersion,
|
||||
};
|
||||
use aether_data_contracts::repository::usage::UsageRepository;
|
||||
use aether_routing_core::RoutingGroupConfig;
|
||||
|
||||
use super::{
|
||||
AnnouncementReadRepository, AnnouncementWriteRepository, AuthApiKeyReadRepository,
|
||||
@@ -213,6 +218,21 @@ impl GatewayDataState {
|
||||
self
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn with_cached_provider_catalog_reader_for_tests<T>(
|
||||
mut self,
|
||||
repository: Arc<T>,
|
||||
) -> Self
|
||||
where
|
||||
T: ProviderCatalogReadRepository + 'static,
|
||||
{
|
||||
let inner: Arc<dyn ProviderCatalogReadRepository> = repository;
|
||||
self.provider_catalog_reader = Some(Arc::new(
|
||||
super::provider_catalog_cache::CachedProviderCatalogReadRepository::new(inner),
|
||||
));
|
||||
self
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn with_request_candidate_reader(
|
||||
mut self,
|
||||
@@ -877,6 +897,30 @@ impl GatewayDataState {
|
||||
self
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn with_system_default_routing_group_for_tests(self) -> Self {
|
||||
let now = 1;
|
||||
let repository = Arc::new(InMemoryRoutingGroupRepository::seed(
|
||||
[StoredRoutingGroup {
|
||||
id: "system-default".to_string(),
|
||||
name: "system-default".to_string(),
|
||||
description: Some("test system default routing strategy".to_string()),
|
||||
enabled: true,
|
||||
is_system_default: true,
|
||||
sort_order: 0,
|
||||
config_json: serde_json::to_value(RoutingGroupConfig::default())
|
||||
.expect("default routing config should serialize"),
|
||||
version: 1,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
published_at: Some(now),
|
||||
}],
|
||||
std::iter::empty::<StoredRoutingGroupBinding>(),
|
||||
std::iter::empty::<StoredRoutingGroupVersion>(),
|
||||
));
|
||||
self.with_routing_group_repository_for_tests(repository)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn with_auth_api_key_reader(
|
||||
mut self,
|
||||
@@ -1746,6 +1790,18 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn attach_auth_api_key_repository_for_tests<T>(mut self, repository: Arc<T>) -> Self
|
||||
where
|
||||
T: aether_data::repository::auth::AuthRepository + 'static,
|
||||
{
|
||||
let auth_api_key_reader: Arc<dyn AuthApiKeyReadRepository> = repository.clone();
|
||||
let auth_api_key_writer: Arc<dyn AuthApiKeyWriteRepository> = repository;
|
||||
self.auth_api_key_reader = Some(auth_api_key_reader);
|
||||
self.auth_api_key_writer = Some(auth_api_key_writer);
|
||||
self
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn with_decision_trace_readers_for_tests(
|
||||
request_candidate_repository: Arc<dyn RequestCandidateReadRepository>,
|
||||
@@ -2335,6 +2391,36 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn with_auth_candidate_selection_provider_catalog_request_candidate_and_gemini_file_mapping_repositories_for_tests<
|
||||
T,
|
||||
U,
|
||||
V,
|
||||
>(
|
||||
auth_api_key_repository: Arc<dyn AuthApiKeyReadRepository>,
|
||||
candidate_selection_repository: Arc<dyn MinimalCandidateSelectionReadRepository>,
|
||||
provider_catalog_repository: Arc<U>,
|
||||
request_candidate_repository: Arc<T>,
|
||||
gemini_file_mapping_repository: Arc<V>,
|
||||
encryption_key: impl Into<String>,
|
||||
) -> Self
|
||||
where
|
||||
T: RequestCandidateRepository + 'static,
|
||||
U: ProviderCatalogReadRepository + ProviderCatalogWriteRepository + 'static,
|
||||
V: aether_data::repository::gemini_file_mappings::GeminiFileMappingRepository + 'static,
|
||||
{
|
||||
let mut state = Self::with_auth_candidate_selection_provider_catalog_and_request_candidate_repository_for_tests(
|
||||
auth_api_key_repository,
|
||||
candidate_selection_repository,
|
||||
provider_catalog_repository,
|
||||
request_candidate_repository,
|
||||
encryption_key,
|
||||
);
|
||||
state.gemini_file_mapping_reader = Some(gemini_file_mapping_repository.clone());
|
||||
state.gemini_file_mapping_writer = Some(gemini_file_mapping_repository);
|
||||
state
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn with_auth_candidate_selection_provider_catalog_request_candidates_for_tests<
|
||||
T,
|
||||
|
||||
@@ -12,10 +12,113 @@ use aether_data_contracts::repository::usage::{
|
||||
};
|
||||
|
||||
use aether_data::repository::auth::AuthApiKeyReadRepository;
|
||||
use aether_data::repository::management_tokens::{
|
||||
InMemoryManagementTokenRepository, ManagementTokenReadRepository,
|
||||
ManagementTokenWriteRepository, StoredManagementToken, StoredManagementTokenUserSummary,
|
||||
StoredManagementTokenWithUser,
|
||||
};
|
||||
use aether_data::repository::proxy_nodes::{InMemoryProxyNodeRepository, StoredProxyNode};
|
||||
use aether_data::repository::proxy_nodes::{ProxyNodeReadRepository, ProxyNodeWriteRepository};
|
||||
use aether_data::repository::users::{
|
||||
InMemoryUserReadRepository, StoredUserAuthRecord, UserReadRepository,
|
||||
};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::{GatewayDataConfig, GatewayDataState};
|
||||
|
||||
impl GatewayDataState {
|
||||
pub(crate) fn with_tunnel_management_auth_for_testkit(
|
||||
node_id: &str,
|
||||
tunnel_generation: &str,
|
||||
raw_token: &str,
|
||||
encryption_key: impl Into<String>,
|
||||
) -> Result<Self, aether_data::DataLayerError> {
|
||||
const TOKEN_ID: &str = "token-tunnel-harness";
|
||||
const USER_ID: &str = "user-tunnel-harness";
|
||||
|
||||
let node = StoredProxyNode::new(
|
||||
node_id.to_string(),
|
||||
"tunnel harness node".to_string(),
|
||||
"127.0.0.1".to_string(),
|
||||
0,
|
||||
false,
|
||||
"offline".to_string(),
|
||||
30,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
true,
|
||||
false,
|
||||
0,
|
||||
)?
|
||||
.with_tunnel_generation(tunnel_generation.to_string());
|
||||
let proxy_repository = Arc::new(InMemoryProxyNodeRepository::seed([node]));
|
||||
|
||||
let user_summary = StoredManagementTokenUserSummary::new(
|
||||
USER_ID.to_string(),
|
||||
Some("[email protected]".to_string()),
|
||||
"tunnel_harness_admin".to_string(),
|
||||
"admin".to_string(),
|
||||
)?;
|
||||
let token = StoredManagementToken::new(
|
||||
TOKEN_ID.to_string(),
|
||||
USER_ID.to_string(),
|
||||
"tunnel harness token".to_string(),
|
||||
)?
|
||||
.with_permissions(Some(serde_json::json!(["admin:proxy_nodes:admin"])));
|
||||
let token_hash = format!("{:x}", Sha256::digest(raw_token.as_bytes()));
|
||||
let token_repository = Arc::new(InMemoryManagementTokenRepository::seed_with_hashes(
|
||||
[StoredManagementTokenWithUser::new(token, user_summary)],
|
||||
[(token_hash, TOKEN_ID.to_string())],
|
||||
));
|
||||
let token_reader: Arc<dyn ManagementTokenReadRepository> = token_repository.clone();
|
||||
let token_writer: Arc<dyn ManagementTokenWriteRepository> = token_repository;
|
||||
|
||||
let user = StoredUserAuthRecord::new(
|
||||
USER_ID.to_string(),
|
||||
Some("[email protected]".to_string()),
|
||||
true,
|
||||
"tunnel_harness_admin".to_string(),
|
||||
None,
|
||||
"admin".to_string(),
|
||||
"local".to_string(),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
true,
|
||||
false,
|
||||
None,
|
||||
None,
|
||||
)?;
|
||||
let user_reader: Arc<dyn UserReadRepository> =
|
||||
Arc::new(InMemoryUserReadRepository::seed_auth_users([user]));
|
||||
|
||||
let mut state =
|
||||
Self::with_proxy_node_repository_for_testkit(proxy_repository, encryption_key);
|
||||
state.management_token_reader = Some(token_reader);
|
||||
state.management_token_writer = Some(token_writer);
|
||||
state.user_reader = Some(user_reader);
|
||||
Ok(state)
|
||||
}
|
||||
|
||||
pub(crate) fn with_proxy_node_repository_for_testkit<T>(
|
||||
repository: Arc<T>,
|
||||
encryption_key: impl Into<String>,
|
||||
) -> Self
|
||||
where
|
||||
T: ProxyNodeReadRepository + ProxyNodeWriteRepository + 'static,
|
||||
{
|
||||
let proxy_node_reader: Arc<dyn ProxyNodeReadRepository> = repository.clone();
|
||||
let proxy_node_writer: Arc<dyn ProxyNodeWriteRepository> = repository;
|
||||
let mut state = Self::disabled();
|
||||
state.config = GatewayDataConfig::disabled().with_encryption_key(encryption_key);
|
||||
state.proxy_node_reader = Some(proxy_node_reader);
|
||||
state.proxy_node_writer = Some(proxy_node_writer);
|
||||
state
|
||||
}
|
||||
|
||||
pub(crate) fn with_openai_chat_pressure_repositories_for_testkit<T, U, V>(
|
||||
auth_api_key_repository: Arc<dyn AuthApiKeyReadRepository>,
|
||||
candidate_selection_repository: Arc<dyn MinimalCandidateSelectionReadRepository>,
|
||||
|
||||
@@ -254,40 +254,6 @@ async fn app_state_wires_gateway_data_state_from_config() {
|
||||
assert!(state.data.has_video_task_reader());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn app_state_prepares_sqlite_database_startup() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let mut pool = SqlPoolConfig::default();
|
||||
pool.min_connections = 0;
|
||||
pool.max_connections = 1;
|
||||
let database = SqlDatabaseConfig::new(DatabaseDriver::Sqlite, "sqlite::memory:", pool)?;
|
||||
let state =
|
||||
AppState::new()?.with_data_config(GatewayDataConfig::from_database_config(database))?;
|
||||
|
||||
let pending = state
|
||||
.prepare_database_for_startup()
|
||||
.await?
|
||||
.expect("sqlite database should expose migration state");
|
||||
assert!(
|
||||
!pending.is_empty(),
|
||||
"fresh sqlite gateway databases should report pending migrations"
|
||||
);
|
||||
|
||||
assert!(
|
||||
state.run_database_migrations().await?,
|
||||
"sqlite gateway database should run migrations"
|
||||
);
|
||||
let pending = state
|
||||
.prepare_database_for_startup()
|
||||
.await?
|
||||
.expect("sqlite database should expose migration state");
|
||||
assert!(
|
||||
pending.is_empty(),
|
||||
"sqlite gateway databases should be current after migrations"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn data_state_checks_user_uniqueness_through_user_reader() {
|
||||
let user = StoredUserAuthRecord::new(
|
||||
@@ -312,7 +278,7 @@ async fn data_state_checks_user_uniqueness_through_user_reader() {
|
||||
Some("[email protected]".to_string()),
|
||||
true,
|
||||
"admin".to_string(),
|
||||
Some(format!("$2b$12${}", "a".repeat(53))),
|
||||
Some("$2b$12$4qL4tdcsFwVaDTw5Ck3xzu8GpNdre56DiNR6Dnw7t6gCXaEnqAe7G".to_string()),
|
||||
"admin".to_string(),
|
||||
"local".to_string(),
|
||||
None,
|
||||
|
||||
@@ -318,7 +318,9 @@ fn active_probe_member_is_unschedulable_for_request(
|
||||
}) {
|
||||
return true;
|
||||
}
|
||||
key_context.is_some_and(|context| context.account_blocked || context.quota_exhausted)
|
||||
key_context.is_some_and(|context| {
|
||||
context.account_blocked || context.quota_exhausted || context.quota_hard_blocked
|
||||
})
|
||||
}
|
||||
|
||||
async fn expand_pool_group_candidate(
|
||||
@@ -1456,13 +1458,30 @@ async fn read_pool_catalog_key_contexts_by_id(
|
||||
key_count = key_ids.len(),
|
||||
"gateway pool scheduler: failed to read catalog key metadata"
|
||||
);
|
||||
return BTreeMap::new();
|
||||
// Do not fail open when the quota metadata read is unavailable. A
|
||||
// missing context must never turn an exhausted account into an
|
||||
// eligible candidate and produce another upstream 429. The caller
|
||||
// treats this marker as a pool quota skip and the next request will
|
||||
// retry the metadata read.
|
||||
return key_ids
|
||||
.into_iter()
|
||||
.map(|key_id| {
|
||||
(
|
||||
key_id,
|
||||
PoolCatalogKeyContext {
|
||||
quota_hard_blocked: true,
|
||||
..PoolCatalogKeyContext::default()
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
}
|
||||
};
|
||||
|
||||
let provider_pool_service = ProviderPoolService::with_builtin_adapters();
|
||||
|
||||
keys.into_iter()
|
||||
let mut contexts = keys
|
||||
.into_iter()
|
||||
.map(|key| {
|
||||
let provider_type = provider_type_by_key_id
|
||||
.get(&key.id)
|
||||
@@ -1479,7 +1498,19 @@ async fn read_pool_catalog_key_contexts_by_id(
|
||||
),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
// A key can disappear between the candidate-row and catalog reads. Keep
|
||||
// the snapshot non-empty and fail closed for those IDs so the caller does
|
||||
// not interpret an incomplete read as "all accounts are healthy".
|
||||
for key_id in key_ids {
|
||||
contexts
|
||||
.entry(key_id)
|
||||
.or_insert_with(|| PoolCatalogKeyContext {
|
||||
quota_hard_blocked: true,
|
||||
..PoolCatalogKeyContext::default()
|
||||
});
|
||||
}
|
||||
contexts
|
||||
}
|
||||
|
||||
fn build_pool_catalog_key_context(
|
||||
@@ -1724,7 +1755,21 @@ fn run_local_execution_pool_scheduler_with_runtime_map(
|
||||
let key_context = key_context_by_id
|
||||
.get(&candidate.candidate.key_id)
|
||||
.cloned()
|
||||
.unwrap_or_default();
|
||||
.unwrap_or_else(|| {
|
||||
// An explicitly non-empty metadata snapshot should contain
|
||||
// every catalog key in this page. If one disappeared between
|
||||
// reads, fail closed for that key instead of sending traffic
|
||||
// with an unknown quota state. Empty maps are retained for
|
||||
// callers/tests that intentionally provide no runtime context.
|
||||
if key_context_by_id.is_empty() {
|
||||
PoolCatalogKeyContext::default()
|
||||
} else {
|
||||
PoolCatalogKeyContext {
|
||||
quota_hard_blocked: true,
|
||||
..PoolCatalogKeyContext::default()
|
||||
}
|
||||
}
|
||||
});
|
||||
let admin_pool_config = effective_pool_config_by_provider
|
||||
.get(&candidate.candidate.provider_id)
|
||||
.cloned()
|
||||
@@ -2000,7 +2045,7 @@ mod tests {
|
||||
use aether_data_contracts::repository::provider_catalog::{
|
||||
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
|
||||
};
|
||||
use aether_pool_core::PoolSchedulingPreset;
|
||||
use aether_pool_core::{PoolSchedulingPreset, POOL_ACCOUNT_EXHAUSTED_SKIP_REASON};
|
||||
use aether_provider_pool::ProviderPoolService;
|
||||
use aether_provider_transport::snapshot::{
|
||||
GatewayProviderTransportEndpoint, GatewayProviderTransportKey,
|
||||
@@ -2112,6 +2157,55 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_scheduler_skips_quota_exhausted_key_when_flag_is_false() {
|
||||
let ready = sample_eligible_candidate(
|
||||
"provider-pool",
|
||||
"endpoint-1",
|
||||
"key-ready",
|
||||
10,
|
||||
Some(json!({ "pool_advanced": {} })),
|
||||
);
|
||||
let exhausted = sample_eligible_candidate(
|
||||
"provider-pool",
|
||||
"endpoint-1",
|
||||
"key-exhausted",
|
||||
10,
|
||||
Some(json!({ "pool_advanced": { "skip_exhausted_accounts": false } })),
|
||||
);
|
||||
let key_context_by_id = BTreeMap::from([
|
||||
("key-ready".to_string(), PoolCatalogKeyContext::default()),
|
||||
(
|
||||
"key-exhausted".to_string(),
|
||||
PoolCatalogKeyContext {
|
||||
quota_exhausted: true,
|
||||
..PoolCatalogKeyContext::default()
|
||||
},
|
||||
),
|
||||
]);
|
||||
|
||||
let (scheduled, skipped) = apply_local_execution_pool_scheduler_with_runtime_map(
|
||||
vec![ready, exhausted],
|
||||
&BTreeMap::new(),
|
||||
&key_context_by_id,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
scheduled
|
||||
.iter()
|
||||
.map(|item| item.candidate.key_id.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["key-ready"]
|
||||
);
|
||||
assert_eq!(
|
||||
skipped
|
||||
.iter()
|
||||
.map(|item| (item.candidate.key_id.as_str(), item.skip_reason))
|
||||
.collect::<Vec<_>>(),
|
||||
vec![("key-exhausted", POOL_ACCOUNT_EXHAUSTED_SKIP_REASON)]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_scheduler_attaches_group_and_pool_metadata_to_ranked_candidates() {
|
||||
let pool_first = sample_eligible_candidate(
|
||||
@@ -4712,6 +4806,15 @@ mod tests {
|
||||
))
|
||||
}
|
||||
|
||||
fn provider_catalog_credential_state() -> AppState {
|
||||
AppState::new()
|
||||
.expect("credential state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::disabled()
|
||||
.with_encryption_key_for_tests(aether_crypto::DEVELOPMENT_ENCRYPTION_KEY),
|
||||
)
|
||||
}
|
||||
|
||||
fn large_pool_fixture(
|
||||
key_count: usize,
|
||||
provider_config: Option<serde_json::Value>,
|
||||
@@ -4762,10 +4865,18 @@ mod tests {
|
||||
)
|
||||
.expect("endpoint transport should build");
|
||||
|
||||
let credential_state = provider_catalog_credential_state();
|
||||
let mut keys = Vec::with_capacity(key_count);
|
||||
let mut rows = Vec::with_capacity(key_count);
|
||||
for index in 0..key_count {
|
||||
let key_id = format!("key-{index:05}");
|
||||
let encrypted_api_key = credential_state
|
||||
.seal_provider_catalog_key_api_key(
|
||||
"provider-pool",
|
||||
&key_id,
|
||||
&format!("secret-{index}"),
|
||||
)
|
||||
.expect("api key should encrypt");
|
||||
let mut key = StoredProviderCatalogKey::new(
|
||||
key_id.clone(),
|
||||
"provider-pool".to_string(),
|
||||
@@ -4777,7 +4888,7 @@ mod tests {
|
||||
.expect("key should build")
|
||||
.with_transport_fields(
|
||||
Some(json!(["openai:chat"])),
|
||||
Some(format!("secret-{index}")),
|
||||
encrypted_api_key,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
@@ -4913,6 +5024,9 @@ mod tests {
|
||||
}
|
||||
|
||||
fn sample_codex_pool_key(provider_id: &str, key_id: &str) -> StoredProviderCatalogKey {
|
||||
let encrypted_api_key = provider_catalog_credential_state()
|
||||
.seal_provider_catalog_key_api_key(provider_id, key_id, &format!("secret-{key_id}"))
|
||||
.expect("api key should encrypt");
|
||||
let mut key = StoredProviderCatalogKey::new(
|
||||
key_id.to_string(),
|
||||
provider_id.to_string(),
|
||||
@@ -4924,7 +5038,7 @@ mod tests {
|
||||
.expect("key should build")
|
||||
.with_transport_fields(
|
||||
Some(json!(["openai:responses"])),
|
||||
Some(format!("secret-{key_id}")),
|
||||
encrypted_api_key,
|
||||
None,
|
||||
None,
|
||||
Some(json!({"openai:responses": 1})),
|
||||
|
||||
@@ -1,13 +1,28 @@
|
||||
use base64::Engine;
|
||||
|
||||
use crate::handlers::shared::{
|
||||
decrypt_catalog_secret_with_fallbacks, system_config_bool, system_config_string,
|
||||
decrypt_or_migrate_smtp_password, smtp_password_binding, system_config_bool,
|
||||
};
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
const SMTP_TIMEOUT_SECS: u64 = 30;
|
||||
const SMTP_MAX_HOST_BYTES: usize = 255;
|
||||
const SMTP_MAX_ADDRESS_BYTES: usize = 320;
|
||||
const SMTP_MAX_HEADER_VALUE_BYTES: usize = 512;
|
||||
const SMTP_MAX_USERNAME_BYTES: usize = 320;
|
||||
const SMTP_MAX_PASSWORD_BYTES: usize = 16 * 1024;
|
||||
const SMTP_MAX_STORED_PASSWORD_BYTES: usize = 64 * 1024;
|
||||
const SMTP_MAX_BODY_BYTES: usize = 2 * 1024 * 1024;
|
||||
const SMTP_MAX_MESSAGE_BYTES: usize = 8 * 1024 * 1024;
|
||||
const SMTP_MAX_DIAGNOSTIC_BYTES: usize = 4096;
|
||||
// SMTP servers normally emit short ASCII status lines. Keep parser buffers
|
||||
// bounded even when the peer is untrusted or compromised; these limits apply
|
||||
// only to control responses, not to the message body being submitted.
|
||||
const SMTP_MAX_RESPONSE_LINE_BYTES: usize = 16 * 1024;
|
||||
const SMTP_MAX_RESPONSE_BYTES: usize = 256 * 1024;
|
||||
const SMTP_MAX_RESPONSE_LINES: usize = 128;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct SmtpDeliveryConfig {
|
||||
pub(crate) host: String,
|
||||
pub(crate) port: u16,
|
||||
@@ -19,7 +34,7 @@ pub(crate) struct SmtpDeliveryConfig {
|
||||
pub(crate) from_name: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct ComposedEmail {
|
||||
pub(crate) to_email: String,
|
||||
pub(crate) subject: String,
|
||||
@@ -27,6 +42,26 @@ pub(crate) struct ComposedEmail {
|
||||
pub(crate) text_body: String,
|
||||
}
|
||||
|
||||
fn bounded_system_config_string(
|
||||
field: &str,
|
||||
value: Option<&serde_json::Value>,
|
||||
max_bytes: usize,
|
||||
) -> Result<Option<String>, GatewayError> {
|
||||
let Some(serde_json::Value::String(raw)) = value else {
|
||||
return Ok(None);
|
||||
};
|
||||
let value = raw.trim();
|
||||
if value.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
if value.len() > max_bytes {
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"smtp {field} exceeds the allowed size"
|
||||
)));
|
||||
}
|
||||
Ok(Some(value.to_string()))
|
||||
}
|
||||
|
||||
pub(crate) async fn read_smtp_delivery_config(
|
||||
state: &AppState,
|
||||
) -> Result<Option<SmtpDeliveryConfig>, GatewayError> {
|
||||
@@ -34,10 +69,16 @@ pub(crate) async fn read_smtp_delivery_config(
|
||||
let smtp_from_email = state
|
||||
.read_system_config_json_value("smtp_from_email")
|
||||
.await?;
|
||||
let Some(host) = system_config_string(smtp_host.as_ref()) else {
|
||||
let Some(host) = bounded_system_config_string("host", smtp_host.as_ref(), SMTP_MAX_HOST_BYTES)?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(from_email) = system_config_string(smtp_from_email.as_ref()) else {
|
||||
let Some(from_email) = bounded_system_config_string(
|
||||
"from_email",
|
||||
smtp_from_email.as_ref(),
|
||||
SMTP_MAX_ADDRESS_BYTES,
|
||||
)?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
@@ -50,20 +91,43 @@ pub(crate) async fn read_smtp_delivery_config(
|
||||
.read_system_config_json_value("smtp_from_name")
|
||||
.await?;
|
||||
|
||||
let password = system_config_string(smtp_password.as_ref()).map(|value| {
|
||||
decrypt_catalog_secret_with_fallbacks(state.encryption_key(), &value).unwrap_or(value)
|
||||
});
|
||||
let port = system_config_u16(smtp_port.as_ref(), 587);
|
||||
let user = bounded_system_config_string("user", smtp_user.as_ref(), SMTP_MAX_USERNAME_BYTES)?;
|
||||
let use_tls = system_config_bool(smtp_use_tls.as_ref(), true);
|
||||
let use_ssl = system_config_bool(smtp_use_ssl.as_ref(), false);
|
||||
let password = match (
|
||||
bounded_system_config_string(
|
||||
"stored_password",
|
||||
smtp_password.as_ref(),
|
||||
SMTP_MAX_STORED_PASSWORD_BYTES,
|
||||
)?,
|
||||
smtp_password_binding(&host, port, user.as_deref(), use_tls, use_ssl),
|
||||
) {
|
||||
(Some(value), Some(binding)) => {
|
||||
Some(decrypt_or_migrate_smtp_password(state, &binding, value).await?)
|
||||
}
|
||||
(Some(_), None) => {
|
||||
return Err(GatewayError::Internal(
|
||||
"SMTP password binding is invalid".to_string(),
|
||||
));
|
||||
}
|
||||
(None, _) => None,
|
||||
};
|
||||
|
||||
Ok(Some(SmtpDeliveryConfig {
|
||||
host,
|
||||
port: system_config_u16(smtp_port.as_ref(), 587),
|
||||
user: system_config_string(smtp_user.as_ref()),
|
||||
port,
|
||||
user,
|
||||
password,
|
||||
use_tls: system_config_bool(smtp_use_tls.as_ref(), true),
|
||||
use_ssl: system_config_bool(smtp_use_ssl.as_ref(), false),
|
||||
use_tls,
|
||||
use_ssl,
|
||||
from_email,
|
||||
from_name: system_config_string(smtp_from_name.as_ref())
|
||||
.unwrap_or_else(|| "Aether".to_string()),
|
||||
from_name: bounded_system_config_string(
|
||||
"from_name",
|
||||
smtp_from_name.as_ref(),
|
||||
SMTP_MAX_HEADER_VALUE_BYTES,
|
||||
)?
|
||||
.unwrap_or_else(|| "Aether".to_string()),
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -71,17 +135,150 @@ pub(crate) async fn send_smtp_email(
|
||||
config: SmtpDeliveryConfig,
|
||||
email: ComposedEmail,
|
||||
) -> Result<(), GatewayError> {
|
||||
validate_smtp_delivery_inputs(&config, &email)?;
|
||||
tokio::task::spawn_blocking(move || send_smtp_email_blocking(config, email))
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||
}
|
||||
|
||||
pub(crate) async fn probe_smtp_connection(config: SmtpDeliveryConfig) -> Result<(), GatewayError> {
|
||||
validate_smtp_config(&config)?;
|
||||
tokio::task::spawn_blocking(move || probe_smtp_connection_blocking(config))
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||
}
|
||||
|
||||
fn validate_smtp_control_field(field: &str, value: &str) -> Result<(), GatewayError> {
|
||||
if value.bytes().any(|byte| byte < 0x20 || byte == 0x7f) {
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"smtp {field} contains forbidden control characters"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_smtp_bounded_field(
|
||||
field: &str,
|
||||
value: &str,
|
||||
max_bytes: usize,
|
||||
) -> Result<(), GatewayError> {
|
||||
validate_smtp_control_field(field, value)?;
|
||||
if value.len() > max_bytes {
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"smtp {field} exceeds the allowed size"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_smtp_body_field(field: &str, value: &str) -> Result<(), GatewayError> {
|
||||
// Bodies are base64 encoded before DATA is written, so line breaks and
|
||||
// tabs are valid content. NUL is still rejected because it is not valid
|
||||
// textual mail content and can confuse downstream gateways.
|
||||
if value.bytes().any(|byte| byte == 0) {
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"smtp {field} contains a forbidden NUL byte"
|
||||
)));
|
||||
}
|
||||
if value.len() > SMTP_MAX_BODY_BYTES {
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"smtp {field} exceeds the allowed size"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_smtp_address(field: &str, value: &str) -> Result<(), GatewayError> {
|
||||
validate_smtp_bounded_field(field, value, SMTP_MAX_ADDRESS_BYTES)?;
|
||||
if value.is_empty() || value.trim() != value || value.chars().any(char::is_whitespace) {
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"smtp {field} must be a single mailbox address"
|
||||
)));
|
||||
}
|
||||
// Addresses are inserted inside SMTP angle brackets. Reject delimiters
|
||||
// that could turn one envelope/header value into multiple fields.
|
||||
if value
|
||||
.bytes()
|
||||
.any(|byte| matches!(byte, b'<' | b'>' | b',' | b';' | b'"' | b'\\'))
|
||||
{
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"smtp {field} contains invalid mailbox delimiters"
|
||||
)));
|
||||
}
|
||||
let Some((local, domain)) = value.split_once('@') else {
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"smtp {field} must contain a mailbox domain"
|
||||
)));
|
||||
};
|
||||
if local.is_empty() || domain.is_empty() || domain.contains('@') {
|
||||
return Err(GatewayError::Internal(format!(
|
||||
"smtp {field} must contain a valid mailbox domain"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_smtp_auth_config(config: &SmtpDeliveryConfig) -> Result<(), GatewayError> {
|
||||
let username = config.user.as_deref().map(str::trim);
|
||||
let has_username = username.is_some_and(|value| !value.is_empty());
|
||||
let has_password = config.password.is_some();
|
||||
if has_username != has_password {
|
||||
return Err(GatewayError::Internal(
|
||||
"smtp username and password must be configured together".to_string(),
|
||||
));
|
||||
}
|
||||
if let Some(username) = username.filter(|value| !value.is_empty()) {
|
||||
validate_smtp_bounded_field("user", username, SMTP_MAX_USERNAME_BYTES)?;
|
||||
if !config.use_tls && !config.use_ssl {
|
||||
return Err(GatewayError::Internal(
|
||||
"smtp authentication requires TLS or SSL encryption".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
if let Some(password) = config.password.as_deref() {
|
||||
validate_smtp_bounded_field("password", password, SMTP_MAX_PASSWORD_BYTES)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_smtp_config(config: &SmtpDeliveryConfig) -> Result<(), GatewayError> {
|
||||
validate_smtp_bounded_field("host", &config.host, SMTP_MAX_HOST_BYTES)?;
|
||||
if config.host.is_empty() || config.host.trim() != config.host {
|
||||
return Err(GatewayError::Internal(
|
||||
"smtp host must not be empty or padded".to_string(),
|
||||
));
|
||||
}
|
||||
if config.host.chars().any(char::is_whitespace) {
|
||||
return Err(GatewayError::Internal(
|
||||
"smtp host contains invalid whitespace".to_string(),
|
||||
));
|
||||
}
|
||||
if config.port == 0 {
|
||||
return Err(GatewayError::Internal(
|
||||
"smtp port must be non-zero".to_string(),
|
||||
));
|
||||
}
|
||||
if config.use_tls && config.use_ssl {
|
||||
return Err(GatewayError::Internal(
|
||||
"smtp TLS and SSL modes cannot both be enabled".to_string(),
|
||||
));
|
||||
}
|
||||
validate_smtp_address("from_email", &config.from_email)?;
|
||||
validate_smtp_bounded_field("from_name", &config.from_name, SMTP_MAX_HEADER_VALUE_BYTES)?;
|
||||
validate_smtp_auth_config(config)
|
||||
}
|
||||
|
||||
fn validate_smtp_delivery_inputs(
|
||||
config: &SmtpDeliveryConfig,
|
||||
email: &ComposedEmail,
|
||||
) -> Result<(), GatewayError> {
|
||||
validate_smtp_config(config)?;
|
||||
validate_smtp_address("to_email", &email.to_email)?;
|
||||
validate_smtp_bounded_field("subject", &email.subject, SMTP_MAX_HEADER_VALUE_BYTES)?;
|
||||
validate_smtp_body_field("html_body", &email.html_body)?;
|
||||
validate_smtp_body_field("text_body", &email.text_body)
|
||||
}
|
||||
|
||||
pub(crate) fn system_config_u16(value: Option<&serde_json::Value>, default: u16) -> u16 {
|
||||
match value {
|
||||
Some(serde_json::Value::Number(value)) => value
|
||||
@@ -132,8 +329,42 @@ fn resolve_server_name(host: &str) -> Result<rustls::pki_types::ServerName<'stat
|
||||
}
|
||||
|
||||
fn connect_tcp_stream(config: &SmtpDeliveryConfig) -> Result<std::net::TcpStream, GatewayError> {
|
||||
let stream = std::net::TcpStream::connect((config.host.as_str(), config.port))
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
use std::net::ToSocketAddrs;
|
||||
let addresses = (config.host.as_str(), config.port)
|
||||
.to_socket_addrs()
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||
.take(16)
|
||||
.collect::<Vec<_>>();
|
||||
if addresses.is_empty() {
|
||||
return Err(GatewayError::Internal(
|
||||
"smtp host did not resolve to an address".to_string(),
|
||||
));
|
||||
}
|
||||
let deadline = std::time::Instant::now()
|
||||
.checked_add(std::time::Duration::from_secs(SMTP_TIMEOUT_SECS))
|
||||
.unwrap_or_else(std::time::Instant::now);
|
||||
let mut last_error = None;
|
||||
let mut stream = None;
|
||||
for address in addresses {
|
||||
let remaining = deadline.saturating_duration_since(std::time::Instant::now());
|
||||
if remaining.is_zero() {
|
||||
break;
|
||||
}
|
||||
match std::net::TcpStream::connect_timeout(&address, remaining) {
|
||||
Ok(candidate) => {
|
||||
stream = Some(candidate);
|
||||
break;
|
||||
}
|
||||
Err(err) => last_error = Some(err),
|
||||
}
|
||||
}
|
||||
let stream = stream.ok_or_else(|| {
|
||||
GatewayError::Internal(
|
||||
last_error
|
||||
.map(|err| err.to_string())
|
||||
.unwrap_or_else(|| "smtp connection timed out".to_string()),
|
||||
)
|
||||
})?;
|
||||
stream
|
||||
.set_read_timeout(Some(std::time::Duration::from_secs(SMTP_TIMEOUT_SECS)))
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
@@ -155,38 +386,109 @@ fn wrap_tls_stream(
|
||||
|
||||
fn smtp_read_response<T: std::io::BufRead>(reader: &mut T) -> Result<(u16, String), GatewayError> {
|
||||
let mut message = String::new();
|
||||
let code = loop {
|
||||
let parsed_code;
|
||||
let continuation;
|
||||
let trimmed;
|
||||
{
|
||||
let mut line = String::new();
|
||||
let bytes = reader
|
||||
.read_line(&mut line)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
if bytes == 0 {
|
||||
let mut expected_code = None;
|
||||
for line_number in 0..SMTP_MAX_RESPONSE_LINES {
|
||||
let mut line = Vec::new();
|
||||
let bytes = read_smtp_response_line(reader, &mut line)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
if bytes == 0 {
|
||||
return Err(GatewayError::Internal(
|
||||
"smtp connection closed unexpectedly".to_string(),
|
||||
));
|
||||
}
|
||||
if line.len() > SMTP_MAX_RESPONSE_LINE_BYTES {
|
||||
return Err(GatewayError::Internal(
|
||||
"smtp response line exceeds the allowed size".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
// Strip only the protocol line ending. The remaining bytes are kept
|
||||
// for diagnostics after validating that they are UTF-8.
|
||||
while matches!(line.last(), Some(b'\r' | b'\n')) {
|
||||
line.pop();
|
||||
}
|
||||
if line.len() < 3 || !line[..3].iter().all(|byte| byte.is_ascii_digit()) {
|
||||
return Err(GatewayError::Internal("invalid smtp response".to_string()));
|
||||
}
|
||||
let parsed_code = u16::from(line[0] - b'0') * 100
|
||||
+ u16::from(line[1] - b'0') * 10
|
||||
+ u16::from(line[2] - b'0');
|
||||
if let Some(expected_code) = expected_code {
|
||||
if parsed_code != expected_code {
|
||||
return Err(GatewayError::Internal(
|
||||
"smtp connection closed unexpectedly".to_string(),
|
||||
"smtp response continuation code changed".to_string(),
|
||||
));
|
||||
}
|
||||
trimmed = line.trim_end_matches(['\r', '\n']).to_string();
|
||||
if trimmed.len() < 3 {
|
||||
return Err(GatewayError::Internal("invalid smtp response".to_string()));
|
||||
}
|
||||
parsed_code = trimmed[..3]
|
||||
.parse::<u16>()
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
continuation = trimmed.as_bytes().get(3).copied() == Some(b'-');
|
||||
} else {
|
||||
expected_code = Some(parsed_code);
|
||||
}
|
||||
let separator = line.get(3).copied().unwrap_or(b' ');
|
||||
if separator != b'-' && separator != b' ' {
|
||||
return Err(GatewayError::Internal("invalid smtp response".to_string()));
|
||||
}
|
||||
|
||||
let trimmed = std::str::from_utf8(&line)
|
||||
.map_err(|_| GatewayError::Internal("smtp response is not valid UTF-8".to_string()))?;
|
||||
let additional = trimmed.len() + usize::from(!message.is_empty());
|
||||
if message
|
||||
.len()
|
||||
.checked_add(additional)
|
||||
.is_none_or(|length| length > SMTP_MAX_RESPONSE_BYTES)
|
||||
{
|
||||
return Err(GatewayError::Internal(
|
||||
"smtp response exceeds the allowed size".to_string(),
|
||||
));
|
||||
}
|
||||
if !message.is_empty() {
|
||||
message.push('\n');
|
||||
}
|
||||
message.push_str(&trimmed);
|
||||
if !continuation {
|
||||
break parsed_code;
|
||||
message.push_str(trimmed);
|
||||
|
||||
if separator != b'-' {
|
||||
return Ok((parsed_code, message));
|
||||
}
|
||||
};
|
||||
Ok((code, message))
|
||||
|
||||
if line_number + 1 == SMTP_MAX_RESPONSE_LINES {
|
||||
return Err(GatewayError::Internal(
|
||||
"smtp response has too many continuation lines".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
Err(GatewayError::Internal(
|
||||
"smtp response has too many continuation lines".to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
/// Read one SMTP response line without allowing `BufRead::read_until` to
|
||||
/// allocate an attacker-controlled amount of memory before a size check.
|
||||
fn read_smtp_response_line<T: std::io::BufRead>(
|
||||
reader: &mut T,
|
||||
line: &mut Vec<u8>,
|
||||
) -> std::io::Result<usize> {
|
||||
loop {
|
||||
let buffered = reader.fill_buf()?;
|
||||
if buffered.is_empty() {
|
||||
return Ok(line.len());
|
||||
}
|
||||
let newline = buffered.iter().position(|byte| *byte == b'\n');
|
||||
let take = newline.map_or(buffered.len(), |index| index + 1);
|
||||
if line
|
||||
.len()
|
||||
.checked_add(take)
|
||||
.is_none_or(|length| length > SMTP_MAX_RESPONSE_LINE_BYTES)
|
||||
{
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"smtp response line exceeds the allowed size",
|
||||
));
|
||||
}
|
||||
line.extend_from_slice(&buffered[..take]);
|
||||
reader.consume(take);
|
||||
if newline.is_some() {
|
||||
return Ok(line.len());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn smtp_expect<T: std::io::BufRead>(
|
||||
@@ -197,11 +499,42 @@ fn smtp_expect<T: std::io::BufRead>(
|
||||
if allowed_codes.contains(&code) {
|
||||
return Ok(message);
|
||||
}
|
||||
let message = sanitize_smtp_diagnostic(&message);
|
||||
Err(GatewayError::Internal(format!(
|
||||
"unexpected smtp response {code}: {message}"
|
||||
)))
|
||||
}
|
||||
|
||||
/// SMTP responses are controlled by a remote server. Keep diagnostics useful
|
||||
/// while preventing terminal escapes, log/UI line injection, and oversized
|
||||
/// error payloads from crossing the API boundary.
|
||||
fn sanitize_smtp_diagnostic(message: &str) -> String {
|
||||
let mut sanitized = String::new();
|
||||
let mut previous_space = false;
|
||||
for character in message.chars() {
|
||||
if character == '\u{1b}' || character.is_control() {
|
||||
if !previous_space {
|
||||
sanitized.push(' ');
|
||||
previous_space = true;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if sanitized.len() + character.len_utf8() > SMTP_MAX_DIAGNOSTIC_BYTES {
|
||||
break;
|
||||
}
|
||||
if character.is_whitespace() {
|
||||
if !previous_space {
|
||||
sanitized.push(' ');
|
||||
previous_space = true;
|
||||
}
|
||||
} else {
|
||||
sanitized.push(character);
|
||||
previous_space = false;
|
||||
}
|
||||
}
|
||||
sanitized.trim().to_string()
|
||||
}
|
||||
|
||||
fn smtp_write_line<T: std::io::Write>(writer: &mut T, line: &str) -> Result<(), GatewayError> {
|
||||
writer
|
||||
.write_all(line.as_bytes())
|
||||
@@ -223,7 +556,11 @@ fn smtp_send_command<S: std::io::Read + std::io::Write>(
|
||||
smtp_expect(reader, allowed_codes)
|
||||
}
|
||||
|
||||
fn build_email_message(config: &SmtpDeliveryConfig, email: &ComposedEmail) -> String {
|
||||
fn build_email_message(
|
||||
config: &SmtpDeliveryConfig,
|
||||
email: &ComposedEmail,
|
||||
) -> Result<String, GatewayError> {
|
||||
validate_smtp_delivery_inputs(config, email)?;
|
||||
let boundary = format!("aether-{}", uuid::Uuid::new_v4().simple());
|
||||
let text_body =
|
||||
wrap_base64(&base64::engine::general_purpose::STANDARD.encode(email.text_body.as_bytes()));
|
||||
@@ -238,11 +575,17 @@ fn build_email_message(config: &SmtpDeliveryConfig, email: &ComposedEmail) -> St
|
||||
config.from_email
|
||||
)
|
||||
};
|
||||
format!(
|
||||
let message = format!(
|
||||
"From: {from_header}\r\nTo: <{to_email}>\r\nSubject: {subject}\r\nMIME-Version: 1.0\r\nContent-Type: multipart/alternative; boundary=\"{boundary}\"\r\n\r\n--{boundary}\r\nContent-Type: text/plain; charset=\"utf-8\"\r\nContent-Transfer-Encoding: base64\r\n\r\n{text_body}--{boundary}\r\nContent-Type: text/html; charset=\"utf-8\"\r\nContent-Transfer-Encoding: base64\r\n\r\n{html_body}--{boundary}--\r\n",
|
||||
to_email = email.to_email,
|
||||
subject = encode_mime_header(&email.subject),
|
||||
)
|
||||
);
|
||||
if message.len() > SMTP_MAX_MESSAGE_BYTES {
|
||||
return Err(GatewayError::Internal(
|
||||
"smtp message exceeds the allowed size".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(message)
|
||||
}
|
||||
|
||||
fn smtp_authenticate<S: std::io::Read + std::io::Write>(
|
||||
@@ -257,6 +600,11 @@ fn smtp_authenticate<S: std::io::Read + std::io::Write>(
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
if !config.use_tls && !config.use_ssl {
|
||||
return Err(GatewayError::Internal(
|
||||
"smtp authentication requires TLS or SSL encryption".to_string(),
|
||||
));
|
||||
}
|
||||
let password = config.password.as_deref().unwrap_or("");
|
||||
smtp_send_command(reader, "AUTH LOGIN", &[334])?;
|
||||
smtp_send_command(
|
||||
@@ -277,6 +625,9 @@ fn smtp_deliver_message<S: std::io::Read + std::io::Write>(
|
||||
config: &SmtpDeliveryConfig,
|
||||
email: &ComposedEmail,
|
||||
) -> Result<(), GatewayError> {
|
||||
// Keep this check next to command construction for callers that bypass
|
||||
// the async delivery wrapper.
|
||||
validate_smtp_delivery_inputs(config, email)?;
|
||||
smtp_send_command(
|
||||
reader,
|
||||
&format!("MAIL FROM:<{}>", config.from_email),
|
||||
@@ -288,7 +639,7 @@ fn smtp_deliver_message<S: std::io::Read + std::io::Write>(
|
||||
&[250, 251],
|
||||
)?;
|
||||
smtp_send_command(reader, "DATA", &[354])?;
|
||||
let message = build_email_message(config, email);
|
||||
let message = build_email_message(config, email)?;
|
||||
reader
|
||||
.get_mut()
|
||||
.write_all(message.as_bytes())
|
||||
@@ -379,3 +730,135 @@ fn probe_smtp_connection_blocking(config: SmtpDeliveryConfig) -> Result<(), Gate
|
||||
let _ = smtp_send_command(&mut reader, "QUIT", &[221]);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn config() -> SmtpDeliveryConfig {
|
||||
SmtpDeliveryConfig {
|
||||
host: "smtp.example.com".to_string(),
|
||||
port: 587,
|
||||
user: Some("[email protected]".to_string()),
|
||||
password: Some("password".to_string()),
|
||||
use_tls: true,
|
||||
use_ssl: false,
|
||||
from_email: "[email protected]".to_string(),
|
||||
from_name: "Aether".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn email() -> ComposedEmail {
|
||||
ComposedEmail {
|
||||
to_email: "[email protected]".to_string(),
|
||||
subject: "Subject".to_string(),
|
||||
html_body: "<p>hello</p>".to_string(),
|
||||
text_body: "hello".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_crlf_in_smtp_envelope_and_header_fields() {
|
||||
let mut malicious_config = config();
|
||||
malicious_config.from_email =
|
||||
"[email protected]\r\nRCPT TO:<[email protected]>".to_string();
|
||||
let error = validate_smtp_delivery_inputs(&malicious_config, &email())
|
||||
.expect_err("CRLF in an envelope address must be rejected");
|
||||
assert!(format!("{error:?}").contains("from_email"));
|
||||
|
||||
let mut malicious_email = email();
|
||||
malicious_email.subject = "Subject\nX-Injected: yes".to_string();
|
||||
let error = validate_smtp_delivery_inputs(&config(), &malicious_email)
|
||||
.expect_err("CRLF in a header value must be rejected");
|
||||
assert!(format!("{error:?}").contains("subject"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn allows_normal_smtp_values() {
|
||||
assert!(validate_smtp_delivery_inputs(&config(), &email()).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_authentication_over_plaintext_smtp() {
|
||||
let mut insecure = config();
|
||||
insecure.use_tls = false;
|
||||
insecure.use_ssl = false;
|
||||
let error = validate_smtp_delivery_inputs(&insecure, &email())
|
||||
.expect_err("SMTP credentials must never be sent over plaintext");
|
||||
assert!(format!("{error:?}").contains("requires TLS or SSL"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_malformed_mailboxes_and_non_protocol_delimiters() {
|
||||
let mut malicious = email();
|
||||
malicious.to_email = "[email protected]>\x01RCPT TO:<[email protected]>".to_string();
|
||||
let error = validate_smtp_delivery_inputs(&config(), &malicious)
|
||||
.expect_err("control characters and envelope delimiters must be rejected");
|
||||
assert!(format!("{error:?}").contains("to_email"));
|
||||
|
||||
let mut malformed = email();
|
||||
malformed.to_email = "not-an-email".to_string();
|
||||
assert!(validate_smtp_delivery_inputs(&config(), &malformed).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounds_message_bodies_before_smtp_submission() {
|
||||
let mut oversized = email();
|
||||
oversized.html_body = "x".repeat(SMTP_MAX_BODY_BYTES + 1);
|
||||
let error = validate_smtp_delivery_inputs(&config(), &oversized)
|
||||
.expect_err("oversized message bodies must be rejected");
|
||||
assert!(format!("{error:?}").contains("html_body"));
|
||||
|
||||
let mut textual = email();
|
||||
textual.text_body = "line one\nline two\t✓".to_string();
|
||||
assert!(validate_smtp_delivery_inputs(&config(), &textual).is_ok());
|
||||
|
||||
textual.text_body.push('\0');
|
||||
assert!(validate_smtp_delivery_inputs(&config(), &textual).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sanitizes_remote_response_diagnostics() {
|
||||
let mut reader = std::io::BufReader::new("550 bad\u{1b}[31m\r\n".as_bytes());
|
||||
let error = smtp_expect(&mut reader, &[250]).expect_err("unexpected response must fail");
|
||||
let GatewayError::Internal(message) = error else {
|
||||
panic!("expected internal SMTP error");
|
||||
};
|
||||
assert!(!message.contains('\u{1b}'));
|
||||
assert!(!message.contains('\n'));
|
||||
assert!(message.len() < SMTP_MAX_DIAGNOSTIC_BYTES);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn smtp_response_rejects_non_ascii_status_prefix_without_panicking() {
|
||||
let mut reader = std::io::BufReader::new("é00 greeting\r\n".as_bytes());
|
||||
let error = smtp_read_response(&mut reader)
|
||||
.expect_err("a non-ASCII status prefix must be rejected");
|
||||
assert!(format!("{error:?}").contains("invalid smtp response"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn smtp_response_rejects_oversized_lines_before_allocating_unbounded_memory() {
|
||||
let mut input = vec![b'2'; SMTP_MAX_RESPONSE_LINE_BYTES + 1];
|
||||
input.push(b'\n');
|
||||
let mut reader = std::io::BufReader::new(input.as_slice());
|
||||
let error = smtp_read_response(&mut reader).expect_err("oversized line must be rejected");
|
||||
assert!(format!("{error:?}").contains("response line exceeds"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn smtp_response_bounds_continuation_lines_and_code_changes() {
|
||||
let repeated = (0..=SMTP_MAX_RESPONSE_LINES)
|
||||
.map(|_| "250-more\r\n")
|
||||
.collect::<String>();
|
||||
let mut reader = std::io::BufReader::new(repeated.as_bytes());
|
||||
let error = smtp_read_response(&mut reader)
|
||||
.expect_err("too many continuation lines must be rejected");
|
||||
assert!(format!("{error:?}").contains("too many continuation lines"));
|
||||
|
||||
let mut reader = std::io::BufReader::new("250-more\r\n550 done\r\n".as_bytes());
|
||||
let error = smtp_read_response(&mut reader)
|
||||
.expect_err("continuation response code changes must be rejected");
|
||||
assert!(format!("{error:?}").contains("continuation code changed"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ use axum::http::{Response, StatusCode};
|
||||
use axum::response::IntoResponse;
|
||||
use axum::Json;
|
||||
use serde_json::json;
|
||||
use sha2::{Digest, Sha256};
|
||||
use tracing::warn;
|
||||
|
||||
use crate::ai_serving::AiSurfaceFinalizeError;
|
||||
@@ -33,6 +34,9 @@ pub(crate) enum GatewayError {
|
||||
status: StatusCode,
|
||||
message: String,
|
||||
},
|
||||
PlanUsageLimited(crate::plan_usage_policy::PlanUsagePolicyRejection),
|
||||
LastActiveAdminUpdateDenied,
|
||||
LastActiveAdminDeleteDenied,
|
||||
Internal(String),
|
||||
}
|
||||
|
||||
@@ -43,6 +47,10 @@ impl GatewayError {
|
||||
| Self::ControlUnavailable { message, .. }
|
||||
| Self::Client { message, .. }
|
||||
| Self::Internal(message) => message,
|
||||
Self::PlanUsageLimited(rejection) => format!(
|
||||
"subscription plan {} limit {} reached for {} window; retry after {} seconds",
|
||||
rejection.metric, rejection.limit, rejection.window, rejection.retry_after
|
||||
),
|
||||
Self::LocalExecutionPlanningTimeout {
|
||||
phase, timeout_ms, ..
|
||||
} => {
|
||||
@@ -55,6 +63,8 @@ impl GatewayError {
|
||||
} => {
|
||||
format!("gateway admission gate {gate} timed out after {queue_budget_ms}ms")
|
||||
}
|
||||
Self::LastActiveAdminUpdateDenied => "不能降级或停用最后一个管理员账户".to_string(),
|
||||
Self::LastActiveAdminDeleteDenied => "不能删除最后一个管理员账户".to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -63,7 +73,13 @@ impl IntoResponse for GatewayError {
|
||||
fn into_response(self) -> Response<Body> {
|
||||
match self {
|
||||
Self::UpstreamUnavailable { trace_id, message } => {
|
||||
warn!(trace_id = %trace_id, error = %message, "gateway proxy unavailable");
|
||||
let error_fingerprint = gateway_error_fingerprint(&message);
|
||||
warn!(
|
||||
trace_id = %trace_id,
|
||||
error_fingerprint,
|
||||
error_length = message.len(),
|
||||
"gateway proxy unavailable"
|
||||
);
|
||||
let body = Json(json!({
|
||||
"error": {
|
||||
"message": "gateway proxy unavailable",
|
||||
@@ -81,7 +97,13 @@ impl IntoResponse for GatewayError {
|
||||
response
|
||||
}
|
||||
Self::ControlUnavailable { trace_id, message } => {
|
||||
warn!(trace_id = %trace_id, error = %message, "gateway control unavailable");
|
||||
let error_fingerprint = gateway_error_fingerprint(&message);
|
||||
warn!(
|
||||
trace_id = %trace_id,
|
||||
error_fingerprint,
|
||||
error_length = message.len(),
|
||||
"gateway control unavailable"
|
||||
);
|
||||
let body = Json(json!({
|
||||
"error": {
|
||||
"message": "gateway control unavailable",
|
||||
@@ -162,19 +184,59 @@ impl IntoResponse for GatewayError {
|
||||
})),
|
||||
)
|
||||
.into_response(),
|
||||
Self::Internal(message) => (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
Self::PlanUsageLimited(rejection) => (
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
Json(json!({
|
||||
"error": {
|
||||
"message": message,
|
||||
"type": "plan_usage_limit_exceeded",
|
||||
"message": "套餐使用限制已达到上限,请稍后重试",
|
||||
"details": {
|
||||
"metric": rejection.metric,
|
||||
"window": rejection.window,
|
||||
"limit": rejection.limit,
|
||||
"retry_after": rejection.retry_after,
|
||||
}
|
||||
}
|
||||
})),
|
||||
)
|
||||
.into_response(),
|
||||
Self::LastActiveAdminUpdateDenied => (
|
||||
StatusCode::BAD_REQUEST,
|
||||
Json(json!({ "detail": "不能降级或停用最后一个管理员账户" })),
|
||||
)
|
||||
.into_response(),
|
||||
Self::LastActiveAdminDeleteDenied => (
|
||||
StatusCode::BAD_REQUEST,
|
||||
Json(json!({ "detail": "不能删除最后一个管理员账户" })),
|
||||
)
|
||||
.into_response(),
|
||||
Self::Internal(message) => {
|
||||
let error_fingerprint = gateway_error_fingerprint(&message);
|
||||
tracing::error!(
|
||||
event_name = "gateway_internal_error",
|
||||
error_fingerprint,
|
||||
error_length = message.len(),
|
||||
"internal gateway error hidden from client"
|
||||
);
|
||||
(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
Json(json!({
|
||||
"error": {
|
||||
"message": "internal server error",
|
||||
}
|
||||
})),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn gateway_error_fingerprint(message: &str) -> String {
|
||||
let digest = Sha256::digest(message.as_bytes());
|
||||
format!("{:x}", digest)[..16].to_string()
|
||||
}
|
||||
|
||||
impl From<AiSurfaceFinalizeError> for GatewayError {
|
||||
fn from(error: AiSurfaceFinalizeError) -> Self {
|
||||
GatewayError::Internal(error.0)
|
||||
@@ -183,12 +245,41 @@ impl From<AiSurfaceFinalizeError> for GatewayError {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use axum::body::to_bytes;
|
||||
use axum::http::{header::RETRY_AFTER, StatusCode};
|
||||
use axum::response::IntoResponse;
|
||||
|
||||
use crate::constants::TRACE_ID_HEADER;
|
||||
|
||||
use super::GatewayError;
|
||||
use super::{gateway_error_fingerprint, GatewayError};
|
||||
|
||||
#[tokio::test]
|
||||
async fn internal_errors_do_not_expose_internal_details() {
|
||||
let response = GatewayError::Internal(
|
||||
"database connection failed: password=internal-secret".to_string(),
|
||||
)
|
||||
.into_response();
|
||||
|
||||
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
|
||||
let body = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("internal error response body should read");
|
||||
let payload: serde_json::Value =
|
||||
serde_json::from_slice(&body).expect("internal error response should be JSON");
|
||||
assert_eq!(payload["error"]["message"], "internal server error");
|
||||
assert!(!String::from_utf8_lossy(&body).contains("internal-secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_fingerprints_are_stable_without_containing_source_text() {
|
||||
let secret_error = "postgresql://admin:[email protected]/aether";
|
||||
let fingerprint = gateway_error_fingerprint(secret_error);
|
||||
|
||||
assert_eq!(fingerprint, gateway_error_fingerprint(secret_error));
|
||||
assert_eq!(fingerprint.len(), 16);
|
||||
assert!(fingerprint.bytes().all(|byte| byte.is_ascii_hexdigit()));
|
||||
assert!(!fingerprint.contains("database-secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn admission_timeout_returns_429_with_retry_after_without_panicking() {
|
||||
|
||||
@@ -0,0 +1,567 @@
|
||||
//! Terminal settlement for a local stream attempt whose future is dropped
|
||||
//! mid-flight.
|
||||
//!
|
||||
//! A local stream attempt writes its `usage` row and its `request_candidates`
|
||||
//! slot as `pending` before it dispatches to the provider, then keeps running
|
||||
//! inside the downstream request future. When the client disconnects, axum drops
|
||||
//! that future: the remaining `.await`s never resume and nothing settles either
|
||||
//! row. They stay `pending` until the maintenance sweeper rewrites them as a 504
|
||||
//! timeout roughly ten minutes later, which loses the real outcome and the real
|
||||
//! latency.
|
||||
//!
|
||||
//! The stream transport therefore keeps a guard alive across the window between
|
||||
//! the `pending` write and terminal settlement, and settles the attempt from
|
||||
//! `Drop` when that window is left by cancellation instead of by a terminal
|
||||
//! state.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use aether_contracts::ExecutionPlan;
|
||||
use aether_data_contracts::repository::candidates::RequestCandidateStatus;
|
||||
use aether_scheduler_core::SchedulerRequestCandidateStatusUpdate;
|
||||
use aether_usage_runtime::{
|
||||
build_usage_event_data_seed_describing_request_bodies, UsageEvent, UsageEventData,
|
||||
UsageEventType,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
use tracing::warn;
|
||||
|
||||
use crate::clock::current_unix_ms as current_request_candidate_unix_ms;
|
||||
use crate::execution_runtime::attempt_lifecycle::CLIENT_CANCELLED_STATUS_CODE;
|
||||
use crate::execution_runtime::transport_failure::StreamCandidateWatchdogProgress;
|
||||
use crate::log_ids::short_request_id;
|
||||
use crate::request_candidate_runtime::{
|
||||
record_local_request_candidate_status_snapshot, LocalRequestCandidateStatusSnapshot,
|
||||
};
|
||||
use crate::request_diagnostics::{
|
||||
attach_request_diagnostics_to_report_context, current_request_diagnostics, RequestDiagnostics,
|
||||
};
|
||||
use crate::AppState;
|
||||
|
||||
fn elapsed_ms_since(started_at: Instant) -> u64 {
|
||||
started_at.elapsed().as_millis().min(u128::from(u64::MAX)) as u64
|
||||
}
|
||||
|
||||
/// The facts the guard needs to settle the attempt it is watching.
|
||||
///
|
||||
/// This is held for the whole attempt, so it is deliberately free of request
|
||||
/// bodies. A request body can be megabytes, and holding one per in-flight
|
||||
/// attempt would cost far more than the row it settles: the usage seed is built
|
||||
/// with [`build_usage_event_data_seed_describing_request_bodies`], which derives
|
||||
/// every capture state, body reference and derived request fact from the real
|
||||
/// plan and report context but keeps neither body. The terminal write it
|
||||
/// produces therefore preserves the capture the `pending` write recorded instead
|
||||
/// of clearing it.
|
||||
struct ArmedAttempt {
|
||||
request_id: String,
|
||||
candidate_id: Option<String>,
|
||||
candidate: Option<LocalRequestCandidateStatusSnapshot>,
|
||||
// Boxed: the guard lives inside the stream request future, which is already
|
||||
// very large, and `UsageEventData` is a wide struct.
|
||||
usage_seed: Option<Box<UsageEventData>>,
|
||||
request_diagnostics: Option<Arc<RequestDiagnostics>>,
|
||||
candidate_started_unix_ms: u64,
|
||||
candidate_started_at: Instant,
|
||||
}
|
||||
|
||||
/// Settles an attempt as cancelled when its future is dropped before the
|
||||
/// transport reaches a terminal state.
|
||||
///
|
||||
/// The guard is created disarmed and stays inert until [`Self::arm`] is called,
|
||||
/// so an attempt that is dropped before it owns any `pending` row does not grow
|
||||
/// a settlement row it never had. The owner disarms it as soon as the attempt
|
||||
/// completes, whichever way it completes: from that point terminal settlement
|
||||
/// belongs to the transport (for streams, to the stream finalizer that lives in
|
||||
/// the response body), and the guard must not write a second terminal state.
|
||||
///
|
||||
/// A stream candidate also runs under a first-byte watchdog that drops the
|
||||
/// attempt future when it gives up. That drop is not a client disconnect and the
|
||||
/// watchdog settles the attempt itself, so the guard stands down for it.
|
||||
pub(crate) struct AttemptCancellationGuard {
|
||||
state: AppState,
|
||||
error_type: &'static str,
|
||||
error_message: &'static str,
|
||||
watchdog: Option<Arc<StreamCandidateWatchdogProgress>>,
|
||||
armed: Option<ArmedAttempt>,
|
||||
}
|
||||
|
||||
impl AttemptCancellationGuard {
|
||||
pub(crate) fn disarmed(
|
||||
state: &AppState,
|
||||
error_type: &'static str,
|
||||
error_message: &'static str,
|
||||
) -> Self {
|
||||
Self {
|
||||
state: state.clone(),
|
||||
error_type,
|
||||
error_message,
|
||||
watchdog: StreamCandidateWatchdogProgress::current(),
|
||||
armed: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Takes ownership of the attempt's settlement until it is disarmed.
|
||||
pub(crate) fn arm(
|
||||
&mut self,
|
||||
plan: &ExecutionPlan,
|
||||
report_context: Option<&Value>,
|
||||
candidate: Option<&LocalRequestCandidateStatusSnapshot>,
|
||||
candidate_started_unix_ms: u64,
|
||||
candidate_started_at: Instant,
|
||||
) {
|
||||
let usage_seed = self.state.usage_runtime.is_enabled().then(|| {
|
||||
Box::new(build_usage_event_data_seed_describing_request_bodies(
|
||||
plan,
|
||||
report_context,
|
||||
))
|
||||
});
|
||||
self.armed = Some(ArmedAttempt {
|
||||
request_id: plan.request_id.clone(),
|
||||
candidate_id: plan.candidate_id.clone(),
|
||||
candidate: candidate.cloned(),
|
||||
usage_seed,
|
||||
request_diagnostics: current_request_diagnostics(),
|
||||
candidate_started_unix_ms,
|
||||
candidate_started_at,
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn disarm(&mut self) {
|
||||
self.armed = None;
|
||||
}
|
||||
}
|
||||
|
||||
/// Writes the candidate terminal row and the terminal usage event for an attempt
|
||||
/// that never reached its own terminal path.
|
||||
async fn settle_cancelled_attempt(
|
||||
state: AppState,
|
||||
armed: ArmedAttempt,
|
||||
error_type: &'static str,
|
||||
error_message: &'static str,
|
||||
) {
|
||||
let ArmedAttempt {
|
||||
request_id,
|
||||
candidate_id: _,
|
||||
candidate,
|
||||
usage_seed,
|
||||
request_diagnostics,
|
||||
candidate_started_unix_ms,
|
||||
candidate_started_at,
|
||||
} = armed;
|
||||
let terminal_unix_ms = current_request_candidate_unix_ms();
|
||||
let latency_ms = elapsed_ms_since(candidate_started_at);
|
||||
|
||||
if let Some(candidate) = candidate.as_ref() {
|
||||
record_local_request_candidate_status_snapshot(
|
||||
&state,
|
||||
candidate,
|
||||
SchedulerRequestCandidateStatusUpdate {
|
||||
status: RequestCandidateStatus::Cancelled,
|
||||
status_code: Some(CLIENT_CANCELLED_STATUS_CODE),
|
||||
error_type: Some(error_type.to_string()),
|
||||
error_message: Some(error_message.to_string()),
|
||||
latency_ms: Some(latency_ms),
|
||||
started_at_unix_ms: Some(candidate_started_unix_ms),
|
||||
finished_at_unix_ms: Some(terminal_unix_ms),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
let Some(usage_data) = usage_seed else {
|
||||
return;
|
||||
};
|
||||
let mut usage_data = *usage_data;
|
||||
// The seed was built when the attempt was armed, so it predates the
|
||||
// diagnostics it should carry. Attaching them to the seed's metadata is the
|
||||
// same write the report context would have carried into a seed built here:
|
||||
// both land the same keys in the same object.
|
||||
usage_data.request_metadata = attach_request_diagnostics_to_report_context(
|
||||
usage_data.request_metadata.take(),
|
||||
request_diagnostics.as_ref(),
|
||||
);
|
||||
usage_data.status_code = Some(CLIENT_CANCELLED_STATUS_CODE);
|
||||
usage_data.error_message = Some(error_message.to_string());
|
||||
usage_data.error_category = Some("cancelled".to_string());
|
||||
usage_data.response_time_ms = Some(latency_ms);
|
||||
let error_body = json!({
|
||||
"error": {
|
||||
"type": error_type,
|
||||
"message": error_message,
|
||||
"code": CLIENT_CANCELLED_STATUS_CODE
|
||||
}
|
||||
});
|
||||
usage_data.response_headers = Some(json!({"content-type": "application/json"}));
|
||||
usage_data.response_body = Some(error_body.clone());
|
||||
usage_data.client_response_headers = Some(json!({"content-type": "application/json"}));
|
||||
usage_data.client_response_body = Some(error_body);
|
||||
|
||||
state
|
||||
.usage_runtime
|
||||
.record_terminal_event_direct(
|
||||
state.usage_lifecycle_data_state().as_ref(),
|
||||
UsageEvent::new(UsageEventType::Cancelled, request_id, usage_data),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
impl Drop for AttemptCancellationGuard {
|
||||
fn drop(&mut self) {
|
||||
let Some(armed) = self.armed.take() else {
|
||||
return;
|
||||
};
|
||||
if self
|
||||
.watchdog
|
||||
.as_ref()
|
||||
.is_some_and(|watchdog| watchdog.abandoned())
|
||||
{
|
||||
return;
|
||||
}
|
||||
let state = self.state.clone();
|
||||
let error_type = self.error_type;
|
||||
let error_message = self.error_message;
|
||||
// `Drop` cannot await, and the settlement writes touch the database.
|
||||
// Hand them to the runtime so they survive the dropped request future.
|
||||
let Ok(handle) = tokio::runtime::Handle::try_current() else {
|
||||
warn!(
|
||||
event_name = "local_attempt_cancellation_guard_no_runtime",
|
||||
log_type = "ops",
|
||||
request_id = %short_request_id(armed.request_id.as_str()),
|
||||
candidate_id = ?armed.candidate_id,
|
||||
error_type,
|
||||
"gateway could not settle dropped local attempt because no Tokio runtime is available"
|
||||
);
|
||||
return;
|
||||
};
|
||||
handle.spawn(async move {
|
||||
settle_cancelled_attempt(state, armed, error_type, error_message).await;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use aether_contracts::RequestBody;
|
||||
use aether_data::repository::candidates::InMemoryRequestCandidateRepository;
|
||||
use aether_data::repository::usage::InMemoryUsageReadRepository;
|
||||
use aether_data_contracts::repository::candidates::RequestCandidateReadRepository;
|
||||
use aether_data_contracts::repository::usage::{
|
||||
StoredRequestUsageAudit, UsageBodyCaptureState, UsageReadRepository, UsageWriteRepository,
|
||||
};
|
||||
use aether_usage_runtime::{
|
||||
build_lifecycle_usage_seed, build_pending_usage_record, UsageRuntimeConfig,
|
||||
};
|
||||
use std::collections::BTreeMap;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::request_candidate_runtime::{
|
||||
ensure_execution_request_candidate_slot, snapshot_local_request_candidate_status,
|
||||
};
|
||||
|
||||
const TEST_ERROR_TYPE: &str = "local_stream_attempt_cancelled";
|
||||
const TEST_ERROR_MESSAGE: &str =
|
||||
"Local stream attempt was dropped before terminal finalization.";
|
||||
|
||||
fn test_stream_plan(request_id: &str) -> ExecutionPlan {
|
||||
ExecutionPlan {
|
||||
request_id: request_id.to_string(),
|
||||
candidate_id: None,
|
||||
provider_name: Some("Anthropic".to_string()),
|
||||
provider_id: "provider-1".to_string(),
|
||||
endpoint_id: "endpoint-1".to_string(),
|
||||
key_id: "key-1".to_string(),
|
||||
method: "POST".to_string(),
|
||||
url: "https://example.test/v1/messages".to_string(),
|
||||
headers: BTreeMap::new(),
|
||||
content_type: Some("application/json".to_string()),
|
||||
content_encoding: None,
|
||||
body: RequestBody::from_json(json!({"stream": true, "service_tier": "priority"})),
|
||||
stream: true,
|
||||
client_api_format: "claude:messages".to_string(),
|
||||
provider_api_format: "claude:messages".to_string(),
|
||||
model_name: Some("claude-sonnet-4-5".to_string()),
|
||||
proxy: None,
|
||||
transport_profile: None,
|
||||
timeouts: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn test_report_context() -> Option<Value> {
|
||||
Some(json!({
|
||||
"candidate_index": 0,
|
||||
"retry_index": 0,
|
||||
"user_id": "user-cancel",
|
||||
"api_key_id": "api-key-cancel",
|
||||
"client_api_format": "claude:messages",
|
||||
"provider_api_format": "claude:messages",
|
||||
"request_path": "/v1/messages",
|
||||
"request_path_and_query": "/v1/messages?beta=true",
|
||||
"upstream_url": "https://example.test/v1/messages",
|
||||
"mapped_model": "claude-sonnet-4-5",
|
||||
"original_request_body": {"stream": true, "messages": []},
|
||||
}))
|
||||
}
|
||||
|
||||
fn test_state(
|
||||
usage_repository: &Arc<InMemoryUsageReadRepository>,
|
||||
request_candidate_repository: &Arc<InMemoryRequestCandidateRepository>,
|
||||
) -> AppState {
|
||||
AppState::new()
|
||||
.expect("gateway state should build")
|
||||
.with_data_state_for_tests(
|
||||
crate::data::GatewayDataState::with_request_candidate_and_usage_repository_for_tests(
|
||||
Arc::clone(request_candidate_repository),
|
||||
Arc::clone(usage_repository),
|
||||
),
|
||||
)
|
||||
.with_usage_runtime_for_tests(UsageRuntimeConfig {
|
||||
enabled: true,
|
||||
..UsageRuntimeConfig::default()
|
||||
})
|
||||
}
|
||||
|
||||
/// Writes the `pending` rows the same way a stream attempt does before it
|
||||
/// dispatches to the provider, and returns the candidate slot snapshot the
|
||||
/// attempt owns from that point on.
|
||||
async fn record_pending_attempt(
|
||||
state: &AppState,
|
||||
plan: &mut ExecutionPlan,
|
||||
report_context: &mut Option<Value>,
|
||||
candidate_started_unix_ms: u64,
|
||||
) -> LocalRequestCandidateStatusSnapshot {
|
||||
ensure_execution_request_candidate_slot(state, plan, report_context).await;
|
||||
state.usage_runtime.record_pending(
|
||||
state.usage_lifecycle_data_state().as_ref(),
|
||||
build_lifecycle_usage_seed(plan, report_context.as_ref()),
|
||||
);
|
||||
let snapshot = snapshot_local_request_candidate_status(plan, report_context.as_ref())
|
||||
.expect("attempt should own a candidate slot");
|
||||
record_local_request_candidate_status_snapshot(
|
||||
state,
|
||||
&snapshot,
|
||||
SchedulerRequestCandidateStatusUpdate {
|
||||
status: RequestCandidateStatus::Pending,
|
||||
status_code: None,
|
||||
error_type: None,
|
||||
error_message: None,
|
||||
latency_ms: None,
|
||||
started_at_unix_ms: Some(candidate_started_unix_ms),
|
||||
finished_at_unix_ms: None,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
snapshot
|
||||
}
|
||||
|
||||
async fn wait_for_usage_status(
|
||||
usage_repository: &InMemoryUsageReadRepository,
|
||||
request_id: &str,
|
||||
status: &str,
|
||||
) -> Option<StoredRequestUsageAudit> {
|
||||
for _ in 0..50 {
|
||||
if let Some(usage) = usage_repository
|
||||
.find_by_request_id(request_id)
|
||||
.await
|
||||
.expect("usage should read")
|
||||
{
|
||||
if usage.status == status {
|
||||
return Some(usage);
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn armed_guard_settles_a_dropped_attempt_as_cancelled() {
|
||||
let usage_repository = Arc::new(InMemoryUsageReadRepository::default());
|
||||
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
|
||||
let state = test_state(&usage_repository, &request_candidate_repository);
|
||||
let mut plan = test_stream_plan("stream-cancel-guard-request");
|
||||
let mut report_context = test_report_context();
|
||||
let candidate_started_unix_ms = current_request_candidate_unix_ms();
|
||||
let snapshot = record_pending_attempt(
|
||||
&state,
|
||||
&mut plan,
|
||||
&mut report_context,
|
||||
candidate_started_unix_ms,
|
||||
)
|
||||
.await;
|
||||
|
||||
{
|
||||
let mut guard =
|
||||
AttemptCancellationGuard::disarmed(&state, TEST_ERROR_TYPE, TEST_ERROR_MESSAGE);
|
||||
guard.arm(
|
||||
&plan,
|
||||
report_context.as_ref(),
|
||||
Some(&snapshot),
|
||||
candidate_started_unix_ms,
|
||||
Instant::now(),
|
||||
);
|
||||
}
|
||||
|
||||
let usage = wait_for_usage_status(
|
||||
usage_repository.as_ref(),
|
||||
"stream-cancel-guard-request",
|
||||
"cancelled",
|
||||
)
|
||||
.await
|
||||
.expect("cancelled usage should be recorded");
|
||||
assert_eq!(usage.billing_status, "void");
|
||||
assert_eq!(usage.status_code, Some(CLIENT_CANCELLED_STATUS_CODE));
|
||||
assert_eq!(usage.error_category.as_deref(), Some("cancelled"));
|
||||
assert!(usage.response_time_ms.is_some());
|
||||
|
||||
let candidates = request_candidate_repository
|
||||
.list_by_request_id("stream-cancel-guard-request")
|
||||
.await
|
||||
.expect("candidates should read");
|
||||
let candidate = candidates.first().expect("candidate row should exist");
|
||||
assert_eq!(candidate.status, RequestCandidateStatus::Cancelled);
|
||||
assert_eq!(candidate.status_code, Some(CLIENT_CANCELLED_STATUS_CODE));
|
||||
assert_eq!(candidate.error_type.as_deref(), Some(TEST_ERROR_TYPE));
|
||||
assert!(candidate.finished_at_unix_ms.is_some());
|
||||
}
|
||||
|
||||
/// The guard holds no request body, and the persistence boundary intentionally
|
||||
/// rejects request/response capture material. A dropped-attempt settlement
|
||||
/// must not re-introduce an inline body or a caller-controlled body reference.
|
||||
#[tokio::test]
|
||||
async fn settling_a_dropped_attempt_does_not_reintroduce_request_body_capture() {
|
||||
let usage_repository = Arc::new(InMemoryUsageReadRepository::default());
|
||||
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
|
||||
let state = test_state(&usage_repository, &request_candidate_repository);
|
||||
let mut plan = test_stream_plan("stream-cancel-guard-capture");
|
||||
let mut report_context = test_report_context();
|
||||
let candidate_started_unix_ms = current_request_candidate_unix_ms();
|
||||
let snapshot = record_pending_attempt(
|
||||
&state,
|
||||
&mut plan,
|
||||
&mut report_context,
|
||||
candidate_started_unix_ms,
|
||||
)
|
||||
.await;
|
||||
// This deliberately supplies capture material to prove that the usage
|
||||
// persistence boundary strips it before either lifecycle write stores it.
|
||||
let captured_body = json!({"stream": true, "service_tier": "priority"});
|
||||
let mut capture = build_pending_usage_record(
|
||||
&plan,
|
||||
report_context.as_ref(),
|
||||
current_request_candidate_unix_ms() / 1_000,
|
||||
)
|
||||
.expect("pending usage record should build");
|
||||
capture.provider_request_body = Some(captured_body.clone());
|
||||
capture.provider_request_body_state = Some(UsageBodyCaptureState::Inline);
|
||||
usage_repository
|
||||
.upsert(capture)
|
||||
.await
|
||||
.expect("captured request body should upsert");
|
||||
|
||||
{
|
||||
let mut guard =
|
||||
AttemptCancellationGuard::disarmed(&state, TEST_ERROR_TYPE, TEST_ERROR_MESSAGE);
|
||||
guard.arm(
|
||||
&plan,
|
||||
report_context.as_ref(),
|
||||
Some(&snapshot),
|
||||
candidate_started_unix_ms,
|
||||
Instant::now(),
|
||||
);
|
||||
}
|
||||
|
||||
let usage = wait_for_usage_status(
|
||||
usage_repository.as_ref(),
|
||||
"stream-cancel-guard-capture",
|
||||
"cancelled",
|
||||
)
|
||||
.await
|
||||
.expect("cancelled usage should be recorded");
|
||||
assert_eq!(usage.provider_request_body, None);
|
||||
assert_eq!(usage.provider_request_body_ref, None);
|
||||
assert_eq!(usage.provider_request_body_state, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn guard_stands_down_when_the_watchdog_abandons_the_attempt() {
|
||||
let usage_repository = Arc::new(InMemoryUsageReadRepository::default());
|
||||
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
|
||||
let state = test_state(&usage_repository, &request_candidate_repository);
|
||||
let mut plan = test_stream_plan("stream-watchdog-guard-request");
|
||||
let mut report_context = test_report_context();
|
||||
let candidate_started_unix_ms = current_request_candidate_unix_ms();
|
||||
let snapshot = record_pending_attempt(
|
||||
&state,
|
||||
&mut plan,
|
||||
&mut report_context,
|
||||
candidate_started_unix_ms,
|
||||
)
|
||||
.await;
|
||||
|
||||
let watchdog = StreamCandidateWatchdogProgress::shared();
|
||||
Arc::clone(&watchdog)
|
||||
.scope(async {
|
||||
let mut guard =
|
||||
AttemptCancellationGuard::disarmed(&state, TEST_ERROR_TYPE, TEST_ERROR_MESSAGE);
|
||||
guard.arm(
|
||||
&plan,
|
||||
report_context.as_ref(),
|
||||
Some(&snapshot),
|
||||
candidate_started_unix_ms,
|
||||
Instant::now(),
|
||||
);
|
||||
// The watchdog gives up and takes over settlement before the
|
||||
// abandoned attempt is dropped.
|
||||
watchdog.mark_abandoned();
|
||||
})
|
||||
.await;
|
||||
|
||||
assert!(wait_for_usage_status(
|
||||
usage_repository.as_ref(),
|
||||
"stream-watchdog-guard-request",
|
||||
"cancelled",
|
||||
)
|
||||
.await
|
||||
.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disarmed_guard_leaves_the_attempt_pending() {
|
||||
let usage_repository = Arc::new(InMemoryUsageReadRepository::default());
|
||||
let request_candidate_repository = Arc::new(InMemoryRequestCandidateRepository::default());
|
||||
let state = test_state(&usage_repository, &request_candidate_repository);
|
||||
let mut plan = test_stream_plan("stream-disarmed-guard-request");
|
||||
let mut report_context = test_report_context();
|
||||
let candidate_started_unix_ms = current_request_candidate_unix_ms();
|
||||
let snapshot = record_pending_attempt(
|
||||
&state,
|
||||
&mut plan,
|
||||
&mut report_context,
|
||||
candidate_started_unix_ms,
|
||||
)
|
||||
.await;
|
||||
|
||||
{
|
||||
let mut guard =
|
||||
AttemptCancellationGuard::disarmed(&state, TEST_ERROR_TYPE, TEST_ERROR_MESSAGE);
|
||||
guard.arm(
|
||||
&plan,
|
||||
report_context.as_ref(),
|
||||
Some(&snapshot),
|
||||
candidate_started_unix_ms,
|
||||
Instant::now(),
|
||||
);
|
||||
guard.disarm();
|
||||
}
|
||||
|
||||
assert!(wait_for_usage_status(
|
||||
usage_repository.as_ref(),
|
||||
"stream-disarmed-guard-request",
|
||||
"cancelled",
|
||||
)
|
||||
.await
|
||||
.is_none());
|
||||
}
|
||||
}
|
||||
@@ -36,10 +36,11 @@ use aether_contracts::{ExecutionPlan, ExecutionStreamTerminalSummary, ExecutionT
|
||||
use aether_data_contracts::repository::candidates::RequestCandidateStatus;
|
||||
use aether_data_contracts::repository::usage::UsageBodyCaptureState;
|
||||
use aether_scheduler_core::SchedulerRequestCandidateStatusUpdate;
|
||||
#[cfg(test)]
|
||||
use aether_usage_runtime::DEFAULT_USAGE_RESPONSE_BODY_CAPTURE_LIMIT_BYTES;
|
||||
use aether_usage_runtime::{
|
||||
build_lifecycle_usage_seed, build_stream_terminal_usage_payload_seed,
|
||||
build_terminal_usage_context_seed, stream_report_represents_failure,
|
||||
DEFAULT_USAGE_RESPONSE_BODY_CAPTURE_LIMIT_BYTES,
|
||||
};
|
||||
use base64::Engine as _;
|
||||
use serde_json::Value;
|
||||
@@ -433,7 +434,7 @@ impl AttemptBodyCapture {
|
||||
if bytes.is_empty() || self.truncated {
|
||||
return;
|
||||
}
|
||||
let max_bytes = DEFAULT_USAGE_RESPONSE_BODY_CAPTURE_LIMIT_BYTES;
|
||||
let max_bytes = crate::execution_runtime::MAX_STREAM_BODY_CAPTURE_BYTES;
|
||||
if self.buffer.len() >= max_bytes {
|
||||
self.truncated = true;
|
||||
return;
|
||||
@@ -447,11 +448,19 @@ impl AttemptBodyCapture {
|
||||
}
|
||||
|
||||
pub(crate) fn encode(&self) -> (Option<String>, Option<UsageBodyCaptureState>) {
|
||||
let body = (!self.buffer.is_empty())
|
||||
.then(|| base64::engine::general_purpose::STANDARD.encode(&self.buffer));
|
||||
let state = if self.truncated {
|
||||
self.encode_with_limit(crate::execution_runtime::MAX_STREAM_BODY_CAPTURE_BYTES)
|
||||
}
|
||||
|
||||
fn encode_with_limit(
|
||||
&self,
|
||||
max_bytes: usize,
|
||||
) -> (Option<String>, Option<UsageBodyCaptureState>) {
|
||||
let captured = &self.buffer[..self.buffer.len().min(max_bytes)];
|
||||
let body = (!captured.is_empty())
|
||||
.then(|| base64::engine::general_purpose::STANDARD.encode(captured));
|
||||
let state = if self.truncated || captured.len() < self.buffer.len() {
|
||||
UsageBodyCaptureState::Truncated
|
||||
} else if self.buffer.is_empty() {
|
||||
} else if captured.is_empty() {
|
||||
UsageBodyCaptureState::None
|
||||
} else {
|
||||
UsageBodyCaptureState::Inline
|
||||
@@ -1418,15 +1427,13 @@ mod stage_tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// body capture 的编码状态。截断分支这里到不了:共享的
|
||||
/// `DEFAULT_USAGE_RESPONSE_BODY_CAPTURE_LIMIT_BYTES` 是 `usize::MAX`,
|
||||
/// 也就是默认不限长;截断只在 usage 侧把上限调低后才可能发生。
|
||||
/// body capture 的编码状态。Full 记录级别仍受 gateway 的硬上限约束,
|
||||
/// 这样长连接不会把审计副本无限累积。
|
||||
#[test]
|
||||
fn body_capture_encodes_inline_and_empty_states() {
|
||||
assert_eq!(
|
||||
super::DEFAULT_USAGE_RESPONSE_BODY_CAPTURE_LIMIT_BYTES,
|
||||
usize::MAX,
|
||||
"the default capture limit is unbounded; truncation is not reachable here"
|
||||
crate::execution_runtime::MAX_STREAM_BODY_CAPTURE_BYTES,
|
||||
crate::execution_runtime::MAX_STREAM_BODY_CAPTURE_BYTES
|
||||
);
|
||||
|
||||
let mut capture = AttemptBodyCapture::default();
|
||||
@@ -1454,6 +1461,20 @@ mod stage_tests {
|
||||
state,
|
||||
Some(aether_data_contracts::repository::usage::UsageBodyCaptureState::None)
|
||||
);
|
||||
|
||||
let defensive = AttemptBodyCapture {
|
||||
buffer: b"abcdef".to_vec(),
|
||||
truncated: false,
|
||||
};
|
||||
let (body, state) = defensive.encode_with_limit(3);
|
||||
let decoded = base64::engine::general_purpose::STANDARD
|
||||
.decode(body.expect("bounded capture should be encoded"))
|
||||
.expect("capture is valid base64");
|
||||
assert_eq!(decoded, b"abc");
|
||||
assert_eq!(
|
||||
state,
|
||||
Some(aether_data_contracts::repository::usage::UsageBodyCaptureState::Truncated)
|
||||
);
|
||||
}
|
||||
|
||||
/// candidate 行的 error_type 映射:投递失败与供应商侧失败必须各有名字。
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -3,3 +3,13 @@
|
||||
pub(crate) use aether_gateway_execution::{
|
||||
MAX_ERROR_BODY_BYTES, MAX_STREAM_PREFETCH_BYTES, MAX_STREAM_PREFETCH_FRAMES,
|
||||
};
|
||||
|
||||
// Usage/audit captures are secondary copies of the stream. Keep a hard
|
||||
// ceiling even when the configurable "full" record level is otherwise
|
||||
// unbounded; this does not limit bytes forwarded to the client.
|
||||
pub(crate) const MAX_STREAM_BODY_CAPTURE_BYTES: usize = 64 * 1024 * 1024;
|
||||
|
||||
// Stream frames are newline-delimited JSON. Binary response chunks are base64
|
||||
// encoded before framing, so this must be larger than the normal 64 MiB raw
|
||||
// response limit while still bounding an attacker-controlled unterminated line.
|
||||
pub(crate) const MAX_EXECUTION_STREAM_FRAME_LINE_BYTES: usize = 128 * 1024 * 1024;
|
||||
|
||||
@@ -156,7 +156,22 @@ pub(crate) fn should_fallback_to_control_sync(
|
||||
return true;
|
||||
};
|
||||
|
||||
body_json.get("error").is_some()
|
||||
sync_body_has_embedded_error(Some(body_json))
|
||||
}
|
||||
|
||||
/// Mirrors the error-like body markers used by the formats layer. Successful OpenAI Responses
|
||||
/// bodies contain `"error": null`, which must not route them through error finalization.
|
||||
fn sync_body_has_embedded_error(body_json: Option<&serde_json::Value>) -> bool {
|
||||
let Some(object) = body_json.and_then(serde_json::Value::as_object) else {
|
||||
return false;
|
||||
};
|
||||
|
||||
object.get("error").is_some_and(|error| !error.is_null())
|
||||
|| object.get("status").and_then(serde_json::Value::as_str) == Some("failed")
|
||||
|| object
|
||||
.get("type")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.is_some_and(|value| value == "error")
|
||||
}
|
||||
|
||||
pub(crate) fn should_finalize_sync_response(report_kind: Option<&str>) -> bool {
|
||||
@@ -168,7 +183,7 @@ pub(crate) fn resolve_core_sync_error_finalize_report_kind(
|
||||
result: &ExecutionResult,
|
||||
body_json: Option<&serde_json::Value>,
|
||||
) -> Option<String> {
|
||||
let has_embedded_error = body_json.is_some_and(|value| value.get("error").is_some());
|
||||
let has_embedded_error = sync_body_has_embedded_error(body_json);
|
||||
if result.status_code < 400 && !has_embedded_error {
|
||||
return None;
|
||||
}
|
||||
@@ -355,6 +370,7 @@ pub(crate) fn resolve_core_stream_direct_finalize_report_kind(plan_kind: &str) -
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use aether_contracts::{ExecutionError, ExecutionErrorKind, ExecutionPhase, ExecutionResult};
|
||||
@@ -435,6 +451,15 @@ mod tests {
|
||||
}
|
||||
|
||||
fn sample_key() -> StoredProviderCatalogKey {
|
||||
let credential_state = AppState::new()
|
||||
.expect("credential state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::disabled()
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY),
|
||||
);
|
||||
let encrypted_api_key = credential_state
|
||||
.seal_provider_catalog_key_api_key("provider-1", "key-1", "plain-upstream-key")
|
||||
.expect("api key should encrypt");
|
||||
StoredProviderCatalogKey::new(
|
||||
"key-1".to_string(),
|
||||
"provider-1".to_string(),
|
||||
@@ -446,7 +471,7 @@ mod tests {
|
||||
.expect("key should build")
|
||||
.with_transport_fields(
|
||||
Some(serde_json::json!(["openai:chat"])),
|
||||
"plain-upstream-key".to_string(),
|
||||
encrypted_api_key,
|
||||
None,
|
||||
None,
|
||||
Some(serde_json::json!({"openai:chat": 1})),
|
||||
@@ -466,7 +491,7 @@ mod tests {
|
||||
);
|
||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||
std::sync::Arc::new(provider_catalog),
|
||||
"development-key",
|
||||
DEVELOPMENT_ENCRYPTION_KEY,
|
||||
);
|
||||
AppState::new()
|
||||
.expect("state should build")
|
||||
@@ -500,6 +525,74 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn successful_responses_body_with_null_error_stays_on_success_path() {
|
||||
let result = ExecutionResult {
|
||||
request_id: "req-1".to_string(),
|
||||
candidate_id: None,
|
||||
status_code: 200,
|
||||
headers: Default::default(),
|
||||
response_observation: None,
|
||||
body: None,
|
||||
telemetry: None,
|
||||
error: None,
|
||||
};
|
||||
let body_json = serde_json::json!({
|
||||
"id": "resp_1",
|
||||
"object": "response",
|
||||
"status": "completed",
|
||||
"error": null,
|
||||
"output": [],
|
||||
});
|
||||
|
||||
assert_eq!(
|
||||
resolve_core_sync_error_finalize_report_kind(
|
||||
"openai_responses_sync",
|
||||
&result,
|
||||
Some(&body_json)
|
||||
),
|
||||
None
|
||||
);
|
||||
assert!(!should_fallback_to_control_sync(
|
||||
"openai_responses_sync",
|
||||
&result,
|
||||
Some(&body_json),
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_like_success_status_bodies_still_map_to_error_finalize() {
|
||||
let result = ExecutionResult {
|
||||
request_id: "req-1".to_string(),
|
||||
candidate_id: None,
|
||||
status_code: 200,
|
||||
headers: Default::default(),
|
||||
response_observation: None,
|
||||
body: None,
|
||||
telemetry: None,
|
||||
error: None,
|
||||
};
|
||||
|
||||
for body_json in [
|
||||
serde_json::json!({"status": "failed", "error": null}),
|
||||
serde_json::json!({"type": "error"}),
|
||||
serde_json::json!({"error": {"message": "boom"}}),
|
||||
] {
|
||||
assert_eq!(
|
||||
resolve_core_sync_error_finalize_report_kind(
|
||||
"openai_responses_sync",
|
||||
&result,
|
||||
Some(&body_json)
|
||||
),
|
||||
Some("openai_responses_sync_finalize".to_string()),
|
||||
"error-like body must not escape through the success path: {body_json}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stream_failover_marks_chat_errors() {
|
||||
assert!(should_fallback_to_control_stream(
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -21,11 +21,13 @@ use crate::execution_runtime::kiro_cache::{
|
||||
};
|
||||
use crate::execution_runtime::ndjson::encode_stream_frame_ndjson;
|
||||
use crate::execution_runtime::transport::{
|
||||
DirectSyncExecutionRuntime, ExecutionRuntimeTransportError,
|
||||
apply_upstream_response_body_limit, decode_base64_body_with_limit,
|
||||
json_value_fits_serialized_limit, DirectSyncExecutionRuntime, ExecutionRuntimeTransportError,
|
||||
};
|
||||
use crate::AppState;
|
||||
|
||||
const WEB_SEARCH_TOOL_NAME: &str = "web_search";
|
||||
const MAX_KIRO_MCP_JSON_BYTES: usize = 8 * 1024 * 1024;
|
||||
const WEB_SEARCH_TOOL_TYPE_PREFIX: &str = "web_search";
|
||||
const WEB_SEARCH_QUERY_PREFIX: &str = "Perform a web search for the query: ";
|
||||
|
||||
@@ -73,8 +75,6 @@ struct McpResponse {
|
||||
struct McpError {
|
||||
#[serde(default)]
|
||||
code: Option<i64>,
|
||||
#[serde(default)]
|
||||
message: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
@@ -146,14 +146,14 @@ pub(crate) async fn maybe_execute_kiro_web_search_stream(
|
||||
request_id = %plan.request_id,
|
||||
candidate_id = ?plan.candidate_id,
|
||||
status_code = mcp_execution.result.status_code,
|
||||
mcp_url = %mcp_execution.url,
|
||||
mcp_origin = %crate::handlers::shared::security_log_url_origin(&mcp_execution.url),
|
||||
profile_arn_present = mcp_execution.profile_arn_present,
|
||||
"gateway executed Kiro web_search through MCP endpoint"
|
||||
);
|
||||
|
||||
if !(200..300).contains(&mcp_execution.result.status_code) {
|
||||
return Ok(Some(KiroWebSearchStream {
|
||||
frame_stream: execution_result_frame_stream(&mcp_execution.result),
|
||||
frame_stream: execution_result_failure_frame_stream(&mcp_execution.result),
|
||||
report_context: report_context.cloned(),
|
||||
}));
|
||||
}
|
||||
@@ -183,7 +183,7 @@ pub(crate) async fn maybe_execute_kiro_web_search_stream(
|
||||
cache_usage,
|
||||
)
|
||||
.map_err(ExecutionRuntimeTransportError::BodyEncode)?;
|
||||
let mut synthetic_context = synthetic_report_context(report_context, mcp_execution.url);
|
||||
let mut synthetic_context = synthetic_report_context(report_context);
|
||||
if let Some(context) = synthetic_context.as_mut().and_then(Value::as_object_mut) {
|
||||
context.insert("kiro_web_search_mcp".to_string(), Value::Bool(true));
|
||||
}
|
||||
@@ -214,13 +214,13 @@ async fn kiro_simulated_cache_enabled(state: &AppState, plan: &ExecutionPlan) ->
|
||||
.is_some_and(|provider| {
|
||||
kiro_simulated_cache_enabled_from_provider_config(provider.config.as_ref())
|
||||
}),
|
||||
Err(err) => {
|
||||
Err(_) => {
|
||||
warn!(
|
||||
event_name = "kiro_simulated_cache_config_read_failed",
|
||||
log_type = "event",
|
||||
request_id = %plan.request_id,
|
||||
provider_id = %plan.provider_id,
|
||||
error = ?err,
|
||||
error_category = "provider_catalog_read_failed",
|
||||
"failed to read Kiro simulated cache provider config; defaulting disabled"
|
||||
);
|
||||
false
|
||||
@@ -228,30 +228,28 @@ async fn kiro_simulated_cache_enabled(state: &AppState, plan: &ExecutionPlan) ->
|
||||
}
|
||||
}
|
||||
|
||||
fn execute_result_body_bytes(result: &ExecutionResult) -> Vec<u8> {
|
||||
let Some(body) = result.body.as_ref() else {
|
||||
return Vec::new();
|
||||
};
|
||||
if let Some(json_body) = body.json_body.as_ref() {
|
||||
return serde_json::to_vec(json_body).unwrap_or_default();
|
||||
}
|
||||
body.body_bytes_b64
|
||||
.as_deref()
|
||||
.and_then(|body| base64::engine::general_purpose::STANDARD.decode(body).ok())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn execution_result_frame_stream(
|
||||
fn execution_result_failure_frame_stream(
|
||||
result: &ExecutionResult,
|
||||
) -> BoxStream<'static, Result<Bytes, IoError>> {
|
||||
let body = serde_json::to_vec(&kiro_mcp_failure_body(result.status_code)).unwrap_or_default();
|
||||
raw_response_frame_stream(
|
||||
result.status_code,
|
||||
result.headers.clone(),
|
||||
Bytes::from(execute_result_body_bytes(result)),
|
||||
BTreeMap::from([("content-type".to_string(), "application/json".to_string())]),
|
||||
Bytes::from(body),
|
||||
result.telemetry.clone(),
|
||||
)
|
||||
}
|
||||
|
||||
fn kiro_mcp_failure_body(status_code: u16) -> Value {
|
||||
json!({
|
||||
"error": {
|
||||
"type": "kiro_web_search_error",
|
||||
"message": "Kiro web search request failed",
|
||||
"code": status_code,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn sse_frame_stream(body: Bytes) -> BoxStream<'static, Result<Bytes, IoError>> {
|
||||
raw_response_frame_stream(
|
||||
200,
|
||||
@@ -328,12 +326,7 @@ async fn execute_mcp_request(
|
||||
request: &McpRequest,
|
||||
) -> Result<KiroMcpExecution, ExecutionRuntimeTransportError> {
|
||||
let mcp_url = aether_provider_transport::kiro::build_kiro_mcp_url_from_resolved_url(&plan.url)
|
||||
.ok_or_else(|| {
|
||||
ExecutionRuntimeTransportError::UpstreamRequest(format!(
|
||||
"failed to build Kiro MCP url from {}",
|
||||
plan.url
|
||||
))
|
||||
})?;
|
||||
.ok_or_else(kiro_mcp_url_build_error)?;
|
||||
let mut request_context = build_mcp_request_context(state, plan).await;
|
||||
if !request_context.profile_arn_present {
|
||||
if let Some(profile_arn) = discover_kiro_profile_arn(state, plan, &request_context).await? {
|
||||
@@ -346,7 +339,7 @@ async fn execute_mcp_request(
|
||||
}
|
||||
let body_json =
|
||||
serde_json::to_value(request).map_err(ExecutionRuntimeTransportError::BodyEncode)?;
|
||||
let mcp_plan = ExecutionPlan {
|
||||
let mut mcp_plan = ExecutionPlan {
|
||||
request_id: plan.request_id.clone(),
|
||||
candidate_id: plan.candidate_id.clone(),
|
||||
provider_name: plan.provider_name.clone(),
|
||||
@@ -367,6 +360,7 @@ async fn execute_mcp_request(
|
||||
transport_profile: plan.transport_profile.clone(),
|
||||
timeouts: plan.timeouts.clone(),
|
||||
};
|
||||
apply_upstream_response_body_limit(&mut mcp_plan, MAX_KIRO_MCP_JSON_BYTES);
|
||||
let result = DirectSyncExecutionRuntime::new()
|
||||
.execute_sync(&mcp_plan)
|
||||
.await?;
|
||||
@@ -390,7 +384,7 @@ async fn build_mcp_request_context(
|
||||
{
|
||||
Ok(Some(transport)) => transport,
|
||||
Ok(None) => return fallback(),
|
||||
Err(err) => {
|
||||
Err(_) => {
|
||||
warn!(
|
||||
event_name = "kiro_web_search_transport_snapshot_unavailable",
|
||||
log_type = "ops",
|
||||
@@ -399,7 +393,7 @@ async fn build_mcp_request_context(
|
||||
provider_id = %plan.provider_id,
|
||||
endpoint_id = %plan.endpoint_id,
|
||||
key_id = %plan.key_id,
|
||||
error = ?err,
|
||||
error_category = "transport_snapshot_read_failed",
|
||||
"gateway could not read Kiro transport snapshot for web_search MCP"
|
||||
);
|
||||
return fallback();
|
||||
@@ -574,7 +568,7 @@ async fn discover_kiro_profile_arn_in_region(
|
||||
if let Some(token) = next_token.as_deref() {
|
||||
body.insert("nextToken".to_string(), Value::String(token.to_string()));
|
||||
}
|
||||
let list_plan = ExecutionPlan {
|
||||
let mut list_plan = ExecutionPlan {
|
||||
request_id: plan.request_id.clone(),
|
||||
candidate_id: plan.candidate_id.clone(),
|
||||
provider_name: plan.provider_name.clone(),
|
||||
@@ -598,6 +592,7 @@ async fn discover_kiro_profile_arn_in_region(
|
||||
transport_profile: plan.transport_profile.clone(),
|
||||
timeouts: plan.timeouts.clone(),
|
||||
};
|
||||
apply_upstream_response_body_limit(&mut list_plan, MAX_KIRO_MCP_JSON_BYTES);
|
||||
let result = DirectSyncExecutionRuntime::new()
|
||||
.execute_sync(&list_plan)
|
||||
.await?;
|
||||
@@ -676,6 +671,10 @@ fn kiro_runtime_base_url_for_region(region: &str) -> String {
|
||||
}
|
||||
|
||||
fn kiro_runtime_host_for_region(region: &str) -> String {
|
||||
// Region values can originate in persisted OAuth metadata. Normalize
|
||||
// before interpolation so a crafted value cannot turn the host header or
|
||||
// URL into an attacker-controlled origin.
|
||||
let region = aether_provider_transport::kiro::normalize_kiro_region(region);
|
||||
match region {
|
||||
"us-gov-east-1" | "us-gov-west-1" => format!("q-fips.{region}.amazonaws.com"),
|
||||
"us-iso-east-1" => "q.us-iso-east-1.c2s.ic.gov".to_string(),
|
||||
@@ -959,7 +958,6 @@ fn parse_mcp_search_results(result: &ExecutionResult) -> Option<WebSearchResults
|
||||
event_name = "kiro_web_search_mcp_error",
|
||||
log_type = "event",
|
||||
code = error.code.unwrap_or_default(),
|
||||
message = error.message.as_deref().unwrap_or("unknown"),
|
||||
"Kiro MCP web_search returned JSON-RPC error"
|
||||
);
|
||||
return None;
|
||||
@@ -983,12 +981,18 @@ fn parse_mcp_search_results(result: &ExecutionResult) -> Option<WebSearchResults
|
||||
fn execution_result_body_json(result: &ExecutionResult) -> Option<Value> {
|
||||
let body = result.body.as_ref()?;
|
||||
if let Some(json_body) = body.json_body.as_ref() {
|
||||
return Some(json_body.clone());
|
||||
return json_value_fits_serialized_limit(json_body, MAX_KIRO_MCP_JSON_BYTES)
|
||||
.then(|| json_body.clone());
|
||||
}
|
||||
let body = body.body_bytes_b64.as_deref()?;
|
||||
let bytes = base64::engine::general_purpose::STANDARD
|
||||
.decode(body)
|
||||
.ok()?;
|
||||
// MCP responses are parsed into an owned JSON tree. Keep a fixed ceiling
|
||||
// here even if the operator disables the general internal body cap, so a
|
||||
// forged execution result cannot trigger an unbounded base64 allocation.
|
||||
let bytes = decode_base64_body_with_limit(
|
||||
body,
|
||||
crate::headers::max_internal_buffered_body_bytes().min(MAX_KIRO_MCP_JSON_BYTES),
|
||||
)
|
||||
.ok()?;
|
||||
serde_json::from_slice(&bytes).ok()
|
||||
}
|
||||
|
||||
@@ -1198,13 +1202,14 @@ fn generate_search_summary(query: &str, results: Option<&WebSearchResults>) -> S
|
||||
}
|
||||
summary.push_str(&format!(" Source: {}\n\n", result.url));
|
||||
}
|
||||
if let Some(error) = results
|
||||
if results
|
||||
.error
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.is_some()
|
||||
{
|
||||
summary.push_str(&format!("Search warning: {error}\n\n"));
|
||||
summary.push_str("Search warning: the provider reported an incomplete result.\n\n");
|
||||
}
|
||||
}
|
||||
_ => summary.push_str("No results found.\n"),
|
||||
@@ -1237,12 +1242,15 @@ fn estimate_text_tokens(text: &str) -> u64 {
|
||||
((text.len() as u64 + 3) / 4).max(1)
|
||||
}
|
||||
|
||||
fn synthetic_report_context(report_context: Option<&Value>, mcp_url: String) -> Option<Value> {
|
||||
fn kiro_mcp_url_build_error() -> ExecutionRuntimeTransportError {
|
||||
ExecutionRuntimeTransportError::UpstreamRequest("failed to build Kiro MCP URL".to_string())
|
||||
}
|
||||
|
||||
fn synthetic_report_context(report_context: Option<&Value>) -> Option<Value> {
|
||||
let mut context = report_context.cloned()?;
|
||||
if let Some(object) = context.as_object_mut() {
|
||||
object.insert("has_envelope".to_string(), Value::Bool(false));
|
||||
object.insert("needs_conversion".to_string(), Value::Bool(false));
|
||||
object.insert("upstream_url".to_string(), Value::String(mcp_url));
|
||||
object.remove("envelope_name");
|
||||
}
|
||||
Some(context)
|
||||
@@ -1257,7 +1265,9 @@ mod tests {
|
||||
|
||||
use super::{
|
||||
build_mcp_headers_from_plan, build_web_search_sse_body, detect_kiro_web_search_request,
|
||||
parse_mcp_search_results, strip_search_query_prefix, KiroPromptCacheUsage,
|
||||
execution_result_body_json, generate_search_summary, kiro_mcp_failure_body,
|
||||
kiro_mcp_url_build_error, parse_mcp_search_results, strip_search_query_prefix,
|
||||
KiroPromptCacheUsage, WebSearchResult, WebSearchResults,
|
||||
};
|
||||
|
||||
fn sample_plan(body: serde_json::Value) -> ExecutionPlan {
|
||||
@@ -1443,6 +1453,43 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kiro_mcp_url_build_error_omits_resolved_url() {
|
||||
let sensitive_url = "https://token:[email protected]/path";
|
||||
let message = kiro_mcp_url_build_error().to_string();
|
||||
|
||||
assert_eq!(
|
||||
message,
|
||||
"failed to execute upstream request: failed to build Kiro MCP URL"
|
||||
);
|
||||
assert!(!message.contains(sensitive_url));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kiro_mcp_errors_and_search_warnings_do_not_copy_upstream_secrets() {
|
||||
let secret = "authorization=Bearer upstream-secret";
|
||||
let failure = kiro_mcp_failure_body(502).to_string();
|
||||
let summary = generate_search_summary(
|
||||
"test",
|
||||
Some(&WebSearchResults {
|
||||
results: vec![WebSearchResult {
|
||||
title: "Safe result".to_string(),
|
||||
url: "https://example.test/result".to_string(),
|
||||
snippet: None,
|
||||
published_date: None,
|
||||
}],
|
||||
total_results: Some(1),
|
||||
query: None,
|
||||
error: Some(secret.to_string()),
|
||||
}),
|
||||
);
|
||||
|
||||
assert!(failure.contains("Kiro web search request failed"));
|
||||
assert!(summary.contains("provider reported an incomplete result"));
|
||||
assert!(!failure.contains("upstream-secret"));
|
||||
assert!(!summary.contains("upstream-secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_mcp_search_result_text_payload() {
|
||||
let result = aether_contracts::ExecutionResult {
|
||||
@@ -1474,6 +1521,29 @@ mod tests {
|
||||
assert_eq!(parsed.results[0].title, "Example");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kiro_mcp_result_rejects_oversized_base64_before_decode() {
|
||||
let encoded_limit =
|
||||
crate::execution_runtime::transport::maximum_base64_len_for_decoded_limit(
|
||||
super::MAX_KIRO_MCP_JSON_BYTES,
|
||||
);
|
||||
let result = aether_contracts::ExecutionResult {
|
||||
request_id: "req-oversized".to_string(),
|
||||
candidate_id: None,
|
||||
status_code: 200,
|
||||
headers: BTreeMap::new(),
|
||||
response_observation: None,
|
||||
body: Some(aether_contracts::ResponseBody {
|
||||
json_body: None,
|
||||
body_bytes_b64: Some("A".repeat(encoded_limit + 1)),
|
||||
}),
|
||||
telemetry: None,
|
||||
error: None,
|
||||
};
|
||||
|
||||
assert!(execution_result_body_json(&result).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_anthropic_web_search_sse() {
|
||||
let sse = build_web_search_sse_body(
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user