mirror of
https://github.com/fawney19/Aether.git
synced 2026-10-05 00:47:48 +08:00
Compare commits
150
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8b766930b0 | ||
|
|
c7e403b410 | ||
|
|
cf8ea19856 | ||
|
|
7113d04f8a | ||
|
|
099b810a2f | ||
|
|
7aa0c89244 | ||
|
|
7847ae98c6 | ||
|
|
a90d564931 | ||
|
|
a5c3699ae9 | ||
|
|
7b8048c6ae | ||
|
|
ec95f2ca1f | ||
|
|
aa7dbe67d3 | ||
|
|
a26680f460 | ||
|
|
522b979052 | ||
|
|
808946312a | ||
|
|
741107bf71 | ||
|
|
6962731220 | ||
|
|
062e111c03 | ||
|
|
470c59e197 | ||
|
|
2f929e74c7 | ||
|
|
fc0417ceb9 | ||
|
|
44174a31e0 | ||
|
|
b599fb7354 | ||
|
|
14f96c9fa0 | ||
|
|
6948852992 | ||
|
|
1b01b08c31 | ||
|
|
2281f2b754 | ||
|
|
b5ed802277 | ||
|
|
d1b5eb08ee | ||
|
|
dba5e6e9e9 | ||
|
|
c125e78c5f | ||
|
|
d1cb0ebecf | ||
|
|
9d7a0665c0 | ||
|
|
882bb43125 | ||
|
|
db6c522d60 | ||
|
|
e29442a06a | ||
|
|
e15ea0d5d3 | ||
|
|
2f374d6af2 | ||
|
|
4a356f4ea5 | ||
|
|
c7676d567d | ||
|
|
5ca4f87951 | ||
|
|
1fee8954cc | ||
|
|
f69b770f5e | ||
|
|
856accdced | ||
|
|
92749b4d6e | ||
|
|
f08c2e6729 | ||
|
|
e420bc6324 | ||
|
|
d723fb92d3 | ||
|
|
5b1de5f921 | ||
|
|
7ed48e7b58 | ||
|
|
af712ebdbf | ||
|
|
33d5cd5993 | ||
|
|
f5e1420ee6 | ||
|
|
b37b252b14 | ||
|
|
0097ea89ad | ||
|
|
10e63507f0 | ||
|
|
9ff4d73d5c | ||
|
|
0e3bd7eff4 | ||
|
|
1c89b5f9ab | ||
|
|
cdbbda40a6 | ||
|
|
29a9d608d9 | ||
|
|
a6dc43d5f6 | ||
|
|
c6718754d3 | ||
|
|
afdd033745 | ||
|
|
d5f54ffe8b | ||
|
|
f5ec76c5c8 | ||
|
|
784a1e0611 | ||
|
|
507cb33089 | ||
|
|
b08fa3bdb6 | ||
|
|
018af84d7d | ||
|
|
27b0381a9a | ||
|
|
57cdef4b8d | ||
|
|
36e9d21e3f | ||
|
|
b72b6ab137 | ||
|
|
30b2c8548a | ||
|
|
7c5cce4b3c | ||
|
|
9362c34fcd | ||
|
|
344b3031e9 | ||
|
|
e89c3aa674 | ||
|
|
ddbbf835af | ||
|
|
cb58a63ee3 | ||
|
|
6b1074cfcd | ||
|
|
d6894b5532 | ||
|
|
635c6765d9 | ||
|
|
86f7cc0d58 | ||
|
|
206995645b | ||
|
|
9282cce1d6 | ||
|
|
c005700a7e | ||
|
|
14744abd57 | ||
|
|
66d6c17d2d | ||
|
|
c142d39951 | ||
|
|
1eb2d10dec | ||
|
|
dabaeb8dfa | ||
|
|
2d17d4b73f | ||
|
|
18d78dd6c9 | ||
|
|
499942e3e7 | ||
|
|
ba11a72214 | ||
|
|
12571764bc | ||
|
|
1e13fa032c | ||
|
|
47b21a25d3 | ||
|
|
45a3ba8829 | ||
|
|
03f2914044 | ||
|
|
c8d1ae3e7e | ||
|
|
c5ae9c2c77 | ||
|
|
4e47c00154 | ||
|
|
313a637982 | ||
|
|
fe8ff268df | ||
|
|
bac6d6866a | ||
|
|
579f2c7cc1 | ||
|
|
ddcbeb3ae9 | ||
|
|
e25fc984af | ||
|
|
4cf47b1dee | ||
|
|
95cbd43097 | ||
|
|
09005939bf | ||
|
|
7b612b8b5a | ||
|
|
670d5e8d33 | ||
|
|
1de2e70d41 | ||
|
|
89b57464d2 | ||
|
|
09ef3adf70 | ||
|
|
3dfc15963c | ||
|
|
f6884eb8c4 | ||
|
|
f8b4382a54 | ||
|
|
d78b5a81fb | ||
|
|
89fe9e9f0a | ||
|
|
4291a91dc0 | ||
|
|
4c6bafe255 | ||
|
|
979dbc4b33 | ||
|
|
9309ad844f | ||
|
|
d672ba2068 | ||
|
|
587486ab0c | ||
|
|
40a5e1470d | ||
|
|
058660ec2e | ||
|
|
668bf5e40f | ||
|
|
77f93c638d | ||
|
|
d0c0996b9f | ||
|
|
2cb4d554aa | ||
|
|
76fb8905c9 | ||
|
|
f822df6cce | ||
|
|
45c840b8d3 | ||
|
|
e8d9877b79 | ||
|
|
cae9aa4134 | ||
|
|
e3644c6142 | ||
|
|
cc6f5e89b6 | ||
|
|
2ed2cc66ef | ||
|
|
b1bf7837cf | ||
|
|
77229943d1 | ||
|
|
a0369cf49a | ||
|
|
dbbe7b22ab | ||
|
|
d5f34b2ee2 | ||
|
|
5059093d29 |
+49
-12
@@ -26,21 +26,25 @@ RUST_LOG=aether_gateway=info
|
|||||||
# 示例: http://localhost:5173,https://app.example.com
|
# 示例: http://localhost:5173,https://app.example.com
|
||||||
# CORS_ORIGINS=http://localhost:5173
|
# CORS_ORIGINS=http://localhost:5173
|
||||||
# CORS_ALLOW_CREDENTIALS=true
|
# CORS_ALLOW_CREDENTIALS=true
|
||||||
# 如果前后端跨站并依赖登录刷新 Cookie,还要配合:
|
# 登录刷新 Cookie 对同源浏览器请求和可信反代自动适配 HTTP/HTTPS。
|
||||||
|
# HTTP 自动使用兼容的 SameSite=Lax(显式 Strict 保留);HTTPS 保留原有 SameSite 配置。
|
||||||
|
# 无法确认访问协议时保留安全默认值;HTTPS 反代请正确传递 X-Forwarded-Proto。
|
||||||
|
# AUTH_REFRESH_COOKIE_SECURE 可显式覆盖自动判断,公网部署仍建议使用 HTTPS。
|
||||||
|
# 如果前后端跨站并依赖登录刷新 Cookie,必须使用 HTTPS,并配合:
|
||||||
# AUTH_REFRESH_COOKIE_SAMESITE=None
|
# AUTH_REFRESH_COOKIE_SAMESITE=None
|
||||||
# AUTH_REFRESH_COOKIE_SECURE=true
|
# AUTH_REFRESH_COOKIE_SECURE=true
|
||||||
|
|
||||||
# 数据库配置
|
# 数据库配置(仅支持 PostgreSQL)
|
||||||
DB_HOST=localhost
|
DB_HOST=localhost
|
||||||
DB_PORT=5432
|
DB_PORT=5432
|
||||||
DB_USER=postgres
|
DB_USER=postgres
|
||||||
DB_NAME=aether
|
DB_NAME=aether
|
||||||
DB_PASSWORD=aether
|
DB_PASSWORD=
|
||||||
|
|
||||||
# Redis 配置
|
# Redis 配置
|
||||||
REDIS_HOST=localhost
|
REDIS_HOST=localhost
|
||||||
REDIS_PORT=6379
|
REDIS_PORT=6379
|
||||||
REDIS_PASSWORD=aether
|
REDIS_PASSWORD=
|
||||||
|
|
||||||
# JWT密钥(使用 ./generate_keys.sh 生成)
|
# JWT密钥(使用 ./generate_keys.sh 生成)
|
||||||
# 用于用户登录 token 签名,更换后所有用户需重新登录
|
# 用于用户登录 token 签名,更换后所有用户需重新登录
|
||||||
@@ -50,8 +54,12 @@ JWT_SECRET_KEY=change-this-to-a-secure-random-string
|
|||||||
# 注意:更换此密钥后需要在管理面板重新配置所有 Provider API Key
|
# 注意:更换此密钥后需要在管理面板重新配置所有 Provider API Key
|
||||||
ENCRYPTION_KEY=change-this-to-another-secure-random-string
|
ENCRYPTION_KEY=change-this-to-another-secure-random-string
|
||||||
|
|
||||||
|
# S3 备份的独立加密密钥(推荐)。未配置时为兼容旧部署,会回退到 ENCRYPTION_KEY。
|
||||||
|
# 密钥轮换前必须保留旧值,离线恢复工具需要它解密历史备份。
|
||||||
|
# AETHER_BACKUP_ENCRYPTION_KEY=change-this-to-a-dedicated-secure-random-string
|
||||||
|
|
||||||
# 启动自举管理员(仅在当前库里还没有活动管理员时生效)
|
# 启动自举管理员(仅在当前库里还没有活动管理员时生效)
|
||||||
# 手动部署时取消注释并设置;install.sh 首次生成配置时会提示输入。
|
# 首次启动前必须设置 ADMIN_PASSWORD;install.sh 首次生成配置时会提示输入。
|
||||||
ADMIN_EMAIL=[email protected]
|
ADMIN_EMAIL=[email protected]
|
||||||
ADMIN_USERNAME=admin123456
|
ADMIN_USERNAME=admin123456
|
||||||
# ADMIN_PASSWORD=
|
# ADMIN_PASSWORD=
|
||||||
@@ -69,21 +77,50 @@ ADMIN_USERNAME=admin123456
|
|||||||
# AETHER_VSCODEX_PUBLIC_WS_URL=wss://aether.example.com/api/vscodex/ws
|
# AETHER_VSCODEX_PUBLIC_WS_URL=wss://aether.example.com/api/vscodex/ws
|
||||||
# AETHER_VSCODEX_ALLOWED_ORIGINS=https://aether.example.com
|
# AETHER_VSCODEX_ALLOWED_ORIGINS=https://aether.example.com
|
||||||
|
|
||||||
# docker compose 下 app 启动前自动执行 pending migration/backfill(默认 true)
|
# 启动时的数据库准备策略:auto(默认)或 verify-only
|
||||||
# AETHER_GATEWAY_AUTO_PREPARE_DATABASE=true
|
# AETHER_GATEWAY_DATABASE_MODE=auto
|
||||||
|
|
||||||
# PostgreSQL 连接池配置(默认每核 4 条、总池至少 32 条且最多 100 条;多实例部署应显式分配每实例预算)
|
# PostgreSQL 连接池配置(默认每核 4 条、总池至少 32 条且最多 100 条;多实例部署应显式分配每实例预算)
|
||||||
# AETHER_GATEWAY_DATA_POSTGRES_MIN_CONNECTIONS=12
|
# AETHER_GATEWAY_DATA_POSTGRES_MIN_CONNECTIONS=12
|
||||||
# AETHER_GATEWAY_DATA_POSTGRES_MAX_CONNECTIONS=80
|
# AETHER_GATEWAY_DATA_POSTGRES_MAX_CONNECTIONS=80
|
||||||
# AETHER_GATEWAY_MAX_IN_FLIGHT_REQUESTS=2048
|
# AETHER_GATEWAY_MAX_IN_FLIGHT_REQUESTS=2048
|
||||||
# AETHER_GATEWAY_REQUEST_BODY_BUFFER_BUDGET_MB=256
|
# AETHER_GATEWAY_REQUEST_BODY_BUFFER_BUDGET_MB=256
|
||||||
# AETHER_GATEWAY_REQUEST_BODY_READ_TIMEOUT_MS=120000
|
# 请求体完整读取总超时默认关闭;确需限制时配置 1000-600000 毫秒的非零值。
|
||||||
# 可选的 Payload 上限(MiB);默认及 0 均表示不限制。
|
# AETHER_GATEWAY_REQUEST_BODY_READ_TIMEOUT_MS=0
|
||||||
# AETHER_MAX_REQUEST_BODY_MB=0
|
# 单请求解压后 Payload 上限(MiB),默认 256;显式设为 0 才表示不限制。
|
||||||
|
# AETHER_MAX_REQUEST_BODY_MB=256
|
||||||
# AETHER_GATEWAY_SECURITY_CACHE_TTL_MS=1000
|
# AETHER_GATEWAY_SECURITY_CACHE_TTL_MS=1000
|
||||||
# AETHER_MAX_REDACTED_SYNC_RESPONSE_BODY_MB=0
|
# AETHER_MAX_REDACTED_SYNC_RESPONSE_BODY_MB=64
|
||||||
# AETHER_MAX_INTERNAL_BUFFERED_BODY_MB=0
|
# AETHER_MAX_INTERNAL_BUFFERED_BODY_MB=64
|
||||||
# AETHER_TUNNEL_NODE_STATUS_QUEUE_CAPACITY=1024
|
# AETHER_TUNNEL_NODE_STATUS_QUEUE_CAPACITY=1024
|
||||||
|
# Tunnel relay 使用的独立 HMAC 密钥。启用 HTTP tunnel relay 或多网关 owner 转发时必须配置,
|
||||||
|
# 所有网关实例必须使用同一个至少 32 字节的随机值;不要复用 JWT 或数据加密密钥。
|
||||||
|
# AETHER_TUNNEL_RELAY_AUTH_SECRET=
|
||||||
|
# 旧版 /api/internal/gateway/* 控制面默认关闭。确需独立服务调用时,配置至少 32 字节的
|
||||||
|
# 独立 HMAC 密钥;不要复用 JWT、数据加密或 tunnel relay 密钥。多节点必须使用相同值和共享 Redis。
|
||||||
|
# AETHER_INTERNAL_GATEWAY_AUTH_SECRET=
|
||||||
|
# 远程 relay 地址必须使用 HTTPS;HTTP 仅允许 localhost 或回环 IP。
|
||||||
|
# AETHER_TUNNEL_RELAY_BASE_URL=https://gateway-a.example.com
|
||||||
|
# 跨网关 relay 解析到受控私有地址时才显式开启;默认关闭以防止被篡改的 attachment
|
||||||
|
# 记录诱导网关向内网转发 relay 凭据。该开关不放宽普通 provider 的目标地址策略。
|
||||||
|
# AETHER_TUNNEL_RELAY_ALLOW_PRIVATE_TARGETS=false
|
||||||
|
# 更推荐按 relay 主机名精确放行私网部署(逗号分隔,大小写不敏感);不支持通配符/后缀。
|
||||||
|
# AETHER_TUNNEL_RELAY_PRIVATE_HOST_ALLOWLIST=gateway-a.internal,gateway-b.internal
|
||||||
|
# Bark 自建服务默认仅允许公网 HTTPS。确需明文 HTTP 或内网目标时分别显式开启:
|
||||||
|
# AETHER_BARK_ALLOW_HTTP=false
|
||||||
|
# AETHER_BARK_ALLOW_PRIVATE_TARGETS=false
|
||||||
|
|
||||||
|
# 普通 Provider 反代(包括 Provider OAuth)不按 DNS 地址过滤上游,兼容任意
|
||||||
|
# Fake-IP 域名及内网 DNS。仅信任管理员配置的上游;没有严格 DNS 过滤开关。
|
||||||
|
# URL 协议、字面 IP、TLS 证书,以及隧道中继和登录 OAuth 的校验仍保留。
|
||||||
|
|
||||||
|
# 可选 Provider OAuth 客户端。Gemini CLI 和 Antigravity 默认使用内置 native-app
|
||||||
|
# 客户端凭据;自定义 client ID 时必须同时配置对应的 client secret。
|
||||||
|
# 显式配置的 client secret 优先于默认值。
|
||||||
|
# AETHER_GEMINI_CLI_OAUTH_CLIENT_ID=
|
||||||
|
# AETHER_GEMINI_CLI_OAUTH_CLIENT_SECRET=
|
||||||
|
# AETHER_ANTIGRAVITY_OAUTH_CLIENT_ID=
|
||||||
|
# AETHER_ANTIGRAVITY_OAUTH_CLIENT_SECRET=
|
||||||
|
|
||||||
# PostgreSQL 容器调优:docker-compose.yml 已内置通用默认值,通常不用配置。
|
# PostgreSQL 容器调优:docker-compose.yml 已内置通用默认值,通常不用配置。
|
||||||
# 只有在 Postgres 独占大内存、或压测显示 DB 缓存/排序/维护任务成为瓶颈时再覆盖。
|
# 只有在 Postgres 独占大内存、或压测显示 DB 缓存/排序/维护任务成为瓶颈时再覆盖。
|
||||||
|
|||||||
@@ -6,7 +6,8 @@ on:
|
|||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
contents: write
|
actions: read
|
||||||
|
contents: read
|
||||||
|
|
||||||
concurrency:
|
concurrency:
|
||||||
group: build-tunnel-${{ github.ref }}
|
group: build-tunnel-${{ github.ref }}
|
||||||
@@ -17,7 +18,7 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
if: startsWith(github.ref, 'refs/tags/')
|
if: startsWith(github.ref, 'refs/tags/')
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Ensure tunnel tag matches Cargo version
|
- name: Ensure tunnel tag matches Cargo version
|
||||||
shell: bash
|
shell: bash
|
||||||
@@ -78,10 +79,10 @@ jobs:
|
|||||||
use_cross: false
|
use_cross: false
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
- name: Install Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||||
with:
|
with:
|
||||||
targets: ${{ matrix.target }}
|
targets: ${{ matrix.target }}
|
||||||
|
|
||||||
@@ -89,14 +90,14 @@ jobs:
|
|||||||
run: rustup target add ${{ matrix.target }}
|
run: rustup target add ${{ matrix.target }}
|
||||||
|
|
||||||
- name: Rust cache
|
- name: Rust cache
|
||||||
uses: Swatinem/rust-cache@v2
|
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||||
with:
|
with:
|
||||||
workspaces: apps/aether-tunnel -> target
|
workspaces: apps/aether-tunnel -> target
|
||||||
key: ${{ matrix.target }}
|
key: ${{ matrix.target }}
|
||||||
|
|
||||||
- name: Install cross
|
- name: Install cross
|
||||||
if: matrix.use_cross
|
if: matrix.use_cross
|
||||||
uses: taiki-e/install-action@cross
|
uses: taiki-e/install-action@1ae7257be536a92d9218a6b343dc6e6ba650f7e1 # cross
|
||||||
|
|
||||||
- name: Build
|
- name: Build
|
||||||
working-directory: apps/aether-tunnel
|
working-directory: apps/aether-tunnel
|
||||||
@@ -122,9 +123,10 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
cd target/${{ matrix.target }}/release
|
cd target/${{ matrix.target }}/release
|
||||||
7z a ../../../aether-tunnel-${{ matrix.name }}.zip aether-tunnel.exe
|
7z a ../../../aether-tunnel-${{ matrix.name }}.zip aether-tunnel.exe
|
||||||
|
tar czf ../../../aether-tunnel-${{ matrix.name }}.tar.gz aether-tunnel.exe
|
||||||
|
|
||||||
- name: Upload artifact
|
- name: Upload artifact
|
||||||
uses: actions/upload-artifact@v5
|
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||||
with:
|
with:
|
||||||
name: aether-tunnel-${{ matrix.name }}
|
name: aether-tunnel-${{ matrix.name }}
|
||||||
path: |
|
path: |
|
||||||
@@ -137,9 +139,14 @@ jobs:
|
|||||||
needs: build
|
needs: build
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
if: startsWith(github.ref, 'refs/tags/')
|
if: startsWith(github.ref, 'refs/tags/')
|
||||||
|
permissions:
|
||||||
|
actions: read
|
||||||
|
attestations: write
|
||||||
|
contents: write
|
||||||
|
id-token: write
|
||||||
steps:
|
steps:
|
||||||
- name: Download all artifacts
|
- name: Download all artifacts
|
||||||
uses: actions/download-artifact@v5
|
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||||
with:
|
with:
|
||||||
merge-multiple: true
|
merge-multiple: true
|
||||||
path: artifacts
|
path: artifacts
|
||||||
@@ -148,6 +155,20 @@ jobs:
|
|||||||
working-directory: artifacts
|
working-directory: artifacts
|
||||||
run: sha256sum aether-tunnel-* > SHA256SUMS.txt
|
run: sha256sum aether-tunnel-* > SHA256SUMS.txt
|
||||||
|
|
||||||
|
- name: Attest tunnel release provenance
|
||||||
|
id: attest-release
|
||||||
|
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||||
|
with:
|
||||||
|
subject-path: |
|
||||||
|
artifacts/aether-tunnel-*.tar.gz
|
||||||
|
artifacts/aether-tunnel-*.zip
|
||||||
|
artifacts/SHA256SUMS.txt
|
||||||
|
|
||||||
|
- name: Bundle tunnel release provenance
|
||||||
|
env:
|
||||||
|
ATTESTATION_BUNDLE: ${{ steps.attest-release.outputs.bundle-path }}
|
||||||
|
run: install -m 0644 "${ATTESTATION_BUNDLE}" artifacts/AETHER_TUNNEL_RELEASE_PROVENANCE.sigstore.json
|
||||||
|
|
||||||
- name: Delete stale draft releases for tag
|
- name: Delete stale draft releases for tag
|
||||||
env:
|
env:
|
||||||
GH_TOKEN: ${{ github.token }}
|
GH_TOKEN: ${{ github.token }}
|
||||||
@@ -170,12 +191,13 @@ jobs:
|
|||||||
done <<< "${draft_ids}"
|
done <<< "${draft_ids}"
|
||||||
|
|
||||||
- name: Create GitHub Release
|
- name: Create GitHub Release
|
||||||
uses: softprops/action-gh-release@v2
|
uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2
|
||||||
with:
|
with:
|
||||||
name: "${{ github.ref_name }}"
|
name: "${{ github.ref_name }}"
|
||||||
generate_release_notes: true
|
generate_release_notes: true
|
||||||
files: |
|
files: |
|
||||||
artifacts/aether-tunnel-*
|
artifacts/aether-tunnel-*
|
||||||
|
artifacts/AETHER_TUNNEL_RELEASE_PROVENANCE.sigstore.json
|
||||||
artifacts/SHA256SUMS.txt
|
artifacts/SHA256SUMS.txt
|
||||||
fail_on_unmatched_files: true
|
fail_on_unmatched_files: true
|
||||||
|
|
||||||
@@ -183,8 +205,10 @@ jobs:
|
|||||||
needs: release
|
needs: release
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
if: startsWith(github.ref, 'refs/tags/')
|
if: startsWith(github.ref, 'refs/tags/')
|
||||||
|
permissions:
|
||||||
|
contents: write
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
with:
|
with:
|
||||||
ref: main
|
ref: main
|
||||||
|
|
||||||
|
|||||||
@@ -7,8 +7,6 @@ on:
|
|||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
contents: read
|
contents: read
|
||||||
pages: write
|
|
||||||
id-token: write
|
|
||||||
|
|
||||||
concurrency:
|
concurrency:
|
||||||
group: pages
|
group: pages
|
||||||
@@ -46,10 +44,10 @@ jobs:
|
|||||||
if: needs.preflight.outputs.deploy_pages == 'true'
|
if: needs.preflight.outputs.deploy_pages == 'true'
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Setup Node.js
|
- name: Setup Node.js
|
||||||
uses: actions/setup-node@v5
|
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5
|
||||||
with:
|
with:
|
||||||
node-version: '22'
|
node-version: '22'
|
||||||
cache: 'npm'
|
cache: 'npm'
|
||||||
@@ -77,10 +75,10 @@ jobs:
|
|||||||
run: cp frontend/dist/index.html frontend/dist/404.html
|
run: cp frontend/dist/index.html frontend/dist/404.html
|
||||||
|
|
||||||
- name: Setup Pages
|
- name: Setup Pages
|
||||||
uses: actions/configure-pages@v5
|
uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5
|
||||||
|
|
||||||
- name: Upload artifact
|
- name: Upload artifact
|
||||||
uses: actions/upload-pages-artifact@v3
|
uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3
|
||||||
with:
|
with:
|
||||||
path: frontend/dist
|
path: frontend/dist
|
||||||
|
|
||||||
@@ -90,7 +88,10 @@ jobs:
|
|||||||
url: ${{ steps.deployment.outputs.page_url }}
|
url: ${{ steps.deployment.outputs.page_url }}
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs: build
|
needs: build
|
||||||
|
permissions:
|
||||||
|
id-token: write
|
||||||
|
pages: write
|
||||||
steps:
|
steps:
|
||||||
- name: Deploy to GitHub Pages
|
- name: Deploy to GitHub Pages
|
||||||
id: deployment
|
id: deployment
|
||||||
uses: actions/deploy-pages@v4
|
uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4
|
||||||
|
|||||||
@@ -25,7 +25,6 @@ env:
|
|||||||
CARGO_PROFILE_TEST_DEBUG: '0'
|
CARGO_PROFILE_TEST_DEBUG: '0'
|
||||||
CARGO_TERM_COLOR: always
|
CARGO_TERM_COLOR: always
|
||||||
RUST_BACKTRACE: '1'
|
RUST_BACKTRACE: '1'
|
||||||
GHCR_IMAGE: ghcr.io/fawney19/aether
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
source:
|
source:
|
||||||
@@ -36,6 +35,7 @@ jobs:
|
|||||||
sha: ${{ steps.snapshot.outputs.sha }}
|
sha: ${{ steps.snapshot.outputs.sha }}
|
||||||
short_sha: ${{ steps.snapshot.outputs.short_sha }}
|
short_sha: ${{ steps.snapshot.outputs.short_sha }}
|
||||||
date: ${{ steps.snapshot.outputs.date }}
|
date: ${{ steps.snapshot.outputs.date }}
|
||||||
|
ghcr_image: ${{ steps.snapshot.outputs.ghcr_image }}
|
||||||
steps:
|
steps:
|
||||||
- name: Require main branch
|
- name: Require main branch
|
||||||
id: snapshot
|
id: snapshot
|
||||||
@@ -49,9 +49,13 @@ jobs:
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
sha="${GITHUB_SHA}"
|
sha="${GITHUB_SHA}"
|
||||||
|
# Docker 镜像仓库名必须全小写;GitHub owner 可能保留大写,先统一规范化。
|
||||||
|
repository_owner="${GITHUB_REPOSITORY%%/*}"
|
||||||
|
repository_owner="${repository_owner,,}"
|
||||||
echo "sha=${sha}" >> "${GITHUB_OUTPUT}"
|
echo "sha=${sha}" >> "${GITHUB_OUTPUT}"
|
||||||
echo "short_sha=${sha:0:7}" >> "${GITHUB_OUTPUT}"
|
echo "short_sha=${sha:0:7}" >> "${GITHUB_OUTPUT}"
|
||||||
echo "date=$(date -u +'%Y-%m-%d')" >> "${GITHUB_OUTPUT}"
|
echo "date=$(date -u +'%Y-%m-%d')" >> "${GITHUB_OUTPUT}"
|
||||||
|
echo "ghcr_image=ghcr.io/${repository_owner}/aether" >> "${GITHUB_OUTPUT}"
|
||||||
echo "Building main at ${sha}."
|
echo "Building main at ${sha}."
|
||||||
|
|
||||||
# Keep the scheduled backend coverage in one place so it cannot drift from PR CI.
|
# Keep the scheduled backend coverage in one place so it cannot drift from PR CI.
|
||||||
@@ -66,12 +70,12 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 90
|
timeout-minutes: 90
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
with:
|
with:
|
||||||
ref: ${{ needs.source.outputs.sha }}
|
ref: ${{ needs.source.outputs.sha }}
|
||||||
|
|
||||||
- name: Install pinned Rust toolchain
|
- name: Install pinned Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||||
with:
|
with:
|
||||||
toolchain: 1.95.0
|
toolchain: 1.95.0
|
||||||
|
|
||||||
@@ -79,13 +83,13 @@ jobs:
|
|||||||
run: rustc -Vv
|
run: rustc -Vv
|
||||||
|
|
||||||
- name: Rust cache
|
- name: Rust cache
|
||||||
uses: Swatinem/rust-cache@v2
|
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||||
with:
|
with:
|
||||||
shared-key: nightly-rust-1.95-${{ runner.os }}
|
shared-key: nightly-rust-1.95-${{ runner.os }}
|
||||||
workspaces: . -> target
|
workspaces: . -> target
|
||||||
|
|
||||||
- name: Setup sccache
|
- name: Setup sccache
|
||||||
uses: mozilla-actions/[email protected]
|
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||||
|
|
||||||
- name: Check all workspace targets
|
- name: Check all workspace targets
|
||||||
env:
|
env:
|
||||||
@@ -112,12 +116,12 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 30
|
timeout-minutes: 30
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
with:
|
with:
|
||||||
ref: ${{ needs.source.outputs.sha }}
|
ref: ${{ needs.source.outputs.sha }}
|
||||||
|
|
||||||
- name: Setup Node.js
|
- name: Setup Node.js
|
||||||
uses: actions/setup-node@v5
|
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5
|
||||||
with:
|
with:
|
||||||
node-version: '22'
|
node-version: '22'
|
||||||
cache: npm
|
cache: npm
|
||||||
@@ -157,7 +161,7 @@ jobs:
|
|||||||
run: npm run build
|
run: npm run build
|
||||||
|
|
||||||
- name: Upload frontend artifact
|
- name: Upload frontend artifact
|
||||||
uses: actions/upload-artifact@v5
|
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||||
with:
|
with:
|
||||||
name: nightly-frontend-dist
|
name: nightly-frontend-dist
|
||||||
path: frontend/dist/
|
path: frontend/dist/
|
||||||
@@ -171,12 +175,12 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 10
|
timeout-minutes: 10
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
with:
|
with:
|
||||||
ref: ${{ needs.source.outputs.sha }}
|
ref: ${{ needs.source.outputs.sha }}
|
||||||
|
|
||||||
- name: Setup Node.js
|
- name: Setup Node.js
|
||||||
uses: actions/setup-node@v5
|
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5
|
||||||
with:
|
with:
|
||||||
node-version: '22'
|
node-version: '22'
|
||||||
|
|
||||||
@@ -260,25 +264,25 @@ jobs:
|
|||||||
os: macos-15
|
os: macos-15
|
||||||
use_cross: false
|
use_cross: false
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
with:
|
with:
|
||||||
ref: ${{ needs.source.outputs.sha }}
|
ref: ${{ needs.source.outputs.sha }}
|
||||||
|
|
||||||
- name: Install pinned Rust toolchain
|
- name: Install pinned Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||||
with:
|
with:
|
||||||
toolchain: 1.95.0
|
toolchain: 1.95.0
|
||||||
targets: ${{ matrix.target }}
|
targets: ${{ matrix.target }}
|
||||||
|
|
||||||
- name: Rust cache
|
- name: Rust cache
|
||||||
uses: Swatinem/rust-cache@v2
|
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||||
with:
|
with:
|
||||||
shared-key: nightly-release-${{ matrix.target }}
|
shared-key: nightly-release-${{ matrix.target }}
|
||||||
workspaces: . -> target
|
workspaces: . -> target
|
||||||
|
|
||||||
- name: Install cross
|
- name: Install cross
|
||||||
if: matrix.use_cross
|
if: matrix.use_cross
|
||||||
uses: taiki-e/install-action@cross
|
uses: taiki-e/install-action@1ae7257be536a92d9218a6b343dc6e6ba650f7e1 # cross
|
||||||
|
|
||||||
- name: Build release binary
|
- name: Build release binary
|
||||||
env:
|
env:
|
||||||
@@ -295,7 +299,7 @@ jobs:
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
- name: Upload binary artifact
|
- name: Upload binary artifact
|
||||||
uses: actions/upload-artifact@v5
|
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||||
with:
|
with:
|
||||||
name: nightly-gateway-${{ matrix.platform }}-${{ matrix.arch }}
|
name: nightly-gateway-${{ matrix.platform }}-${{ matrix.arch }}
|
||||||
path: target/${{ matrix.target }}/release/aether-gateway
|
path: target/${{ matrix.target }}/release/aether-gateway
|
||||||
@@ -308,17 +312,19 @@ jobs:
|
|||||||
needs: [source, checks, build]
|
needs: [source, checks, build]
|
||||||
if: ${{ needs.checks.result == 'success' && needs.build.result == 'success' }}
|
if: ${{ needs.checks.result == 'success' && needs.build.result == 'success' }}
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
env:
|
||||||
|
GHCR_IMAGE: ${{ needs.source.outputs.ghcr_image }}
|
||||||
permissions:
|
permissions:
|
||||||
actions: read
|
actions: read
|
||||||
contents: read
|
contents: read
|
||||||
packages: write
|
packages: write
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
with:
|
with:
|
||||||
ref: ${{ needs.source.outputs.sha }}
|
ref: ${{ needs.source.outputs.sha }}
|
||||||
|
|
||||||
- name: Download Linux binaries and frontend
|
- name: Download Linux binaries and frontend
|
||||||
uses: actions/download-artifact@v5
|
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||||
with:
|
with:
|
||||||
pattern: nightly-*
|
pattern: nightly-*
|
||||||
path: artifacts
|
path: artifacts
|
||||||
@@ -335,20 +341,20 @@ jobs:
|
|||||||
cp -R artifacts/nightly-frontend-dist/. dist/frontend/
|
cp -R artifacts/nightly-frontend-dist/. dist/frontend/
|
||||||
|
|
||||||
- name: Set up QEMU
|
- name: Set up QEMU
|
||||||
uses: docker/setup-qemu-action@v3
|
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
- name: Set up Docker Buildx
|
||||||
uses: docker/setup-buildx-action@v3
|
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6 # v3
|
||||||
|
|
||||||
- name: Log in to GHCR
|
- name: Log in to GHCR
|
||||||
uses: docker/login-action@v3
|
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||||
with:
|
with:
|
||||||
registry: ghcr.io
|
registry: ghcr.io
|
||||||
username: ${{ github.actor }}
|
username: ${{ github.actor }}
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
- name: Build and push nightly image
|
- name: Build and push nightly image
|
||||||
uses: docker/build-push-action@v6
|
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
|
||||||
with:
|
with:
|
||||||
context: .
|
context: .
|
||||||
file: ./Dockerfile.app
|
file: ./Dockerfile.app
|
||||||
@@ -372,12 +378,12 @@ jobs:
|
|||||||
actions: read
|
actions: read
|
||||||
contents: read
|
contents: read
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
with:
|
with:
|
||||||
ref: ${{ needs.source.outputs.sha }}
|
ref: ${{ needs.source.outputs.sha }}
|
||||||
|
|
||||||
- name: Download nightly artifacts
|
- name: Download nightly artifacts
|
||||||
uses: actions/download-artifact@v5
|
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||||
with:
|
with:
|
||||||
pattern: nightly-*
|
pattern: nightly-*
|
||||||
path: artifacts
|
path: artifacts
|
||||||
@@ -434,7 +440,7 @@ jobs:
|
|||||||
done
|
done
|
||||||
|
|
||||||
- name: Upload nightly package artifact
|
- name: Upload nightly package artifact
|
||||||
uses: actions/upload-artifact@v5
|
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||||
with:
|
with:
|
||||||
name: nightly-release-assets
|
name: nightly-release-assets
|
||||||
path: release-assets/*
|
path: release-assets/*
|
||||||
@@ -452,7 +458,7 @@ jobs:
|
|||||||
contents: write
|
contents: write
|
||||||
steps:
|
steps:
|
||||||
- name: Download nightly package artifact
|
- name: Download nightly package artifact
|
||||||
uses: actions/download-artifact@v5
|
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||||
with:
|
with:
|
||||||
name: nightly-release-assets
|
name: nightly-release-assets
|
||||||
path: release-assets
|
path: release-assets
|
||||||
@@ -466,6 +472,7 @@ jobs:
|
|||||||
SOURCE_SHA: ${{ needs.source.outputs.sha }}
|
SOURCE_SHA: ${{ needs.source.outputs.sha }}
|
||||||
SOURCE_SHORT_SHA: ${{ needs.source.outputs.short_sha }}
|
SOURCE_SHORT_SHA: ${{ needs.source.outputs.short_sha }}
|
||||||
RELEASE_DATE: ${{ needs.source.outputs.date }}
|
RELEASE_DATE: ${{ needs.source.outputs.date }}
|
||||||
|
GHCR_IMAGE: ${{ needs.source.outputs.ghcr_image }}
|
||||||
run: |
|
run: |
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
|
|||||||
@@ -6,8 +6,8 @@ on:
|
|||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
contents: write
|
actions: read
|
||||||
packages: write
|
contents: read
|
||||||
|
|
||||||
concurrency:
|
concurrency:
|
||||||
group: release-aether-${{ github.ref }}
|
group: release-aether-${{ github.ref }}
|
||||||
@@ -70,10 +70,10 @@ jobs:
|
|||||||
needs: preflight
|
needs: preflight
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Setup Node.js
|
- name: Setup Node.js
|
||||||
uses: actions/setup-node@v4
|
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
|
||||||
with:
|
with:
|
||||||
node-version: 22
|
node-version: 22
|
||||||
cache: npm
|
cache: npm
|
||||||
@@ -94,7 +94,7 @@ jobs:
|
|||||||
npm run build
|
npm run build
|
||||||
|
|
||||||
- name: Upload frontend artifact
|
- name: Upload frontend artifact
|
||||||
uses: actions/upload-artifact@v5
|
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||||
with:
|
with:
|
||||||
name: frontend-dist
|
name: frontend-dist
|
||||||
path: frontend/dist/
|
path: frontend/dist/
|
||||||
@@ -106,10 +106,10 @@ jobs:
|
|||||||
needs: preflight
|
needs: preflight
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Setup Node.js
|
- name: Setup Node.js
|
||||||
uses: actions/setup-node@v4
|
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
|
||||||
with:
|
with:
|
||||||
node-version: 22
|
node-version: 22
|
||||||
cache: npm
|
cache: npm
|
||||||
@@ -159,7 +159,7 @@ jobs:
|
|||||||
unzip -l "aether-vscodex-${version}.vsix" | grep 'extension/node_modules/ws/index.js' >/dev/null
|
unzip -l "aether-vscodex-${version}.vsix" | grep 'extension/node_modules/ws/index.js' >/dev/null
|
||||||
|
|
||||||
- name: Upload VSIX artifact
|
- name: Upload VSIX artifact
|
||||||
uses: actions/upload-artifact@v5
|
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||||
with:
|
with:
|
||||||
name: aether-vscodex-vsix
|
name: aether-vscodex-vsix
|
||||||
path: aether-vscodex/vscode-extension/aether-vscodex-*.vsix
|
path: aether-vscodex/vscode-extension/aether-vscodex-*.vsix
|
||||||
@@ -199,22 +199,22 @@ jobs:
|
|||||||
os: macos-15
|
os: macos-15
|
||||||
use_cross: false
|
use_cross: false
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
- name: Install Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||||
with:
|
with:
|
||||||
targets: ${{ matrix.target }}
|
targets: ${{ matrix.target }}
|
||||||
|
|
||||||
- name: Rust cache
|
- name: Rust cache
|
||||||
uses: Swatinem/rust-cache@v2
|
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||||
with:
|
with:
|
||||||
shared-key: release-${{ matrix.target }}
|
shared-key: release-${{ matrix.target }}
|
||||||
workspaces: . -> target
|
workspaces: . -> target
|
||||||
|
|
||||||
- name: Install cross
|
- name: Install cross
|
||||||
if: matrix.use_cross
|
if: matrix.use_cross
|
||||||
uses: taiki-e/install-action@cross
|
uses: taiki-e/install-action@1ae7257be536a92d9218a6b343dc6e6ba650f7e1 # cross
|
||||||
|
|
||||||
- name: Build
|
- name: Build
|
||||||
env:
|
env:
|
||||||
@@ -230,7 +230,7 @@ jobs:
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
- name: Upload binary artifact
|
- name: Upload binary artifact
|
||||||
uses: actions/upload-artifact@v5
|
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||||
with:
|
with:
|
||||||
name: aether-gateway-${{ matrix.platform }}-${{ matrix.arch }}
|
name: aether-gateway-${{ matrix.platform }}-${{ matrix.arch }}
|
||||||
path: target/${{ matrix.target }}/release/aether-gateway
|
path: target/${{ matrix.target }}/release/aether-gateway
|
||||||
@@ -242,11 +242,17 @@ jobs:
|
|||||||
needs: [preflight, frontend, build]
|
needs: [preflight, frontend, build]
|
||||||
if: needs.preflight.outputs.publish == 'true'
|
if: needs.preflight.outputs.publish == 'true'
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
actions: read
|
||||||
|
attestations: write
|
||||||
|
contents: read
|
||||||
|
id-token: write
|
||||||
|
packages: write
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Download all artifacts
|
- name: Download all artifacts
|
||||||
uses: actions/download-artifact@v5
|
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||||
with:
|
with:
|
||||||
path: artifacts
|
path: artifacts
|
||||||
|
|
||||||
@@ -259,27 +265,27 @@ jobs:
|
|||||||
cp -r artifacts/frontend-dist dist/frontend
|
cp -r artifacts/frontend-dist dist/frontend
|
||||||
|
|
||||||
- name: Set up QEMU
|
- name: Set up QEMU
|
||||||
uses: docker/setup-qemu-action@v3
|
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
- name: Set up Docker Buildx
|
||||||
uses: docker/setup-buildx-action@v3
|
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
|
||||||
|
|
||||||
- name: Log in to GHCR
|
- name: Log in to GHCR
|
||||||
uses: docker/login-action@v3
|
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||||
with:
|
with:
|
||||||
registry: ${{ env.REGISTRY }}
|
registry: ${{ env.REGISTRY }}
|
||||||
username: ${{ github.actor }}
|
username: ${{ github.actor }}
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
- name: Log in to Docker Hub
|
- name: Log in to Docker Hub
|
||||||
uses: docker/login-action@v3
|
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||||
with:
|
with:
|
||||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||||
|
|
||||||
- name: Extract metadata
|
- name: Extract metadata
|
||||||
id: meta
|
id: meta
|
||||||
uses: docker/metadata-action@v5
|
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5
|
||||||
with:
|
with:
|
||||||
images: |
|
images: |
|
||||||
${{ env.REGISTRY }}/${{ env.GHCR_IMAGE }}
|
${{ env.REGISTRY }}/${{ env.GHCR_IMAGE }}
|
||||||
@@ -295,7 +301,8 @@ jobs:
|
|||||||
latest=false
|
latest=false
|
||||||
|
|
||||||
- name: Build and push
|
- name: Build and push
|
||||||
uses: docker/build-push-action@v6
|
id: push
|
||||||
|
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
|
||||||
with:
|
with:
|
||||||
context: .
|
context: .
|
||||||
file: ./Dockerfile.app
|
file: ./Dockerfile.app
|
||||||
@@ -304,15 +311,36 @@ jobs:
|
|||||||
labels: ${{ steps.meta.outputs.labels }}
|
labels: ${{ steps.meta.outputs.labels }}
|
||||||
platforms: linux/amd64,linux/arm64
|
platforms: linux/amd64,linux/arm64
|
||||||
|
|
||||||
|
- name: Attest GHCR image provenance
|
||||||
|
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||||
|
with:
|
||||||
|
subject-name: ${{ env.REGISTRY }}/${{ env.GHCR_IMAGE }}
|
||||||
|
subject-digest: ${{ steps.push.outputs.digest }}
|
||||||
|
push-to-registry: true
|
||||||
|
create-storage-record: false
|
||||||
|
|
||||||
|
- name: Attest Docker Hub image provenance
|
||||||
|
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||||
|
with:
|
||||||
|
subject-name: docker.io/${{ env.DOCKERHUB_IMAGE }}
|
||||||
|
subject-digest: ${{ steps.push.outputs.digest }}
|
||||||
|
push-to-registry: true
|
||||||
|
create-storage-record: false
|
||||||
|
|
||||||
package:
|
package:
|
||||||
name: Release tarballs
|
name: Release tarballs
|
||||||
needs: [preflight, frontend, build]
|
needs: [preflight, frontend, build]
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
actions: read
|
||||||
|
attestations: write
|
||||||
|
contents: read
|
||||||
|
id-token: write
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Download all artifacts
|
- name: Download all artifacts
|
||||||
uses: actions/download-artifact@v5
|
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||||
with:
|
with:
|
||||||
path: artifacts
|
path: artifacts
|
||||||
|
|
||||||
@@ -362,8 +390,24 @@ jobs:
|
|||||||
chmod +x release-assets/install.sh
|
chmod +x release-assets/install.sh
|
||||||
(cd release-assets && sha256sum *.tar.gz > SHA256SUMS)
|
(cd release-assets && sha256sum *.tar.gz > SHA256SUMS)
|
||||||
|
|
||||||
|
- name: Attest release package provenance
|
||||||
|
id: attest-release
|
||||||
|
if: needs.preflight.outputs.publish == 'true'
|
||||||
|
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
|
||||||
|
with:
|
||||||
|
subject-path: |
|
||||||
|
release-assets/*.tar.gz
|
||||||
|
release-assets/install.sh
|
||||||
|
release-assets/SHA256SUMS
|
||||||
|
|
||||||
|
- name: Bundle release package provenance
|
||||||
|
if: needs.preflight.outputs.publish == 'true'
|
||||||
|
env:
|
||||||
|
ATTESTATION_BUNDLE: ${{ steps.attest-release.outputs.bundle-path }}
|
||||||
|
run: install -m 0644 "${ATTESTATION_BUNDLE}" release-assets/AETHER_RELEASE_PROVENANCE.sigstore.json
|
||||||
|
|
||||||
- name: Upload release package artifact
|
- name: Upload release package artifact
|
||||||
uses: actions/upload-artifact@v5
|
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
|
||||||
with:
|
with:
|
||||||
name: release-assets
|
name: release-assets
|
||||||
path: release-assets/*
|
path: release-assets/*
|
||||||
@@ -375,15 +419,18 @@ jobs:
|
|||||||
needs: [preflight, docker, package, vscodex]
|
needs: [preflight, docker, package, vscodex]
|
||||||
if: needs.preflight.outputs.publish == 'true'
|
if: needs.preflight.outputs.publish == 'true'
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
actions: read
|
||||||
|
contents: write
|
||||||
steps:
|
steps:
|
||||||
- name: Download release package artifact
|
- name: Download release package artifact
|
||||||
uses: actions/download-artifact@v5
|
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||||
with:
|
with:
|
||||||
name: release-assets
|
name: release-assets
|
||||||
path: release-assets
|
path: release-assets
|
||||||
|
|
||||||
- name: Download VSIX artifact
|
- name: Download VSIX artifact
|
||||||
uses: actions/download-artifact@v5
|
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
|
||||||
with:
|
with:
|
||||||
name: aether-vscodex-vsix
|
name: aether-vscodex-vsix
|
||||||
path: release-assets
|
path: release-assets
|
||||||
@@ -410,13 +457,14 @@ jobs:
|
|||||||
done <<< "${draft_ids}"
|
done <<< "${draft_ids}"
|
||||||
|
|
||||||
- name: Publish GitHub Release assets
|
- name: Publish GitHub Release assets
|
||||||
uses: softprops/action-gh-release@v2
|
uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2
|
||||||
with:
|
with:
|
||||||
generate_release_notes: true
|
generate_release_notes: true
|
||||||
prerelease: ${{ needs.preflight.outputs.prerelease }}
|
prerelease: ${{ needs.preflight.outputs.prerelease }}
|
||||||
make_latest: ${{ needs.preflight.outputs.make_latest }}
|
make_latest: ${{ needs.preflight.outputs.make_latest }}
|
||||||
files: |
|
files: |
|
||||||
release-assets/*.tar.gz
|
release-assets/*.tar.gz
|
||||||
|
release-assets/AETHER_RELEASE_PROVENANCE.sigstore.json
|
||||||
release-assets/SHA256SUMS
|
release-assets/SHA256SUMS
|
||||||
release-assets/install.sh
|
release-assets/install.sh
|
||||||
release-assets/*.vsix
|
release-assets/*.vsix
|
||||||
|
|||||||
+122
-197
@@ -11,6 +11,26 @@ on:
|
|||||||
- "Cargo.lock"
|
- "Cargo.lock"
|
||||||
- "crates/**"
|
- "crates/**"
|
||||||
- "apps/**"
|
- "apps/**"
|
||||||
|
- "install.sh"
|
||||||
|
- "deploy.sh"
|
||||||
|
- "update.sh"
|
||||||
|
- "generate_keys.sh"
|
||||||
|
- ".env.example"
|
||||||
|
- "README.md"
|
||||||
|
- "Dockerfile.app"
|
||||||
|
- "docker-compose.yml"
|
||||||
|
- "docker-compose.single-node.yml"
|
||||||
|
- "docker-compose.local.yml"
|
||||||
|
- "docker-compose.release-local.yml"
|
||||||
|
- "tests/compose_database_config_test.py"
|
||||||
|
- "tests/install_*_test.sh"
|
||||||
|
- "tests/deploy_*_test.sh"
|
||||||
|
- "tests/update_*_test.sh"
|
||||||
|
- "tests/release_supply_chain_test.sh"
|
||||||
|
- "tests/tunnel_installer_config_security_test.sh"
|
||||||
|
- ".github/workflows/build-tunnel.yml"
|
||||||
|
- ".github/workflows/deploy-pages.yml"
|
||||||
|
- ".github/workflows/release.yml"
|
||||||
- ".github/workflows/rust-ci.yml"
|
- ".github/workflows/rust-ci.yml"
|
||||||
- ".github/workflows/nightly.yml"
|
- ".github/workflows/nightly.yml"
|
||||||
pull_request:
|
pull_request:
|
||||||
@@ -19,6 +39,26 @@ on:
|
|||||||
- "Cargo.lock"
|
- "Cargo.lock"
|
||||||
- "crates/**"
|
- "crates/**"
|
||||||
- "apps/**"
|
- "apps/**"
|
||||||
|
- "install.sh"
|
||||||
|
- "deploy.sh"
|
||||||
|
- "update.sh"
|
||||||
|
- "generate_keys.sh"
|
||||||
|
- ".env.example"
|
||||||
|
- "README.md"
|
||||||
|
- "Dockerfile.app"
|
||||||
|
- "docker-compose.yml"
|
||||||
|
- "docker-compose.single-node.yml"
|
||||||
|
- "docker-compose.local.yml"
|
||||||
|
- "docker-compose.release-local.yml"
|
||||||
|
- "tests/compose_database_config_test.py"
|
||||||
|
- "tests/install_*_test.sh"
|
||||||
|
- "tests/deploy_*_test.sh"
|
||||||
|
- "tests/update_*_test.sh"
|
||||||
|
- "tests/release_supply_chain_test.sh"
|
||||||
|
- "tests/tunnel_installer_config_security_test.sh"
|
||||||
|
- ".github/workflows/build-tunnel.yml"
|
||||||
|
- ".github/workflows/deploy-pages.yml"
|
||||||
|
- ".github/workflows/release.yml"
|
||||||
- ".github/workflows/rust-ci.yml"
|
- ".github/workflows/rust-ci.yml"
|
||||||
- ".github/workflows/nightly.yml"
|
- ".github/workflows/nightly.yml"
|
||||||
|
|
||||||
@@ -36,14 +76,35 @@ env:
|
|||||||
CARGO_TERM_COLOR: always
|
CARGO_TERM_COLOR: always
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
|
shell_security:
|
||||||
|
name: Shell security fixtures
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
|
- name: Run installer and supply-chain fixtures
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
python3 tests/compose_database_config_test.py
|
||||||
|
bash tests/deploy_state_safety_test.sh
|
||||||
|
bash tests/install_archive_safety_test.sh
|
||||||
|
bash tests/install_container_runtime_security_test.sh
|
||||||
|
bash tests/install_current_release_link_test.sh
|
||||||
|
bash tests/install_local_bundle_safety_test.sh
|
||||||
|
bash tests/install_privileged_write_safety_test.sh
|
||||||
|
bash tests/install_source_trust_test.sh
|
||||||
|
bash tests/release_supply_chain_test.sh
|
||||||
|
bash tests/update_compose_safety_test.sh
|
||||||
|
bash tests/tunnel_installer_config_security_test.sh
|
||||||
|
|
||||||
fmt:
|
fmt:
|
||||||
name: Format
|
name: Format
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
- name: Install Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||||
with:
|
with:
|
||||||
toolchain: 1.95.0
|
toolchain: 1.95.0
|
||||||
components: rustfmt
|
components: rustfmt
|
||||||
@@ -55,22 +116,22 @@ jobs:
|
|||||||
name: Clippy (Gateway)
|
name: Clippy (Gateway)
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
- name: Install Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||||
with:
|
with:
|
||||||
toolchain: 1.95.0
|
toolchain: 1.95.0
|
||||||
components: clippy
|
components: clippy
|
||||||
|
|
||||||
- name: Rust cache
|
- name: Rust cache
|
||||||
uses: Swatinem/rust-cache@v2
|
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||||
with:
|
with:
|
||||||
shared-key: rust-ci-${{ runner.os }}
|
shared-key: rust-ci-${{ runner.os }}
|
||||||
workspaces: . -> target
|
workspaces: . -> target
|
||||||
|
|
||||||
- name: Setup sccache
|
- name: Setup sccache
|
||||||
uses: mozilla-actions/[email protected]
|
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||||
|
|
||||||
- name: Clippy
|
- name: Clippy
|
||||||
env:
|
env:
|
||||||
@@ -89,22 +150,22 @@ jobs:
|
|||||||
name: Clippy (Data)
|
name: Clippy (Data)
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
- name: Install Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||||
with:
|
with:
|
||||||
toolchain: 1.95.0
|
toolchain: 1.95.0
|
||||||
components: clippy
|
components: clippy
|
||||||
|
|
||||||
- name: Rust cache
|
- name: Rust cache
|
||||||
uses: Swatinem/rust-cache@v2
|
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||||
with:
|
with:
|
||||||
shared-key: rust-ci-${{ runner.os }}
|
shared-key: rust-ci-${{ runner.os }}
|
||||||
workspaces: . -> target
|
workspaces: . -> target
|
||||||
|
|
||||||
- name: Setup sccache
|
- name: Setup sccache
|
||||||
uses: mozilla-actions/[email protected]
|
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||||
|
|
||||||
- name: Clippy
|
- name: Clippy
|
||||||
env:
|
env:
|
||||||
@@ -123,22 +184,22 @@ jobs:
|
|||||||
name: Clippy (Workspace Rest)
|
name: Clippy (Workspace Rest)
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
- name: Install Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||||
with:
|
with:
|
||||||
toolchain: 1.95.0
|
toolchain: 1.95.0
|
||||||
components: clippy
|
components: clippy
|
||||||
|
|
||||||
- name: Rust cache
|
- name: Rust cache
|
||||||
uses: Swatinem/rust-cache@v2
|
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||||
with:
|
with:
|
||||||
shared-key: rust-ci-${{ runner.os }}
|
shared-key: rust-ci-${{ runner.os }}
|
||||||
workspaces: . -> target
|
workspaces: . -> target
|
||||||
|
|
||||||
- name: Setup sccache
|
- name: Setup sccache
|
||||||
uses: mozilla-actions/[email protected]
|
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||||
|
|
||||||
- name: Clippy
|
- name: Clippy
|
||||||
env:
|
env:
|
||||||
@@ -175,28 +236,31 @@ jobs:
|
|||||||
name: Test (Gateway)
|
name: Test (Gateway)
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
- name: Install Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||||
|
|
||||||
- name: Show Rust toolchain
|
- name: Show Rust toolchain
|
||||||
run: rustup show active-toolchain
|
run: rustup show active-toolchain
|
||||||
|
|
||||||
- name: Rust cache
|
- name: Rust cache
|
||||||
uses: Swatinem/rust-cache@v2
|
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||||
with:
|
with:
|
||||||
shared-key: rust-ci-${{ runner.os }}
|
shared-key: rust-ci-${{ runner.os }}
|
||||||
workspaces: . -> target
|
workspaces: . -> target
|
||||||
|
|
||||||
- name: Setup sccache
|
- name: Setup sccache
|
||||||
uses: mozilla-actions/[email protected]
|
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||||
|
|
||||||
- name: Setup mold
|
- name: Setup mold
|
||||||
uses: rui314/setup-mold@v1
|
uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1
|
||||||
|
|
||||||
- name: Install nextest
|
- name: Install nextest
|
||||||
uses: taiki-e/install-action@nextest
|
uses: taiki-e/install-action@d5f9268ff7620505a81ada10ddf18cdd72240185 # nextest
|
||||||
|
|
||||||
|
- name: Expose PostgreSQL test binaries
|
||||||
|
run: pg_config --bindir >> "$GITHUB_PATH"
|
||||||
|
|
||||||
- name: Test lib
|
- name: Test lib
|
||||||
env:
|
env:
|
||||||
@@ -225,30 +289,34 @@ jobs:
|
|||||||
name: Test (Data)
|
name: Test (Data)
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
- name: Install Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||||
|
|
||||||
- name: Show Rust toolchain
|
- name: Show Rust toolchain
|
||||||
run: rustup show active-toolchain
|
run: rustup show active-toolchain
|
||||||
|
|
||||||
- name: Rust cache
|
- name: Rust cache
|
||||||
uses: Swatinem/rust-cache@v2
|
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||||
with:
|
with:
|
||||||
shared-key: rust-ci-${{ runner.os }}
|
shared-key: rust-ci-${{ runner.os }}
|
||||||
workspaces: . -> target
|
workspaces: . -> target
|
||||||
|
|
||||||
- name: Setup sccache
|
- name: Setup sccache
|
||||||
uses: mozilla-actions/[email protected]
|
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||||
|
|
||||||
- name: Install nextest
|
- name: Install nextest
|
||||||
uses: taiki-e/install-action@nextest
|
uses: taiki-e/install-action@d5f9268ff7620505a81ada10ddf18cdd72240185 # nextest
|
||||||
|
|
||||||
|
- name: Expose PostgreSQL test binaries
|
||||||
|
run: pg_config --bindir >> "$GITHUB_PATH"
|
||||||
|
|
||||||
- name: Test
|
- name: Test
|
||||||
env:
|
env:
|
||||||
RUSTC_WRAPPER: sccache
|
RUSTC_WRAPPER: sccache
|
||||||
SCCACHE_GHA_ENABLED: "true"
|
SCCACHE_GHA_ENABLED: "true"
|
||||||
|
AETHER_REQUIRE_LOCAL_POSTGRES_TESTS: "true"
|
||||||
run: cargo nextest run -p aether-data
|
run: cargo nextest run -p aether-data
|
||||||
|
|
||||||
- name: Show sccache stats
|
- name: Show sccache stats
|
||||||
@@ -266,23 +334,21 @@ jobs:
|
|||||||
matrix:
|
matrix:
|
||||||
feature:
|
feature:
|
||||||
- postgres
|
- postgres
|
||||||
- mysql
|
|
||||||
- sqlite
|
|
||||||
- all-drivers
|
- all-drivers
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
- name: Install Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||||
|
|
||||||
- name: Rust cache
|
- name: Rust cache
|
||||||
uses: Swatinem/rust-cache@v2
|
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||||
with:
|
with:
|
||||||
shared-key: rust-ci-${{ runner.os }}
|
shared-key: rust-ci-${{ runner.os }}
|
||||||
workspaces: . -> target
|
workspaces: . -> target
|
||||||
|
|
||||||
- name: Setup sccache
|
- name: Setup sccache
|
||||||
uses: mozilla-actions/[email protected]
|
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||||
|
|
||||||
- name: Check selected data driver
|
- name: Check selected data driver
|
||||||
env:
|
env:
|
||||||
@@ -301,25 +367,25 @@ jobs:
|
|||||||
name: Test (Workspace Rest)
|
name: Test (Workspace Rest)
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
- name: Install Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||||
|
|
||||||
- name: Show Rust toolchain
|
- name: Show Rust toolchain
|
||||||
run: rustup show active-toolchain
|
run: rustup show active-toolchain
|
||||||
|
|
||||||
- name: Rust cache
|
- name: Rust cache
|
||||||
uses: Swatinem/rust-cache@v2
|
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||||
with:
|
with:
|
||||||
shared-key: rust-ci-${{ runner.os }}
|
shared-key: rust-ci-${{ runner.os }}
|
||||||
workspaces: . -> target
|
workspaces: . -> target
|
||||||
|
|
||||||
- name: Setup sccache
|
- name: Setup sccache
|
||||||
uses: mozilla-actions/[email protected]
|
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||||
|
|
||||||
- name: Install nextest
|
- name: Install nextest
|
||||||
uses: taiki-e/install-action@nextest
|
uses: taiki-e/install-action@d5f9268ff7620505a81ada10ddf18cdd72240185 # nextest
|
||||||
|
|
||||||
- name: Test
|
- name: Test
|
||||||
env:
|
env:
|
||||||
@@ -342,25 +408,23 @@ jobs:
|
|||||||
matrix:
|
matrix:
|
||||||
package:
|
package:
|
||||||
- aether-data-postgres
|
- aether-data-postgres
|
||||||
- aether-data-mysql
|
|
||||||
- aether-data-sqlite
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
- name: Install Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||||
|
|
||||||
- name: Rust cache
|
- name: Rust cache
|
||||||
uses: Swatinem/rust-cache@v2
|
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||||
with:
|
with:
|
||||||
shared-key: rust-ci-${{ runner.os }}
|
shared-key: rust-ci-${{ runner.os }}
|
||||||
workspaces: . -> target
|
workspaces: . -> target
|
||||||
|
|
||||||
- name: Setup sccache
|
- name: Setup sccache
|
||||||
uses: mozilla-actions/[email protected]
|
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||||
|
|
||||||
- name: Install nextest
|
- name: Install nextest
|
||||||
uses: taiki-e/install-action@nextest
|
uses: taiki-e/install-action@d5f9268ff7620505a81ada10ddf18cdd72240185 # nextest
|
||||||
|
|
||||||
- name: Test adapter
|
- name: Test adapter
|
||||||
env:
|
env:
|
||||||
@@ -379,19 +443,22 @@ jobs:
|
|||||||
name: Test (Integration Scenarios)
|
name: Test (Integration Scenarios)
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
- name: Install Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||||
|
|
||||||
- name: Rust cache
|
- name: Rust cache
|
||||||
uses: Swatinem/rust-cache@v2
|
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||||
with:
|
with:
|
||||||
shared-key: rust-ci-${{ runner.os }}
|
shared-key: rust-ci-${{ runner.os }}
|
||||||
workspaces: . -> target
|
workspaces: . -> target
|
||||||
|
|
||||||
- name: Setup sccache
|
- name: Setup sccache
|
||||||
uses: mozilla-actions/[email protected]
|
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||||
|
|
||||||
|
- name: Expose PostgreSQL test binaries
|
||||||
|
run: pg_config --bindir >> "$GITHUB_PATH"
|
||||||
|
|
||||||
- name: Test scenario binaries and end-to-end suites
|
- name: Test scenario binaries and end-to-end suites
|
||||||
env:
|
env:
|
||||||
@@ -430,40 +497,6 @@ jobs:
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
data_db_smoke_sqlite:
|
|
||||||
name: Data DB Smoke (SQLite)
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v5
|
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
|
||||||
uses: dtolnay/rust-toolchain@stable
|
|
||||||
|
|
||||||
- name: Show Rust toolchain
|
|
||||||
run: rustup show active-toolchain
|
|
||||||
|
|
||||||
- name: Rust cache
|
|
||||||
uses: Swatinem/rust-cache@v2
|
|
||||||
with:
|
|
||||||
shared-key: rust-ci-${{ runner.os }}
|
|
||||||
workspaces: . -> target
|
|
||||||
|
|
||||||
- name: Setup sccache
|
|
||||||
uses: mozilla-actions/[email protected]
|
|
||||||
|
|
||||||
- name: Run SQLite data smoke tests
|
|
||||||
env:
|
|
||||||
RUSTC_WRAPPER: sccache
|
|
||||||
SCCACHE_GHA_ENABLED: "true"
|
|
||||||
run: cargo test -p aether-data --all-features sqlite --lib
|
|
||||||
|
|
||||||
- name: Show sccache stats
|
|
||||||
if: always()
|
|
||||||
env:
|
|
||||||
RUSTC_WRAPPER: sccache
|
|
||||||
SCCACHE_GHA_ENABLED: "true"
|
|
||||||
run: sccache --show-stats
|
|
||||||
|
|
||||||
data_db_smoke_postgres:
|
data_db_smoke_postgres:
|
||||||
name: Data DB Smoke (Postgres)
|
name: Data DB Smoke (Postgres)
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
@@ -482,22 +515,22 @@ jobs:
|
|||||||
--health-timeout=5s
|
--health-timeout=5s
|
||||||
--health-retries=20
|
--health-retries=20
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
- name: Install Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||||
|
|
||||||
- name: Show Rust toolchain
|
- name: Show Rust toolchain
|
||||||
run: rustup show active-toolchain
|
run: rustup show active-toolchain
|
||||||
|
|
||||||
- name: Rust cache
|
- name: Rust cache
|
||||||
uses: Swatinem/rust-cache@v2
|
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||||
with:
|
with:
|
||||||
shared-key: rust-ci-${{ runner.os }}
|
shared-key: rust-ci-${{ runner.os }}
|
||||||
workspaces: . -> target
|
workspaces: . -> target
|
||||||
|
|
||||||
- name: Setup sccache
|
- name: Setup sccache
|
||||||
uses: mozilla-actions/[email protected]
|
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||||
|
|
||||||
- name: Add PostgreSQL server binaries to PATH
|
- name: Add PostgreSQL server binaries to PATH
|
||||||
run: echo "$(pg_config --bindir)" >> "$GITHUB_PATH"
|
run: echo "$(pg_config --bindir)" >> "$GITHUB_PATH"
|
||||||
@@ -535,112 +568,6 @@ jobs:
|
|||||||
AETHER_TEST_POSTGRES_URL: postgres://aether:[email protected]:5432/aether_test
|
AETHER_TEST_POSTGRES_URL: postgres://aether:[email protected]:5432/aether_test
|
||||||
run: cargo test -p aether-data --all-features postgres_core_export_reads_migrated_database_rows_when_url_is_set --lib -- --nocapture
|
run: cargo test -p aether-data --all-features postgres_core_export_reads_migrated_database_rows_when_url_is_set --lib -- --nocapture
|
||||||
|
|
||||||
- name: Run SQLite-to-Postgres import smoke test
|
|
||||||
env:
|
|
||||||
RUSTC_WRAPPER: sccache
|
|
||||||
SCCACHE_GHA_ENABLED: "true"
|
|
||||||
AETHER_TEST_POSTGRES_URL: postgres://aether:[email protected]:5432/aether_test
|
|
||||||
run: cargo test -p aether-data --all-features sqlite_core_export_reads_migrated_database_rows --lib -- --nocapture
|
|
||||||
|
|
||||||
- name: Show sccache stats
|
|
||||||
if: always()
|
|
||||||
env:
|
|
||||||
RUSTC_WRAPPER: sccache
|
|
||||||
SCCACHE_GHA_ENABLED: "true"
|
|
||||||
run: sccache --show-stats
|
|
||||||
|
|
||||||
data_db_smoke_mysql:
|
|
||||||
name: Data DB Smoke (MySQL)
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
services:
|
|
||||||
mysql:
|
|
||||||
image: mysql:8.0
|
|
||||||
env:
|
|
||||||
MYSQL_DATABASE: aether_test
|
|
||||||
MYSQL_USER: aether
|
|
||||||
MYSQL_PASSWORD: aether
|
|
||||||
MYSQL_ROOT_PASSWORD: aether_root
|
|
||||||
ports:
|
|
||||||
- 3306:3306
|
|
||||||
options: >-
|
|
||||||
--health-cmd="mysqladmin ping -h 127.0.0.1 -uaether -paether --silent"
|
|
||||||
--health-interval=5s
|
|
||||||
--health-timeout=5s
|
|
||||||
--health-retries=20
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v5
|
|
||||||
|
|
||||||
- name: Install Rust toolchain
|
|
||||||
uses: dtolnay/rust-toolchain@stable
|
|
||||||
|
|
||||||
- name: Show Rust toolchain
|
|
||||||
run: rustup show active-toolchain
|
|
||||||
|
|
||||||
- name: Rust cache
|
|
||||||
uses: Swatinem/rust-cache@v2
|
|
||||||
with:
|
|
||||||
shared-key: rust-ci-${{ runner.os }}
|
|
||||||
workspaces: . -> target
|
|
||||||
|
|
||||||
- name: Setup sccache
|
|
||||||
uses: mozilla-actions/[email protected]
|
|
||||||
|
|
||||||
- name: Run MySQL migration smoke test
|
|
||||||
env:
|
|
||||||
RUSTC_WRAPPER: sccache
|
|
||||||
SCCACHE_GHA_ENABLED: "true"
|
|
||||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
|
||||||
run: cargo test -p aether-data --all-features mysql_migrations_create_core_config_tables_when_url_is_set --lib -- --nocapture
|
|
||||||
|
|
||||||
- name: Run MySQL usage write smoke test
|
|
||||||
env:
|
|
||||||
RUSTC_WRAPPER: sccache
|
|
||||||
SCCACHE_GHA_ENABLED: "true"
|
|
||||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
|
||||||
run: cargo test -p aether-data-mysql mysql_usage_write_repository_upserts_and_flushes_counters_when_url_is_set --lib -- --nocapture
|
|
||||||
|
|
||||||
- name: Run MySQL usage read smoke test
|
|
||||||
env:
|
|
||||||
RUSTC_WRAPPER: sccache
|
|
||||||
SCCACHE_GHA_ENABLED: "true"
|
|
||||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
|
||||||
run: cargo test -p aether-data-mysql mysql_usage_read_repository_reads_usage_contract_views_when_url_is_set --lib -- --nocapture
|
|
||||||
|
|
||||||
- name: Run MySQL provider catalog smoke test
|
|
||||||
env:
|
|
||||||
RUSTC_WRAPPER: sccache
|
|
||||||
SCCACHE_GHA_ENABLED: "true"
|
|
||||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
|
||||||
run: cargo test -p aether-data-mysql mysql_provider_catalog_repository_round_trips_when_url_is_set --lib -- --nocapture
|
|
||||||
|
|
||||||
- name: Run MySQL core export smoke test
|
|
||||||
env:
|
|
||||||
RUSTC_WRAPPER: sccache
|
|
||||||
SCCACHE_GHA_ENABLED: "true"
|
|
||||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
|
||||||
run: cargo test -p aether-data --all-features mysql_core_export_reads_migrated_database_rows_when_url_is_set --lib -- --nocapture
|
|
||||||
|
|
||||||
- name: Run MySQL wallet read smoke test
|
|
||||||
env:
|
|
||||||
RUSTC_WRAPPER: sccache
|
|
||||||
SCCACHE_GHA_ENABLED: "true"
|
|
||||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
|
||||||
run: cargo test -p aether-data-mysql mysql_wallet_read_repository_reads_wallet_contract_views --lib -- --nocapture
|
|
||||||
|
|
||||||
- name: Run MySQL wallet daily usage aggregation smoke test
|
|
||||||
env:
|
|
||||||
RUSTC_WRAPPER: sccache
|
|
||||||
SCCACHE_GHA_ENABLED: "true"
|
|
||||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
|
||||||
run: cargo test -p aether-data --all-features mysql_wallet_daily_usage_aggregation_uses_settlement_wallets_when_url_is_set --lib -- --nocapture
|
|
||||||
|
|
||||||
- name: Run MySQL stats aggregation smoke test
|
|
||||||
env:
|
|
||||||
RUSTC_WRAPPER: sccache
|
|
||||||
SCCACHE_GHA_ENABLED: "true"
|
|
||||||
AETHER_TEST_MYSQL_URL: mysql://aether:[email protected]:3306/aether_test
|
|
||||||
run: cargo test -p aether-data --all-features mysql_stats_aggregation_runs_after_mysql_migrations_when_url_is_set --lib -- --nocapture
|
|
||||||
|
|
||||||
- name: Show sccache stats
|
- name: Show sccache stats
|
||||||
if: always()
|
if: always()
|
||||||
env:
|
env:
|
||||||
@@ -652,16 +579,12 @@ jobs:
|
|||||||
name: Data DB Smoke
|
name: Data DB Smoke
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs:
|
needs:
|
||||||
- data_db_smoke_sqlite
|
|
||||||
- data_db_smoke_postgres
|
- data_db_smoke_postgres
|
||||||
- data_db_smoke_mysql
|
|
||||||
if: ${{ always() }}
|
if: ${{ always() }}
|
||||||
steps:
|
steps:
|
||||||
- name: Verify database smoke jobs
|
- name: Verify database smoke jobs
|
||||||
run: |
|
run: |
|
||||||
if [ "${{ needs.data_db_smoke_sqlite.result }}" != "success" ] || \
|
if [ "${{ needs.data_db_smoke_postgres.result }}" != "success" ]; then
|
||||||
[ "${{ needs.data_db_smoke_postgres.result }}" != "success" ] || \
|
|
||||||
[ "${{ needs.data_db_smoke_mysql.result }}" != "success" ]; then
|
|
||||||
echo "Data DB smoke failed"
|
echo "Data DB smoke failed"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
@@ -674,6 +597,7 @@ jobs:
|
|||||||
- clippy
|
- clippy
|
||||||
- test
|
- test
|
||||||
- data_db_smoke
|
- data_db_smoke
|
||||||
|
- shell_security
|
||||||
if: ${{ always() }}
|
if: ${{ always() }}
|
||||||
steps:
|
steps:
|
||||||
- name: Verify required jobs
|
- name: Verify required jobs
|
||||||
@@ -681,7 +605,8 @@ jobs:
|
|||||||
if [ "${{ needs.fmt.result }}" != "success" ] || \
|
if [ "${{ needs.fmt.result }}" != "success" ] || \
|
||||||
[ "${{ needs.clippy.result }}" != "success" ] || \
|
[ "${{ needs.clippy.result }}" != "success" ] || \
|
||||||
[ "${{ needs.test.result }}" != "success" ] || \
|
[ "${{ needs.test.result }}" != "success" ] || \
|
||||||
[ "${{ needs.data_db_smoke.result }}" != "success" ]; then
|
[ "${{ needs.data_db_smoke.result }}" != "success" ] || \
|
||||||
|
[ "${{ needs.shell_security.result }}" != "success" ]; then
|
||||||
echo "Rust CI failed"
|
echo "Rust CI failed"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -13,6 +13,10 @@
|
|||||||
.plans
|
.plans
|
||||||
.playwright-mcp/
|
.playwright-mcp/
|
||||||
|
|
||||||
|
docs/architecture
|
||||||
|
!docs/architecture/architecture-dark.svg
|
||||||
|
!docs/architecture/architecture-light.svg
|
||||||
|
|
||||||
### Python ###
|
### Python ###
|
||||||
*.db
|
*.db
|
||||||
*.db-*
|
*.db-*
|
||||||
|
|||||||
Generated
+595
-386
File diff suppressed because it is too large
Load Diff
+3
-5
@@ -11,8 +11,6 @@ members = [
|
|||||||
"crates/aether-routing-core",
|
"crates/aether-routing-core",
|
||||||
"crates/aether-data/contracts",
|
"crates/aether-data/contracts",
|
||||||
"crates/aether-data/adapters/postgres",
|
"crates/aether-data/adapters/postgres",
|
||||||
"crates/aether-data/adapters/mysql",
|
|
||||||
"crates/aether-data/adapters/sqlite",
|
|
||||||
"crates/aether-data/query",
|
"crates/aether-data/query",
|
||||||
"crates/aether-data/schema",
|
"crates/aether-data/schema",
|
||||||
"crates/aether-dispatch-core",
|
"crates/aether-dispatch-core",
|
||||||
@@ -66,8 +64,6 @@ aether-provider-pool = { path = "crates/aether-provider/pool" }
|
|||||||
aether-routing-core = { path = "crates/aether-routing-core" }
|
aether-routing-core = { path = "crates/aether-routing-core" }
|
||||||
aether-data-contracts = { path = "crates/aether-data/contracts" }
|
aether-data-contracts = { path = "crates/aether-data/contracts" }
|
||||||
aether-data-postgres = { path = "crates/aether-data/adapters/postgres" }
|
aether-data-postgres = { path = "crates/aether-data/adapters/postgres" }
|
||||||
aether-data-mysql = { path = "crates/aether-data/adapters/mysql" }
|
|
||||||
aether-data-sqlite = { path = "crates/aether-data/adapters/sqlite" }
|
|
||||||
aether-data-query = { path = "crates/aether-data/query" }
|
aether-data-query = { path = "crates/aether-data/query" }
|
||||||
aether-data-schema = { path = "crates/aether-data/schema" }
|
aether-data-schema = { path = "crates/aether-data/schema" }
|
||||||
aether-dispatch-core = { path = "crates/aether-dispatch-core" }
|
aether-dispatch-core = { path = "crates/aether-dispatch-core" }
|
||||||
@@ -101,6 +97,7 @@ aether-runtime = { path = "crates/aether-runtime/base" }
|
|||||||
aether-testkit = { path = "crates/aether-testing/testkit" }
|
aether-testkit = { path = "crates/aether-testing/testkit" }
|
||||||
aes = "0.8"
|
aes = "0.8"
|
||||||
aes-gcm = "0.10"
|
aes-gcm = "0.10"
|
||||||
|
aws-lc-rs = { version = "1.16.2", default-features = false, features = ["alloc", "aws-lc-sys"] }
|
||||||
async-stream = "0.3"
|
async-stream = "0.3"
|
||||||
async-trait = "0.1"
|
async-trait = "0.1"
|
||||||
axum = "0.8"
|
axum = "0.8"
|
||||||
@@ -117,8 +114,9 @@ flate2 = "1"
|
|||||||
futures-util = "0.3"
|
futures-util = "0.3"
|
||||||
hmac = "0.12"
|
hmac = "0.12"
|
||||||
http = "1"
|
http = "1"
|
||||||
object_store = { version = "0.12", default-features = false, features = ["aws"] }
|
object_store = { version = "0.14.1", default-features = false, features = ["aws"] }
|
||||||
pbkdf2 = { version = "0.12", default-features = false, features = ["hmac"] }
|
pbkdf2 = { version = "0.12", default-features = false, features = ["hmac"] }
|
||||||
|
percent-encoding = "2"
|
||||||
reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "rustls-tls", "http2", "socks"] }
|
reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "rustls-tls", "http2", "socks"] }
|
||||||
redis = { version = "0.28", default-features = false, features = ["tokio-comp", "script", "streams", "connection-manager"] }
|
redis = { version = "0.28", default-features = false, features = ["tokio-comp", "script", "streams", "connection-manager"] }
|
||||||
regex = "1"
|
regex = "1"
|
||||||
|
|||||||
+9
-4
@@ -10,20 +10,24 @@
|
|||||||
|
|
||||||
# --- layout stage: create /opt/aether directory structure with symlink ---
|
# --- layout stage: create /opt/aether directory structure with symlink ---
|
||||||
# distroless has no shell, so we use busybox to set up the symlink.
|
# distroless has no shell, so we use busybox to set up the symlink.
|
||||||
FROM busybox:1.37-musl AS layout
|
FROM busybox:1.37.0-musl@sha256:fc6dddc4c44b1bfe37f41cae8e67d1693828e8f42a91862816d7953e2c9d3f23 AS layout
|
||||||
|
|
||||||
ARG TARGETARCH
|
ARG TARGETARCH
|
||||||
|
|
||||||
RUN mkdir -p /opt/aether/releases/image/bin /opt/aether/releases/image/frontend /opt/aether/logs
|
RUN mkdir -p /opt/aether/releases/image/bin /opt/aether/releases/image/frontend /opt/aether/logs
|
||||||
|
|
||||||
COPY dist/aether-gateway-${TARGETARCH} /opt/aether/releases/image/bin/aether-gateway
|
COPY dist/aether-gateway-${TARGETARCH} /opt/aether/releases/image/bin/aether-gateway
|
||||||
RUN chmod 0755 /opt/aether/releases/image/bin/aether-gateway
|
|
||||||
COPY dist/frontend/ /opt/aether/releases/image/frontend/
|
COPY dist/frontend/ /opt/aether/releases/image/frontend/
|
||||||
|
|
||||||
|
# Keep the immutable release root-owned while guaranteeing that the runtime
|
||||||
|
# identity can traverse and read every packaged asset.
|
||||||
|
RUN chmod -R u=rwX,go=rX /opt/aether/releases/image \
|
||||||
|
&& chmod 0755 /opt/aether/releases/image/bin/aether-gateway
|
||||||
|
|
||||||
RUN ln -s /opt/aether/releases/image /opt/aether/current
|
RUN ln -s /opt/aether/releases/image /opt/aether/current
|
||||||
|
|
||||||
# --- final stage: distroless runtime ---
|
# --- final stage: distroless runtime ---
|
||||||
FROM gcr.io/distroless/static-debian12
|
FROM gcr.io/distroless/static-debian12@sha256:6447365a6337c3732f412d1b74357b30a633831955b2bc45552b0086be907687
|
||||||
|
|
||||||
COPY --from=layout /opt/aether /opt/aether
|
COPY --from=layout /opt/aether /opt/aether
|
||||||
|
|
||||||
@@ -31,6 +35,7 @@ WORKDIR /opt/aether
|
|||||||
|
|
||||||
ENV RUST_LOG=aether_gateway=info \
|
ENV RUST_LOG=aether_gateway=info \
|
||||||
APP_PORT=8084 \
|
APP_PORT=8084 \
|
||||||
|
HOME=/tmp/aether-home \
|
||||||
AETHER_UPDATE_STRATEGY=docker \
|
AETHER_UPDATE_STRATEGY=docker \
|
||||||
AETHER_GATEWAY_STATIC_DIR=/opt/aether/current/frontend
|
AETHER_GATEWAY_STATIC_DIR=/opt/aether/current/frontend
|
||||||
|
|
||||||
@@ -39,5 +44,5 @@ EXPOSE 8084
|
|||||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
||||||
CMD ["/opt/aether/current/bin/aether-gateway", "--healthcheck"]
|
CMD ["/opt/aether/current/bin/aether-gateway", "--healthcheck"]
|
||||||
|
|
||||||
USER root
|
USER 0:0
|
||||||
ENTRYPOINT ["/opt/aether/current/bin/aether-gateway"]
|
ENTRYPOINT ["/opt/aether/current/bin/aether-gateway"]
|
||||||
|
|||||||
@@ -157,4 +157,5 @@ EXPOSE 8084
|
|||||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
||||||
CMD ["/usr/local/bin/aether-gateway", "--healthcheck"]
|
CMD ["/usr/local/bin/aether-gateway", "--healthcheck"]
|
||||||
|
|
||||||
|
USER 0:0
|
||||||
ENTRYPOINT ["/usr/local/bin/aether-gateway"]
|
ENTRYPOINT ["/usr/local/bin/aether-gateway"]
|
||||||
|
|||||||
@@ -156,4 +156,5 @@ EXPOSE 8084
|
|||||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
||||||
CMD ["/opt/aether/current/bin/aether-gateway", "--healthcheck"]
|
CMD ["/opt/aether/current/bin/aether-gateway", "--healthcheck"]
|
||||||
|
|
||||||
|
USER 0:0
|
||||||
ENTRYPOINT ["/opt/aether/current/bin/aether-gateway"]
|
ENTRYPOINT ["/opt/aether/current/bin/aether-gateway"]
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ DEV_RUST_LOG := $(RUST_LOG)
|
|||||||
endif
|
endif
|
||||||
export DEV_RUST_LOG
|
export DEV_RUST_LOG
|
||||||
|
|
||||||
.PHONY: dev dev-backend dev-frontend migration backfill
|
.PHONY: dev dev-backend dev-frontend db-status db-prepare migration backfill
|
||||||
|
|
||||||
define DEV_BACKEND_SCRIPT
|
define DEV_BACKEND_SCRIPT
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
@@ -20,6 +20,13 @@ set -a
|
|||||||
source .env
|
source .env
|
||||||
set +a
|
set +a
|
||||||
|
|
||||||
|
if [[ -n "$${ADMIN_EMAIL:-}" || -n "$${ADMIN_USERNAME:-}" || -n "$${ADMIN_PASSWORD:-}" ]]; then
|
||||||
|
if [[ -z "$${ADMIN_USERNAME:-}" || -z "$${ADMIN_PASSWORD:-}" ]]; then
|
||||||
|
echo "=> 管理员自举配置不完整,请在 .env 中设置 ADMIN_USERNAME 和 ADMIN_PASSWORD"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
dotenv_has_key() {
|
dotenv_has_key() {
|
||||||
local key="$$1"
|
local key="$$1"
|
||||||
grep -Eq "^[[:space:]]*$${key}=" .env
|
grep -Eq "^[[:space:]]*$${key}=" .env
|
||||||
@@ -29,15 +36,6 @@ lowercase() {
|
|||||||
printf '%s' "$$1" | tr '[:upper:]' '[:lower:]'
|
printf '%s' "$$1" | tr '[:upper:]' '[:lower:]'
|
||||||
}
|
}
|
||||||
|
|
||||||
dev_uses_sqlite_database() {
|
|
||||||
local driver
|
|
||||||
local url
|
|
||||||
driver="$$(lowercase "$${AETHER_DATABASE_DRIVER:-}")"
|
|
||||||
url="$${AETHER_DATABASE_URL:-$${DATABASE_URL:-}}"
|
|
||||||
|
|
||||||
[[ "$${driver}" == "sqlite" || "$${url}" == sqlite:* ]]
|
|
||||||
}
|
|
||||||
|
|
||||||
dev_uses_postgres_database() {
|
dev_uses_postgres_database() {
|
||||||
local driver
|
local driver
|
||||||
local url
|
local url
|
||||||
@@ -61,9 +59,6 @@ dev_uses_redis_runtime() {
|
|||||||
if [[ "$${backend}" == "redis" ]]; then
|
if [[ "$${backend}" == "redis" ]]; then
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
if dev_uses_sqlite_database; then
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
@@ -201,12 +196,17 @@ print_startup_failure_hint() {
|
|||||||
|
|
||||||
if [ -n "$${log_file}" ] && [ -f "$${log_file}" ]; then
|
if [ -n "$${log_file}" ] && [ -f "$${log_file}" ]; then
|
||||||
if grep -Eq "database schema is behind" "$${log_file}"; then
|
if grep -Eq "database schema is behind" "$${log_file}"; then
|
||||||
echo "=> 检测到数据库 schema 落后,请执行: make migration"
|
echo "=> 检测到数据库尚未准备完成,请执行: make db-prepare"
|
||||||
return
|
return
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if grep -Eq "database backfills are behind" "$${log_file}"; then
|
if grep -Eq "database backfills are behind" "$${log_file}"; then
|
||||||
echo "=> 检测到待执行 backfills,请执行: make backfill"
|
echo "=> 检测到数据库尚未准备完成,请执行: make db-prepare"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
if grep -Eq "bootstrap admin env is partially configured.*ADMIN_PASSWORD" "$${log_file}"; then
|
||||||
|
echo "=> 首次启动需要管理员密码,请在 .env 中设置 ADMIN_PASSWORD"
|
||||||
return
|
return
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
@@ -344,6 +344,9 @@ if ! ensure_dev_infra; then
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
echo "=> 编译 aether-gateway..."
|
||||||
|
cargo build -p aether-gateway --bin aether-gateway
|
||||||
|
|
||||||
GATEWAY_PID=""
|
GATEWAY_PID=""
|
||||||
GATEWAY_LOG_DIR=""
|
GATEWAY_LOG_DIR=""
|
||||||
GATEWAY_LOG_FILE=""
|
GATEWAY_LOG_FILE=""
|
||||||
@@ -352,8 +355,8 @@ create_gateway_log_file
|
|||||||
|
|
||||||
echo "=> 启动 aether-gateway (Rust frontdoor: 0.0.0.0:$${APP_PORT})..."
|
echo "=> 启动 aether-gateway (Rust frontdoor: 0.0.0.0:$${APP_PORT})..."
|
||||||
echo "=> 日志过滤: $${RUST_LOG}"
|
echo "=> 日志过滤: $${RUST_LOG}"
|
||||||
echo "=> 执行命令: cargo run -p aether-gateway -- --app-port $${APP_PORT}"
|
echo "=> 执行命令: target/debug/aether-gateway --app-port $${APP_PORT}"
|
||||||
cargo run -p aether-gateway -- --app-port "$${APP_PORT}" > >(
|
target/debug/aether-gateway --app-port "$${APP_PORT}" > >(
|
||||||
tee -a "$${GATEWAY_LOG_FILE}"
|
tee -a "$${GATEWAY_LOG_FILE}"
|
||||||
) 2>&1 &
|
) 2>&1 &
|
||||||
GATEWAY_PID=$$!
|
GATEWAY_PID=$$!
|
||||||
@@ -444,7 +447,7 @@ if [ -f .env ]; then
|
|||||||
fi
|
fi
|
||||||
export APP_PORT="$${APP_PORT:-8084}"
|
export APP_PORT="$${APP_PORT:-8084}"
|
||||||
|
|
||||||
echo "=> 启动后端: RUST_LOG=$${DEV_RUST_LOG} cargo run -p aether-gateway -- --app-port $${APP_PORT:-8084}"
|
echo "=> 启动后端: 先编译 aether-gateway,再运行 target/debug/aether-gateway --app-port $${APP_PORT:-8084}"
|
||||||
/bin/bash -euo pipefail -c "$$DEV_BACKEND_SCRIPT" &
|
/bin/bash -euo pipefail -c "$$DEV_BACKEND_SCRIPT" &
|
||||||
backend_pid=$$!
|
backend_pid=$$!
|
||||||
|
|
||||||
@@ -494,8 +497,14 @@ export DEV_SCRIPT
|
|||||||
define DB_TASK_SCRIPT
|
define DB_TASK_SCRIPT
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
if [ -z "$${DB_TASK_FLAG:-}" ] || [ -z "$${DB_TASK_LABEL:-}" ]; then
|
if [ -z "$${DB_TASK_COMMAND:-}" ] || [ -z "$${DB_TASK_LABEL:-}" ]; then
|
||||||
echo "=> 内部错误: DB_TASK_FLAG / DB_TASK_LABEL 未设置"
|
echo "=> 内部错误: DB_TASK_COMMAND / DB_TASK_LABEL 未设置"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
read -r -a db_task_args <<< "$${DB_TASK_COMMAND}"
|
||||||
|
if [ "$${#db_task_args[@]}" -eq 0 ]; then
|
||||||
|
echo "=> 内部错误: DB_TASK_COMMAND 为空"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@@ -546,8 +555,8 @@ if ! command -v cargo >/dev/null 2>&1; then
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "=> 执行 $${DB_TASK_LABEL}: cargo run -p aether-gateway -- $${DB_TASK_FLAG}"
|
echo "=> 执行 $${DB_TASK_LABEL}: cargo run -p aether-gateway --bin aether-gateway -- $${db_task_args[*]}"
|
||||||
exec cargo run -p aether-gateway -- "$${DB_TASK_FLAG}"
|
exec cargo run -p aether-gateway --bin aether-gateway -- "$${db_task_args[@]}"
|
||||||
endef
|
endef
|
||||||
export DB_TASK_SCRIPT
|
export DB_TASK_SCRIPT
|
||||||
|
|
||||||
@@ -560,8 +569,14 @@ dev-backend:
|
|||||||
dev-frontend:
|
dev-frontend:
|
||||||
@cd frontend && npm run dev
|
@cd frontend && npm run dev
|
||||||
|
|
||||||
|
db-status:
|
||||||
|
@DB_TASK_COMMAND="db status" DB_TASK_LABEL="数据库状态检查" $(SHELL) -euo pipefail -c "$$DB_TASK_SCRIPT"
|
||||||
|
|
||||||
|
db-prepare:
|
||||||
|
@DB_TASK_COMMAND="db prepare" DB_TASK_LABEL="数据库准备" $(SHELL) -euo pipefail -c "$$DB_TASK_SCRIPT"
|
||||||
|
|
||||||
migration:
|
migration:
|
||||||
@DB_TASK_FLAG=--migrate DB_TASK_LABEL="数据库迁移" $(SHELL) -euo pipefail -c "$$DB_TASK_SCRIPT"
|
@DB_TASK_COMMAND="--migrate" DB_TASK_LABEL="数据库迁移" $(SHELL) -euo pipefail -c "$$DB_TASK_SCRIPT"
|
||||||
|
|
||||||
backfill:
|
backfill:
|
||||||
@DB_TASK_FLAG=--apply-backfills DB_TASK_LABEL="数据库 backfill" $(SHELL) -euo pipefail -c "$$DB_TASK_SCRIPT"
|
@DB_TASK_COMMAND="--apply-backfills" DB_TASK_LABEL="数据库 backfill" $(SHELL) -euo pipefail -c "$$DB_TASK_SCRIPT"
|
||||||
|
|||||||
@@ -44,90 +44,42 @@ cd Aether
|
|||||||
|
|
||||||
# 2. 配置环境变量
|
# 2. 配置环境变量
|
||||||
cp .env.example .env
|
cp .env.example .env
|
||||||
# 生成 JWT_SECRET_KEY / ENCRYPTION_KEY, 并填入 .env
|
# .env 包含数据库、JWT 和数据加密密钥,先限制为仅当前用户可读写
|
||||||
|
chmod 600 .env
|
||||||
|
# 生成 JWT / 加密 / Postgres / Redis 独立随机密钥,并填入 .env
|
||||||
./generate_keys.sh
|
./generate_keys.sh
|
||||||
# 编辑 .env 设置 ADMIN_PASSWORD
|
# 编辑 .env 设置 ADMIN_PASSWORD
|
||||||
|
|
||||||
# 3. 首次部署 / 更新 (从以下部署形态任选其一)
|
# 3. Docker 部署 / 更新(PostgreSQL + Redis)
|
||||||
# Postgres + Redis (适用于企业或多人使用)
|
|
||||||
docker compose pull && docker compose up -d
|
docker compose pull && docker compose up -d
|
||||||
# Single Node (适用于个人用户或朋友分享)
|
|
||||||
docker compose -f docker-compose.single-node.yml pull && docker compose -f docker-compose.single-node.yml up -d
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### 一键更新
|
### 一键安装(PostgreSQL + Redis)
|
||||||
|
|
||||||
Docker Compose 部署后,可在部署目录直接执行:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./update.sh
|
|
||||||
```
|
|
||||||
|
|
||||||
`update.sh` 会拉取最新 `app` 镜像并重建 `app` 容器,Docker named volumes、`./data` 和 `./logs` 不会被删除。Single Node 部署也可显式指定:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./update.sh --mode single-node
|
|
||||||
```
|
|
||||||
|
|
||||||
仓库自带的 Docker Compose 默认把应用日志输出到容器 `stdout/stderr`,直接用 `docker compose logs -f app` 查看,并由 Docker 轮转日志,避免正式发布镜像切换到非 root 用户后再被宿主机挂载日志目录的权限问题拖垮启动。如果你确实需要文件日志,需要在 compose 里把 `AETHER_LOG_DESTINATION` 改成 `file|both`,并额外挂载一个容器用户可写的目录到 `/opt/aether/logs`。
|
|
||||||
|
|
||||||
管理后台右上角“版本信息”会检测新版本。Docker Compose 部署只提示版本,实际更新继续执行 `./update.sh`;systemd / launchd / 二进制部署才使用后台自更新,流程是下载对应平台的 GitHub Release 包、强制校验 `SHA256SUMS`、解压到 `/opt/aether/releases/<version>`,再切换 `/opt/aether/current` 并退出进程,交给 systemd / launchd 拉起新版本。
|
|
||||||
|
|
||||||
源码或本地构建版本不会启用后台在线更新,请继续使用源码更新流程。Docker Compose 用户如果希望“容器重建后也保持镜像层面的新版本”,仍建议定期运行 `./update.sh` 拉取并重建 app 镜像。服务器访问 GitHub 需要代理时,可设置 `AETHER_UPDATE_PROXY_URL`,也兼容 `UPDATE_PROXY_URL`、`HTTPS_PROXY`、`ALL_PROXY`、`HTTP_PROXY` 以及 `NO_PROXY`。共享出口触发 GitHub API 限流时,可设置只读 `AETHER_UPDATE_GITHUB_TOKEN`,也兼容 `GITHUB_TOKEN` / `GH_TOKEN`。下载总超时默认 600 秒,连续无响应/无数据默认 30 秒,可通过 `AETHER_UPDATE_DOWNLOAD_TIMEOUT_SECS` 和 `AETHER_UPDATE_DOWNLOAD_IDLE_TIMEOUT_SECS` 调整。
|
|
||||||
|
|
||||||
标准 Docker Compose 使用 Docker named volumes 存放 Postgres/Redis/MySQL 数据;Single Node 使用部署目录下的 `./data` 存放 SQLite 数据。
|
|
||||||
|
|
||||||
如果是本地源码构建镜像的部署,继续使用:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./deploy.sh
|
|
||||||
```
|
|
||||||
|
|
||||||
如果要在本机联调“管理后台在线更新”本身,可启动仓库内置的 release-layout 测试环境:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
docker compose -f docker-compose.release-local.yml up -d --build
|
|
||||||
```
|
|
||||||
|
|
||||||
这套环境会用当前源码构建一个本地测试镜像,但编译为 `release` 类型,并默认伪装成 `v0.7.0`,这样后台会按正式发布版逻辑开放“立即更新”。默认监听 `http://127.0.0.1:18085`,数据目录使用 `./data-release-local`;日志默认走 `docker logs`,不会影响你正在跑的源码构建容器。
|
|
||||||
|
|
||||||
如果这套容器在 `prepare-update` 时访问 GitHub 失败,而你本机是通过代理出网,请在 `.env` 里把 `AETHER_UPDATE_PROXY_URL` 写成宿主机地址,例如 `http://host.docker.internal:7890`;容器内的 `127.0.0.1` 指向容器自身,不是宿主机。
|
|
||||||
|
|
||||||
如果想重置这套联调环境(包括 `/opt/aether/current` 和已下载的历史版本),执行:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
docker compose -f docker-compose.release-local.yml down -v
|
|
||||||
```
|
|
||||||
|
|
||||||
可选变量:
|
|
||||||
|
|
||||||
- `AETHER_RELEASE_LOCAL_VERSION`:本地联调镜像对外声明的当前版本,默认 `v0.7.0`
|
|
||||||
- `AETHER_RELEASE_LOCAL_PORT`:本地联调端口,默认 `18085`
|
|
||||||
- `LOCAL_RELEASE_APP_IMAGE`:本地联调镜像名,默认 `aether-app:release-local`
|
|
||||||
|
|
||||||
### 一键安装(默认 Single Node:Linux systemd / macOS launchd + SQLite)
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git clone https://github.com/fawney19/Aether.git
|
git clone https://github.com/fawney19/Aether.git
|
||||||
cd Aether
|
cd Aether
|
||||||
curl -fsSL https://raw.githubusercontent.com/fawney19/Aether/main/install.sh | sudo bash
|
curl -fsSL https://raw.githubusercontent.com/fawney19/Aether/main/install.sh | sudo bash -s -- --mode compose
|
||||||
```
|
```
|
||||||
|
|
||||||
|
原生 Linux systemd / macOS launchd 安装需先准备 PostgreSQL,将连接串通过 `DATABASE_URL` 传给安装进程,并选择 `--mode single-node`;不再自动创建本地数据库文件。
|
||||||
|
|
||||||
### Nightly(每日 main 构建)
|
### Nightly(每日 main 构建)
|
||||||
|
|
||||||
Nightly workflow 每天从 `main` 的固定 commit 构建并发布滚动的 GitHub Release `nightly`,同时推送多架构 GHCR 镜像 `ghcr.io/fawney19/aether:nightly`。Nightly 是预发布版本,适合验证最新代码,不保证与正式版相同的稳定性。滚动 Release 需要仓库保持关闭 GitHub Release immutability。
|
Nightly workflow 每天从 `main` 的固定 commit 构建并发布滚动的 GitHub Release `nightly`,同时推送多架构 GHCR 镜像 `ghcr.io/fawney19/aether:nightly`。Nightly 是预发布版本,适合验证最新代码,不保证与正式版相同的稳定性。滚动 Release 需要仓库保持关闭 GitHub Release immutability。
|
||||||
|
|
||||||
安装最新 nightly(Linux systemd / macOS launchd + SQLite):
|
安装最新 nightly(PostgreSQL + Redis):
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
curl -fsSL https://raw.githubusercontent.com/fawney19/Aether/main/install.sh | sudo bash -s -- --channel nightly
|
curl -fsSL https://raw.githubusercontent.com/fawney19/Aether/main/install.sh | sudo bash -s -- --mode compose --channel nightly
|
||||||
```
|
```
|
||||||
|
|
||||||
Docker Compose 用户可在部署目录的 `.env` 中设置 `APP_IMAGE=ghcr.io/fawney19/aether:nightly`,然后运行 `./update.sh` 获取下一次 nightly。二进制方式可重新执行上述安装命令升级;当前管理后台的在线更新列表只跟踪正式版/RC/Beta,不会自动提示下一次 nightly。
|
Docker Compose 用户可在部署目录的 `.env` 中设置 `APP_IMAGE=ghcr.io/fawney19/aether:nightly`,然后运行 `./update.sh` 获取下一次 nightly。二进制部署请沿用已有 PostgreSQL 环境配置,并使用 `--mode single-node --channel nightly` 重新运行安装脚本升级;当前管理后台的在线更新列表只跟踪正式版/RC/Beta,不会自动提示下一次 nightly。
|
||||||
|
|
||||||
## 本地开发
|
## 本地开发
|
||||||
|
|
||||||
依赖 Docker、Rust toolchain、Node.js 和 make。
|
依赖 Docker、Rust toolchain、Node.js 和 make。
|
||||||
|
首次启动前需要在 `.env` 中设置 `ADMIN_PASSWORD`,用于创建本地管理员。
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
make dev
|
make dev
|
||||||
@@ -135,6 +87,12 @@ make dev
|
|||||||
|
|
||||||
`make dev` 会同时启动后端 `aether-gateway` 和前端 `frontend` 的 Vite dev server。需要单独启动时可使用 `make dev-backend` 或 `make dev-frontend`。
|
`make dev` 会同时启动后端 `aether-gateway` 和前端 `frontend` 的 Vite dev server。需要单独启动时可使用 `make dev-backend` 或 `make dev-frontend`。
|
||||||
Postgres / Redis 本地依赖未就绪时,`make dev` 会自动执行 `docker compose up -d postgres redis`。
|
Postgres / Redis 本地依赖未就绪时,`make dev` 会自动执行 `docker compose up -d postgres redis`。
|
||||||
|
`make dev` 会先完成后端编译,再开始计算服务健康检查超时。数据库 schema 和必要的派生数据准备也会在启动时自动完成;通常不需要手动区分 migration 与 backfill。升级不会主动重写或清除已有业务历史记录,新写入会直接遵循当前的数据持久化策略。排查或部署前预执行时可使用:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make db-status
|
||||||
|
make db-prepare
|
||||||
|
```
|
||||||
|
|
||||||
## Codex 远程协同
|
## Codex 远程协同
|
||||||
|
|
||||||
@@ -161,25 +119,46 @@ Aether Tunnel 是配套的正向代理节点,部署在海外 VPS 上,为墙
|
|||||||
## 环境变量
|
## 环境变量
|
||||||
|
|
||||||
- `APP_PORT`:`aether-gateway` 唯一监听端口,固定绑定 `0.0.0.0:${APP_PORT}`
|
- `APP_PORT`:`aether-gateway` 唯一监听端口,固定绑定 `0.0.0.0:${APP_PORT}`
|
||||||
- `DATABASE_URL`:数据库连接串;SQLite 例如 `sqlite:///opt/aether/data/aether.db`,Postgres 例如 `postgresql://postgres:aether@postgres:5432/aether`
|
- `DATABASE_URL`:PostgreSQL 连接串,例如 `postgresql://USER:PASSWORD@HOST:5432/aether`
|
||||||
- `AETHER_GATEWAY_DATA_POSTGRES_MIN_CONNECTIONS` / `AETHER_GATEWAY_DATA_POSTGRES_MAX_CONNECTIONS`:数据库连接池手动覆盖值;未配置时 SQLite 固定 `1/1`,Postgres/MySQL 按每核 `4` 条自动推导,总池范围为 `32-100`。该预算按进程计算,多实例部署应按数据库连接上限显式分配
|
- `AETHER_GATEWAY_DATA_POSTGRES_MIN_CONNECTIONS` / `AETHER_GATEWAY_DATA_POSTGRES_MAX_CONNECTIONS`:数据库连接池手动覆盖值;未配置时 PostgreSQL 按每核 `4` 条自动推导,总池范围为 `32-100`。该预算按进程计算,多实例部署应按数据库连接上限显式分配
|
||||||
- `AETHER_GATEWAY_MAX_IN_FLIGHT_REQUESTS`:单实例请求并发上限;未配置时按 CPU 自动推导(基础范围 `512-65536`),低文件描述符预算时会进一步下调
|
- `AETHER_GATEWAY_MAX_IN_FLIGHT_REQUESTS`:单实例请求并发上限;未配置时按 CPU 自动推导(基础范围 `512-65536`),低文件描述符预算时会进一步下调
|
||||||
- `AETHER_GATEWAY_REQUEST_BODY_BUFFER_BUDGET_MB`:单实例同时读取和解压请求体的加权内存预算,默认 `256MB`
|
- `AETHER_GATEWAY_REQUEST_BODY_BUFFER_BUDGET_MB`:单实例同时读取和解压请求体的加权内存预算,默认 `256MB`
|
||||||
- `AETHER_GATEWAY_REQUEST_BODY_READ_TIMEOUT_MS`:请求体完整读取超时,默认 `120000ms`
|
- `AETHER_GATEWAY_REQUEST_BODY_READ_TIMEOUT_MS`:可选的请求体完整读取超时;默认或显式设为 `0` 时关闭,非零值限制在 `1000-600000ms`
|
||||||
- `AETHER_MAX_REQUEST_BODY_MB`:可选的单请求解压后请求体上限;未配置或设为 `0` 时不限制
|
- `AETHER_MAX_REQUEST_BODY_MB`:单请求解压后请求体上限,默认 `256MB`;显式设为 `0` 表示不再收紧默认值,但仍受 `256MB` 安全硬上限约束
|
||||||
- `AETHER_MAX_INTERNAL_BUFFERED_BODY_MB`:可选的 heartbeat、管理探测等内部整包响应体上限;未配置或设为 `0` 时不限制
|
- `AETHER_MAX_INTERNAL_BUFFERED_BODY_MB`:heartbeat、管理探测等内部整包响应体上限,默认 `64MB`;显式设为 `0` 表示不再收紧默认值,但仍受 `256MB` 安全硬上限约束
|
||||||
- `AETHER_TUNNEL_NODE_STATUS_QUEUE_CAPACITY`:隧道节点状态上报队列容量,默认 `1024`;满载时拒绝新事件,避免控制面故障导致无界内存增长
|
- `AETHER_TUNNEL_NODE_STATUS_QUEUE_CAPACITY`:隧道节点状态上报队列容量,默认 `1024`;满载时拒绝新事件,避免控制面故障导致无界内存增长
|
||||||
|
- `AETHER_TUNNEL_RELAY_ALLOW_PRIVATE_TARGETS`:跨网关 owner relay 解析到私有/保留地址时的显式运维开关,默认关闭;仅当多网关 relay URL 是受控的内网 HTTPS 地址时设置为 `true`。它不改变普通 provider 请求的 DNS/代理策略,也不允许明文 HTTP 非 loopback relay
|
||||||
|
- `AETHER_TUNNEL_RELAY_PRIVATE_HOST_ALLOWLIST`:更窄的 owner relay 私网例外,填写逗号分隔的精确主机名(例如 `gateway-a.internal,gateway-b.internal`,忽略大小写和末尾点);仅这些主机解析出的私有地址会被允许,并且请求仍使用解析后地址 pin。不要填写通配符或 `.internal` 这类后缀
|
||||||
|
- `AETHER_INTERNAL_GATEWAY_AUTH_SECRET`:旧版 `/api/internal/gateway/*` 高权限控制面的独立 HMAC 密钥,至少 `32` 字节;未配置时该控制面返回 `404`。不要复用 JWT、数据加密或 tunnel relay 密钥,多节点必须使用同一值及共享 Redis 防重放
|
||||||
- `AETHER_GATEWAY_SECURITY_CACHE_TTL_MS`:IP 黑白名单本地缓存时间,默认 `1000ms`,写操作会主动失效相关缓存
|
- `AETHER_GATEWAY_SECURITY_CACHE_TTL_MS`:IP 黑白名单本地缓存时间,默认 `1000ms`,写操作会主动失效相关缓存
|
||||||
- `AETHER_MAX_REDACTED_SYNC_RESPONSE_BODY_MB`:可选的 PII 恢复同步响应缓冲上限;未配置或设为 `0` 时不限制
|
- `AETHER_MAX_REDACTED_SYNC_RESPONSE_BODY_MB`:PII 恢复同步响应缓冲上限,默认 `64MB`;显式设为 `0` 表示不再收紧默认值,但仍受 `256MB` 安全硬上限约束
|
||||||
- `REDIS_URL`:Redis 连接串;仅 Postgres + Redis 的 Docker Compose 部署需要配置
|
- `REDIS_URL`:Redis 连接串;仅 Postgres + Redis 的 Docker Compose 部署需要配置
|
||||||
- `AETHER_RUNTIME_BACKEND=memory|redis`:运行时缓存/协调后端。SQLite 默认用 `memory`,不会连接 Redis;多节点部署和需要跨 gateway 重启恢复 OpenAI Responses continuation history 的部署必须使用共享 Redis
|
- `AETHER_RUNTIME_BACKEND=memory|redis`:运行时缓存/协调后端。配置 Redis 时使用 `redis`,否则使用 `memory`;多节点部署和需要跨 gateway 重启恢复 OpenAI Responses continuation history 的部署必须使用共享 Redis
|
||||||
- `AETHER_GATEWAY_AUTO_PREPARE_DATABASE`:常规启动前自动执行挂起的 schema migration 和 backfill;仓库自带的 `docker-compose.yml` 默认开启
|
- `AETHER_GATEWAY_DATABASE_MODE=auto|verify-only`:数据库启动策略,默认 `auto`,自动完成挂起的 schema migration 和 backfill;`verify-only` 仅检查并在数据库落后时拒绝启动
|
||||||
|
- `AETHER_GATEWAY_AUTO_PREPARE_DATABASE`:旧版兼容开关;新配置请使用 `AETHER_GATEWAY_DATABASE_MODE`
|
||||||
- `JWT_SECRET_KEY` / `ENCRYPTION_KEY`:认证和敏感数据加密所需密钥
|
- `JWT_SECRET_KEY` / `ENCRYPTION_KEY`:认证和敏感数据加密所需密钥
|
||||||
|
- `AETHER_BACKUP_ENCRYPTION_KEY`:推荐的 S3 备份独立加密密钥;缺省回退到 `ENCRYPTION_KEY`。新备份使用带 key ID 的 AES-256-GCM v2 envelope,轮换前必须保留旧密钥
|
||||||
- `API_KEY_PREFIX`:用户和管理员新建 API Key 时使用的前缀,默认 `sk`
|
- `API_KEY_PREFIX`:用户和管理员新建 API Key 时使用的前缀,默认 `sk`
|
||||||
- `ADMIN_USERNAME` / `ADMIN_PASSWORD` / `ADMIN_EMAIL`:首次启动时自举首个本地管理员;`install.sh` 会提示输入管理员密码
|
- `ADMIN_USERNAME` / `ADMIN_PASSWORD` / `ADMIN_EMAIL`:首次启动时自举首个本地管理员;`install.sh` 会提示输入管理员密码
|
||||||
- `CORS_ORIGINS` / `CORS_ALLOW_CREDENTIALS`:前端跨域来源控制;如果要跨域带登录 Cookie,`CORS_ORIGINS` 不能写 `*`
|
- `CORS_ORIGINS` / `CORS_ALLOW_CREDENTIALS`:前端跨域来源控制;如果要跨域带登录 Cookie,`CORS_ORIGINS` 不能写 `*`
|
||||||
- `RUST_LOG`:Rust 日志过滤,例如 `aether_gateway=info`、`aether_gateway=debug,sqlx=warn`
|
- `RUST_LOG`:Rust 日志过滤,例如 `aether_gateway=info`、`aether_gateway=debug,sqlx=warn`
|
||||||
- Docker Compose 的 `DB_PASSWORD` / `REDIS_PASSWORD` 默认使用 `aether`
|
- `DB_PASSWORD` / `REDIS_PASSWORD`:Docker Compose 后端密码,首次安装时分别随机生成;手工部署必须替换示例占位值,不要互相复用
|
||||||
|
|
||||||
|
### S3 备份离线恢复
|
||||||
|
|
||||||
|
先从 S3 下载完整的 `.json.zst.aes256gcm` 对象,再使用原始的完整 S3 object key 做认证解密。恢复工具只验证并输出本地 JSON,不会直接写数据库;数据库导入仍应在维护窗口通过管理端完成。
|
||||||
|
|
||||||
|
```bash
|
||||||
|
AETHER_BACKUP_ENCRYPTION_KEY='原备份密钥' \
|
||||||
|
cargo run -p aether-gateway --bin aether-backup-restore -- \
|
||||||
|
--input ./backup.json.zst.aes256gcm \
|
||||||
|
--object-key 'aether/backups/aether-data-backup-20260822-010000.json.zst.aes256gcm' \
|
||||||
|
--output ./restored-backup.json
|
||||||
|
```
|
||||||
|
|
||||||
|
工具默认拒绝覆盖,输出采用原子写并在 Unix 上设置为 `0600`;Unix 可用 `--overwrite` 原子替换,Windows 为避免非原子删除窗口会要求选择新输出路径。密钥不能作为命令行参数。可使用 `AETHER_BACKUP_ENCRYPTION_KEY`、兼容用 `AETHER_GATEWAY_DATA_ENCRYPTION_KEY` / `ENCRYPTION_KEY`、受保护的 `--key-file`,或 `AETHER_BACKUP_KEYRING_FILE`。Keyring JSON 格式为 `{"version":1,"keys":["当前或历史 v2 secret"],"legacy_v1":["旧 v1 secret"]}`;条目也可写成 `{"secret":"..."}`(兼容字段名 `key`)。也可由 `AETHER_BACKUP_HISTORICAL_KEYS_JSON` 提供同一结构。密钥文件必须是非符号链接的普通文件,Unix 下权限需为 `0600` 或更严格。
|
||||||
|
|
||||||
|
默认限制密文为 `512MiB`、解压后 JSON 为 `1GiB`,可通过受限的 `--max-encrypted-mib` / `--max-json-mib` 调整。网关最多扫描同一备份前缀下 10,000 个对象,并且不会自动删除 S3 对象:`backup_s3_retention_count` 只用于报告超出保留数量的清理候选。旧明文备份在创建并验证加密副本后仍会保留,必须通过 bucket lifecycle 或支持版本条件的外部清理工具移除;启用 Versioning 时还需清理 noncurrent versions,Object Lock/retention 可能阻止物理删除。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
@@ -92,7 +92,7 @@ npm --prefix aether-vscodex/web run build
|
|||||||
`vscodex_data`,短期配对码、60 秒一次性浏览器票据和在线房间保存在内存。不要在未引入
|
`vscodex_data`,短期配对码、60 秒一次性浏览器票据和在线房间保存在内存。不要在未引入
|
||||||
共享连接目录前横向扩容 sidecar。
|
共享连接目录前横向扩容 sidecar。
|
||||||
|
|
||||||
登录 Aether 后打开“Codex 远程控制”,生成一次性配对码。然后在 VS Code 命令面板执行
|
登录 Aether 后打开“远程控制”,生成一次性配对码。然后在 VS Code 命令面板执行
|
||||||
**Codex Remote: Pair with Aether**,填写 Aether 地址和配对码。插件会把设备凭据写入
|
**Codex Remote: Pair with Aether**,填写 Aether 地址和配对码。插件会把设备凭据写入
|
||||||
VS Code SecretStorage,并同时保持本机控制台连接。
|
VS Code SecretStorage,并同时保持本机控制台连接。
|
||||||
|
|
||||||
|
|||||||
@@ -65,14 +65,14 @@ http.workspace = true
|
|||||||
http-body-util = "0.1"
|
http-body-util = "0.1"
|
||||||
hyper = { version = "1", features = ["client", "server", "http1", "http2"] }
|
hyper = { version = "1", features = ["client", "server", "http1", "http2"] }
|
||||||
hyper-util = { version = "0.1", features = ["client-legacy", "client-pool", "server-auto", "service", "tokio"] }
|
hyper-util = { version = "0.1", features = ["client-legacy", "client-pool", "server-auto", "service", "tokio"] }
|
||||||
ldap3 = { version = "0.11", default-features = false, features = ["sync", "tls-rustls"] }
|
ldap3 = { version = "0.12.1", default-features = false, features = ["sync", "tls-rustls-ring"] }
|
||||||
libc = "0.2"
|
libc = "0.2"
|
||||||
md-5 = "0.10"
|
md-5 = "0.10"
|
||||||
object_store.workspace = true
|
object_store.workspace = true
|
||||||
parking_lot = "0.12"
|
parking_lot = "0.12"
|
||||||
|
percent-encoding.workspace = true
|
||||||
regex.workspace = true
|
regex.workspace = true
|
||||||
reqwest.workspace = true
|
reqwest.workspace = true
|
||||||
rsa = "0.9.10"
|
|
||||||
rustls.workspace = true
|
rustls.workspace = true
|
||||||
serde.workspace = true
|
serde.workspace = true
|
||||||
serde_json.workspace = true
|
serde_json.workspace = true
|
||||||
@@ -81,7 +81,7 @@ sha1 = "0.10"
|
|||||||
sha2 = { workspace = true, features = ["oid"] }
|
sha2 = { workspace = true, features = ["oid"] }
|
||||||
socket2.workspace = true
|
socket2.workspace = true
|
||||||
tar.workspace = true
|
tar.workspace = true
|
||||||
sqlx = { workspace = true, features = ["postgres", "mysql", "sqlite", "migrate"] }
|
sqlx = { workspace = true, features = ["postgres", "migrate"] }
|
||||||
sysinfo = "0.32"
|
sysinfo = "0.32"
|
||||||
thiserror.workspace = true
|
thiserror.workspace = true
|
||||||
tokio.workspace = true
|
tokio.workspace = true
|
||||||
@@ -102,5 +102,7 @@ tikv-jemallocator = { version = "0.6", optional = true }
|
|||||||
tikv-jemalloc-sys = { version = "0.6", optional = true }
|
tikv-jemalloc-sys = { version = "0.6", optional = true }
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
|
aether-testkit = { workspace = true, features = ["postgres"] }
|
||||||
aether-test-support.workspace = true
|
aether-test-support.workspace = true
|
||||||
|
aws-lc-rs.workspace = true
|
||||||
tracing-subscriber.workspace = true
|
tracing-subscriber.workspace = true
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ struct Args {
|
|||||||
#[arg(
|
#[arg(
|
||||||
long,
|
long,
|
||||||
env = "AETHER_EXECUTION_RUNTIME_UNIX_SOCKET",
|
env = "AETHER_EXECUTION_RUNTIME_UNIX_SOCKET",
|
||||||
default_value = "/tmp/aether-execution-runtime.sock"
|
default_value = "/tmp/aether-execution-runtime/aether-execution-runtime.sock"
|
||||||
)]
|
)]
|
||||||
unix_socket: PathBuf,
|
unix_socket: PathBuf,
|
||||||
|
|
||||||
|
|||||||
@@ -1,17 +1,18 @@
|
|||||||
pub(crate) use crate::handlers::admin::{
|
pub(crate) use crate::handlers::admin::{
|
||||||
admin_provider_ops_local_action_response, admin_provider_pool_config,
|
admin_provider_ops_local_action_response, admin_provider_pool_config,
|
||||||
build_internal_control_error_response, create_provider_oauth_catalog_key,
|
build_internal_control_error_response, create_provider_oauth_catalog_key,
|
||||||
find_duplicate_provider_oauth_key, maybe_build_local_admin_pool_response,
|
execute_admin_system_import_exclusively, find_duplicate_provider_oauth_key,
|
||||||
maybe_build_local_admin_response, persist_provider_quota_refresh_state,
|
maybe_build_local_admin_pool_response, maybe_build_local_admin_response,
|
||||||
provider_oauth_maintenance_endpoint_for_provider, provider_oauth_runtime_endpoint_for_provider,
|
persist_provider_quota_refresh_state, provider_oauth_maintenance_endpoint_for_provider,
|
||||||
provider_quota_refresh_endpoint_for_provider, provider_type_supports_quota_refresh,
|
provider_oauth_runtime_endpoint_for_provider, provider_quota_refresh_endpoint_for_provider,
|
||||||
reconcile_admin_fixed_provider_template_endpoints,
|
provider_type_supports_quota_refresh, reconcile_admin_fixed_provider_template_endpoints,
|
||||||
refresh_provider_oauth_account_state_after_update, refresh_provider_pool_quota_locally,
|
refresh_provider_oauth_account_state_after_update, refresh_provider_pool_quota_locally,
|
||||||
store_admin_provider_ops_balance_cache, update_existing_provider_oauth_catalog_key,
|
release_admin_system_import_lease, store_admin_provider_ops_balance_cache,
|
||||||
|
try_acquire_admin_system_import_lease, update_existing_provider_oauth_catalog_key,
|
||||||
AdminAppState, AdminGatewayProviderTransportSnapshot, AdminLocalOAuthRefreshError,
|
AdminAppState, AdminGatewayProviderTransportSnapshot, AdminLocalOAuthRefreshError,
|
||||||
AdminRequestContext, AdminRouteRequest, AdminRouteResponse, AdminRouteResult,
|
AdminRequestContext, AdminRouteRequest, AdminRouteResponse, AdminRouteResult,
|
||||||
AdminStatsTimeRange, AdminStatsUsageFilter, OAUTH_ACCOUNT_BLOCK_PREFIX,
|
AdminStatsTimeRange, AdminStatsUsageFilter, AdminSystemImportLockError, SystemExportMode,
|
||||||
OAUTH_REQUEST_FAILED_PREFIX,
|
OAUTH_ACCOUNT_BLOCK_PREFIX, OAUTH_REQUEST_FAILED_PREFIX,
|
||||||
};
|
};
|
||||||
|
|
||||||
use crate::handlers::admin::{
|
use crate::handlers::admin::{
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
use aether_usage_runtime::decode_internal_report_body_base64;
|
||||||
use base64::Engine as _;
|
use base64::Engine as _;
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
|
|
||||||
@@ -43,9 +44,8 @@ pub(crate) fn maybe_normalize_provider_private_sync_report_payload(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if let Some(body_base64) = payload.body_base64.as_deref() {
|
if let Some(body_base64) = payload.body_base64.as_deref() {
|
||||||
let body_bytes = base64::engine::general_purpose::STANDARD
|
let body_bytes =
|
||||||
.decode(body_base64)
|
decode_internal_report_body_base64(body_base64).map_err(GatewayError::Internal)?;
|
||||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
|
||||||
let Some(normalized_bytes) =
|
let Some(normalized_bytes) =
|
||||||
normalize_provider_private_stream_bytes(report_context, &body_bytes)?
|
normalize_provider_private_stream_bytes(report_context, &body_bytes)?
|
||||||
else {
|
else {
|
||||||
|
|||||||
@@ -1,15 +1,15 @@
|
|||||||
use std::sync::{Arc, OnceLock};
|
use std::sync::{Arc, OnceLock};
|
||||||
use std::time::{SystemTime, UNIX_EPOCH};
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
use aether_provider_transport::CodexFingerprintConvergenceContext;
|
|
||||||
use http::{request::Parts, HeaderMap};
|
use http::{request::Parts, HeaderMap};
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
use crate::ai_serving::transport::ProviderOutboundRequestContext;
|
||||||
use crate::client_session_affinity::codex_request_signals_from_request;
|
use crate::client_session_affinity::codex_request_signals_from_request;
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub(crate) struct CodexFingerprintContextSlot(Arc<OnceLock<CodexFingerprintConvergenceContext>>);
|
pub(crate) struct CodexFingerprintContextSlot(Arc<OnceLock<ProviderOutboundRequestContext>>);
|
||||||
|
|
||||||
impl Default for CodexFingerprintContextSlot {
|
impl Default for CodexFingerprintContextSlot {
|
||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
@@ -18,11 +18,7 @@ impl Default for CodexFingerprintContextSlot {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl CodexFingerprintContextSlot {
|
impl CodexFingerprintContextSlot {
|
||||||
fn resolve(
|
fn resolve(&self, headers: &HeaderMap, body_json: &Value) -> ProviderOutboundRequestContext {
|
||||||
&self,
|
|
||||||
headers: &HeaderMap,
|
|
||||||
body_json: &Value,
|
|
||||||
) -> CodexFingerprintConvergenceContext {
|
|
||||||
self.0
|
self.0
|
||||||
.get_or_init(|| {
|
.get_or_init(|| {
|
||||||
build_codex_fingerprint_context(headers, body_json, Uuid::now_v7().to_string())
|
build_codex_fingerprint_context(headers, body_json, Uuid::now_v7().to_string())
|
||||||
@@ -34,10 +30,10 @@ impl CodexFingerprintContextSlot {
|
|||||||
pub(crate) fn resolve_codex_fingerprint_context(
|
pub(crate) fn resolve_codex_fingerprint_context(
|
||||||
parts: &Parts,
|
parts: &Parts,
|
||||||
body_json: &Value,
|
body_json: &Value,
|
||||||
) -> CodexFingerprintConvergenceContext {
|
) -> ProviderOutboundRequestContext {
|
||||||
if let Some(context) = parts
|
if let Some(context) = parts
|
||||||
.extensions
|
.extensions
|
||||||
.get::<CodexFingerprintConvergenceContext>()
|
.get::<ProviderOutboundRequestContext>()
|
||||||
.cloned()
|
.cloned()
|
||||||
{
|
{
|
||||||
return context;
|
return context;
|
||||||
@@ -51,7 +47,7 @@ pub(crate) fn resolve_codex_fingerprint_context(
|
|||||||
pub(crate) fn install_codex_fingerprint_context_slot(parts: &mut Parts) {
|
pub(crate) fn install_codex_fingerprint_context_slot(parts: &mut Parts) {
|
||||||
if parts
|
if parts
|
||||||
.extensions
|
.extensions
|
||||||
.get::<CodexFingerprintConvergenceContext>()
|
.get::<ProviderOutboundRequestContext>()
|
||||||
.is_none()
|
.is_none()
|
||||||
&& parts
|
&& parts
|
||||||
.extensions
|
.extensions
|
||||||
@@ -67,11 +63,11 @@ pub(crate) fn install_codex_fingerprint_context_slot(parts: &mut Parts) {
|
|||||||
pub(crate) fn ensure_codex_fingerprint_context(
|
pub(crate) fn ensure_codex_fingerprint_context(
|
||||||
parts: &mut Parts,
|
parts: &mut Parts,
|
||||||
body_json: &Value,
|
body_json: &Value,
|
||||||
) -> CodexFingerprintConvergenceContext {
|
) -> ProviderOutboundRequestContext {
|
||||||
let context = resolve_codex_fingerprint_context(parts, body_json);
|
let context = resolve_codex_fingerprint_context(parts, body_json);
|
||||||
if parts
|
if parts
|
||||||
.extensions
|
.extensions
|
||||||
.get::<CodexFingerprintConvergenceContext>()
|
.get::<ProviderOutboundRequestContext>()
|
||||||
.is_none()
|
.is_none()
|
||||||
{
|
{
|
||||||
parts.extensions.remove::<CodexFingerprintContextSlot>();
|
parts.extensions.remove::<CodexFingerprintContextSlot>();
|
||||||
@@ -84,7 +80,7 @@ pub(crate) fn attach_codex_logical_turn_context(
|
|||||||
parts: &mut Parts,
|
parts: &mut Parts,
|
||||||
body_json: &Value,
|
body_json: &Value,
|
||||||
logical_turn_id: &str,
|
logical_turn_id: &str,
|
||||||
) -> CodexFingerprintConvergenceContext {
|
) -> ProviderOutboundRequestContext {
|
||||||
let context =
|
let context =
|
||||||
build_codex_fingerprint_context(&parts.headers, body_json, logical_turn_id.to_string());
|
build_codex_fingerprint_context(&parts.headers, body_json, logical_turn_id.to_string());
|
||||||
parts.extensions.remove::<CodexFingerprintContextSlot>();
|
parts.extensions.remove::<CodexFingerprintContextSlot>();
|
||||||
@@ -94,7 +90,7 @@ pub(crate) fn attach_codex_logical_turn_context(
|
|||||||
|
|
||||||
pub(crate) fn restore_codex_logical_turn_context(
|
pub(crate) fn restore_codex_logical_turn_context(
|
||||||
parts: &mut Parts,
|
parts: &mut Parts,
|
||||||
context: &CodexFingerprintConvergenceContext,
|
context: &ProviderOutboundRequestContext,
|
||||||
) {
|
) {
|
||||||
parts.extensions.remove::<CodexFingerprintContextSlot>();
|
parts.extensions.remove::<CodexFingerprintContextSlot>();
|
||||||
parts.extensions.insert(context.clone());
|
parts.extensions.insert(context.clone());
|
||||||
@@ -104,10 +100,9 @@ fn build_codex_fingerprint_context(
|
|||||||
headers: &HeaderMap,
|
headers: &HeaderMap,
|
||||||
body_json: &Value,
|
body_json: &Value,
|
||||||
logical_turn_id: String,
|
logical_turn_id: String,
|
||||||
) -> CodexFingerprintConvergenceContext {
|
) -> ProviderOutboundRequestContext {
|
||||||
let signals = codex_request_signals_from_request(headers, Some(body_json));
|
let signals = codex_request_signals_from_request(headers, Some(body_json));
|
||||||
let mut context =
|
let mut context = ProviderOutboundRequestContext::new(logical_turn_id, current_unix_millis());
|
||||||
CodexFingerprintConvergenceContext::new(logical_turn_id, current_unix_millis());
|
|
||||||
|
|
||||||
if let Some(turn_id) = signals.turn_id {
|
if let Some(turn_id) = signals.turn_id {
|
||||||
context = context.with_original_turn_id(turn_id);
|
context = context.with_original_turn_id(turn_id);
|
||||||
@@ -160,14 +155,14 @@ mod tests {
|
|||||||
assert_eq!(context.original_client_session_id(), Some("header-thread"));
|
assert_eq!(context.original_client_session_id(), Some("header-thread"));
|
||||||
assert_eq!(context.original_prompt_cache_key(), Some("client-cache"));
|
assert_eq!(context.original_prompt_cache_key(), Some("client-cache"));
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
parts.extensions.get::<CodexFingerprintConvergenceContext>(),
|
parts.extensions.get::<ProviderOutboundRequestContext>(),
|
||||||
Some(&context)
|
Some(&context)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn restored_context_wins_over_retry_request_signals() {
|
fn restored_context_wins_over_retry_request_signals() {
|
||||||
let original = CodexFingerprintConvergenceContext::new("logical-turn", 1234)
|
let original = ProviderOutboundRequestContext::new("logical-turn", 1234)
|
||||||
.with_original_turn_id("original-turn")
|
.with_original_turn_id("original-turn")
|
||||||
.with_original_client_session_id("original-thread")
|
.with_original_client_session_id("original-thread")
|
||||||
.with_original_prompt_cache_key("original-cache");
|
.with_original_prompt_cache_key("original-cache");
|
||||||
|
|||||||
@@ -11,10 +11,10 @@ use super::{
|
|||||||
convert_gemini_chat_response_to_openai_chat, convert_gemini_response_to_openai_responses,
|
convert_gemini_chat_response_to_openai_chat, convert_gemini_response_to_openai_responses,
|
||||||
maybe_build_local_core_sync_finalize_response,
|
maybe_build_local_core_sync_finalize_response,
|
||||||
};
|
};
|
||||||
use crate::ai_serving::GatewayControlDecision;
|
|
||||||
use crate::ai_serving::{
|
use crate::ai_serving::{
|
||||||
convert_openai_chat_response_to_openai_responses,
|
convert_openai_chat_response_to_openai_responses,
|
||||||
convert_openai_responses_response_to_openai_chat,
|
convert_openai_responses_response_to_openai_chat, openai_responses_message_item_id,
|
||||||
|
GatewayControlDecision,
|
||||||
};
|
};
|
||||||
use crate::usage::GatewaySyncReportRequest;
|
use crate::usage::GatewaySyncReportRequest;
|
||||||
|
|
||||||
@@ -192,7 +192,7 @@ fn aggregates_openai_responses_stream_completed_event_to_final_response() {
|
|||||||
"output_text": "Hello",
|
"output_text": "Hello",
|
||||||
"output": [{
|
"output": [{
|
||||||
"type": "message",
|
"type": "message",
|
||||||
"id": "resp_123_msg",
|
"id": openai_responses_message_item_id("resp_123", 0),
|
||||||
"role": "assistant",
|
"role": "assistant",
|
||||||
"status": "completed",
|
"status": "completed",
|
||||||
"content": [{
|
"content": [{
|
||||||
@@ -843,7 +843,7 @@ fn converts_claude_cli_response_to_openai_responses_response() {
|
|||||||
"output_text": "Hello Claude CLI",
|
"output_text": "Hello Claude CLI",
|
||||||
"output": [{
|
"output": [{
|
||||||
"type": "message",
|
"type": "message",
|
||||||
"id": "msg_cli_123_msg",
|
"id": openai_responses_message_item_id("msg_cli_123", 0),
|
||||||
"role": "assistant",
|
"role": "assistant",
|
||||||
"status": "completed",
|
"status": "completed",
|
||||||
"content": [{
|
"content": [{
|
||||||
@@ -907,7 +907,7 @@ fn converts_claude_cli_tool_use_to_openai_responses_function_call() {
|
|||||||
"output": [
|
"output": [
|
||||||
{
|
{
|
||||||
"type": "message",
|
"type": "message",
|
||||||
"id": "msg_cli_tool_123_msg",
|
"id": openai_responses_message_item_id("msg_cli_tool_123", 0),
|
||||||
"role": "assistant",
|
"role": "assistant",
|
||||||
"status": "completed",
|
"status": "completed",
|
||||||
"content": [{
|
"content": [{
|
||||||
@@ -977,7 +977,7 @@ fn converts_gemini_cli_response_to_openai_responses_response() {
|
|||||||
"output_text": "Hello Gemini CLI",
|
"output_text": "Hello Gemini CLI",
|
||||||
"output": [{
|
"output": [{
|
||||||
"type": "message",
|
"type": "message",
|
||||||
"id": "resp_cli_123_msg",
|
"id": openai_responses_message_item_id("resp_cli_123", 0),
|
||||||
"role": "assistant",
|
"role": "assistant",
|
||||||
"status": "completed",
|
"status": "completed",
|
||||||
"content": [{
|
"content": [{
|
||||||
@@ -1046,7 +1046,7 @@ fn converts_gemini_cli_function_call_to_openai_responses_function_call() {
|
|||||||
"output": [
|
"output": [
|
||||||
{
|
{
|
||||||
"type": "message",
|
"type": "message",
|
||||||
"id": "resp_cli_tool_123_msg",
|
"id": openai_responses_message_item_id("resp_cli_tool_123", 0),
|
||||||
"role": "assistant",
|
"role": "assistant",
|
||||||
"status": "completed",
|
"status": "completed",
|
||||||
"content": [{
|
"content": [{
|
||||||
@@ -1252,7 +1252,7 @@ fn local_finalize_handles_openai_responses_openai_family_sync_response_even_when
|
|||||||
"model": "gpt-5",
|
"model": "gpt-5",
|
||||||
"output": [{
|
"output": [{
|
||||||
"type": "message",
|
"type": "message",
|
||||||
"id": "resp_cli_family_123_msg",
|
"id": openai_responses_message_item_id("resp_cli_family_123", 0),
|
||||||
"role": "assistant",
|
"role": "assistant",
|
||||||
"status": "completed",
|
"status": "completed",
|
||||||
"content": [{
|
"content": [{
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ use aether_ai_serving::{
|
|||||||
use aether_dispatch_core::{DispatchSequence, DispatchSequenceItem};
|
use aether_dispatch_core::{DispatchSequence, DispatchSequenceItem};
|
||||||
use aether_routing_core::{
|
use aether_routing_core::{
|
||||||
rank_vector_for_candidate, CandidateKind, ResolvedRoutingPolicy, RoutingCandidateFacts,
|
rank_vector_for_candidate, CandidateKind, ResolvedRoutingPolicy, RoutingCandidateFacts,
|
||||||
RoutingCandidateTrace, RoutingDecisionTrace,
|
RoutingCandidateTrace, RoutingDecisionTrace, RoutingExecutionPolicy,
|
||||||
};
|
};
|
||||||
use aether_scheduler_core::{
|
use aether_scheduler_core::{
|
||||||
ClientSessionAffinity, SchedulerMinimalCandidateSelectionCandidate, SchedulerRankingOutcome,
|
ClientSessionAffinity, SchedulerMinimalCandidateSelectionCandidate, SchedulerRankingOutcome,
|
||||||
@@ -79,6 +79,13 @@ type DecorateSkippedCandidateFn<'a> = Arc<
|
|||||||
pub(crate) trait LocalExecutionAttemptSource<T>: Send {
|
pub(crate) trait LocalExecutionAttemptSource<T>: Send {
|
||||||
async fn next_execution_attempt(&mut self) -> Result<Option<T>, GatewayError>;
|
async fn next_execution_attempt(&mut self) -> Result<Option<T>, GatewayError>;
|
||||||
|
|
||||||
|
/// Returns the request-scoped execution behaviour selected by routing.
|
||||||
|
/// Execution wrappers use this snapshot before consuming the first
|
||||||
|
/// attempt, avoiding a second lookup against mutable system settings.
|
||||||
|
fn routing_execution_policy(&self) -> Option<RoutingExecutionPolicy> {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
async fn drain_execution_attempts(&mut self) -> Result<Vec<T>, GatewayError>;
|
async fn drain_execution_attempts(&mut self) -> Result<Vec<T>, GatewayError>;
|
||||||
|
|
||||||
async fn skip_credential(&mut self, key_id: &str) -> Result<(), GatewayError>;
|
async fn skip_credential(&mut self, key_id: &str) -> Result<(), GatewayError>;
|
||||||
@@ -1237,9 +1244,7 @@ async fn scheduler_cache_affinity_enabled(
|
|||||||
state: PlannerAppState<'_>,
|
state: PlannerAppState<'_>,
|
||||||
routing_policy: Option<&ResolvedRoutingPolicy>,
|
routing_policy: Option<&ResolvedRoutingPolicy>,
|
||||||
) -> bool {
|
) -> bool {
|
||||||
scheduler_ordering_config_for_routing_policy(state, routing_policy)
|
scheduler_ordering_config_for_routing_policy(routing_policy).scheduling_mode
|
||||||
.await
|
|
||||||
.scheduling_mode
|
|
||||||
== SchedulerSchedulingMode::CacheAffinity
|
== SchedulerSchedulingMode::CacheAffinity
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2320,16 +2325,11 @@ mod tests {
|
|||||||
assert_eq!(stored.len(), 1);
|
assert_eq!(stored.len(), 1);
|
||||||
assert_eq!(stored[0].key_id.as_deref(), Some("normal-key"));
|
assert_eq!(stored[0].key_id.as_deref(), Some("normal-key"));
|
||||||
assert_eq!(stored[0].candidate_index, 2);
|
assert_eq!(stored[0].candidate_index, 2);
|
||||||
assert_eq!(
|
assert!(stored[0]
|
||||||
stored[0]
|
.extra_data
|
||||||
.extra_data
|
.as_ref()
|
||||||
.as_ref()
|
.and_then(|value| value.get("dispatch_ref"))
|
||||||
.and_then(|value| value.get("dispatch_ref"))
|
.is_none());
|
||||||
.and_then(|value| value.get("SingleKey"))
|
|
||||||
.and_then(|value| value.get("key"))
|
|
||||||
.and_then(|value| value.get("key_id")),
|
|
||||||
Some(&json!("normal-key"))
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -2477,14 +2477,23 @@ mod tests {
|
|||||||
|
|
||||||
assert!(should_cache_resolved_candidate_page(&cursor));
|
assert!(should_cache_resolved_candidate_page(&cursor));
|
||||||
|
|
||||||
let fixed_order_app = AppState::new()
|
let fixed_order_app = AppState::new().expect("state should build");
|
||||||
.expect("state should build")
|
let fixed_order_policy = ResolvedRoutingPolicy {
|
||||||
.with_data_state_for_tests(
|
group_id: Some("routing-group-fixed-order".to_string()),
|
||||||
GatewayDataState::disabled().with_system_config_values_for_tests([(
|
group_version: Some(1),
|
||||||
"scheduling_mode".to_string(),
|
selection_source: "test".to_string(),
|
||||||
json!("fixed_order"),
|
requested_model: "gpt-5".to_string(),
|
||||||
)]),
|
resolved_model: "gpt-5".to_string(),
|
||||||
);
|
priority_mode: aether_routing_core::RoutingSetPriorityMode::Provider,
|
||||||
|
scheduling_mode: aether_routing_core::RoutingSchedulingMode::FixedOrder,
|
||||||
|
keep_priority_on_conversion: false,
|
||||||
|
sticky_key_attempts: aether_routing_core::DEFAULT_STICKY_KEY_ATTEMPTS,
|
||||||
|
execution_policy: Default::default(),
|
||||||
|
ranking_overlay: Default::default(),
|
||||||
|
mutation_plan: Default::default(),
|
||||||
|
pool_policy_overrides: Default::default(),
|
||||||
|
matched_rules: Vec::new(),
|
||||||
|
};
|
||||||
let mut page_cursor = LocalCandidatePreselectionPageCursor::new(
|
let mut page_cursor = LocalCandidatePreselectionPageCursor::new(
|
||||||
PlannerAppState::new(&fixed_order_app),
|
PlannerAppState::new(&fixed_order_app),
|
||||||
&model_directive_policy,
|
&model_directive_policy,
|
||||||
@@ -2494,7 +2503,7 @@ mod tests {
|
|||||||
true,
|
true,
|
||||||
None,
|
None,
|
||||||
&auth_snapshot,
|
&auth_snapshot,
|
||||||
None,
|
Some(&fixed_order_policy),
|
||||||
None,
|
None,
|
||||||
None,
|
None,
|
||||||
false,
|
false,
|
||||||
@@ -2512,7 +2521,7 @@ mod tests {
|
|||||||
auth_snapshot,
|
auth_snapshot,
|
||||||
client_session_affinity: None,
|
client_session_affinity: None,
|
||||||
required_capabilities: None,
|
required_capabilities: None,
|
||||||
routing_policy: None,
|
routing_policy: Some(fixed_order_policy),
|
||||||
sticky_session_token: None,
|
sticky_session_token: None,
|
||||||
request_auth_channel: None,
|
request_auth_channel: None,
|
||||||
skipped_user_id: "user-1".to_string(),
|
skipped_user_id: "user-1".to_string(),
|
||||||
@@ -2610,16 +2619,11 @@ mod tests {
|
|||||||
);
|
);
|
||||||
assert_eq!(stored[1].key_id.as_deref(), Some("normal-key"));
|
assert_eq!(stored[1].key_id.as_deref(), Some("normal-key"));
|
||||||
assert_eq!(stored[1].candidate_index, 1);
|
assert_eq!(stored[1].candidate_index, 1);
|
||||||
assert_eq!(
|
assert!(stored[1]
|
||||||
stored[1]
|
.extra_data
|
||||||
.extra_data
|
.as_ref()
|
||||||
.as_ref()
|
.and_then(|value| value.get("dispatch_ref"))
|
||||||
.and_then(|value| value.get("dispatch_ref"))
|
.is_none());
|
||||||
.and_then(|value| value.get("SingleKey"))
|
|
||||||
.and_then(|value| value.get("key"))
|
|
||||||
.and_then(|value| value.get("key_id")),
|
|
||||||
Some(&json!("normal-key"))
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -2689,7 +2693,7 @@ mod tests {
|
|||||||
.as_ref()
|
.as_ref()
|
||||||
.and_then(serde_json::Value::as_object)
|
.and_then(serde_json::Value::as_object)
|
||||||
.expect("ranking metadata should persist as object extra data");
|
.expect("ranking metadata should persist as object extra data");
|
||||||
assert_eq!(extra_data.get("existing"), Some(&json!("value")));
|
assert!(extra_data.get("existing").is_none());
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
extra_data.get("ranking_mode"),
|
extra_data.get("ranking_mode"),
|
||||||
Some(&json!("CacheAffinity"))
|
Some(&json!("CacheAffinity"))
|
||||||
@@ -2702,14 +2706,7 @@ mod tests {
|
|||||||
Some(&json!("cached_affinity"))
|
Some(&json!("cached_affinity"))
|
||||||
);
|
);
|
||||||
assert_eq!(extra_data.get("demoted_by"), Some(&json!("cross_format")));
|
assert_eq!(extra_data.get("demoted_by"), Some(&json!("cross_format")));
|
||||||
assert_eq!(
|
assert!(extra_data.get("dispatch_ref").is_none());
|
||||||
extra_data
|
|
||||||
.get("dispatch_ref")
|
|
||||||
.and_then(|value| value.get("SingleKey"))
|
|
||||||
.and_then(|value| value.get("key"))
|
|
||||||
.and_then(|value| value.get("key_id")),
|
|
||||||
Some(&json!("ranked-key"))
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -3047,7 +3044,7 @@ mod tests {
|
|||||||
.as_ref()
|
.as_ref()
|
||||||
.and_then(serde_json::Value::as_object)
|
.and_then(serde_json::Value::as_object)
|
||||||
.expect("skipped ranking metadata should persist");
|
.expect("skipped ranking metadata should persist");
|
||||||
assert_eq!(extra_data.get("existing"), Some(&json!("value")));
|
assert!(extra_data.get("existing").is_none());
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
extra_data.get("ranking_mode"),
|
extra_data.get("ranking_mode"),
|
||||||
Some(&json!("CacheAffinity"))
|
Some(&json!("CacheAffinity"))
|
||||||
|
|||||||
@@ -278,13 +278,21 @@ mod tests {
|
|||||||
|
|
||||||
assert_eq!(metadata["transport_diagnostics"]["provider_type"], "codex");
|
assert_eq!(metadata["transport_diagnostics"]["provider_type"], "codex");
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
metadata["transport_diagnostics"]["fingerprint"]["transport_profile"]["profile_id"],
|
metadata["transport_diagnostics"]["key_fingerprint_configured"],
|
||||||
"chrome_136"
|
Value::Bool(true)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
metadata["transport_diagnostics"]["key_transport_profile_configured"],
|
||||||
|
Value::Bool(true)
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
metadata["transport_diagnostics"]["resolved_transport_profile_id"],
|
metadata["transport_diagnostics"]["resolved_transport_profile_id"],
|
||||||
"chrome_136"
|
"chrome_136"
|
||||||
);
|
);
|
||||||
|
assert_eq!(
|
||||||
|
metadata["transport_diagnostics"]["resolved_transport_profile"]["profile_id"],
|
||||||
|
"chrome_136"
|
||||||
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
metadata["transport_diagnostics"]["request_pair"]["conversion_enabled"],
|
metadata["transport_diagnostics"]["request_pair"]["conversion_enabled"],
|
||||||
Value::Bool(true)
|
Value::Bool(true)
|
||||||
|
|||||||
@@ -6,14 +6,11 @@ use aether_ai_serving::{
|
|||||||
use aether_routing_core::ResolvedRoutingPolicy;
|
use aether_routing_core::ResolvedRoutingPolicy;
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use tokio::sync::Mutex;
|
use tokio::sync::Mutex;
|
||||||
use tracing::warn;
|
|
||||||
|
|
||||||
use crate::ai_serving::{GatewayAuthApiKeySnapshot, PlannerAppState};
|
use crate::ai_serving::{GatewayAuthApiKeySnapshot, PlannerAppState};
|
||||||
use crate::clock::current_unix_ms;
|
use crate::clock::current_unix_ms;
|
||||||
use crate::handlers::shared::provider_pool::admin_provider_pool_config_from_config_value;
|
use crate::handlers::shared::provider_pool::admin_provider_pool_config_from_config_value;
|
||||||
use crate::scheduler::config::{
|
use crate::scheduler::config::{SchedulerOrderingConfig, SchedulerSchedulingMode};
|
||||||
read_scheduler_ordering_config, SchedulerOrderingConfig, SchedulerSchedulingMode,
|
|
||||||
};
|
|
||||||
use aether_scheduler_core::{
|
use aether_scheduler_core::{
|
||||||
matches_affinity_target, ClientSessionAffinity, SchedulerAffinityTarget,
|
matches_affinity_target, ClientSessionAffinity, SchedulerAffinityTarget,
|
||||||
SchedulerMinimalCandidateSelectionCandidate, SchedulerPriorityMode, SchedulerRankableCandidate,
|
SchedulerMinimalCandidateSelectionCandidate, SchedulerPriorityMode, SchedulerRankableCandidate,
|
||||||
@@ -133,7 +130,7 @@ pub(crate) async fn rank_eligible_local_execution_candidates(
|
|||||||
required_capabilities: Option<&serde_json::Value>,
|
required_capabilities: Option<&serde_json::Value>,
|
||||||
routing_policy: Option<&ResolvedRoutingPolicy>,
|
routing_policy: Option<&ResolvedRoutingPolicy>,
|
||||||
) -> Vec<EligibleLocalExecutionCandidate> {
|
) -> Vec<EligibleLocalExecutionCandidate> {
|
||||||
let ordering_config = scheduler_ordering_config_for_routing_policy(state, routing_policy).await;
|
let ordering_config = scheduler_ordering_config_for_routing_policy(routing_policy);
|
||||||
let port = GatewayLocalCandidateRankingPort {
|
let port = GatewayLocalCandidateRankingPort {
|
||||||
state,
|
state,
|
||||||
requested_model,
|
requested_model,
|
||||||
@@ -184,16 +181,24 @@ fn ai_ranking_scheduling_mode(mode: SchedulerSchedulingMode) -> AiRankingSchedul
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Ordering config for a request. A resolved routing policy is authoritative
|
/// Return the immutable scheduler snapshot carried by a resolved routing
|
||||||
/// and is never merged with legacy system-config values; without a policy the
|
/// policy. A missing policy is a programming error in production request
|
||||||
/// effective default (system-default routing group, then legacy keys) applies.
|
/// paths; unit tests may use the scheduler default for isolated ranking tests.
|
||||||
pub(crate) async fn scheduler_ordering_config_for_routing_policy(
|
pub(crate) fn scheduler_ordering_config_for_routing_policy(
|
||||||
state: PlannerAppState<'_>,
|
|
||||||
routing_policy: Option<&ResolvedRoutingPolicy>,
|
routing_policy: Option<&ResolvedRoutingPolicy>,
|
||||||
) -> SchedulerOrderingConfig {
|
) -> SchedulerOrderingConfig {
|
||||||
match routing_policy {
|
match routing_policy {
|
||||||
Some(policy) => SchedulerOrderingConfig::from_routing_policy(policy),
|
Some(policy) => SchedulerOrderingConfig::from_routing_policy(policy),
|
||||||
None => read_scheduler_ordering_config_or_default(state).await,
|
None => {
|
||||||
|
#[cfg(test)]
|
||||||
|
{
|
||||||
|
SchedulerOrderingConfig::default()
|
||||||
|
}
|
||||||
|
#[cfg(not(test))]
|
||||||
|
{
|
||||||
|
panic!("resolved routing policy is required before candidate scheduling")
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -238,23 +243,6 @@ fn routing_overlaid_candidate(
|
|||||||
overlaid
|
overlaid
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn read_scheduler_ordering_config_or_default(
|
|
||||||
state: PlannerAppState<'_>,
|
|
||||||
) -> SchedulerOrderingConfig {
|
|
||||||
match read_scheduler_ordering_config(state.app()).await {
|
|
||||||
Ok(config) => config,
|
|
||||||
Err(error) => {
|
|
||||||
warn!(
|
|
||||||
event_name = "planner_scheduler_ordering_config_load_failed",
|
|
||||||
log_type = "event",
|
|
||||||
error = ?error,
|
|
||||||
"failed to load scheduler ordering config while ranking local execution candidates"
|
|
||||||
);
|
|
||||||
SchedulerOrderingConfig::default()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
@@ -263,10 +251,17 @@ mod tests {
|
|||||||
use aether_ai_serving::{
|
use aether_ai_serving::{
|
||||||
ai_ranking_context, build_ai_rankable_candidate, AiRankableCandidateParts,
|
ai_ranking_context, build_ai_rankable_candidate, AiRankableCandidateParts,
|
||||||
};
|
};
|
||||||
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||||
|
use aether_data::repository::{
|
||||||
|
provider_catalog::InMemoryProviderCatalogReadRepository,
|
||||||
|
routing_profiles::InMemoryRoutingGroupRepository,
|
||||||
|
};
|
||||||
use aether_data_contracts::repository::provider_catalog::{
|
use aether_data_contracts::repository::provider_catalog::{
|
||||||
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
|
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
|
||||||
};
|
};
|
||||||
|
use aether_data_contracts::repository::routing_profiles::{
|
||||||
|
CreateRoutingGroupRecord, RoutingGroupWriteRepository,
|
||||||
|
};
|
||||||
use aether_scheduler_core::{
|
use aether_scheduler_core::{
|
||||||
apply_scheduler_candidate_ranking,
|
apply_scheduler_candidate_ranking,
|
||||||
build_scheduler_affinity_cache_key_for_api_key_id_with_client_session,
|
build_scheduler_affinity_cache_key_for_api_key_id_with_client_session,
|
||||||
@@ -296,7 +291,11 @@ mod tests {
|
|||||||
required_capabilities: Option<&serde_json::Value>,
|
required_capabilities: Option<&serde_json::Value>,
|
||||||
) -> Vec<SchedulerMinimalCandidateSelectionCandidate> {
|
) -> Vec<SchedulerMinimalCandidateSelectionCandidate> {
|
||||||
let normalized_client_api_format = client_api_format.trim().to_ascii_lowercase();
|
let normalized_client_api_format = client_api_format.trim().to_ascii_lowercase();
|
||||||
let ordering_config = super::read_scheduler_ordering_config_or_default(state).await;
|
let ordering_config =
|
||||||
|
crate::scheduler::config::read_system_default_routing_ordering_config(state.app())
|
||||||
|
.await
|
||||||
|
.expect("routing strategy should load")
|
||||||
|
.unwrap_or_default();
|
||||||
let mut candidates = candidates;
|
let mut candidates = candidates;
|
||||||
let mut rankables = Vec::with_capacity(candidates.len());
|
let mut rankables = Vec::with_capacity(candidates.len());
|
||||||
let mut ordering_cache = CandidateTransportRankingFactsCache::default();
|
let mut ordering_cache = CandidateTransportRankingFactsCache::default();
|
||||||
@@ -372,6 +371,7 @@ mod tests {
|
|||||||
scheduling_mode: aether_routing_core::RoutingSchedulingMode::CacheAffinity,
|
scheduling_mode: aether_routing_core::RoutingSchedulingMode::CacheAffinity,
|
||||||
keep_priority_on_conversion: false,
|
keep_priority_on_conversion: false,
|
||||||
sticky_key_attempts: aether_routing_core::DEFAULT_STICKY_KEY_ATTEMPTS,
|
sticky_key_attempts: aether_routing_core::DEFAULT_STICKY_KEY_ATTEMPTS,
|
||||||
|
execution_policy: Default::default(),
|
||||||
ranking_overlay: aether_routing_core::RankingOverlay::default(),
|
ranking_overlay: aether_routing_core::RankingOverlay::default(),
|
||||||
mutation_plan: Default::default(),
|
mutation_plan: Default::default(),
|
||||||
pool_policy_overrides: BTreeMap::new(),
|
pool_policy_overrides: BTreeMap::new(),
|
||||||
@@ -408,17 +408,14 @@ mod tests {
|
|||||||
scheduling_mode: aether_routing_core::RoutingSchedulingMode::FixedOrder,
|
scheduling_mode: aether_routing_core::RoutingSchedulingMode::FixedOrder,
|
||||||
keep_priority_on_conversion: false,
|
keep_priority_on_conversion: false,
|
||||||
sticky_key_attempts: aether_routing_core::DEFAULT_STICKY_KEY_ATTEMPTS,
|
sticky_key_attempts: aether_routing_core::DEFAULT_STICKY_KEY_ATTEMPTS,
|
||||||
|
execution_policy: Default::default(),
|
||||||
ranking_overlay: Default::default(),
|
ranking_overlay: Default::default(),
|
||||||
mutation_plan: Default::default(),
|
mutation_plan: Default::default(),
|
||||||
pool_policy_overrides: Default::default(),
|
pool_policy_overrides: Default::default(),
|
||||||
matched_rules: Vec::new(),
|
matched_rules: Vec::new(),
|
||||||
};
|
};
|
||||||
|
|
||||||
let ordering = super::scheduler_ordering_config_for_routing_policy(
|
let ordering = super::scheduler_ordering_config_for_routing_policy(Some(&policy));
|
||||||
PlannerAppState::new(&state),
|
|
||||||
Some(&policy),
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
ordering.scheduling_mode,
|
ordering.scheduling_mode,
|
||||||
@@ -446,6 +443,7 @@ mod tests {
|
|||||||
scheduling_mode: aether_routing_core::RoutingSchedulingMode::CacheAffinity,
|
scheduling_mode: aether_routing_core::RoutingSchedulingMode::CacheAffinity,
|
||||||
keep_priority_on_conversion: false,
|
keep_priority_on_conversion: false,
|
||||||
sticky_key_attempts: aether_routing_core::DEFAULT_STICKY_KEY_ATTEMPTS,
|
sticky_key_attempts: aether_routing_core::DEFAULT_STICKY_KEY_ATTEMPTS,
|
||||||
|
execution_policy: Default::default(),
|
||||||
ranking_overlay: aether_routing_core::RankingOverlay {
|
ranking_overlay: aether_routing_core::RankingOverlay {
|
||||||
pool_priority_overrides: BTreeMap::from([("provider-1".to_string(), 4)]),
|
pool_priority_overrides: BTreeMap::from([("provider-1".to_string(), 4)]),
|
||||||
key_priority_overrides: BTreeMap::from([("representative-key".to_string(), 1)]),
|
key_priority_overrides: BTreeMap::from([("representative-key".to_string(), 1)]),
|
||||||
@@ -569,6 +567,15 @@ mod tests {
|
|||||||
api_formats: Option<serde_json::Value>,
|
api_formats: Option<serde_json::Value>,
|
||||||
allowed_models: Option<serde_json::Value>,
|
allowed_models: Option<serde_json::Value>,
|
||||||
) -> StoredProviderCatalogKey {
|
) -> StoredProviderCatalogKey {
|
||||||
|
let credential_state = AppState::new()
|
||||||
|
.expect("credential state should build")
|
||||||
|
.with_data_state_for_tests(
|
||||||
|
GatewayDataState::disabled()
|
||||||
|
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY),
|
||||||
|
);
|
||||||
|
let encrypted_api_key = credential_state
|
||||||
|
.seal_provider_catalog_key_api_key(provider_id, id, "plain-upstream-key")
|
||||||
|
.expect("api key should encrypt");
|
||||||
StoredProviderCatalogKey::new(
|
StoredProviderCatalogKey::new(
|
||||||
id.to_string(),
|
id.to_string(),
|
||||||
provider_id.to_string(),
|
provider_id.to_string(),
|
||||||
@@ -580,7 +587,7 @@ mod tests {
|
|||||||
.expect("key should build")
|
.expect("key should build")
|
||||||
.with_transport_fields(
|
.with_transport_fields(
|
||||||
api_formats,
|
api_formats,
|
||||||
"plain-upstream-key".to_string(),
|
encrypted_api_key,
|
||||||
None,
|
None,
|
||||||
None,
|
None,
|
||||||
Some(json!({"openai:chat": 1})),
|
Some(json!({"openai:chat": 1})),
|
||||||
@@ -694,7 +701,7 @@ mod tests {
|
|||||||
let observed_at_unix_secs = current_unix_secs();
|
let observed_at_unix_secs = current_unix_secs();
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
)
|
)
|
||||||
.with_system_config_values_for_tests(vec![
|
.with_system_config_values_for_tests(vec![
|
||||||
("provider_priority_mode".to_string(), json!("provider")),
|
("provider_priority_mode".to_string(), json!("provider")),
|
||||||
@@ -703,6 +710,7 @@ mod tests {
|
|||||||
serde_json::to_value(TunnelAttachmentRecord {
|
serde_json::to_value(TunnelAttachmentRecord {
|
||||||
gateway_instance_id: "gateway-b".to_string(),
|
gateway_instance_id: "gateway-b".to_string(),
|
||||||
relay_base_url: "http://gateway-b:8080".to_string(),
|
relay_base_url: "http://gateway-b:8080".to_string(),
|
||||||
|
tunnel_generation: "test-generation-remote".to_string(),
|
||||||
conn_count: 1,
|
conn_count: 1,
|
||||||
observed_at_unix_secs,
|
observed_at_unix_secs,
|
||||||
})
|
})
|
||||||
@@ -713,6 +721,7 @@ mod tests {
|
|||||||
serde_json::to_value(TunnelAttachmentRecord {
|
serde_json::to_value(TunnelAttachmentRecord {
|
||||||
gateway_instance_id: "gateway-a".to_string(),
|
gateway_instance_id: "gateway-a".to_string(),
|
||||||
relay_base_url: "http://gateway-a:8080".to_string(),
|
relay_base_url: "http://gateway-a:8080".to_string(),
|
||||||
|
tunnel_generation: "test-generation-local".to_string(),
|
||||||
conn_count: 1,
|
conn_count: 1,
|
||||||
observed_at_unix_secs,
|
observed_at_unix_secs,
|
||||||
})
|
})
|
||||||
@@ -771,7 +780,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
);
|
);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
@@ -824,7 +833,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
)
|
)
|
||||||
.with_system_config_values_for_tests(vec![(
|
.with_system_config_values_for_tests(vec![(
|
||||||
"scheduling_mode".to_string(),
|
"scheduling_mode".to_string(),
|
||||||
@@ -881,7 +890,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
);
|
);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
@@ -917,7 +926,8 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn local_execution_ranking_keeps_cross_format_priority_when_global_override_is_enabled() {
|
async fn local_execution_ranking_keeps_cross_format_priority_when_strategy_override_is_enabled()
|
||||||
|
{
|
||||||
let provider_catalog = InMemoryProviderCatalogReadRepository::seed(
|
let provider_catalog = InMemoryProviderCatalogReadRepository::seed(
|
||||||
vec![
|
vec![
|
||||||
sample_provider_with_options("provider-same", false, 10),
|
sample_provider_with_options("provider-same", false, 10),
|
||||||
@@ -932,14 +942,32 @@ mod tests {
|
|||||||
sample_key_for_provider("provider-cross", "key-cross", ""),
|
sample_key_for_provider("provider-cross", "key-cross", ""),
|
||||||
],
|
],
|
||||||
);
|
);
|
||||||
|
let routing_repository = std::sync::Arc::new(InMemoryRoutingGroupRepository::default());
|
||||||
|
routing_repository
|
||||||
|
.create_routing_group(CreateRoutingGroupRecord {
|
||||||
|
id: "strategy-default".to_string(),
|
||||||
|
name: "strategy-default".to_string(),
|
||||||
|
description: None,
|
||||||
|
enabled: true,
|
||||||
|
is_system_default: true,
|
||||||
|
sort_order: 0,
|
||||||
|
config_json: json!({
|
||||||
|
"default_policy": {
|
||||||
|
"keep_priority_on_conversion": true
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
version: 1,
|
||||||
|
created_at: 1,
|
||||||
|
updated_at: 1,
|
||||||
|
published_at: None,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("routing strategy should be created");
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
)
|
)
|
||||||
.with_system_config_values_for_tests(vec![(
|
.with_routing_group_repository_for_tests(routing_repository);
|
||||||
"keep_priority_on_conversion".to_string(),
|
|
||||||
json!(true),
|
|
||||||
)]);
|
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
.with_data_state_for_tests(data_state);
|
.with_data_state_for_tests(data_state);
|
||||||
@@ -999,7 +1027,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
)
|
)
|
||||||
.with_system_config_values_for_tests(vec![(
|
.with_system_config_values_for_tests(vec![(
|
||||||
"provider_priority_mode".to_string(),
|
"provider_priority_mode".to_string(),
|
||||||
@@ -1065,7 +1093,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
);
|
);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
@@ -1118,7 +1146,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
);
|
);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
@@ -1192,7 +1220,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
);
|
);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
@@ -1272,7 +1300,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
);
|
);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
@@ -1348,7 +1376,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
);
|
);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
@@ -1415,7 +1443,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
);
|
);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
@@ -1498,7 +1526,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
);
|
);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
@@ -1563,7 +1591,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
);
|
);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
@@ -1652,7 +1680,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
);
|
);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
@@ -1738,7 +1766,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
);
|
);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
@@ -1835,7 +1863,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
);
|
);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
@@ -1940,7 +1968,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
);
|
);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
@@ -2034,7 +2062,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
let data_state = GatewayDataState::with_provider_transport_reader_for_tests(
|
||||||
std::sync::Arc::new(provider_catalog),
|
std::sync::Arc::new(provider_catalog),
|
||||||
"development-key",
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
);
|
);
|
||||||
let state = AppState::new()
|
let state = AppState::new()
|
||||||
.expect("state should build")
|
.expect("state should build")
|
||||||
|
|||||||
@@ -384,8 +384,7 @@ async fn resolve_and_rank_local_execution_candidates_with_pool_expansion(
|
|||||||
None
|
None
|
||||||
} else {
|
} else {
|
||||||
Some(
|
Some(
|
||||||
scheduler_ordering_config_for_routing_policy(state, routing_policy)
|
scheduler_ordering_config_for_routing_policy(routing_policy)
|
||||||
.await
|
|
||||||
.sticky_key_attempts,
|
.sticky_key_attempts,
|
||||||
)
|
)
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -174,8 +174,9 @@ impl AiCandidatePreselectionPort for GatewayLocalCandidatePreselectionPort<'_> {
|
|||||||
self.ranking_seed,
|
self.ranking_seed,
|
||||||
false,
|
false,
|
||||||
self.request_operation,
|
self.request_operation,
|
||||||
self.routing_policy
|
super::candidate_ranking::scheduler_ordering_config_for_routing_policy(
|
||||||
.map(SchedulerOrderingConfig::from_routing_policy),
|
self.routing_policy,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
@@ -427,11 +428,7 @@ impl<'a> LocalCandidatePreselectionPageCursor<'a> {
|
|||||||
);
|
);
|
||||||
|
|
||||||
let ordering_config =
|
let ordering_config =
|
||||||
super::candidate_ranking::scheduler_ordering_config_for_routing_policy(
|
super::candidate_ranking::scheduler_ordering_config_for_routing_policy(routing_policy);
|
||||||
state,
|
|
||||||
routing_policy,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
Self {
|
Self {
|
||||||
state,
|
state,
|
||||||
@@ -1293,9 +1290,7 @@ impl<'a> LocalCandidatePreselectionPageCursor<'a> {
|
|||||||
.then_some(self.client_session_affinity.as_ref())
|
.then_some(self.client_session_affinity.as_ref())
|
||||||
.flatten(),
|
.flatten(),
|
||||||
self.ranking_seed,
|
self.ranking_seed,
|
||||||
self.routing_policy
|
self.ordering_config,
|
||||||
.as_ref()
|
|
||||||
.map(SchedulerOrderingConfig::from_routing_policy),
|
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
let skipped_candidates = skipped_candidates
|
let skipped_candidates = skipped_candidates
|
||||||
@@ -1478,6 +1473,7 @@ mod tests {
|
|||||||
use super::*;
|
use super::*;
|
||||||
use crate::data::GatewayDataState;
|
use crate::data::GatewayDataState;
|
||||||
use crate::AppState;
|
use crate::AppState;
|
||||||
|
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||||
use aether_data::repository::candidate_selection::InMemoryMinimalCandidateSelectionReadRepository;
|
use aether_data::repository::candidate_selection::InMemoryMinimalCandidateSelectionReadRepository;
|
||||||
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
||||||
use aether_data::DataLayerError;
|
use aether_data::DataLayerError;
|
||||||
@@ -1890,6 +1886,7 @@ mod tests {
|
|||||||
scheduling_mode: aether_routing_core::RoutingSchedulingMode::FixedOrder,
|
scheduling_mode: aether_routing_core::RoutingSchedulingMode::FixedOrder,
|
||||||
keep_priority_on_conversion: false,
|
keep_priority_on_conversion: false,
|
||||||
sticky_key_attempts: aether_routing_core::DEFAULT_STICKY_KEY_ATTEMPTS,
|
sticky_key_attempts: aether_routing_core::DEFAULT_STICKY_KEY_ATTEMPTS,
|
||||||
|
execution_policy: Default::default(),
|
||||||
ranking_overlay: Default::default(),
|
ranking_overlay: Default::default(),
|
||||||
mutation_plan: Default::default(),
|
mutation_plan: Default::default(),
|
||||||
pool_policy_overrides: Default::default(),
|
pool_policy_overrides: Default::default(),
|
||||||
@@ -1954,6 +1951,7 @@ mod tests {
|
|||||||
scheduling_mode: aether_routing_core::RoutingSchedulingMode::FixedOrder,
|
scheduling_mode: aether_routing_core::RoutingSchedulingMode::FixedOrder,
|
||||||
keep_priority_on_conversion: false,
|
keep_priority_on_conversion: false,
|
||||||
sticky_key_attempts: aether_routing_core::DEFAULT_STICKY_KEY_ATTEMPTS,
|
sticky_key_attempts: aether_routing_core::DEFAULT_STICKY_KEY_ATTEMPTS,
|
||||||
|
execution_policy: Default::default(),
|
||||||
ranking_overlay: Default::default(),
|
ranking_overlay: Default::default(),
|
||||||
mutation_plan: Default::default(),
|
mutation_plan: Default::default(),
|
||||||
pool_policy_overrides: Default::default(),
|
pool_policy_overrides: Default::default(),
|
||||||
@@ -2177,6 +2175,19 @@ mod tests {
|
|||||||
None,
|
None,
|
||||||
)
|
)
|
||||||
.expect("endpoint transport should build");
|
.expect("endpoint transport should build");
|
||||||
|
let credential_state = AppState::new()
|
||||||
|
.expect("credential state should build")
|
||||||
|
.with_data_state_for_tests(
|
||||||
|
GatewayDataState::disabled()
|
||||||
|
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY),
|
||||||
|
);
|
||||||
|
let encrypted_api_key = credential_state
|
||||||
|
.seal_provider_catalog_key_api_key(
|
||||||
|
row.provider_id.as_str(),
|
||||||
|
row.key_id.as_str(),
|
||||||
|
"plain-upstream-key",
|
||||||
|
)
|
||||||
|
.expect("api key should encrypt");
|
||||||
let key = StoredProviderCatalogKey::new(
|
let key = StoredProviderCatalogKey::new(
|
||||||
row.key_id.clone(),
|
row.key_id.clone(),
|
||||||
row.provider_id.clone(),
|
row.provider_id.clone(),
|
||||||
@@ -2188,7 +2199,7 @@ mod tests {
|
|||||||
.expect("key should build")
|
.expect("key should build")
|
||||||
.with_transport_fields(
|
.with_transport_fields(
|
||||||
Some(serde_json::json!([row.endpoint_api_format.clone()])),
|
Some(serde_json::json!([row.endpoint_api_format.clone()])),
|
||||||
"plain-upstream-key".to_string(),
|
encrypted_api_key,
|
||||||
None,
|
None,
|
||||||
None,
|
None,
|
||||||
None,
|
None,
|
||||||
@@ -2543,7 +2554,7 @@ mod tests {
|
|||||||
provider_repository,
|
provider_repository,
|
||||||
candidate_repository,
|
candidate_repository,
|
||||||
)
|
)
|
||||||
.with_encryption_key_for_tests("development-key");
|
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY);
|
||||||
let app = AppState::new()
|
let app = AppState::new()
|
||||||
.expect("gateway state should build")
|
.expect("gateway state should build")
|
||||||
.with_data_state_for_tests(data_state);
|
.with_data_state_for_tests(data_state);
|
||||||
@@ -2663,7 +2674,7 @@ mod tests {
|
|||||||
provider_repository,
|
provider_repository,
|
||||||
candidate_repository,
|
candidate_repository,
|
||||||
)
|
)
|
||||||
.with_encryption_key_for_tests("development-key")
|
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY)
|
||||||
// Legacy keys deliberately disagree with the routing policy: the
|
// Legacy keys deliberately disagree with the routing policy: the
|
||||||
// resolved policy must be the only source of scheduler ordering.
|
// resolved policy must be the only source of scheduler ordering.
|
||||||
.with_system_config_values_for_tests([
|
.with_system_config_values_for_tests([
|
||||||
@@ -2692,6 +2703,7 @@ mod tests {
|
|||||||
scheduling_mode: aether_routing_core::RoutingSchedulingMode::FixedOrder,
|
scheduling_mode: aether_routing_core::RoutingSchedulingMode::FixedOrder,
|
||||||
keep_priority_on_conversion: true,
|
keep_priority_on_conversion: true,
|
||||||
sticky_key_attempts: aether_routing_core::DEFAULT_STICKY_KEY_ATTEMPTS,
|
sticky_key_attempts: aether_routing_core::DEFAULT_STICKY_KEY_ATTEMPTS,
|
||||||
|
execution_policy: Default::default(),
|
||||||
ranking_overlay: Default::default(),
|
ranking_overlay: Default::default(),
|
||||||
mutation_plan: Default::default(),
|
mutation_plan: Default::default(),
|
||||||
pool_policy_overrides: Default::default(),
|
pool_policy_overrides: Default::default(),
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ use crate::ai_serving::planner::route::{
|
|||||||
is_matching_stream_request, resolve_execution_runtime_stream_plan_kind,
|
is_matching_stream_request, resolve_execution_runtime_stream_plan_kind,
|
||||||
};
|
};
|
||||||
use crate::ai_serving::{resolve_decision_execution_runtime_auth_context, GatewayControlDecision};
|
use crate::ai_serving::{resolve_decision_execution_runtime_auth_context, GatewayControlDecision};
|
||||||
|
use crate::state::VideoTaskRouteAccess;
|
||||||
use crate::{AiExecutionDecision, AppState, GatewayError};
|
use crate::{AiExecutionDecision, AppState, GatewayError};
|
||||||
|
|
||||||
pub(crate) async fn maybe_build_stream_decision_payload(
|
pub(crate) async fn maybe_build_stream_decision_payload(
|
||||||
@@ -155,16 +156,37 @@ async fn maybe_build_local_video_task_content_stream_decision_payload(
|
|||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
|
|
||||||
let _ = state
|
let Some(user_id) = decision
|
||||||
.hydrate_video_task_for_route(decision.route_family.as_deref(), parts.uri.path())
|
.auth_context
|
||||||
.await?;
|
.as_ref()
|
||||||
|
.filter(|auth_context| auth_context.access_allowed)
|
||||||
|
.map(|auth_context| auth_context.user_id.trim())
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
else {
|
||||||
|
return Err(crate::video_tasks::not_found_error());
|
||||||
|
};
|
||||||
|
if state
|
||||||
|
.hydrate_video_task_for_route_for_user(
|
||||||
|
decision.route_family.as_deref(),
|
||||||
|
parts.uri.path(),
|
||||||
|
user_id,
|
||||||
|
)
|
||||||
|
.await?
|
||||||
|
!= VideoTaskRouteAccess::Allowed
|
||||||
|
{
|
||||||
|
return Err(crate::video_tasks::not_found_error());
|
||||||
|
}
|
||||||
|
|
||||||
let Some(action) = state.video_tasks.prepare_openai_content_stream_action(
|
let Some(action) = state
|
||||||
parts.uri.path(),
|
.video_tasks
|
||||||
parts.uri.query(),
|
.prepare_openai_content_stream_action_for_user(
|
||||||
trace_id,
|
parts.uri.path(),
|
||||||
) else {
|
parts.uri.query(),
|
||||||
return Ok(None);
|
trace_id,
|
||||||
|
user_id,
|
||||||
|
)
|
||||||
|
else {
|
||||||
|
return Err(crate::video_tasks::not_found_error());
|
||||||
};
|
};
|
||||||
|
|
||||||
let crate::video_tasks::LocalVideoTaskContentAction::StreamPlan(plan) = action else {
|
let crate::video_tasks::LocalVideoTaskContentAction::StreamPlan(plan) = action else {
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ use crate::ai_serving::{
|
|||||||
build_execution_runtime_auth_context, resolve_execution_runtime_auth_context,
|
build_execution_runtime_auth_context, resolve_execution_runtime_auth_context,
|
||||||
GatewayControlDecision,
|
GatewayControlDecision,
|
||||||
};
|
};
|
||||||
|
use crate::state::VideoTaskRouteAccess;
|
||||||
use crate::{AiExecutionDecision, AppState, GatewayError};
|
use crate::{AiExecutionDecision, AppState, GatewayError};
|
||||||
|
|
||||||
pub(crate) async fn maybe_build_sync_decision_payload(
|
pub(crate) async fn maybe_build_sync_decision_payload(
|
||||||
@@ -191,10 +192,6 @@ async fn maybe_build_local_video_task_follow_up_sync_decision_payload(
|
|||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
|
|
||||||
let _ = state
|
|
||||||
.hydrate_video_task_for_route(decision.route_family.as_deref(), parts.uri.path())
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
let auth_context = resolve_execution_runtime_auth_context(
|
let auth_context = resolve_execution_runtime_auth_context(
|
||||||
state,
|
state,
|
||||||
decision,
|
decision,
|
||||||
@@ -204,16 +201,30 @@ async fn maybe_build_local_video_task_follow_up_sync_decision_payload(
|
|||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
let Some(auth_context) = auth_context else {
|
let Some(auth_context) = auth_context else {
|
||||||
return Ok(None);
|
return Err(crate::video_tasks::not_found_error());
|
||||||
};
|
};
|
||||||
let Some(follow_up) = state.video_tasks.prepare_follow_up_sync_plan(
|
if !auth_context.access_allowed || auth_context.user_id.trim().is_empty() {
|
||||||
|
return Err(crate::video_tasks::not_found_error());
|
||||||
|
}
|
||||||
|
if state
|
||||||
|
.hydrate_video_task_for_route_for_user(
|
||||||
|
decision.route_family.as_deref(),
|
||||||
|
parts.uri.path(),
|
||||||
|
&auth_context.user_id,
|
||||||
|
)
|
||||||
|
.await?
|
||||||
|
!= VideoTaskRouteAccess::Allowed
|
||||||
|
{
|
||||||
|
return Err(crate::video_tasks::not_found_error());
|
||||||
|
}
|
||||||
|
let Some(follow_up) = state.video_tasks.prepare_follow_up_sync_plan_for_user(
|
||||||
plan_kind,
|
plan_kind,
|
||||||
parts.uri.path(),
|
parts.uri.path(),
|
||||||
Some(body_json),
|
Some(body_json),
|
||||||
Some(&auth_context),
|
Some(&auth_context),
|
||||||
trace_id,
|
trace_id,
|
||||||
) else {
|
) else {
|
||||||
return Ok(None);
|
return Err(crate::video_tasks::not_found_error());
|
||||||
};
|
};
|
||||||
|
|
||||||
let aether_video_tasks_core::LocalVideoTaskFollowUpPlan {
|
let aether_video_tasks_core::LocalVideoTaskFollowUpPlan {
|
||||||
@@ -236,8 +247,7 @@ async fn maybe_build_local_video_task_follow_up_sync_decision_payload(
|
|||||||
downstream_path = %parts.uri.path(),
|
downstream_path = %parts.uri.path(),
|
||||||
provider_api_format = %plan.provider_api_format,
|
provider_api_format = %plan.provider_api_format,
|
||||||
client_api_format = %plan.client_api_format,
|
client_api_format = %plan.client_api_format,
|
||||||
upstream_base_url = ?upstream_base_url,
|
upstream_origin = %crate::handlers::shared::security_log_url_origin(&plan.url),
|
||||||
upstream_url = %plan.url,
|
|
||||||
"gateway built local video follow-up sync decision payload"
|
"gateway built local video follow-up sync decision payload"
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ use http::{HeaderMap, HeaderName, HeaderValue};
|
|||||||
use serde_json::{json, Value};
|
use serde_json::{json, Value};
|
||||||
|
|
||||||
use crate::ai_serving::planner::common::extract_standard_requested_model;
|
use crate::ai_serving::planner::common::extract_standard_requested_model;
|
||||||
use crate::ai_serving::transport::CodexFingerprintConvergenceContext;
|
use crate::ai_serving::transport::ProviderOutboundRequestContext;
|
||||||
use crate::ai_serving::{
|
use crate::ai_serving::{
|
||||||
ClientSurface, ExecutionRuntimeAuthContext, GatewayAuthApiKeySnapshot,
|
ClientSurface, ExecutionRuntimeAuthContext, GatewayAuthApiKeySnapshot,
|
||||||
GatewayCredentialCarrier, GatewayProviderTransportSnapshot, PlannerAppState,
|
GatewayCredentialCarrier, GatewayProviderTransportSnapshot, PlannerAppState,
|
||||||
@@ -37,6 +37,10 @@ const ROUTING_GROUP_SELECTION_CACHE_TTL: Duration = Duration::from_secs(30);
|
|||||||
const ROUTING_GROUP_SELECTION_CACHE_STALE_TTL: Duration = Duration::from_secs(120);
|
const ROUTING_GROUP_SELECTION_CACHE_STALE_TTL: Duration = Duration::from_secs(120);
|
||||||
const CODEX_ACCOUNT_ID_HEADER: &str = "chatgpt-account-id";
|
const CODEX_ACCOUNT_ID_HEADER: &str = "chatgpt-account-id";
|
||||||
const CODEX_FEDRAMP_HEADER: &str = "x-openai-fedramp";
|
const CODEX_FEDRAMP_HEADER: &str = "x-openai-fedramp";
|
||||||
|
const INVALID_ROUTING_PROVIDER_CONTRACT_MESSAGE: &str =
|
||||||
|
"routing provider request violates provider contract";
|
||||||
|
const INVALID_ROUTING_PROVIDER_HEADERS_MESSAGE: &str =
|
||||||
|
"invalid provider request headers in routing mutation";
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub(crate) struct ResolvedLocalDecisionAuthInput {
|
pub(crate) struct ResolvedLocalDecisionAuthInput {
|
||||||
@@ -56,7 +60,7 @@ pub(crate) struct LocalRequestedModelDecisionInput {
|
|||||||
pub(crate) client_surface: Option<ClientSurface>,
|
pub(crate) client_surface: Option<ClientSurface>,
|
||||||
pub(crate) gateway_credential_carrier: Option<GatewayCredentialCarrier>,
|
pub(crate) gateway_credential_carrier: Option<GatewayCredentialCarrier>,
|
||||||
pub(crate) client_session_affinity: Option<ClientSessionAffinity>,
|
pub(crate) client_session_affinity: Option<ClientSessionAffinity>,
|
||||||
pub(crate) codex_fingerprint_context: Option<CodexFingerprintConvergenceContext>,
|
pub(crate) provider_outbound_context: Option<ProviderOutboundRequestContext>,
|
||||||
pub(crate) routing_policy: Option<ResolvedRoutingPolicy>,
|
pub(crate) routing_policy: Option<ResolvedRoutingPolicy>,
|
||||||
pub(crate) routing_trace_seed: Option<RoutingDecisionTrace>,
|
pub(crate) routing_trace_seed: Option<RoutingDecisionTrace>,
|
||||||
pub(crate) routing_context: Option<LocalRoutingRequestContext>,
|
pub(crate) routing_context: Option<LocalRoutingRequestContext>,
|
||||||
@@ -168,7 +172,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
|||||||
provider_api_format.as_str(),
|
provider_api_format.as_str(),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
apply_codex_fingerprint_convergence_to_decision(
|
apply_provider_outbound_request_policies_to_decision(
|
||||||
input,
|
input,
|
||||||
decision,
|
decision,
|
||||||
transport,
|
transport,
|
||||||
@@ -231,7 +235,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
|||||||
provider_api_format.as_str(),
|
provider_api_format.as_str(),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
apply_codex_fingerprint_convergence_to_decision(
|
apply_provider_outbound_request_policies_to_decision(
|
||||||
input,
|
input,
|
||||||
decision,
|
decision,
|
||||||
transport,
|
transport,
|
||||||
@@ -312,10 +316,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
.map_err(|violation| GatewayError::Client {
|
.map_err(|_| invalid_routing_provider_contract())?;
|
||||||
status: StatusCode::BAD_REQUEST,
|
|
||||||
message: format!("routing provider_request violates provider contract: {violation:?}"),
|
|
||||||
})?;
|
|
||||||
}
|
}
|
||||||
let provider_model = provider_request_body
|
let provider_model = provider_request_body
|
||||||
.get("model")
|
.get("model")
|
||||||
@@ -357,7 +358,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
|||||||
if original_provider_request_body.is_some() {
|
if original_provider_request_body.is_some() {
|
||||||
decision.provider_request_body = Some(provider_request_body);
|
decision.provider_request_body = Some(provider_request_body);
|
||||||
}
|
}
|
||||||
apply_codex_fingerprint_convergence_to_decision(
|
apply_provider_outbound_request_policies_to_decision(
|
||||||
input,
|
input,
|
||||||
decision,
|
decision,
|
||||||
transport,
|
transport,
|
||||||
@@ -367,7 +368,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn apply_codex_fingerprint_convergence_to_decision(
|
fn apply_provider_outbound_request_policies_to_decision(
|
||||||
input: &LocalRequestedModelDecisionInput,
|
input: &LocalRequestedModelDecisionInput,
|
||||||
decision: &mut AiExecutionDecision,
|
decision: &mut AiExecutionDecision,
|
||||||
transport: Option<&GatewayProviderTransportSnapshot>,
|
transport: Option<&GatewayProviderTransportSnapshot>,
|
||||||
@@ -378,17 +379,17 @@ fn apply_codex_fingerprint_convergence_to_decision(
|
|||||||
else {
|
else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
let Some(context) = input.codex_fingerprint_context.as_ref() else {
|
let Some(context) = input.provider_outbound_context.as_ref() else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
let applied = crate::ai_serving::transport::apply_codex_fingerprint_convergence_with_context(
|
let results = crate::ai_serving::transport::apply_provider_outbound_request_policies(
|
||||||
transport,
|
transport,
|
||||||
provider_api_format,
|
provider_api_format,
|
||||||
context,
|
context,
|
||||||
&mut decision.provider_request_headers,
|
&mut decision.provider_request_headers,
|
||||||
provider_request_body,
|
provider_request_body,
|
||||||
);
|
);
|
||||||
if applied {
|
if results.iter().any(|result| result.was_applied()) {
|
||||||
decision.prompt_cache_key = provider_request_body
|
decision.prompt_cache_key = provider_request_body
|
||||||
.get("prompt_cache_key")
|
.get("prompt_cache_key")
|
||||||
.and_then(Value::as_str)
|
.and_then(Value::as_str)
|
||||||
@@ -396,6 +397,31 @@ fn apply_codex_fingerprint_convergence_to_decision(
|
|||||||
.filter(|value| !value.is_empty())
|
.filter(|value| !value.is_empty())
|
||||||
.map(ToOwned::to_owned);
|
.map(ToOwned::to_owned);
|
||||||
}
|
}
|
||||||
|
if results.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for result in &results {
|
||||||
|
tracing::debug!(
|
||||||
|
event_name = "provider_outbound_policy_evaluated",
|
||||||
|
log_type = "event",
|
||||||
|
policy = ?result.policy,
|
||||||
|
outcome = ?result.outcome,
|
||||||
|
reason = ?result.reason,
|
||||||
|
mutation_scope = ?result.mutation_scope,
|
||||||
|
identity_scope = ?result.identity_scope,
|
||||||
|
"provider outbound request policy evaluated"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let Some(serde_json::Value::Object(report_context)) = decision.report_context.as_mut() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
report_context.insert(
|
||||||
|
"provider_outbound_policies".to_string(),
|
||||||
|
serde_json::json!({
|
||||||
|
"schema_version": 1,
|
||||||
|
"results": results,
|
||||||
|
}),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
struct GatewayAuthenticatedDecisionInputPort<'a> {
|
struct GatewayAuthenticatedDecisionInputPort<'a> {
|
||||||
@@ -484,7 +510,7 @@ pub(crate) fn build_local_requested_model_decision_input(
|
|||||||
client_surface: None,
|
client_surface: None,
|
||||||
gateway_credential_carrier: None,
|
gateway_credential_carrier: None,
|
||||||
client_session_affinity: None,
|
client_session_affinity: None,
|
||||||
codex_fingerprint_context: None,
|
provider_outbound_context: None,
|
||||||
routing_policy: None,
|
routing_policy: None,
|
||||||
routing_trace_seed: None,
|
routing_trace_seed: None,
|
||||||
routing_context: None,
|
routing_context: None,
|
||||||
@@ -499,7 +525,7 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
|||||||
body_json: &Value,
|
body_json: &Value,
|
||||||
client_api_format: &str,
|
client_api_format: &str,
|
||||||
) -> Result<(), GatewayError> {
|
) -> Result<(), GatewayError> {
|
||||||
input.codex_fingerprint_context =
|
input.provider_outbound_context =
|
||||||
Some(crate::ai_serving::codex_context::resolve_codex_fingerprint_context(parts, body_json));
|
Some(crate::ai_serving::codex_context::resolve_codex_fingerprint_context(parts, body_json));
|
||||||
let explicit_group = routing_header_value_str(&parts.headers, ROUTING_GROUP_HEADER);
|
let explicit_group = routing_header_value_str(&parts.headers, ROUTING_GROUP_HEADER);
|
||||||
let selected_group = match state.routing_group_read_repository() {
|
let selected_group = match state.routing_group_read_repository() {
|
||||||
@@ -625,21 +651,17 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
|||||||
GatewayRoutingSelectionError::NotFound(explicit_group.unwrap_or_default()),
|
GatewayRoutingSelectionError::NotFound(explicit_group.unwrap_or_default()),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
None
|
return Err(routing_selection_error(
|
||||||
|
GatewayRoutingSelectionError::NoDefault,
|
||||||
|
));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let Some((group_id, group_version, group_config_json, selection_source)) = selected_group
|
let Some((group_id, group_version, group_config_json, selection_source)) = selected_group
|
||||||
else {
|
else {
|
||||||
input.client_session_affinity = client_session_affinity_from_api_request(
|
return Err(routing_selection_error(
|
||||||
client_api_format,
|
GatewayRoutingSelectionError::NoDefault,
|
||||||
&parts.headers,
|
));
|
||||||
Some(body_json),
|
|
||||||
);
|
|
||||||
input.routing_policy = None;
|
|
||||||
input.routing_trace_seed = None;
|
|
||||||
input.routing_context = None;
|
|
||||||
return Ok(());
|
|
||||||
};
|
};
|
||||||
|
|
||||||
if try_attach_static_default_routing_policy_to_input(
|
if try_attach_static_default_routing_policy_to_input(
|
||||||
@@ -863,10 +885,36 @@ fn routing_selection_error(error: GatewayRoutingSelectionError) -> GatewayError
|
|||||||
GatewayRoutingSelectionError::Repository(message) => {
|
GatewayRoutingSelectionError::Repository(message) => {
|
||||||
GatewayError::Internal(format!("routing group repository lookup failed: {message}"))
|
GatewayError::Internal(format!("routing group repository lookup failed: {message}"))
|
||||||
}
|
}
|
||||||
error => GatewayError::Client {
|
GatewayRoutingSelectionError::NoDefault => GatewayError::Client {
|
||||||
status: StatusCode::FORBIDDEN,
|
status: StatusCode::SERVICE_UNAVAILABLE,
|
||||||
message: error.to_string(),
|
message: "no enabled routing strategy is configured for this request".to_string(),
|
||||||
},
|
},
|
||||||
|
GatewayRoutingSelectionError::NotFound(_) => GatewayError::Client {
|
||||||
|
status: StatusCode::FORBIDDEN,
|
||||||
|
message: "requested routing group was not found".to_string(),
|
||||||
|
},
|
||||||
|
GatewayRoutingSelectionError::Disabled(_) => GatewayError::Client {
|
||||||
|
status: StatusCode::FORBIDDEN,
|
||||||
|
message: "requested routing group is not enabled".to_string(),
|
||||||
|
},
|
||||||
|
GatewayRoutingSelectionError::Forbidden(_) => GatewayError::Client {
|
||||||
|
status: StatusCode::FORBIDDEN,
|
||||||
|
message: "requested routing group is not allowed for this principal".to_string(),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn invalid_routing_provider_contract() -> GatewayError {
|
||||||
|
GatewayError::Client {
|
||||||
|
status: StatusCode::BAD_REQUEST,
|
||||||
|
message: INVALID_ROUTING_PROVIDER_CONTRACT_MESSAGE.to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn invalid_routing_provider_headers() -> GatewayError {
|
||||||
|
GatewayError::Client {
|
||||||
|
status: StatusCode::BAD_REQUEST,
|
||||||
|
message: INVALID_ROUTING_PROVIDER_HEADERS_MESSAGE.to_string(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -921,14 +969,9 @@ fn btree_headers_to_header_map(
|
|||||||
) -> Result<HeaderMap, GatewayError> {
|
) -> Result<HeaderMap, GatewayError> {
|
||||||
let mut output = HeaderMap::new();
|
let mut output = HeaderMap::new();
|
||||||
for (name, value) in headers {
|
for (name, value) in headers {
|
||||||
let name = HeaderName::from_bytes(name.as_bytes()).map_err(|err| GatewayError::Client {
|
let name = HeaderName::from_bytes(name.as_bytes())
|
||||||
status: StatusCode::BAD_REQUEST,
|
.map_err(|_| invalid_routing_provider_headers())?;
|
||||||
message: format!("invalid provider request header name in routing mutation: {err}"),
|
let value = HeaderValue::from_str(value).map_err(|_| invalid_routing_provider_headers())?;
|
||||||
})?;
|
|
||||||
let value = HeaderValue::from_str(value).map_err(|err| GatewayError::Client {
|
|
||||||
status: StatusCode::BAD_REQUEST,
|
|
||||||
message: format!("invalid provider request header value in routing mutation: {err}"),
|
|
||||||
})?;
|
|
||||||
output.insert(name, value);
|
output.insert(name, value);
|
||||||
}
|
}
|
||||||
Ok(output)
|
Ok(output)
|
||||||
@@ -1151,6 +1194,50 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn routing_selection_errors_do_not_echo_explicit_group() {
|
||||||
|
let secret = "private-group?token=Bearer-secret";
|
||||||
|
|
||||||
|
for error in [
|
||||||
|
GatewayRoutingSelectionError::NotFound(secret.to_string()),
|
||||||
|
GatewayRoutingSelectionError::Disabled(secret.to_string()),
|
||||||
|
GatewayRoutingSelectionError::Forbidden(secret.to_string()),
|
||||||
|
] {
|
||||||
|
let error = routing_selection_error(error);
|
||||||
|
assert!(matches!(
|
||||||
|
error,
|
||||||
|
GatewayError::Client {
|
||||||
|
status: StatusCode::FORBIDDEN,
|
||||||
|
ref message,
|
||||||
|
} if !message.contains(secret)
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn routing_provider_errors_do_not_echo_dynamic_details() {
|
||||||
|
let secret = "https://internal.example/?token=Bearer-secret";
|
||||||
|
let contract_error = invalid_routing_provider_contract();
|
||||||
|
let header_error = btree_headers_to_header_map(&BTreeMap::from([(
|
||||||
|
format!("Authorization: {secret}"),
|
||||||
|
secret.to_string(),
|
||||||
|
)]))
|
||||||
|
.expect_err("invalid header should fail");
|
||||||
|
|
||||||
|
for (error, expected_message) in [
|
||||||
|
(contract_error, INVALID_ROUTING_PROVIDER_CONTRACT_MESSAGE),
|
||||||
|
(header_error, INVALID_ROUTING_PROVIDER_HEADERS_MESSAGE),
|
||||||
|
] {
|
||||||
|
assert!(matches!(
|
||||||
|
error,
|
||||||
|
GatewayError::Client {
|
||||||
|
status: StatusCode::BAD_REQUEST,
|
||||||
|
ref message,
|
||||||
|
} if message == expected_message && !message.contains(secret)
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn explicit_auth_snapshot_override_does_not_fall_back_to_the_planner_cache() {
|
async fn explicit_auth_snapshot_override_does_not_fall_back_to_the_planner_cache() {
|
||||||
// AppState::new has no auth snapshot repository. Without the explicit
|
// AppState::new has no auth snapshot repository. Without the explicit
|
||||||
@@ -1207,6 +1294,7 @@ mod tests {
|
|||||||
description: None,
|
description: None,
|
||||||
enabled: true,
|
enabled: true,
|
||||||
is_system_default: false,
|
is_system_default: false,
|
||||||
|
sort_order: 0,
|
||||||
config_json: json!({}),
|
config_json: json!({}),
|
||||||
version: 1,
|
version: 1,
|
||||||
created_at: 1,
|
created_at: 1,
|
||||||
@@ -1327,7 +1415,7 @@ mod tests {
|
|||||||
client_surface: None,
|
client_surface: None,
|
||||||
gateway_credential_carrier: None,
|
gateway_credential_carrier: None,
|
||||||
client_session_affinity: None,
|
client_session_affinity: None,
|
||||||
codex_fingerprint_context: None,
|
provider_outbound_context: None,
|
||||||
routing_policy: None,
|
routing_policy: None,
|
||||||
routing_trace_seed: None,
|
routing_trace_seed: None,
|
||||||
model_directive_policy: Default::default(),
|
model_directive_policy: Default::default(),
|
||||||
@@ -1576,7 +1664,7 @@ mod tests {
|
|||||||
client_surface: None,
|
client_surface: None,
|
||||||
gateway_credential_carrier: None,
|
gateway_credential_carrier: None,
|
||||||
client_session_affinity: None,
|
client_session_affinity: None,
|
||||||
codex_fingerprint_context: None,
|
provider_outbound_context: None,
|
||||||
routing_policy: None,
|
routing_policy: None,
|
||||||
routing_trace_seed: None,
|
routing_trace_seed: None,
|
||||||
model_directive_policy: Default::default(),
|
model_directive_policy: Default::default(),
|
||||||
@@ -1646,7 +1734,7 @@ mod tests {
|
|||||||
client_surface: None,
|
client_surface: None,
|
||||||
gateway_credential_carrier: None,
|
gateway_credential_carrier: None,
|
||||||
client_session_affinity: None,
|
client_session_affinity: None,
|
||||||
codex_fingerprint_context: None,
|
provider_outbound_context: None,
|
||||||
routing_policy: None,
|
routing_policy: None,
|
||||||
routing_trace_seed: None,
|
routing_trace_seed: None,
|
||||||
routing_context: None,
|
routing_context: None,
|
||||||
@@ -1715,6 +1803,35 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn non_codex_provider_outbound_policies_are_terminal_noop() {
|
||||||
|
let mut input = sample_decision_input();
|
||||||
|
input.routing_context = None;
|
||||||
|
input.provider_outbound_context = Some(ProviderOutboundRequestContext::new(
|
||||||
|
"logical-turn",
|
||||||
|
1_700_000_000_123,
|
||||||
|
));
|
||||||
|
let mut decision = sample_codex_fingerprint_decision();
|
||||||
|
decision.provider_type = Some("openai".to_string());
|
||||||
|
decision.provider_api_format = Some("openai:responses".to_string());
|
||||||
|
decision.client_api_format = Some("openai:responses".to_string());
|
||||||
|
let mut transport = sample_codex_fingerprint_transport();
|
||||||
|
transport.provider.provider_type = "openai".to_string();
|
||||||
|
|
||||||
|
let original_headers = decision.provider_request_headers.clone();
|
||||||
|
let original_body = decision.provider_request_body.clone();
|
||||||
|
apply_provider_request_routing_policy_to_decision(&input, &mut decision, Some(&transport))
|
||||||
|
.expect("non-Codex terminal finalization should succeed");
|
||||||
|
|
||||||
|
assert_eq!(decision.provider_request_headers, original_headers);
|
||||||
|
assert_eq!(decision.provider_request_body, original_body);
|
||||||
|
assert!(decision
|
||||||
|
.report_context
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|context| context.get("provider_outbound_policies"))
|
||||||
|
.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn codex_fingerprint_convergence_runs_at_every_provider_routing_success_exit() {
|
fn codex_fingerprint_convergence_runs_at_every_provider_routing_success_exit() {
|
||||||
let transport = sample_codex_fingerprint_transport();
|
let transport = sample_codex_fingerprint_transport();
|
||||||
@@ -1731,8 +1848,8 @@ mod tests {
|
|||||||
});
|
});
|
||||||
let mut with_mutation = sample_decision_input();
|
let mut with_mutation = sample_decision_input();
|
||||||
for input in [&mut no_context, &mut empty_mutation, &mut with_mutation] {
|
for input in [&mut no_context, &mut empty_mutation, &mut with_mutation] {
|
||||||
input.codex_fingerprint_context = Some(
|
input.provider_outbound_context = Some(
|
||||||
CodexFingerprintConvergenceContext::new(
|
ProviderOutboundRequestContext::new(
|
||||||
uuid::Uuid::new_v4().to_string(),
|
uuid::Uuid::new_v4().to_string(),
|
||||||
1_756_668_000_000,
|
1_756_668_000_000,
|
||||||
)
|
)
|
||||||
@@ -1801,6 +1918,25 @@ mod tests {
|
|||||||
);
|
);
|
||||||
assert_eq!(body["client_metadata"]["x-codex-window-id"], window_id);
|
assert_eq!(body["client_metadata"]["x-codex-window-id"], window_id);
|
||||||
|
|
||||||
|
let policy_results = decision
|
||||||
|
.report_context
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|context| context.get("provider_outbound_policies"))
|
||||||
|
.and_then(|policies| policies.get("results"))
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.expect("provider policy results");
|
||||||
|
assert_eq!(
|
||||||
|
policy_results.len(),
|
||||||
|
1,
|
||||||
|
"policy result count at {exit_name}"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
policy_results[0]["policy"],
|
||||||
|
json!("codex_fingerprint_convergence")
|
||||||
|
);
|
||||||
|
assert_eq!(policy_results[0]["outcome"], json!("applied"));
|
||||||
|
assert_eq!(policy_results[0]["reason"], json!("applied"));
|
||||||
|
|
||||||
let header_metadata: Value =
|
let header_metadata: Value =
|
||||||
serde_json::from_str(&decision.provider_request_headers["x-codex-turn-metadata"])
|
serde_json::from_str(&decision.provider_request_headers["x-codex-turn-metadata"])
|
||||||
.expect("header turn metadata");
|
.expect("header turn metadata");
|
||||||
|
|||||||
+6
-9
@@ -26,7 +26,6 @@ use crate::ai_serving::{
|
|||||||
};
|
};
|
||||||
use crate::client_session_affinity::client_session_affinity_from_api_request;
|
use crate::client_session_affinity::client_session_affinity_from_api_request;
|
||||||
use crate::clock::current_unix_secs;
|
use crate::clock::current_unix_secs;
|
||||||
use crate::scheduler::config::SchedulerOrderingConfig;
|
|
||||||
use crate::{AppState, GatewayError};
|
use crate::{AppState, GatewayError};
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
@@ -141,10 +140,9 @@ pub(crate) async fn materialize_local_same_format_provider_candidate_attempts(
|
|||||||
current_unix_secs(),
|
current_unix_secs(),
|
||||||
false,
|
false,
|
||||||
spec.operation.map(|operation| operation.as_str()),
|
spec.operation.map(|operation| operation.as_str()),
|
||||||
input
|
crate::ai_serving::planner::candidate_ranking::scheduler_ordering_config_for_routing_policy(
|
||||||
.routing_policy
|
input.routing_policy.as_ref(),
|
||||||
.as_ref()
|
),
|
||||||
.map(SchedulerOrderingConfig::from_routing_policy),
|
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
let outcome = materialize_local_execution_candidates_with_serving(
|
let outcome = materialize_local_execution_candidates_with_serving(
|
||||||
@@ -251,10 +249,9 @@ pub(crate) async fn build_local_same_format_provider_candidate_attempt_source<'a
|
|||||||
current_unix_secs(),
|
current_unix_secs(),
|
||||||
false,
|
false,
|
||||||
spec.operation.map(|operation| operation.as_str()),
|
spec.operation.map(|operation| operation.as_str()),
|
||||||
input
|
crate::ai_serving::planner::candidate_ranking::scheduler_ordering_config_for_routing_policy(
|
||||||
.routing_policy
|
input.routing_policy.as_ref(),
|
||||||
.as_ref()
|
),
|
||||||
.map(SchedulerOrderingConfig::from_routing_policy),
|
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ use super::{
|
|||||||
LocalSameFormatProviderCandidateAttemptSource, LocalSameFormatProviderDecisionInput,
|
LocalSameFormatProviderCandidateAttemptSource, LocalSameFormatProviderDecisionInput,
|
||||||
LocalSameFormatProviderSpec,
|
LocalSameFormatProviderSpec,
|
||||||
};
|
};
|
||||||
|
use aether_routing_core::RoutingExecutionPolicy;
|
||||||
|
|
||||||
pub(crate) struct LocalSameFormatProviderSyncAttemptSource<'a> {
|
pub(crate) struct LocalSameFormatProviderSyncAttemptSource<'a> {
|
||||||
state: &'a AppState,
|
state: &'a AppState,
|
||||||
@@ -189,6 +190,13 @@ pub(crate) async fn build_local_stream_attempt_source<'a>(
|
|||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalSameFormatProviderSyncAttemptSource<'_> {
|
impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalSameFormatProviderSyncAttemptSource<'_> {
|
||||||
|
fn routing_execution_policy(&self) -> Option<RoutingExecutionPolicy> {
|
||||||
|
self.input
|
||||||
|
.routing_policy
|
||||||
|
.as_ref()
|
||||||
|
.map(|policy| policy.execution_policy)
|
||||||
|
}
|
||||||
|
|
||||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
||||||
while let Some(attempt) = self.candidates.next_attempt().await? {
|
while let Some(attempt) = self.candidates.next_attempt().await? {
|
||||||
match self.build_sync_attempt(attempt).await? {
|
match self.build_sync_attempt(attempt).await? {
|
||||||
@@ -234,6 +242,13 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalSameFormatProviderSyncA
|
|||||||
impl LocalExecutionAttemptSource<AiStreamAttempt>
|
impl LocalExecutionAttemptSource<AiStreamAttempt>
|
||||||
for LocalSameFormatProviderStreamAttemptSource<'_>
|
for LocalSameFormatProviderStreamAttemptSource<'_>
|
||||||
{
|
{
|
||||||
|
fn routing_execution_policy(&self) -> Option<RoutingExecutionPolicy> {
|
||||||
|
self.input
|
||||||
|
.routing_policy
|
||||||
|
.as_ref()
|
||||||
|
.map(|policy| policy.execution_policy)
|
||||||
|
}
|
||||||
|
|
||||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
||||||
while let Some(attempt) = self.candidates.next_attempt().await? {
|
while let Some(attempt) = self.candidates.next_attempt().await? {
|
||||||
match self.build_stream_attempt(attempt).await? {
|
match self.build_stream_attempt(attempt).await? {
|
||||||
|
|||||||
@@ -21,7 +21,8 @@ use crate::client_session_affinity::{
|
|||||||
};
|
};
|
||||||
use crate::orchestration::{
|
use crate::orchestration::{
|
||||||
insert_pool_key_lease_report_context_fields, ExecutionAttemptIdentity,
|
insert_pool_key_lease_report_context_fields, ExecutionAttemptIdentity,
|
||||||
ROUTING_POOL_POLICY_OVERRIDE_REPORT_FIELD, SCHEDULER_AFFINITY_EPOCH_REPORT_FIELD,
|
ROUTING_EXECUTION_POLICY_REPORT_FIELD, ROUTING_POOL_POLICY_OVERRIDE_REPORT_FIELD,
|
||||||
|
SCHEDULER_AFFINITY_EPOCH_REPORT_FIELD,
|
||||||
};
|
};
|
||||||
use crate::scheduler::affinity::insert_scheduler_affinity_policy_report_context_field;
|
use crate::scheduler::affinity::insert_scheduler_affinity_policy_report_context_field;
|
||||||
|
|
||||||
@@ -75,10 +76,12 @@ pub(crate) fn build_local_execution_report_context(
|
|||||||
let RequestOrigin {
|
let RequestOrigin {
|
||||||
client_ip,
|
client_ip,
|
||||||
user_agent,
|
user_agent,
|
||||||
|
forwarded_headers_trusted,
|
||||||
} = parts
|
} = parts
|
||||||
.request_origin
|
.request_origin
|
||||||
.unwrap_or_else(|| request_origin_from_headers(parts.original_headers));
|
.unwrap_or_else(|| request_origin_from_headers(parts.original_headers));
|
||||||
let original_headers = crate::ai_serving::collect_control_headers(parts.original_headers);
|
let original_headers =
|
||||||
|
collect_report_context_original_headers(parts.original_headers, forwarded_headers_trusted);
|
||||||
let original_request_body = crate::ai_serving::build_report_context_original_request_echo(
|
let original_request_body = crate::ai_serving::build_report_context_original_request_echo(
|
||||||
parts.original_request_body_json,
|
parts.original_request_body_json,
|
||||||
parts.original_request_body_base64,
|
parts.original_request_body_base64,
|
||||||
@@ -105,13 +108,20 @@ pub(crate) fn build_local_execution_report_context(
|
|||||||
value,
|
value,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
if let Some(incoming_tls) =
|
if forwarded_headers_trusted {
|
||||||
crate::ai_serving::tls_fingerprint_from_headers(parts.original_headers)
|
if let Some(incoming_tls) =
|
||||||
{
|
crate::ai_serving::tls_fingerprint_from_headers(parts.original_headers)
|
||||||
merge_incoming_tls_fingerprint(&mut extra_fields, incoming_tls);
|
{
|
||||||
|
merge_incoming_tls_fingerprint(&mut extra_fields, incoming_tls);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
insert_pool_key_lease_report_context_fields(&mut extra_fields, parts.pool_key_lease);
|
insert_pool_key_lease_report_context_fields(&mut extra_fields, parts.pool_key_lease);
|
||||||
insert_scheduler_affinity_policy_report_context_field(&mut extra_fields, parts.routing_policy);
|
insert_scheduler_affinity_policy_report_context_field(&mut extra_fields, parts.routing_policy);
|
||||||
|
if let Some(policy) = parts.routing_policy {
|
||||||
|
if let Ok(value) = serde_json::to_value(policy.execution_policy) {
|
||||||
|
extra_fields.insert(ROUTING_EXECUTION_POLICY_REPORT_FIELD.to_string(), value);
|
||||||
|
}
|
||||||
|
}
|
||||||
if let Some(override_policy) = parts
|
if let Some(override_policy) = parts
|
||||||
.routing_policy
|
.routing_policy
|
||||||
.and_then(|policy| policy.pool_policy_overrides.get(parts.provider_id))
|
.and_then(|policy| policy.pool_policy_overrides.get(parts.provider_id))
|
||||||
@@ -183,6 +193,17 @@ pub(crate) fn build_local_execution_report_context(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn collect_report_context_original_headers(
|
||||||
|
headers: &http::HeaderMap,
|
||||||
|
forwarded_headers_trusted: bool,
|
||||||
|
) -> BTreeMap<String, String> {
|
||||||
|
let mut collected = crate::ai_serving::collect_control_headers(headers);
|
||||||
|
if !forwarded_headers_trusted {
|
||||||
|
collected.retain(|name, _| !name.starts_with("x-aether-tls-"));
|
||||||
|
}
|
||||||
|
collected
|
||||||
|
}
|
||||||
|
|
||||||
fn insert_request_path_fields(
|
fn insert_request_path_fields(
|
||||||
extra_fields: &mut Map<String, Value>,
|
extra_fields: &mut Map<String, Value>,
|
||||||
request_path: Option<&str>,
|
request_path: Option<&str>,
|
||||||
@@ -252,8 +273,8 @@ mod tests {
|
|||||||
use serde_json::{json, Map, Value};
|
use serde_json::{json, Map, Value};
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
build_local_execution_report_context, provider_stream_event_api_format_for_provider_type,
|
build_local_execution_report_context, collect_report_context_original_headers,
|
||||||
LocalExecutionReportContextParts,
|
provider_stream_event_api_format_for_provider_type, LocalExecutionReportContextParts,
|
||||||
};
|
};
|
||||||
use crate::ai_serving::ExecutionRuntimeAuthContext;
|
use crate::ai_serving::ExecutionRuntimeAuthContext;
|
||||||
use crate::ai_serving::RequestOrigin;
|
use crate::ai_serving::RequestOrigin;
|
||||||
@@ -283,6 +304,26 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn untrusted_tls_forwarding_headers_are_excluded_from_report_context() {
|
||||||
|
let mut headers = http::HeaderMap::new();
|
||||||
|
headers.insert("x-aether-tls-ja3", "spoofed-ja3".parse().unwrap());
|
||||||
|
headers.insert(http::header::USER_AGENT, "test-client".parse().unwrap());
|
||||||
|
|
||||||
|
let untrusted = collect_report_context_original_headers(&headers, false);
|
||||||
|
assert!(!untrusted.contains_key("x-aether-tls-ja3"));
|
||||||
|
assert_eq!(
|
||||||
|
untrusted.get("user-agent").map(String::as_str),
|
||||||
|
Some("test-client")
|
||||||
|
);
|
||||||
|
|
||||||
|
let trusted = collect_report_context_original_headers(&headers, true);
|
||||||
|
assert_eq!(
|
||||||
|
trusted.get("x-aether-tls-ja3").map(String::as_str),
|
||||||
|
Some("spoofed-ja3")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn local_execution_report_context_records_request_origin_and_session_affinity() {
|
fn local_execution_report_context_records_request_origin_and_session_affinity() {
|
||||||
let auth_context = ExecutionRuntimeAuthContext {
|
let auth_context = ExecutionRuntimeAuthContext {
|
||||||
@@ -333,6 +374,7 @@ mod tests {
|
|||||||
request_origin: Some(RequestOrigin {
|
request_origin: Some(RequestOrigin {
|
||||||
client_ip: Some("203.0.113.8".to_string()),
|
client_ip: Some("203.0.113.8".to_string()),
|
||||||
user_agent: Some("Claude-Code/1.0".to_string()),
|
user_agent: Some("Claude-Code/1.0".to_string()),
|
||||||
|
forwarded_headers_trusted: false,
|
||||||
}),
|
}),
|
||||||
original_request_body_json: Some(&json!({"model": "gpt-5"})),
|
original_request_body_json: Some(&json!({"model": "gpt-5"})),
|
||||||
original_request_body_base64: None,
|
original_request_body_base64: None,
|
||||||
@@ -485,7 +527,11 @@ mod tests {
|
|||||||
original_headers: &original_headers,
|
original_headers: &original_headers,
|
||||||
request_path: None,
|
request_path: None,
|
||||||
request_query_string: None,
|
request_query_string: None,
|
||||||
request_origin: None,
|
request_origin: Some(RequestOrigin {
|
||||||
|
client_ip: None,
|
||||||
|
user_agent: None,
|
||||||
|
forwarded_headers_trusted: true,
|
||||||
|
}),
|
||||||
original_request_body_json: Some(&json!({"model": "gpt-5"})),
|
original_request_body_json: Some(&json!({"model": "gpt-5"})),
|
||||||
original_request_body_base64: None,
|
original_request_body_base64: None,
|
||||||
client_session_affinity: None,
|
client_session_affinity: None,
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ use crate::ai_serving::{
|
|||||||
resolve_gemini_files_sync_spec as resolve_sync_spec, LocalGeminiFilesSpec,
|
resolve_gemini_files_sync_spec as resolve_sync_spec, LocalGeminiFilesSpec,
|
||||||
};
|
};
|
||||||
use crate::{AiExecutionDecision, AppState, GatewayError};
|
use crate::{AiExecutionDecision, AppState, GatewayError};
|
||||||
|
use aether_routing_core::RoutingExecutionPolicy;
|
||||||
|
|
||||||
use self::decision::maybe_build_local_gemini_files_decision_payload_for_candidate;
|
use self::decision::maybe_build_local_gemini_files_decision_payload_for_candidate;
|
||||||
use self::support::{
|
use self::support::{
|
||||||
@@ -174,6 +175,13 @@ pub(crate) async fn build_local_gemini_files_stream_attempt_source_for_kind<'a>(
|
|||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalGeminiFilesSyncAttemptSource<'_> {
|
impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalGeminiFilesSyncAttemptSource<'_> {
|
||||||
|
fn routing_execution_policy(&self) -> Option<RoutingExecutionPolicy> {
|
||||||
|
self.input
|
||||||
|
.routing_policy
|
||||||
|
.as_ref()
|
||||||
|
.map(|policy| policy.execution_policy)
|
||||||
|
}
|
||||||
|
|
||||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
||||||
while let Some(attempt) = self.candidates.next_attempt().await? {
|
while let Some(attempt) = self.candidates.next_attempt().await? {
|
||||||
match self.build_sync_attempt(attempt).await? {
|
match self.build_sync_attempt(attempt).await? {
|
||||||
@@ -212,6 +220,13 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalGeminiFilesSyncAttemptS
|
|||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalGeminiFilesStreamAttemptSource<'_> {
|
impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalGeminiFilesStreamAttemptSource<'_> {
|
||||||
|
fn routing_execution_policy(&self) -> Option<RoutingExecutionPolicy> {
|
||||||
|
self.input
|
||||||
|
.routing_policy
|
||||||
|
.as_ref()
|
||||||
|
.map(|policy| policy.execution_policy)
|
||||||
|
}
|
||||||
|
|
||||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
||||||
while let Some(attempt) = self.candidates.next_attempt().await? {
|
while let Some(attempt) = self.candidates.next_attempt().await? {
|
||||||
match self.build_stream_attempt(attempt).await? {
|
match self.build_stream_attempt(attempt).await? {
|
||||||
|
|||||||
@@ -8,7 +8,10 @@ use crate::ai_serving::transport::{
|
|||||||
GeminiFilesRequestBodyError,
|
GeminiFilesRequestBodyError,
|
||||||
};
|
};
|
||||||
use crate::ai_serving::GEMINI_FILES_UPLOAD_PLAN_KIND;
|
use crate::ai_serving::GEMINI_FILES_UPLOAD_PLAN_KIND;
|
||||||
use crate::ai_serving::{CandidateFailureDiagnostic, GatewayProviderTransportSnapshot};
|
use crate::ai_serving::{
|
||||||
|
CandidateFailureDiagnostic, GatewayProviderTransportSnapshot, GEMINI_FILES_DELETE_PLAN_KIND,
|
||||||
|
GEMINI_FILES_DOWNLOAD_PLAN_KIND, GEMINI_FILES_GET_PLAN_KIND,
|
||||||
|
};
|
||||||
use crate::AppState;
|
use crate::AppState;
|
||||||
|
|
||||||
use super::support::{
|
use super::support::{
|
||||||
@@ -47,6 +50,26 @@ pub(super) async fn resolve_local_gemini_files_candidate_payload_parts(
|
|||||||
let transport = &attempt.eligible.transport;
|
let transport = &attempt.eligible.transport;
|
||||||
let effective_headers = input.effective_headers(&parts.headers);
|
let effective_headers = input.effective_headers(&parts.headers);
|
||||||
|
|
||||||
|
if matches!(
|
||||||
|
spec_metadata.decision_kind,
|
||||||
|
GEMINI_FILES_GET_PLAN_KIND
|
||||||
|
| GEMINI_FILES_DELETE_PLAN_KIND
|
||||||
|
| GEMINI_FILES_DOWNLOAD_PLAN_KIND
|
||||||
|
) && !candidate_matches_owned_gemini_file_mapping(state, parts, input, attempt).await
|
||||||
|
{
|
||||||
|
mark_skipped_local_gemini_files_candidate(
|
||||||
|
state,
|
||||||
|
input,
|
||||||
|
trace_id,
|
||||||
|
candidate,
|
||||||
|
attempt.candidate_index,
|
||||||
|
&attempt.candidate_id,
|
||||||
|
"gemini_file_mapping_mismatch",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
if let Some(skip_reason) =
|
if let Some(skip_reason) =
|
||||||
gemini_files_transport_unsupported_reason(transport, GEMINI_FILES_CANDIDATE_API_FORMAT)
|
gemini_files_transport_unsupported_reason(transport, GEMINI_FILES_CANDIDATE_API_FORMAT)
|
||||||
{
|
{
|
||||||
@@ -191,3 +214,64 @@ pub(super) async fn resolve_local_gemini_files_candidate_payload_parts(
|
|||||||
file_name,
|
file_name,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn candidate_matches_owned_gemini_file_mapping(
|
||||||
|
state: &AppState,
|
||||||
|
parts: &http::request::Parts,
|
||||||
|
input: &LocalGeminiFilesDecisionInput,
|
||||||
|
attempt: &LocalGeminiFilesCandidateAttempt,
|
||||||
|
) -> bool {
|
||||||
|
let Some(file_name) = normalize_gemini_file_name_from_path(parts.uri.path()) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let user_id = input.auth_context.user_id.trim();
|
||||||
|
if user_id.is_empty() || !state.has_gemini_file_mapping_data_reader() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let Ok(Some(mapping)) = state
|
||||||
|
.find_active_gemini_file_mapping_for_owner(
|
||||||
|
file_name.as_str(),
|
||||||
|
&attempt.eligible.transport.key.id,
|
||||||
|
user_id,
|
||||||
|
crate::clock::current_unix_secs(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
|
||||||
|
mapping.user_id.as_deref().map(str::trim) == Some(user_id)
|
||||||
|
&& mapping.key_id == attempt.eligible.transport.key.id
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn normalize_gemini_file_name_from_path(path: &str) -> Option<String> {
|
||||||
|
let suffix = path.strip_prefix("/v1beta/files/")?.trim_matches('/');
|
||||||
|
let suffix = suffix.strip_suffix(":download").unwrap_or(suffix).trim();
|
||||||
|
let suffix = suffix.strip_prefix("files/").unwrap_or(suffix).trim();
|
||||||
|
if suffix.is_empty() || suffix.contains('/') {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some(format!("files/{suffix}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::normalize_gemini_file_name_from_path;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn normalizes_supported_gemini_file_object_paths() {
|
||||||
|
assert_eq!(
|
||||||
|
normalize_gemini_file_name_from_path("/v1beta/files/file-123"),
|
||||||
|
Some("files/file-123".to_string())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
normalize_gemini_file_name_from_path("/v1beta/files/file-123:download"),
|
||||||
|
Some("files/file-123".to_string())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
normalize_gemini_file_name_from_path("/v1beta/files/files/abc-123"),
|
||||||
|
Some("files/abc-123".to_string())
|
||||||
|
);
|
||||||
|
assert_eq!(normalize_gemini_file_name_from_path("/v1beta/files"), None);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -26,7 +26,6 @@ use crate::ai_serving::{
|
|||||||
};
|
};
|
||||||
use crate::client_session_affinity::client_session_affinity_from_parts;
|
use crate::client_session_affinity::client_session_affinity_from_parts;
|
||||||
use crate::clock::current_unix_secs;
|
use crate::clock::current_unix_secs;
|
||||||
use crate::scheduler::config::SchedulerOrderingConfig;
|
|
||||||
use crate::{AppState, GatewayError};
|
use crate::{AppState, GatewayError};
|
||||||
|
|
||||||
pub(super) use crate::ai_serving::planner::candidate_materialization::LocalExecutionCandidateAttempt as LocalGeminiFilesCandidateAttempt;
|
pub(super) use crate::ai_serving::planner::candidate_materialization::LocalExecutionCandidateAttempt as LocalGeminiFilesCandidateAttempt;
|
||||||
@@ -109,10 +108,9 @@ pub(super) async fn materialize_local_gemini_files_candidate_attempts(
|
|||||||
Some(&input.auth_snapshot),
|
Some(&input.auth_snapshot),
|
||||||
input.client_session_affinity.as_ref(),
|
input.client_session_affinity.as_ref(),
|
||||||
current_unix_secs(),
|
current_unix_secs(),
|
||||||
input
|
crate::ai_serving::planner::candidate_ranking::scheduler_ordering_config_for_routing_policy(
|
||||||
.routing_policy
|
input.routing_policy.as_ref(),
|
||||||
.as_ref()
|
),
|
||||||
.map(SchedulerOrderingConfig::from_routing_policy),
|
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
let outcome = materialize_local_execution_candidates_with_serving(
|
let outcome = materialize_local_execution_candidates_with_serving(
|
||||||
@@ -186,10 +184,9 @@ pub(super) async fn build_local_gemini_files_candidate_attempt_source<'a>(
|
|||||||
Some(&input.auth_snapshot),
|
Some(&input.auth_snapshot),
|
||||||
input.client_session_affinity.as_ref(),
|
input.client_session_affinity.as_ref(),
|
||||||
current_unix_secs(),
|
current_unix_secs(),
|
||||||
input
|
crate::ai_serving::planner::candidate_ranking::scheduler_ordering_config_for_routing_policy(
|
||||||
.routing_policy
|
input.routing_policy.as_ref(),
|
||||||
.as_ref()
|
),
|
||||||
.map(SchedulerOrderingConfig::from_routing_policy),
|
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
Ok(build_local_execution_candidate_attempt_source_with_serving(
|
Ok(build_local_execution_candidate_attempt_source_with_serving(
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ use crate::ai_serving::{
|
|||||||
resolve_local_image_sync_spec as resolve_sync_spec,
|
resolve_local_image_sync_spec as resolve_sync_spec,
|
||||||
};
|
};
|
||||||
use crate::{AiExecutionDecision, AppState, GatewayError};
|
use crate::{AiExecutionDecision, AppState, GatewayError};
|
||||||
|
use aether_routing_core::RoutingExecutionPolicy;
|
||||||
|
|
||||||
use self::decision::maybe_build_local_openai_image_decision_payload_for_candidate;
|
use self::decision::maybe_build_local_openai_image_decision_payload_for_candidate;
|
||||||
use self::support::{
|
use self::support::{
|
||||||
@@ -252,6 +253,13 @@ pub(crate) async fn build_local_image_stream_attempt_source_for_kind<'a>(
|
|||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalOpenAiImageSyncAttemptSource<'_> {
|
impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalOpenAiImageSyncAttemptSource<'_> {
|
||||||
|
fn routing_execution_policy(&self) -> Option<RoutingExecutionPolicy> {
|
||||||
|
self.input
|
||||||
|
.routing_policy
|
||||||
|
.as_ref()
|
||||||
|
.map(|policy| policy.execution_policy)
|
||||||
|
}
|
||||||
|
|
||||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
||||||
while let Some(attempt) = self.candidates.next_attempt().await? {
|
while let Some(attempt) = self.candidates.next_attempt().await? {
|
||||||
match self.build_sync_attempt(attempt).await? {
|
match self.build_sync_attempt(attempt).await? {
|
||||||
@@ -290,6 +298,13 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalOpenAiImageSyncAttemptS
|
|||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalOpenAiImageStreamAttemptSource<'_> {
|
impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalOpenAiImageStreamAttemptSource<'_> {
|
||||||
|
fn routing_execution_policy(&self) -> Option<RoutingExecutionPolicy> {
|
||||||
|
self.input
|
||||||
|
.routing_policy
|
||||||
|
.as_ref()
|
||||||
|
.map(|policy| policy.execution_policy)
|
||||||
|
}
|
||||||
|
|
||||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
||||||
while let Some(attempt) = self.candidates.next_attempt().await? {
|
while let Some(attempt) = self.candidates.next_attempt().await? {
|
||||||
match self.build_stream_attempt(attempt).await? {
|
match self.build_stream_attempt(attempt).await? {
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ use crate::ai_serving::planner::candidate_preparation::{
|
|||||||
};
|
};
|
||||||
use crate::ai_serving::planner::spec_metadata::local_openai_image_spec_metadata;
|
use crate::ai_serving::planner::spec_metadata::local_openai_image_spec_metadata;
|
||||||
use crate::ai_serving::pure::normalize_openai_image_request_with_options;
|
use crate::ai_serving::pure::normalize_openai_image_request_with_options;
|
||||||
|
use crate::ai_serving::transport::antigravity::is_antigravity_provider_transport;
|
||||||
use crate::ai_serving::transport::{
|
use crate::ai_serving::transport::{
|
||||||
build_grok_browser_headers, build_grok_upstream_url, build_openai_image_headers,
|
build_grok_browser_headers, build_grok_upstream_url, build_openai_image_headers,
|
||||||
build_openai_image_upstream_url, build_standard_provider_request_headers,
|
build_openai_image_upstream_url, build_standard_provider_request_headers,
|
||||||
@@ -338,6 +339,25 @@ async fn resolve_local_openai_image_to_gemini_candidate_payload_parts(
|
|||||||
let candidate = &attempt.eligible.candidate;
|
let candidate = &attempt.eligible.candidate;
|
||||||
let transport = &attempt.eligible.transport;
|
let transport = &attempt.eligible.transport;
|
||||||
let provider_api_format = "gemini:generate_content";
|
let provider_api_format = "gemini:generate_content";
|
||||||
|
|
||||||
|
// The gemini:generate_content URL hook rewrites an Antigravity endpoint to
|
||||||
|
// /v1internal:, and this image path has no v1internal envelope to match it.
|
||||||
|
// Skip the candidate instead of posting a bare Gemini body that upstream
|
||||||
|
// would only reject.
|
||||||
|
if is_antigravity_provider_transport(transport) {
|
||||||
|
mark_skipped_local_openai_image_candidate(
|
||||||
|
state,
|
||||||
|
input,
|
||||||
|
trace_id,
|
||||||
|
candidate,
|
||||||
|
attempt.candidate_index,
|
||||||
|
&attempt.candidate_id,
|
||||||
|
"transport_unsupported",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
let effective_headers = input.effective_headers(&parts.headers);
|
let effective_headers = input.effective_headers(&parts.headers);
|
||||||
|
|
||||||
let prepared_candidate = match prepare_header_authenticated_candidate(
|
let prepared_candidate = match prepare_header_authenticated_candidate(
|
||||||
|
|||||||
@@ -27,7 +27,6 @@ use crate::ai_serving::{
|
|||||||
};
|
};
|
||||||
use crate::client_session_affinity::client_session_affinity_from_parts;
|
use crate::client_session_affinity::client_session_affinity_from_parts;
|
||||||
use crate::clock::current_unix_secs;
|
use crate::clock::current_unix_secs;
|
||||||
use crate::scheduler::config::SchedulerOrderingConfig;
|
|
||||||
use crate::{AppState, GatewayError};
|
use crate::{AppState, GatewayError};
|
||||||
use aether_scheduler_core::SchedulerMinimalCandidateSelectionCandidate;
|
use aether_scheduler_core::SchedulerMinimalCandidateSelectionCandidate;
|
||||||
|
|
||||||
@@ -128,10 +127,9 @@ pub(super) async fn list_local_openai_image_candidate_attempts(
|
|||||||
input.client_session_affinity.as_ref(),
|
input.client_session_affinity.as_ref(),
|
||||||
current_unix_secs(),
|
current_unix_secs(),
|
||||||
false,
|
false,
|
||||||
input
|
crate::ai_serving::planner::candidate_ranking::scheduler_ordering_config_for_routing_policy(
|
||||||
.routing_policy
|
input.routing_policy.as_ref(),
|
||||||
.as_ref()
|
),
|
||||||
.map(SchedulerOrderingConfig::from_routing_policy),
|
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
@@ -206,10 +204,9 @@ pub(super) async fn build_local_openai_image_candidate_attempt_source<'a>(
|
|||||||
input.client_session_affinity.as_ref(),
|
input.client_session_affinity.as_ref(),
|
||||||
current_unix_secs(),
|
current_unix_secs(),
|
||||||
false,
|
false,
|
||||||
input
|
crate::ai_serving::planner::candidate_ranking::scheduler_ordering_config_for_routing_policy(
|
||||||
.routing_policy
|
input.routing_policy.as_ref(),
|
||||||
.as_ref()
|
),
|
||||||
.map(SchedulerOrderingConfig::from_routing_policy),
|
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ use crate::ai_serving::{
|
|||||||
LocalVideoCreateSpec,
|
LocalVideoCreateSpec,
|
||||||
};
|
};
|
||||||
use crate::{AiExecutionDecision, AppState, GatewayError};
|
use crate::{AiExecutionDecision, AppState, GatewayError};
|
||||||
|
use aether_routing_core::RoutingExecutionPolicy;
|
||||||
|
|
||||||
use self::decision::maybe_build_local_video_create_decision_payload_for_candidate;
|
use self::decision::maybe_build_local_video_create_decision_payload_for_candidate;
|
||||||
use self::support::{
|
use self::support::{
|
||||||
@@ -104,6 +105,13 @@ pub(crate) async fn build_local_video_sync_attempt_source_for_kind<'a>(
|
|||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalVideoCreateSyncAttemptSource<'_> {
|
impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalVideoCreateSyncAttemptSource<'_> {
|
||||||
|
fn routing_execution_policy(&self) -> Option<RoutingExecutionPolicy> {
|
||||||
|
self.input
|
||||||
|
.routing_policy
|
||||||
|
.as_ref()
|
||||||
|
.map(|policy| policy.execution_policy)
|
||||||
|
}
|
||||||
|
|
||||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
||||||
while let Some(attempt) = self.candidates.next_attempt().await? {
|
while let Some(attempt) = self.candidates.next_attempt().await? {
|
||||||
match self.build_sync_attempt(attempt).await? {
|
match self.build_sync_attempt(attempt).await? {
|
||||||
|
|||||||
@@ -29,7 +29,6 @@ use crate::ai_serving::{
|
|||||||
};
|
};
|
||||||
use crate::client_session_affinity::client_session_affinity_from_parts;
|
use crate::client_session_affinity::client_session_affinity_from_parts;
|
||||||
use crate::clock::current_unix_secs;
|
use crate::clock::current_unix_secs;
|
||||||
use crate::scheduler::config::SchedulerOrderingConfig;
|
|
||||||
use crate::{AppState, GatewayError};
|
use crate::{AppState, GatewayError};
|
||||||
|
|
||||||
pub(super) use crate::ai_serving::planner::candidate_materialization::LocalExecutionCandidateAttempt as LocalVideoCreateCandidateAttempt;
|
pub(super) use crate::ai_serving::planner::candidate_materialization::LocalExecutionCandidateAttempt as LocalVideoCreateCandidateAttempt;
|
||||||
@@ -134,10 +133,9 @@ pub(super) async fn list_local_video_create_candidate_attempts(
|
|||||||
input.client_session_affinity.as_ref(),
|
input.client_session_affinity.as_ref(),
|
||||||
current_unix_secs(),
|
current_unix_secs(),
|
||||||
false,
|
false,
|
||||||
input
|
crate::ai_serving::planner::candidate_ranking::scheduler_ordering_config_for_routing_policy(
|
||||||
.routing_policy
|
input.routing_policy.as_ref(),
|
||||||
.as_ref()
|
),
|
||||||
.map(SchedulerOrderingConfig::from_routing_policy),
|
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
@@ -195,10 +193,9 @@ pub(super) async fn build_local_video_create_candidate_attempt_source<'a>(
|
|||||||
input.client_session_affinity.as_ref(),
|
input.client_session_affinity.as_ref(),
|
||||||
current_unix_secs(),
|
current_unix_secs(),
|
||||||
false,
|
false,
|
||||||
input
|
crate::ai_serving::planner::candidate_ranking::scheduler_ordering_config_for_routing_policy(
|
||||||
.routing_policy
|
input.routing_policy.as_ref(),
|
||||||
.as_ref()
|
),
|
||||||
.map(SchedulerOrderingConfig::from_routing_policy),
|
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -504,8 +504,8 @@ fn projects_uuid_prompt_cache_identity_into_missing_session_headers() {
|
|||||||
);
|
);
|
||||||
assert_eq!(headers.get("x-client-request-id"), None);
|
assert_eq!(headers.get("x-client-request-id"), None);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
headers.get("user-agent"),
|
headers.get("user-agent").map(String::as_str),
|
||||||
Some(&"codex_cli_rs/0.144.1".to_string())
|
Some(aether_ai_formats::CODEX_CLIENT_USER_AGENT)
|
||||||
);
|
);
|
||||||
assert_eq!(headers.get("originator"), Some(&"codex_cli_rs".to_string()));
|
assert_eq!(headers.get("originator"), Some(&"codex_cli_rs".to_string()));
|
||||||
assert!(!headers.contains_key("version"));
|
assert!(!headers.contains_key("version"));
|
||||||
@@ -614,8 +614,8 @@ fn injects_only_codex_client_headers_for_images_requests() {
|
|||||||
Some(&"acc-123".to_string())
|
Some(&"acc-123".to_string())
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
headers.get("user-agent"),
|
headers.get("user-agent").map(String::as_str),
|
||||||
Some(&"codex_cli_rs/0.144.1".to_string())
|
Some(aether_ai_formats::CODEX_CLIENT_USER_AGENT)
|
||||||
);
|
);
|
||||||
assert_eq!(headers.get("originator"), Some(&"codex_cli_rs".to_string()));
|
assert_eq!(headers.get("originator"), Some(&"codex_cli_rs".to_string()));
|
||||||
assert!(!headers.contains_key("version"));
|
assert!(!headers.contains_key("version"));
|
||||||
@@ -698,8 +698,8 @@ fn preserves_client_context_headers_and_enforces_codex_provider_identity() {
|
|||||||
Some(&"kept-by-rule-request".to_string())
|
Some(&"kept-by-rule-request".to_string())
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
headers.get("user-agent"),
|
headers.get("user-agent").map(String::as_str),
|
||||||
Some(&"codex_cli_rs/0.144.1".to_string())
|
Some(aether_ai_formats::CODEX_CLIENT_USER_AGENT)
|
||||||
);
|
);
|
||||||
assert_eq!(headers.get("originator"), Some(&"codex_cli_rs".to_string()));
|
assert_eq!(headers.get("originator"), Some(&"codex_cli_rs".to_string()));
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@@ -762,8 +762,8 @@ fn compact_projects_uuid_prompt_cache_identity_into_session_headers() {
|
|||||||
);
|
);
|
||||||
assert_eq!(headers.get("x-client-request-id"), None);
|
assert_eq!(headers.get("x-client-request-id"), None);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
headers.get("user-agent"),
|
headers.get("user-agent").map(String::as_str),
|
||||||
Some(&"codex_cli_rs/0.144.1".to_string())
|
Some(aether_ai_formats::CODEX_CLIENT_USER_AGENT)
|
||||||
);
|
);
|
||||||
assert_eq!(headers.get("originator"), Some(&"codex_cli_rs".to_string()));
|
assert_eq!(headers.get("originator"), Some(&"codex_cli_rs".to_string()));
|
||||||
assert!(!headers.contains_key("version"));
|
assert!(!headers.contains_key("version"));
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ use crate::ai_serving::planner::spec_metadata::{
|
|||||||
};
|
};
|
||||||
use crate::ai_serving::GatewayControlDecision;
|
use crate::ai_serving::GatewayControlDecision;
|
||||||
use crate::{AiExecutionDecision, AppState, GatewayError};
|
use crate::{AiExecutionDecision, AppState, GatewayError};
|
||||||
|
use aether_routing_core::RoutingExecutionPolicy;
|
||||||
|
|
||||||
use super::candidates::{
|
use super::candidates::{
|
||||||
build_local_standard_candidate_attempt_source, resolve_local_standard_decision_input,
|
build_local_standard_candidate_attempt_source, resolve_local_standard_decision_input,
|
||||||
@@ -177,6 +178,13 @@ pub(crate) async fn build_local_stream_attempt_source<'a>(
|
|||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalStandardSyncAttemptSource<'_> {
|
impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalStandardSyncAttemptSource<'_> {
|
||||||
|
fn routing_execution_policy(&self) -> Option<RoutingExecutionPolicy> {
|
||||||
|
self.input
|
||||||
|
.routing_policy
|
||||||
|
.as_ref()
|
||||||
|
.map(|policy| policy.execution_policy)
|
||||||
|
}
|
||||||
|
|
||||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
||||||
while let Some(attempt) = self.candidates.next_attempt().await? {
|
while let Some(attempt) = self.candidates.next_attempt().await? {
|
||||||
match self.build_sync_attempt(attempt).await? {
|
match self.build_sync_attempt(attempt).await? {
|
||||||
@@ -220,6 +228,13 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalStandardSyncAttemptSour
|
|||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalStandardStreamAttemptSource<'_> {
|
impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalStandardStreamAttemptSource<'_> {
|
||||||
|
fn routing_execution_policy(&self) -> Option<RoutingExecutionPolicy> {
|
||||||
|
self.input
|
||||||
|
.routing_policy
|
||||||
|
.as_ref()
|
||||||
|
.map(|policy| policy.execution_policy)
|
||||||
|
}
|
||||||
|
|
||||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
||||||
while let Some(attempt) = self.candidates.next_attempt().await? {
|
while let Some(attempt) = self.candidates.next_attempt().await? {
|
||||||
match self.build_stream_attempt(attempt).await? {
|
match self.build_stream_attempt(attempt).await? {
|
||||||
|
|||||||
@@ -378,7 +378,7 @@ mod tests {
|
|||||||
client_surface: None,
|
client_surface: None,
|
||||||
gateway_credential_carrier: None,
|
gateway_credential_carrier: None,
|
||||||
client_session_affinity: None,
|
client_session_affinity: None,
|
||||||
codex_fingerprint_context: None,
|
provider_outbound_context: None,
|
||||||
routing_policy: None,
|
routing_policy: None,
|
||||||
routing_trace_seed: None,
|
routing_trace_seed: None,
|
||||||
routing_context: None,
|
routing_context: None,
|
||||||
|
|||||||
@@ -4,6 +4,10 @@ use std::sync::Arc;
|
|||||||
use aether_contracts::ResolvedTransportProfile;
|
use aether_contracts::ResolvedTransportProfile;
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
|
|
||||||
|
use crate::ai_serving::planner::antigravity::{
|
||||||
|
build_antigravity_v1internal_provider_request, AntigravityV1InternalRequestError,
|
||||||
|
AntigravityV1InternalRequestInput, ANTIGRAVITY_V1INTERNAL_ENVELOPE_NAME,
|
||||||
|
};
|
||||||
use crate::ai_serving::planner::candidate_preparation::{
|
use crate::ai_serving::planner::candidate_preparation::{
|
||||||
prepare_header_authenticated_candidate, prepare_header_authenticated_candidate_from_auth,
|
prepare_header_authenticated_candidate, prepare_header_authenticated_candidate_from_auth,
|
||||||
OauthPreparationContext,
|
OauthPreparationContext,
|
||||||
@@ -26,6 +30,7 @@ use crate::ai_serving::planner::standard::{
|
|||||||
openai_provider_request_contract_failure_extra_data, openai_responses_reasoning_replay_policy,
|
openai_provider_request_contract_failure_extra_data, openai_responses_reasoning_replay_policy,
|
||||||
request_body_build_failure_extra_data, request_conversion_failure_extra_data,
|
request_body_build_failure_extra_data, request_conversion_failure_extra_data,
|
||||||
};
|
};
|
||||||
|
use crate::ai_serving::transport::antigravity::is_antigravity_provider_transport;
|
||||||
use crate::ai_serving::transport::kiro::{
|
use crate::ai_serving::transport::kiro::{
|
||||||
build_kiro_provider_headers, build_kiro_provider_request_body,
|
build_kiro_provider_headers, build_kiro_provider_request_body,
|
||||||
is_kiro_claude_messages_transport, KiroProviderHeadersInput, KiroRequestAuth,
|
is_kiro_claude_messages_transport, KiroProviderHeadersInput, KiroRequestAuth,
|
||||||
@@ -587,7 +592,7 @@ pub(crate) async fn resolve_local_standard_candidate_payload_parts(
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
crate::ai_serving::hydrate_openai_response_history(
|
crate::ai_serving::hydrate_openai_response_history(
|
||||||
state.runtime_state(),
|
state,
|
||||||
body_json,
|
body_json,
|
||||||
spec_metadata.api_format,
|
spec_metadata.api_format,
|
||||||
provider_api_format,
|
provider_api_format,
|
||||||
@@ -837,6 +842,29 @@ pub(crate) async fn resolve_local_standard_candidate_payload_parts(
|
|||||||
.await);
|
.await);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if normalized_provider_api_format == "gemini:generate_content"
|
||||||
|
&& is_antigravity_provider_transport(transport)
|
||||||
|
{
|
||||||
|
return Ok(build_antigravity_cross_format_payload_parts(
|
||||||
|
state,
|
||||||
|
parts,
|
||||||
|
trace_id,
|
||||||
|
body_json,
|
||||||
|
input,
|
||||||
|
attempt,
|
||||||
|
transport,
|
||||||
|
spec_metadata.api_format,
|
||||||
|
provider_api_format,
|
||||||
|
prepared_candidate.mapped_model,
|
||||||
|
prepared_candidate.auth_header,
|
||||||
|
prepared_candidate.auth_value,
|
||||||
|
provider_request_body,
|
||||||
|
upstream_is_stream,
|
||||||
|
redaction.redacted,
|
||||||
|
)
|
||||||
|
.await);
|
||||||
|
}
|
||||||
|
|
||||||
if normalized_provider_api_format == "gemini:generate_content"
|
if normalized_provider_api_format == "gemini:generate_content"
|
||||||
&& is_gemini_cli_provider_transport(transport)
|
&& is_gemini_cli_provider_transport(transport)
|
||||||
{
|
{
|
||||||
@@ -963,6 +991,145 @@ fn apply_transport_request_body_semantics(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
async fn build_antigravity_cross_format_payload_parts(
|
||||||
|
state: &AppState,
|
||||||
|
parts: &http::request::Parts,
|
||||||
|
trace_id: &str,
|
||||||
|
original_body_json: &serde_json::Value,
|
||||||
|
input: &LocalStandardDecisionInput,
|
||||||
|
attempt: &LocalStandardCandidateAttempt,
|
||||||
|
transport: &Arc<GatewayProviderTransportSnapshot>,
|
||||||
|
client_api_format: &str,
|
||||||
|
provider_api_format: &str,
|
||||||
|
mapped_model: String,
|
||||||
|
auth_header: String,
|
||||||
|
auth_value: String,
|
||||||
|
gemini_request_body: Value,
|
||||||
|
upstream_is_stream: bool,
|
||||||
|
request_redacted: bool,
|
||||||
|
) -> Option<LocalStandardCandidatePayloadParts> {
|
||||||
|
let candidate = &attempt.eligible.candidate;
|
||||||
|
let effective_headers = input.effective_headers(&parts.headers);
|
||||||
|
let resolved =
|
||||||
|
match build_antigravity_v1internal_provider_request(AntigravityV1InternalRequestInput {
|
||||||
|
state,
|
||||||
|
parts,
|
||||||
|
transport,
|
||||||
|
trace_id,
|
||||||
|
mapped_model: &mapped_model,
|
||||||
|
provider_api_format,
|
||||||
|
auth_header: &auth_header,
|
||||||
|
auth_value: &auth_value,
|
||||||
|
request_headers: effective_headers,
|
||||||
|
original_request_body: original_body_json,
|
||||||
|
gemini_request_body: &gemini_request_body,
|
||||||
|
upstream_is_stream,
|
||||||
|
same_format: false,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(resolved) => resolved,
|
||||||
|
Err(AntigravityV1InternalRequestError::TransportUnsupported) => {
|
||||||
|
mark_skipped_local_standard_candidate(
|
||||||
|
state,
|
||||||
|
input,
|
||||||
|
trace_id,
|
||||||
|
candidate,
|
||||||
|
attempt.candidate_index,
|
||||||
|
&attempt.candidate_id,
|
||||||
|
"transport_unsupported",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Err(AntigravityV1InternalRequestError::EnvelopeUnsupported) => {
|
||||||
|
mark_skipped_local_standard_candidate_with_extra_data(
|
||||||
|
state,
|
||||||
|
input,
|
||||||
|
trace_id,
|
||||||
|
candidate,
|
||||||
|
attempt.candidate_index,
|
||||||
|
&attempt.candidate_id,
|
||||||
|
"provider_request_body_build_failed",
|
||||||
|
request_body_build_failure_extra_data(
|
||||||
|
original_body_json,
|
||||||
|
client_api_format,
|
||||||
|
provider_api_format,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Err(AntigravityV1InternalRequestError::UpstreamUrlUnavailable) => {
|
||||||
|
mark_skipped_local_standard_candidate_with_failure_diagnostic(
|
||||||
|
state,
|
||||||
|
input,
|
||||||
|
trace_id,
|
||||||
|
candidate,
|
||||||
|
attempt.candidate_index,
|
||||||
|
&attempt.candidate_id,
|
||||||
|
"upstream_url_missing",
|
||||||
|
CandidateFailureDiagnostic::upstream_url_missing(
|
||||||
|
client_api_format,
|
||||||
|
provider_api_format,
|
||||||
|
"standard_family_antigravity_url",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Err(AntigravityV1InternalRequestError::HeaderRulesApplyFailed) => {
|
||||||
|
mark_skipped_local_standard_candidate_with_failure_diagnostic(
|
||||||
|
state,
|
||||||
|
input,
|
||||||
|
trace_id,
|
||||||
|
candidate,
|
||||||
|
attempt.candidate_index,
|
||||||
|
&attempt.candidate_id,
|
||||||
|
"transport_header_rules_apply_failed",
|
||||||
|
CandidateFailureDiagnostic::header_rules_apply_failed(
|
||||||
|
client_api_format,
|
||||||
|
provider_api_format,
|
||||||
|
"standard_family_antigravity_headers",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut provider_request_headers = resolved.headers.headers;
|
||||||
|
apply_codex_openai_special_headers(
|
||||||
|
&mut provider_request_headers,
|
||||||
|
&resolved.body,
|
||||||
|
effective_headers,
|
||||||
|
resolved.transport.provider.provider_type.as_str(),
|
||||||
|
provider_api_format,
|
||||||
|
Some(trace_id),
|
||||||
|
resolved.transport.key.decrypted_auth_config.as_deref(),
|
||||||
|
);
|
||||||
|
request_identity_response_encoding_when_redacted(
|
||||||
|
&mut provider_request_headers,
|
||||||
|
request_redacted,
|
||||||
|
);
|
||||||
|
|
||||||
|
Some(LocalStandardCandidatePayloadParts {
|
||||||
|
auth_header: resolved.headers.auth_header,
|
||||||
|
auth_value: resolved.headers.auth_value,
|
||||||
|
mapped_model,
|
||||||
|
provider_api_format: provider_api_format.to_string(),
|
||||||
|
provider_request_body: resolved.body,
|
||||||
|
provider_request_headers,
|
||||||
|
upstream_url: resolved.upstream_url,
|
||||||
|
upstream_is_stream,
|
||||||
|
envelope_name: Some(ANTIGRAVITY_V1INTERNAL_ENVELOPE_NAME),
|
||||||
|
transport: resolved.transport,
|
||||||
|
transport_profile: None,
|
||||||
|
request_redacted,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
async fn build_gemini_cli_cross_format_payload_parts(
|
async fn build_gemini_cli_cross_format_payload_parts(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
|
|||||||
@@ -2183,7 +2183,7 @@ mod tests {
|
|||||||
client_surface: None,
|
client_surface: None,
|
||||||
gateway_credential_carrier: None,
|
gateway_credential_carrier: None,
|
||||||
client_session_affinity: None,
|
client_session_affinity: None,
|
||||||
codex_fingerprint_context: None,
|
provider_outbound_context: None,
|
||||||
routing_policy: None,
|
routing_policy: None,
|
||||||
routing_trace_seed: None,
|
routing_trace_seed: None,
|
||||||
routing_context: None,
|
routing_context: None,
|
||||||
@@ -2741,7 +2741,7 @@ mod tests {
|
|||||||
.provider_request_headers
|
.provider_request_headers
|
||||||
.get("x-client-version")
|
.get("x-client-version")
|
||||||
.map(String::as_str),
|
.map(String::as_str),
|
||||||
Some("1.2.3")
|
Some("4.3.0")
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
payload
|
payload
|
||||||
@@ -2761,7 +2761,7 @@ mod tests {
|
|||||||
assert_eq!(payload.provider_request_body["model"], "gemini-2.5-pro");
|
assert_eq!(payload.provider_request_body["model"], "gemini-2.5-pro");
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
payload.provider_request_body["userAgent"],
|
payload.provider_request_body["userAgent"],
|
||||||
"antigravity/cli/1.0.16 (aidev_client; os_type=linux; arch=arm64; auth_method=consumer)"
|
"vscode/1.X.X (Antigravity/4.3.0)"
|
||||||
);
|
);
|
||||||
assert_eq!(payload.provider_request_body["requestType"], "agent");
|
assert_eq!(payload.provider_request_body["requestType"], "agent");
|
||||||
assert!(payload.provider_request_body.get("contents").is_none());
|
assert!(payload.provider_request_body.get("contents").is_none());
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
use aether_routing_core::RoutingExecutionPolicy;
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use std::collections::VecDeque;
|
use std::collections::VecDeque;
|
||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
@@ -119,6 +120,13 @@ pub(crate) async fn build_local_openai_chat_stream_attempt_source<'a>(
|
|||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalOpenAiChatStreamAttemptSource<'_> {
|
impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalOpenAiChatStreamAttemptSource<'_> {
|
||||||
|
fn routing_execution_policy(&self) -> Option<RoutingExecutionPolicy> {
|
||||||
|
self.input
|
||||||
|
.routing_policy
|
||||||
|
.as_ref()
|
||||||
|
.map(|policy| policy.execution_policy)
|
||||||
|
}
|
||||||
|
|
||||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
||||||
let select_started_at = std::time::Instant::now();
|
let select_started_at = std::time::Instant::now();
|
||||||
let selected = self.next_execution_attempt_with_target_select().await?;
|
let selected = self.next_execution_attempt_with_target_select().await?;
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
use aether_routing_core::RoutingExecutionPolicy;
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
|
|
||||||
@@ -92,6 +93,13 @@ pub(crate) async fn build_local_openai_chat_sync_attempt_source<'a>(
|
|||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalOpenAiChatSyncAttemptSource<'_> {
|
impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalOpenAiChatSyncAttemptSource<'_> {
|
||||||
|
fn routing_execution_policy(&self) -> Option<RoutingExecutionPolicy> {
|
||||||
|
self.input
|
||||||
|
.routing_policy
|
||||||
|
.as_ref()
|
||||||
|
.map(|policy| policy.execution_policy)
|
||||||
|
}
|
||||||
|
|
||||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
||||||
while let Some(attempt) = self.candidates.next_attempt().await? {
|
while let Some(attempt) = self.candidates.next_attempt().await? {
|
||||||
match self.build_sync_attempt(attempt).await? {
|
match self.build_sync_attempt(attempt).await? {
|
||||||
|
|||||||
+12
-27
@@ -1,7 +1,6 @@
|
|||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
use aether_contracts::RequestBody;
|
use aether_contracts::RequestBody;
|
||||||
use tracing::debug;
|
|
||||||
|
|
||||||
use super::super::{
|
use super::super::{
|
||||||
augment_sync_report_context, build_ai_execution_plan_from_decision,
|
augment_sync_report_context, build_ai_execution_plan_from_decision,
|
||||||
@@ -10,7 +9,6 @@ use super::super::{
|
|||||||
AiStreamAttempt,
|
AiStreamAttempt,
|
||||||
};
|
};
|
||||||
use crate::ai_serving::planner::common::enforce_provider_body_stream_policy;
|
use crate::ai_serving::planner::common::enforce_provider_body_stream_policy;
|
||||||
use crate::ai_serving::planner::redaction::sanitize_upstream_url_for_log;
|
|
||||||
use crate::ai_serving::provider_adaptation_requires_eventstream_accept;
|
use crate::ai_serving::provider_adaptation_requires_eventstream_accept;
|
||||||
use crate::ai_serving::transport::{
|
use crate::ai_serving::transport::{
|
||||||
build_standard_plan_fallback_headers, build_standard_plan_fallback_openai_chat_url,
|
build_standard_plan_fallback_headers, build_standard_plan_fallback_openai_chat_url,
|
||||||
@@ -157,21 +155,16 @@ pub(crate) fn build_openai_responses_stream_plan_from_decision(
|
|||||||
let Some(auth_pair) = take_ai_upstream_auth_pair(&mut payload) else {
|
let Some(auth_pair) = take_ai_upstream_auth_pair(&mut payload) else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
let (url, url_source) = if let Some(upstream_url) =
|
let url = if let Some(upstream_url) = take_non_empty_string(&mut payload.upstream_url) {
|
||||||
take_non_empty_string(&mut payload.upstream_url)
|
upstream_url
|
||||||
{
|
|
||||||
(upstream_url, "upstream_url")
|
|
||||||
} else {
|
} else {
|
||||||
let Some(upstream_base_url) = take_non_empty_string(&mut payload.upstream_base_url) else {
|
let Some(upstream_base_url) = take_non_empty_string(&mut payload.upstream_base_url) else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
(
|
build_standard_plan_fallback_openai_responses_url(
|
||||||
build_standard_plan_fallback_openai_responses_url(
|
&upstream_base_url,
|
||||||
&upstream_base_url,
|
parts.uri.query(),
|
||||||
parts.uri.query(),
|
compact,
|
||||||
compact,
|
|
||||||
),
|
|
||||||
"upstream_base_url",
|
|
||||||
)
|
)
|
||||||
};
|
};
|
||||||
let Some(provider_request_body_value) = payload.provider_request_body.take() else {
|
let Some(provider_request_body_value) = payload.provider_request_body.take() else {
|
||||||
@@ -238,16 +231,7 @@ pub(crate) fn build_openai_responses_stream_plan_from_decision(
|
|||||||
.uri
|
.uri
|
||||||
.query()
|
.query()
|
||||||
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
||||||
let log_decision_upstream_base_url = payload
|
tracing::debug!(
|
||||||
.upstream_base_url
|
|
||||||
.as_deref()
|
|
||||||
.map(sanitize_upstream_url_for_log);
|
|
||||||
let log_decision_upstream_url = payload
|
|
||||||
.upstream_url
|
|
||||||
.as_deref()
|
|
||||||
.map(sanitize_upstream_url_for_log);
|
|
||||||
let log_plan_url = sanitize_upstream_url_for_log(plan.url.as_str());
|
|
||||||
debug!(
|
|
||||||
event_name = "local_openai_responses_stream_plan_built",
|
event_name = "local_openai_responses_stream_plan_built",
|
||||||
log_type = "debug",
|
log_type = "debug",
|
||||||
request_id = %plan.request_id,
|
request_id = %plan.request_id,
|
||||||
@@ -255,12 +239,13 @@ pub(crate) fn build_openai_responses_stream_plan_from_decision(
|
|||||||
provider_id = %plan.provider_id,
|
provider_id = %plan.provider_id,
|
||||||
endpoint_id = %plan.endpoint_id,
|
endpoint_id = %plan.endpoint_id,
|
||||||
key_id = %plan.key_id,
|
key_id = %plan.key_id,
|
||||||
|
downstream_path_and_query = %crate::ai_serving::pure::sanitize_request_path_and_query(
|
||||||
|
parts.uri.path(),
|
||||||
|
parts.uri.query(),
|
||||||
|
).unwrap_or_else(|| "/".to_string()),
|
||||||
|
upstream_origin = %crate::handlers::shared::security_log_url_origin(&plan.url),
|
||||||
downstream_path = %parts.uri.path(),
|
downstream_path = %parts.uri.path(),
|
||||||
downstream_query = ?log_downstream_query,
|
downstream_query = ?log_downstream_query,
|
||||||
url_source,
|
|
||||||
decision_upstream_base_url = ?log_decision_upstream_base_url,
|
|
||||||
decision_upstream_url = ?log_decision_upstream_url,
|
|
||||||
plan_url = %log_plan_url,
|
|
||||||
client_api_format = %plan.client_api_format,
|
client_api_format = %plan.client_api_format,
|
||||||
provider_api_format = %plan.provider_api_format,
|
provider_api_format = %plan.provider_api_format,
|
||||||
upstream_is_stream = effective_upstream_is_stream,
|
upstream_is_stream = effective_upstream_is_stream,
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
use aether_contracts::RequestBody;
|
use aether_contracts::RequestBody;
|
||||||
use tracing::debug;
|
|
||||||
|
|
||||||
use super::super::{
|
use super::super::{
|
||||||
augment_sync_report_context, build_ai_execution_plan_from_decision,
|
augment_sync_report_context, build_ai_execution_plan_from_decision,
|
||||||
@@ -10,7 +9,6 @@ use super::super::{
|
|||||||
AiSyncAttempt,
|
AiSyncAttempt,
|
||||||
};
|
};
|
||||||
use crate::ai_serving::planner::common::enforce_provider_body_stream_policy;
|
use crate::ai_serving::planner::common::enforce_provider_body_stream_policy;
|
||||||
use crate::ai_serving::planner::redaction::sanitize_upstream_url_for_log;
|
|
||||||
use crate::ai_serving::transport::{
|
use crate::ai_serving::transport::{
|
||||||
build_standard_plan_fallback_headers, build_standard_plan_fallback_openai_chat_url,
|
build_standard_plan_fallback_headers, build_standard_plan_fallback_openai_chat_url,
|
||||||
build_standard_plan_fallback_openai_responses_url, StandardPlanFallbackAcceptPolicy,
|
build_standard_plan_fallback_openai_responses_url, StandardPlanFallbackAcceptPolicy,
|
||||||
@@ -142,21 +140,16 @@ pub(crate) fn build_openai_responses_sync_plan_from_decision(
|
|||||||
let Some(auth_pair) = take_ai_upstream_auth_pair(&mut payload) else {
|
let Some(auth_pair) = take_ai_upstream_auth_pair(&mut payload) else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
let (url, url_source) = if let Some(upstream_url) =
|
let url = if let Some(upstream_url) = take_non_empty_string(&mut payload.upstream_url) {
|
||||||
take_non_empty_string(&mut payload.upstream_url)
|
upstream_url
|
||||||
{
|
|
||||||
(upstream_url, "upstream_url")
|
|
||||||
} else {
|
} else {
|
||||||
let Some(upstream_base_url) = take_non_empty_string(&mut payload.upstream_base_url) else {
|
let Some(upstream_base_url) = take_non_empty_string(&mut payload.upstream_base_url) else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
(
|
build_standard_plan_fallback_openai_responses_url(
|
||||||
build_standard_plan_fallback_openai_responses_url(
|
&upstream_base_url,
|
||||||
&upstream_base_url,
|
parts.uri.query(),
|
||||||
parts.uri.query(),
|
compact,
|
||||||
compact,
|
|
||||||
),
|
|
||||||
"upstream_base_url",
|
|
||||||
)
|
)
|
||||||
};
|
};
|
||||||
let Some(provider_request_body_value) = payload.provider_request_body.take() else {
|
let Some(provider_request_body_value) = payload.provider_request_body.take() else {
|
||||||
@@ -205,16 +198,7 @@ pub(crate) fn build_openai_responses_sync_plan_from_decision(
|
|||||||
.uri
|
.uri
|
||||||
.query()
|
.query()
|
||||||
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
||||||
let log_decision_upstream_base_url = payload
|
tracing::debug!(
|
||||||
.upstream_base_url
|
|
||||||
.as_deref()
|
|
||||||
.map(sanitize_upstream_url_for_log);
|
|
||||||
let log_decision_upstream_url = payload
|
|
||||||
.upstream_url
|
|
||||||
.as_deref()
|
|
||||||
.map(sanitize_upstream_url_for_log);
|
|
||||||
let log_plan_url = sanitize_upstream_url_for_log(plan.url.as_str());
|
|
||||||
debug!(
|
|
||||||
event_name = "local_openai_responses_sync_plan_built",
|
event_name = "local_openai_responses_sync_plan_built",
|
||||||
log_type = "debug",
|
log_type = "debug",
|
||||||
request_id = %plan.request_id,
|
request_id = %plan.request_id,
|
||||||
@@ -222,12 +206,13 @@ pub(crate) fn build_openai_responses_sync_plan_from_decision(
|
|||||||
provider_id = %plan.provider_id,
|
provider_id = %plan.provider_id,
|
||||||
endpoint_id = %plan.endpoint_id,
|
endpoint_id = %plan.endpoint_id,
|
||||||
key_id = %plan.key_id,
|
key_id = %plan.key_id,
|
||||||
|
downstream_path_and_query = %crate::ai_serving::pure::sanitize_request_path_and_query(
|
||||||
|
parts.uri.path(),
|
||||||
|
parts.uri.query(),
|
||||||
|
).unwrap_or_else(|| "/".to_string()),
|
||||||
|
upstream_origin = %crate::handlers::shared::security_log_url_origin(&plan.url),
|
||||||
downstream_path = %parts.uri.path(),
|
downstream_path = %parts.uri.path(),
|
||||||
downstream_query = ?log_downstream_query,
|
downstream_query = ?log_downstream_query,
|
||||||
url_source,
|
|
||||||
decision_upstream_base_url = ?log_decision_upstream_base_url,
|
|
||||||
decision_upstream_url = ?log_decision_upstream_url,
|
|
||||||
plan_url = %log_plan_url,
|
|
||||||
client_api_format = %plan.client_api_format,
|
client_api_format = %plan.client_api_format,
|
||||||
provider_api_format = %plan.provider_api_format,
|
provider_api_format = %plan.provider_api_format,
|
||||||
upstream_is_stream = payload.upstream_is_stream,
|
upstream_is_stream = payload.upstream_is_stream,
|
||||||
|
|||||||
+5
-5
@@ -3,7 +3,6 @@ use tracing::debug;
|
|||||||
|
|
||||||
use crate::ai_serving::build_request_trace_proxy_value;
|
use crate::ai_serving::build_request_trace_proxy_value;
|
||||||
use crate::ai_serving::planner::decision_input::apply_provider_request_routing_policy_to_decision_with_websocket_mode;
|
use crate::ai_serving::planner::decision_input::apply_provider_request_routing_policy_to_decision_with_websocket_mode;
|
||||||
use crate::ai_serving::planner::redaction::sanitize_upstream_url_for_log;
|
|
||||||
use crate::ai_serving::planner::report_context::{
|
use crate::ai_serving::planner::report_context::{
|
||||||
build_local_execution_report_context, insert_native_client_envelope_name,
|
build_local_execution_report_context, insert_native_client_envelope_name,
|
||||||
insert_provider_stream_event_api_format, LocalExecutionReportContextParts,
|
insert_provider_stream_event_api_format, LocalExecutionReportContextParts,
|
||||||
@@ -205,12 +204,10 @@ pub(crate) async fn maybe_build_local_openai_responses_decision_payload_for_cand
|
|||||||
&resolved.transport,
|
&resolved.transport,
|
||||||
);
|
);
|
||||||
|
|
||||||
let log_base_url = sanitize_upstream_url_for_log(resolved.transport.endpoint.base_url.as_str());
|
|
||||||
let log_request_query = parts
|
let log_request_query = parts
|
||||||
.uri
|
.uri
|
||||||
.query()
|
.query()
|
||||||
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
||||||
let log_upstream_url = sanitize_upstream_url_for_log(resolved.upstream_url.as_str());
|
|
||||||
debug!(
|
debug!(
|
||||||
event_name = "local_openai_responses_decision_payload_built",
|
event_name = "local_openai_responses_decision_payload_built",
|
||||||
log_type = "debug",
|
log_type = "debug",
|
||||||
@@ -227,9 +224,12 @@ pub(crate) async fn maybe_build_local_openai_responses_decision_payload_for_cand
|
|||||||
client_api_format = spec_metadata.api_format,
|
client_api_format = spec_metadata.api_format,
|
||||||
provider_api_format = %resolved.provider_api_format,
|
provider_api_format = %resolved.provider_api_format,
|
||||||
request_path = %parts.uri.path(),
|
request_path = %parts.uri.path(),
|
||||||
|
request_path_and_query = %crate::ai_serving::pure::sanitize_request_path_and_query(
|
||||||
|
parts.uri.path(),
|
||||||
|
parts.uri.query(),
|
||||||
|
).unwrap_or_else(|| "/".to_string()),
|
||||||
|
upstream_origin = %crate::handlers::shared::security_log_url_origin(&resolved.upstream_url),
|
||||||
request_query = ?log_request_query,
|
request_query = ?log_request_query,
|
||||||
upstream_base_url = %log_base_url,
|
|
||||||
upstream_url = %log_upstream_url,
|
|
||||||
upstream_is_stream = resolved.upstream_is_stream,
|
upstream_is_stream = resolved.upstream_is_stream,
|
||||||
has_envelope = resolved.envelope_name.is_some(),
|
has_envelope = resolved.envelope_name.is_some(),
|
||||||
"gateway built local openai responses decision payload"
|
"gateway built local openai responses decision payload"
|
||||||
|
|||||||
+7
-15
@@ -24,7 +24,6 @@ use crate::ai_serving::planner::gemini_cli::{
|
|||||||
};
|
};
|
||||||
use crate::ai_serving::planner::redaction::{
|
use crate::ai_serving::planner::redaction::{
|
||||||
request_identity_response_encoding_when_redacted, resolve_provider_chat_pii_redaction,
|
request_identity_response_encoding_when_redacted, resolve_provider_chat_pii_redaction,
|
||||||
sanitize_upstream_url_for_log,
|
|
||||||
};
|
};
|
||||||
use crate::ai_serving::planner::spec_metadata::local_openai_responses_spec_metadata;
|
use crate::ai_serving::planner::spec_metadata::local_openai_responses_spec_metadata;
|
||||||
use crate::ai_serving::planner::standard::{
|
use crate::ai_serving::planner::standard::{
|
||||||
@@ -428,7 +427,7 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts_with_
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
crate::ai_serving::hydrate_openai_response_history(
|
crate::ai_serving::hydrate_openai_response_history(
|
||||||
state.runtime_state(),
|
state,
|
||||||
body_json,
|
body_json,
|
||||||
spec_metadata.api_format,
|
spec_metadata.api_format,
|
||||||
provider_api_format,
|
provider_api_format,
|
||||||
@@ -867,17 +866,10 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts_with_
|
|||||||
|
|
||||||
let (execution_strategy, conversion_mode) =
|
let (execution_strategy, conversion_mode) =
|
||||||
ai_local_execution_contract_for_formats(spec_metadata.api_format, provider_api_format);
|
ai_local_execution_contract_for_formats(spec_metadata.api_format, provider_api_format);
|
||||||
let log_base_url = sanitize_upstream_url_for_log(transport.endpoint.base_url.as_str());
|
|
||||||
let log_custom_path = transport
|
|
||||||
.endpoint
|
|
||||||
.custom_path
|
|
||||||
.as_deref()
|
|
||||||
.map(sanitize_upstream_url_for_log);
|
|
||||||
let log_request_query = parts
|
let log_request_query = parts
|
||||||
.uri
|
.uri
|
||||||
.query()
|
.query()
|
||||||
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
.and_then(crate::ai_serving::api::sanitize_request_query_string);
|
||||||
let log_upstream_url = sanitize_upstream_url_for_log(upstream_url.as_str());
|
|
||||||
|
|
||||||
debug!(
|
debug!(
|
||||||
event_name = "local_openai_responses_upstream_url_resolved",
|
event_name = "local_openai_responses_upstream_url_resolved",
|
||||||
@@ -893,12 +885,14 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts_with_
|
|||||||
provider_api_format = %provider_api_format,
|
provider_api_format = %provider_api_format,
|
||||||
execution_strategy = execution_strategy.as_str(),
|
execution_strategy = execution_strategy.as_str(),
|
||||||
conversion_mode = conversion_mode.as_str(),
|
conversion_mode = conversion_mode.as_str(),
|
||||||
base_url = %log_base_url,
|
request_path_and_query = %crate::ai_serving::pure::sanitize_request_path_and_query(
|
||||||
custom_path = ?log_custom_path,
|
parts.uri.path(),
|
||||||
|
parts.uri.query(),
|
||||||
|
).unwrap_or_else(|| "/".to_string()),
|
||||||
|
upstream_origin = %crate::handlers::shared::security_log_url_origin(&upstream_url),
|
||||||
request_path = %parts.uri.path(),
|
request_path = %parts.uri.path(),
|
||||||
request_query = ?log_request_query,
|
request_query = ?log_request_query,
|
||||||
mapped_model = %mapped_model,
|
mapped_model = %mapped_model,
|
||||||
upstream_url = %log_upstream_url,
|
|
||||||
upstream_is_stream,
|
upstream_is_stream,
|
||||||
"gateway resolved local openai responses upstream url"
|
"gateway resolved local openai responses upstream url"
|
||||||
);
|
);
|
||||||
@@ -2011,8 +2005,6 @@ async fn build_kiro_openai_responses_payload_parts(
|
|||||||
};
|
};
|
||||||
let (execution_strategy, conversion_mode) =
|
let (execution_strategy, conversion_mode) =
|
||||||
ai_local_execution_contract_for_formats(client_api_format, provider_api_format);
|
ai_local_execution_contract_for_formats(client_api_format, provider_api_format);
|
||||||
let log_upstream_url = sanitize_upstream_url_for_log(upstream_url.as_str());
|
|
||||||
|
|
||||||
debug!(
|
debug!(
|
||||||
event_name = "local_openai_responses_kiro_upstream_url_resolved",
|
event_name = "local_openai_responses_kiro_upstream_url_resolved",
|
||||||
log_type = "debug",
|
log_type = "debug",
|
||||||
@@ -2027,7 +2019,7 @@ async fn build_kiro_openai_responses_payload_parts(
|
|||||||
provider_api_format = %provider_api_format,
|
provider_api_format = %provider_api_format,
|
||||||
execution_strategy = execution_strategy.as_str(),
|
execution_strategy = execution_strategy.as_str(),
|
||||||
conversion_mode = conversion_mode.as_str(),
|
conversion_mode = conversion_mode.as_str(),
|
||||||
upstream_url = %log_upstream_url,
|
upstream_origin = %crate::handlers::shared::security_log_url_origin(&upstream_url),
|
||||||
upstream_is_stream,
|
upstream_is_stream,
|
||||||
"gateway resolved local openai responses kiro upstream url"
|
"gateway resolved local openai responses kiro upstream url"
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
use aether_routing_core::RoutingExecutionPolicy;
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
|
|
||||||
@@ -161,6 +162,13 @@ pub(super) async fn build_local_stream_attempt_source<'a>(
|
|||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalOpenAiResponsesSyncAttemptSource<'_> {
|
impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalOpenAiResponsesSyncAttemptSource<'_> {
|
||||||
|
fn routing_execution_policy(&self) -> Option<RoutingExecutionPolicy> {
|
||||||
|
self.input
|
||||||
|
.routing_policy
|
||||||
|
.as_ref()
|
||||||
|
.map(|policy| policy.execution_policy)
|
||||||
|
}
|
||||||
|
|
||||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
async fn next_execution_attempt(&mut self) -> Result<Option<AiSyncAttempt>, GatewayError> {
|
||||||
while let Some(attempt) = self.candidates.next_attempt().await? {
|
while let Some(attempt) = self.candidates.next_attempt().await? {
|
||||||
match self.build_sync_attempt(attempt).await? {
|
match self.build_sync_attempt(attempt).await? {
|
||||||
@@ -204,6 +212,13 @@ impl LocalExecutionAttemptSource<AiSyncAttempt> for LocalOpenAiResponsesSyncAtte
|
|||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalOpenAiResponsesStreamAttemptSource<'_> {
|
impl LocalExecutionAttemptSource<AiStreamAttempt> for LocalOpenAiResponsesStreamAttemptSource<'_> {
|
||||||
|
fn routing_execution_policy(&self) -> Option<RoutingExecutionPolicy> {
|
||||||
|
self.input
|
||||||
|
.routing_policy
|
||||||
|
.as_ref()
|
||||||
|
.map(|policy| policy.execution_policy)
|
||||||
|
}
|
||||||
|
|
||||||
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
async fn next_execution_attempt(&mut self) -> Result<Option<AiStreamAttempt>, GatewayError> {
|
||||||
while let Some(attempt) = self.candidates.next_attempt().await? {
|
while let Some(attempt) = self.candidates.next_attempt().await? {
|
||||||
match self.build_stream_attempt(attempt).await? {
|
match self.build_stream_attempt(attempt).await? {
|
||||||
|
|||||||
@@ -12,9 +12,8 @@ use crate::scheduler::config::SchedulerOrderingConfig;
|
|||||||
use crate::GatewayError;
|
use crate::GatewayError;
|
||||||
|
|
||||||
impl<'a> PlannerAppState<'a> {
|
impl<'a> PlannerAppState<'a> {
|
||||||
/// `ordering_config` is the request's routing-policy derived scheduler
|
/// `ordering_config` is the immutable scheduler snapshot derived from the
|
||||||
/// config (see `SchedulerOrderingConfig::from_routing_policy`). `None`
|
/// request's resolved routing policy.
|
||||||
/// falls back to the runtime default.
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
pub(crate) async fn list_selectable_candidates(
|
pub(crate) async fn list_selectable_candidates(
|
||||||
self,
|
self,
|
||||||
@@ -26,7 +25,7 @@ impl<'a> PlannerAppState<'a> {
|
|||||||
client_session_affinity: Option<&ClientSessionAffinity>,
|
client_session_affinity: Option<&ClientSessionAffinity>,
|
||||||
now_unix_secs: u64,
|
now_unix_secs: u64,
|
||||||
enable_model_directives: bool,
|
enable_model_directives: bool,
|
||||||
ordering_config: Option<SchedulerOrderingConfig>,
|
ordering_config: SchedulerOrderingConfig,
|
||||||
) -> Result<Vec<SchedulerMinimalCandidateSelectionCandidate>, GatewayError> {
|
) -> Result<Vec<SchedulerMinimalCandidateSelectionCandidate>, GatewayError> {
|
||||||
crate::scheduler::candidate::list_selectable_candidates(
|
crate::scheduler::candidate::list_selectable_candidates(
|
||||||
self.app().data.as_ref(),
|
self.app().data.as_ref(),
|
||||||
@@ -55,7 +54,7 @@ impl<'a> PlannerAppState<'a> {
|
|||||||
client_session_affinity: Option<&ClientSessionAffinity>,
|
client_session_affinity: Option<&ClientSessionAffinity>,
|
||||||
now_unix_secs: u64,
|
now_unix_secs: u64,
|
||||||
enable_model_directives: bool,
|
enable_model_directives: bool,
|
||||||
ordering_config: Option<SchedulerOrderingConfig>,
|
ordering_config: SchedulerOrderingConfig,
|
||||||
) -> Result<
|
) -> Result<
|
||||||
(
|
(
|
||||||
Vec<SchedulerMinimalCandidateSelectionCandidate>,
|
Vec<SchedulerMinimalCandidateSelectionCandidate>,
|
||||||
@@ -90,7 +89,7 @@ impl<'a> PlannerAppState<'a> {
|
|||||||
now_unix_secs: u64,
|
now_unix_secs: u64,
|
||||||
enable_model_directives: bool,
|
enable_model_directives: bool,
|
||||||
request_operation: Option<&str>,
|
request_operation: Option<&str>,
|
||||||
ordering_config: Option<SchedulerOrderingConfig>,
|
ordering_config: SchedulerOrderingConfig,
|
||||||
) -> Result<
|
) -> Result<
|
||||||
(
|
(
|
||||||
Vec<SchedulerMinimalCandidateSelectionCandidate>,
|
Vec<SchedulerMinimalCandidateSelectionCandidate>,
|
||||||
@@ -146,7 +145,7 @@ impl<'a> PlannerAppState<'a> {
|
|||||||
auth_snapshot: Option<&GatewayAuthApiKeySnapshot>,
|
auth_snapshot: Option<&GatewayAuthApiKeySnapshot>,
|
||||||
client_session_affinity: Option<&ClientSessionAffinity>,
|
client_session_affinity: Option<&ClientSessionAffinity>,
|
||||||
now_unix_secs: u64,
|
now_unix_secs: u64,
|
||||||
ordering_config: Option<SchedulerOrderingConfig>,
|
ordering_config: SchedulerOrderingConfig,
|
||||||
) -> Result<
|
) -> Result<
|
||||||
(
|
(
|
||||||
Vec<SchedulerMinimalCandidateSelectionCandidate>,
|
Vec<SchedulerMinimalCandidateSelectionCandidate>,
|
||||||
@@ -177,7 +176,7 @@ impl<'a> PlannerAppState<'a> {
|
|||||||
auth_snapshot: Option<&GatewayAuthApiKeySnapshot>,
|
auth_snapshot: Option<&GatewayAuthApiKeySnapshot>,
|
||||||
client_session_affinity: Option<&ClientSessionAffinity>,
|
client_session_affinity: Option<&ClientSessionAffinity>,
|
||||||
now_unix_secs: u64,
|
now_unix_secs: u64,
|
||||||
ordering_config: Option<SchedulerOrderingConfig>,
|
ordering_config: SchedulerOrderingConfig,
|
||||||
) -> Result<Vec<SchedulerMinimalCandidateSelectionCandidate>, GatewayError> {
|
) -> Result<Vec<SchedulerMinimalCandidateSelectionCandidate>, GatewayError> {
|
||||||
let wait_timeout = Duration::from_millis(API_KEY_CONCURRENCY_WAIT_TIMEOUT_MS);
|
let wait_timeout = Duration::from_millis(API_KEY_CONCURRENCY_WAIT_TIMEOUT_MS);
|
||||||
let wait_interval = Duration::from_millis(API_KEY_CONCURRENCY_WAIT_POLL_INTERVAL_MS.max(1));
|
let wait_interval = Duration::from_millis(API_KEY_CONCURRENCY_WAIT_POLL_INTERVAL_MS.max(1));
|
||||||
|
|||||||
@@ -177,6 +177,7 @@ pub(crate) use aether_ai_formats::{
|
|||||||
api_format_defaults_to_client_error_failover, api_format_defaults_to_non_stream,
|
api_format_defaults_to_client_error_failover, api_format_defaults_to_non_stream,
|
||||||
api_format_permission_covers, codex_responses_lite_tool_is_client_executed,
|
api_format_permission_covers, codex_responses_lite_tool_is_client_executed,
|
||||||
intersect_api_format_allowed_lists, is_embedding_api_format, is_rerank_api_format,
|
intersect_api_format_allowed_lists, is_embedding_api_format, is_rerank_api_format,
|
||||||
|
normalize_openai_responses_message_item_ids, openai_responses_message_item_id,
|
||||||
openai_responses_request_operation, openai_responses_synthetic_reasoning_item_id,
|
openai_responses_request_operation, openai_responses_synthetic_reasoning_item_id,
|
||||||
strip_incompatible_openai_responses_reasoning_items,
|
strip_incompatible_openai_responses_reasoning_items,
|
||||||
strip_incompatible_openai_responses_reasoning_items_with_policy, ApiOperation, ClientSurface,
|
strip_incompatible_openai_responses_reasoning_items_with_policy, ApiOperation, ClientSurface,
|
||||||
|
|||||||
@@ -2,14 +2,15 @@ use crate::ai_serving::{
|
|||||||
hydrate_response_history, normalize_api_format_alias, record_converted_response_history,
|
hydrate_response_history, normalize_api_format_alias, record_converted_response_history,
|
||||||
response_history_is_loaded, response_history_storage_key, ResponseHistoryRecord,
|
response_history_is_loaded, response_history_storage_key, ResponseHistoryRecord,
|
||||||
};
|
};
|
||||||
use aether_runtime_state::RuntimeState;
|
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
|
|
||||||
use crate::GatewayError;
|
use crate::{AppState, GatewayError};
|
||||||
|
|
||||||
|
const RESPONSE_HISTORY_SECRET_PURPOSE: &str = "openai-response-history";
|
||||||
|
|
||||||
pub(crate) async fn hydrate_openai_response_history(
|
pub(crate) async fn hydrate_openai_response_history(
|
||||||
runtime_state: &RuntimeState,
|
state: &AppState,
|
||||||
request: &Value,
|
request: &Value,
|
||||||
client_api_format: &str,
|
client_api_format: &str,
|
||||||
provider_api_format: &str,
|
provider_api_format: &str,
|
||||||
@@ -33,6 +34,7 @@ pub(crate) async fn hydrate_openai_response_history(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let storage_key = response_history_storage_key(previous_response_id, Some(history_scope));
|
let storage_key = response_history_storage_key(previous_response_id, Some(history_scope));
|
||||||
|
let runtime_state = state.runtime_state();
|
||||||
let payload = runtime_state.kv_get(&storage_key).await.map_err(|error| {
|
let payload = runtime_state.kv_get(&storage_key).await.map_err(|error| {
|
||||||
warn!(
|
warn!(
|
||||||
event_name = "openai_response_history_read_failed",
|
event_name = "openai_response_history_read_failed",
|
||||||
@@ -46,8 +48,24 @@ pub(crate) async fn hydrate_openai_response_history(
|
|||||||
let Some(payload) = payload else {
|
let Some(payload) = payload else {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
};
|
};
|
||||||
|
let Some(payload) = crate::handlers::shared::open_runtime_secret_payload(
|
||||||
|
state,
|
||||||
|
RESPONSE_HISTORY_SECRET_PURPOSE,
|
||||||
|
&payload,
|
||||||
|
) else {
|
||||||
|
let _ = runtime_state.kv_delete(&storage_key).await;
|
||||||
|
warn!(
|
||||||
|
event_name = "openai_response_history_decryption_failed",
|
||||||
|
log_type = "ops",
|
||||||
|
backend = runtime_state.backend_kind().as_str(),
|
||||||
|
"gateway rejected undecryptable shared OpenAI response history"
|
||||||
|
);
|
||||||
|
return Err(GatewayError::Internal(
|
||||||
|
"OpenAI response history decryption failed".to_string(),
|
||||||
|
));
|
||||||
|
};
|
||||||
if let Err(error) =
|
if let Err(error) =
|
||||||
hydrate_response_history(previous_response_id, Some(history_scope), &payload)
|
hydrate_response_history(previous_response_id, Some(history_scope), payload.as_str())
|
||||||
{
|
{
|
||||||
let _ = runtime_state.kv_delete(&storage_key).await;
|
let _ = runtime_state.kv_delete(&storage_key).await;
|
||||||
warn!(
|
warn!(
|
||||||
@@ -65,11 +83,25 @@ pub(crate) async fn hydrate_openai_response_history(
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn persist_response_history_record(
|
pub(crate) async fn persist_response_history_record(
|
||||||
runtime_state: &RuntimeState,
|
state: &AppState,
|
||||||
record: ResponseHistoryRecord,
|
record: ResponseHistoryRecord,
|
||||||
) {
|
) {
|
||||||
|
let runtime_state = state.runtime_state();
|
||||||
|
let Some(sealed_payload) = crate::handlers::shared::seal_runtime_secret_payload(
|
||||||
|
state,
|
||||||
|
RESPONSE_HISTORY_SECRET_PURPOSE,
|
||||||
|
&record.payload,
|
||||||
|
) else {
|
||||||
|
warn!(
|
||||||
|
event_name = "openai_response_history_encryption_unavailable",
|
||||||
|
log_type = "ops",
|
||||||
|
backend = runtime_state.backend_kind().as_str(),
|
||||||
|
"gateway refused to persist unencrypted OpenAI response history"
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
};
|
||||||
if let Err(error) = runtime_state
|
if let Err(error) = runtime_state
|
||||||
.kv_set(&record.storage_key, record.payload, Some(record.ttl))
|
.kv_set(&record.storage_key, sealed_payload, Some(record.ttl))
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
warn!(
|
warn!(
|
||||||
@@ -83,7 +115,7 @@ pub(crate) async fn persist_response_history_record(
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn persist_converted_response_history(
|
pub(crate) async fn persist_converted_response_history(
|
||||||
runtime_state: &RuntimeState,
|
state: &AppState,
|
||||||
report_context: &Value,
|
report_context: &Value,
|
||||||
response: Option<&Value>,
|
response: Option<&Value>,
|
||||||
) {
|
) {
|
||||||
@@ -91,6 +123,126 @@ pub(crate) async fn persist_converted_response_history(
|
|||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
if let Some(record) = record_converted_response_history(report_context, response) {
|
if let Some(record) = record_converted_response_history(report_context, response) {
|
||||||
persist_response_history_record(runtime_state, record).await;
|
persist_response_history_record(state, record).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
|
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||||
|
use aether_runtime_state::{MemoryRuntimeStateConfig, RuntimeState};
|
||||||
|
use serde_json::json;
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
|
||||||
|
use super::{
|
||||||
|
hydrate_openai_response_history, persist_response_history_record, ResponseHistoryRecord,
|
||||||
|
};
|
||||||
|
use crate::{ai_serving::response_history_storage_key, data::GatewayDataState, AppState};
|
||||||
|
|
||||||
|
fn response_history_test_state() -> AppState {
|
||||||
|
AppState::new()
|
||||||
|
.expect("test state should build")
|
||||||
|
.with_data_state_for_tests(
|
||||||
|
GatewayDataState::disabled()
|
||||||
|
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY),
|
||||||
|
)
|
||||||
|
.with_runtime_state(Arc::new(RuntimeState::memory(
|
||||||
|
MemoryRuntimeStateConfig::default(),
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn response_history_payload(response_id: &str, scope: &str, marker: &str) -> String {
|
||||||
|
let expires_at_unix_secs = SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_secs()
|
||||||
|
.saturating_add(3600);
|
||||||
|
json!({
|
||||||
|
"version": 1,
|
||||||
|
"response_id": response_id,
|
||||||
|
"scope_fingerprint": format!("{:x}", Sha256::digest(scope.trim().as_bytes())),
|
||||||
|
"expires_at_unix_secs": expires_at_unix_secs,
|
||||||
|
"transcript": [{"type": "message", "content": marker}],
|
||||||
|
})
|
||||||
|
.to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn response_history_is_encrypted_at_rest_and_hydrates() {
|
||||||
|
let state = response_history_test_state();
|
||||||
|
let response_id = "resp_gateway_encrypted_history_v1";
|
||||||
|
let scope = "response-history-encrypted-scope";
|
||||||
|
let marker = "private-response-history-marker";
|
||||||
|
let storage_key = response_history_storage_key(response_id, Some(scope));
|
||||||
|
let payload = response_history_payload(response_id, scope, marker);
|
||||||
|
|
||||||
|
persist_response_history_record(
|
||||||
|
&state,
|
||||||
|
ResponseHistoryRecord {
|
||||||
|
storage_key: storage_key.clone(),
|
||||||
|
payload,
|
||||||
|
ttl: Duration::from_secs(6 * 60 * 60),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let stored = state
|
||||||
|
.runtime_kv_get(&storage_key)
|
||||||
|
.await
|
||||||
|
.expect("history lookup should succeed")
|
||||||
|
.expect("history should be persisted");
|
||||||
|
assert!(crate::handlers::shared::runtime_secret_payload_is_sealed(
|
||||||
|
&stored
|
||||||
|
));
|
||||||
|
assert!(!stored.contains(marker));
|
||||||
|
|
||||||
|
hydrate_openai_response_history(
|
||||||
|
&state,
|
||||||
|
&json!({"previous_response_id": response_id}),
|
||||||
|
"openai:responses",
|
||||||
|
"openai:chat",
|
||||||
|
scope,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("encrypted history should hydrate");
|
||||||
|
assert!(crate::ai_serving::response_history_is_loaded(
|
||||||
|
response_id,
|
||||||
|
Some(scope)
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn response_history_reader_rejects_and_deletes_legacy_plaintext() {
|
||||||
|
let state = response_history_test_state();
|
||||||
|
let response_id = "resp_gateway_legacy_history_v1";
|
||||||
|
let scope = "response-history-legacy-scope";
|
||||||
|
let storage_key = response_history_storage_key(response_id, Some(scope));
|
||||||
|
let payload = response_history_payload(response_id, scope, "legacy-private-history");
|
||||||
|
state
|
||||||
|
.runtime_kv_setex(&storage_key, &payload, 6 * 60 * 60)
|
||||||
|
.await
|
||||||
|
.expect("legacy history should store");
|
||||||
|
|
||||||
|
let result = hydrate_openai_response_history(
|
||||||
|
&state,
|
||||||
|
&json!({"previous_response_id": response_id}),
|
||||||
|
"openai:responses",
|
||||||
|
"openai:chat",
|
||||||
|
scope,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(result.is_err());
|
||||||
|
assert!(!crate::ai_serving::response_history_is_loaded(
|
||||||
|
response_id,
|
||||||
|
Some(scope)
|
||||||
|
));
|
||||||
|
assert!(state
|
||||||
|
.runtime_kv_get(&storage_key)
|
||||||
|
.await
|
||||||
|
.expect("history lookup should succeed")
|
||||||
|
.is_none());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -62,7 +62,8 @@ pub(crate) use aether_provider_transport::{
|
|||||||
append_transport_diagnostics_to_value, apply_codex_fingerprint_convergence,
|
append_transport_diagnostics_to_value, apply_codex_fingerprint_convergence,
|
||||||
apply_codex_fingerprint_convergence_with_context, apply_local_auth_config_header_overrides,
|
apply_codex_fingerprint_convergence_with_context, apply_local_auth_config_header_overrides,
|
||||||
apply_local_body_rules, apply_local_body_rules_with_request_headers, apply_local_header_rules,
|
apply_local_body_rules, apply_local_body_rules_with_request_headers, apply_local_header_rules,
|
||||||
apply_local_header_rules_with_request_headers, apply_standard_provider_request_body_rules,
|
apply_local_header_rules_with_request_headers, apply_provider_outbound_request_policies,
|
||||||
|
apply_standard_provider_request_body_rules,
|
||||||
apply_standard_provider_request_body_rules_with_request_headers,
|
apply_standard_provider_request_body_rules_with_request_headers,
|
||||||
apply_transport_request_body_semantics, body_rules_are_locally_supported,
|
apply_transport_request_body_semantics, body_rules_are_locally_supported,
|
||||||
body_rules_handle_path, body_rules_have_enabled_rules,
|
body_rules_handle_path, body_rules_have_enabled_rules,
|
||||||
@@ -112,7 +113,11 @@ pub(crate) use aether_provider_transport::{
|
|||||||
GeminiCliRequestAuthSupport, GeminiCliRequestAuthUnsupportedReason,
|
GeminiCliRequestAuthSupport, GeminiCliRequestAuthUnsupportedReason,
|
||||||
GeminiCliRequestEnvelopeSupport, GeminiFilesHeadersInput, GeminiFilesRequestBodyError,
|
GeminiCliRequestEnvelopeSupport, GeminiFilesHeadersInput, GeminiFilesRequestBodyError,
|
||||||
GeminiFilesRequestBodyParts, GrokHeaderInput, LocalResolvedOAuthRequestAuth,
|
GeminiFilesRequestBodyParts, GrokHeaderInput, LocalResolvedOAuthRequestAuth,
|
||||||
ProviderOpenAiImageHeadersInput, ProviderVideoCreateFamily, ProviderVideoCreateHeadersInput,
|
ProviderOpenAiImageHeadersInput, ProviderOutboundRequestContext,
|
||||||
|
ProviderOutboundRequestIdentityScope, ProviderOutboundRequestMutationScope,
|
||||||
|
ProviderOutboundRequestPolicy, ProviderOutboundRequestPolicyOutcome,
|
||||||
|
ProviderOutboundRequestPolicyReason, ProviderOutboundRequestPolicyResult,
|
||||||
|
ProviderVideoCreateFamily, ProviderVideoCreateHeadersInput,
|
||||||
SameFormatProviderCompatibilityEdit, SameFormatProviderCompatibilityEditAction,
|
SameFormatProviderCompatibilityEdit, SameFormatProviderCompatibilityEditAction,
|
||||||
SameFormatProviderFamily, SameFormatProviderHeadersInput, SameFormatProviderRequestBehavior,
|
SameFormatProviderFamily, SameFormatProviderHeadersInput, SameFormatProviderRequestBehavior,
|
||||||
SameFormatProviderRequestBehaviorParams, SameFormatProviderRequestBodyInput,
|
SameFormatProviderRequestBehaviorParams, SameFormatProviderRequestBodyInput,
|
||||||
@@ -121,5 +126,5 @@ pub(crate) use aether_provider_transport::{
|
|||||||
StandardProviderRequestHeaders, StandardProviderRequestHeadersInput,
|
StandardProviderRequestHeaders, StandardProviderRequestHeadersInput,
|
||||||
TransportRequestBodySemanticsError, TransportRequestUrlParams, GEMINI_CLI_USER_AGENT,
|
TransportRequestBodySemanticsError, TransportRequestUrlParams, GEMINI_CLI_USER_AGENT,
|
||||||
GEMINI_CLI_V1INTERNAL_ENVELOPE_NAME, GROK_CHAT_PATH, GROK_INTERNAL_HEADER,
|
GEMINI_CLI_V1INTERNAL_ENVELOPE_NAME, GROK_CHAT_PATH, GROK_INTERNAL_HEADER,
|
||||||
GROK_RATE_LIMITS_PATH, WINDSURF_ENVELOPE_NAME,
|
GROK_RATE_LIMITS_PATH, PROVIDER_OUTBOUND_CONTEXT_MAX_VALUE_BYTES, WINDSURF_ENVELOPE_NAME,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -133,7 +133,7 @@ pub(crate) async fn frontdoor_manifest(State(state): State<AppState>) -> impl In
|
|||||||
"internal_gateway": {
|
"internal_gateway": {
|
||||||
"route_groups": INTERNAL_GATEWAY_ROUTE_GROUPS,
|
"route_groups": INTERNAL_GATEWAY_ROUTE_GROUPS,
|
||||||
"path_prefixes": INTERNAL_GATEWAY_PATH_PREFIXES,
|
"path_prefixes": INTERNAL_GATEWAY_PATH_PREFIXES,
|
||||||
"status": "rust_native_control_plane",
|
"status": state.internal_gateway_auth_status(),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"features": {
|
"features": {
|
||||||
|
|||||||
@@ -1,5 +1,14 @@
|
|||||||
|
use std::net::SocketAddr;
|
||||||
|
|
||||||
|
use axum::body::Body;
|
||||||
|
use axum::extract::{ConnectInfo, Request, State};
|
||||||
|
use axum::http::{self, HeaderValue, StatusCode};
|
||||||
|
use axum::middleware::{self, Next};
|
||||||
|
use axum::response::{IntoResponse, Response};
|
||||||
use axum::routing::{get, post};
|
use axum::routing::{get, post};
|
||||||
use axum::Router;
|
use axum::{Json, Router};
|
||||||
|
use serde_json::json;
|
||||||
|
use tracing::warn;
|
||||||
|
|
||||||
use crate::async_task::{
|
use crate::async_task::{
|
||||||
cancel_video_task, get_video_task_detail, get_video_task_stats, get_video_task_video,
|
cancel_video_task, get_video_task_detail, get_video_task_stats, get_video_task_video,
|
||||||
@@ -10,8 +19,18 @@ use crate::hooks::{get_request_audit_bundle, get_request_usage_audit};
|
|||||||
use crate::router::metrics;
|
use crate::router::metrics;
|
||||||
use crate::state::AppState;
|
use crate::state::AppState;
|
||||||
|
|
||||||
pub(crate) fn mount_operational_routes(router: Router<AppState>) -> Router<AppState> {
|
#[derive(Clone, Copy)]
|
||||||
router
|
struct OperationalPermission {
|
||||||
|
required_permissions: &'static [&'static str],
|
||||||
|
write: bool,
|
||||||
|
requires_full_admin_role: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn mount_operational_routes(
|
||||||
|
router: Router<AppState>,
|
||||||
|
state: AppState,
|
||||||
|
) -> Router<AppState> {
|
||||||
|
let operational = Router::<AppState>::new()
|
||||||
.route("/_gateway/metrics", get(metrics))
|
.route("/_gateway/metrics", get(metrics))
|
||||||
.route("/_gateway/async-tasks/video-tasks", get(list_video_tasks))
|
.route("/_gateway/async-tasks/video-tasks", get(list_video_tasks))
|
||||||
.route(
|
.route(
|
||||||
@@ -50,4 +69,236 @@ pub(crate) fn mount_operational_routes(router: Router<AppState>) -> Router<AppSt
|
|||||||
"/_gateway/audit/request-usage/{request_id}",
|
"/_gateway/audit/request-usage/{request_id}",
|
||||||
get(get_request_usage_audit),
|
get(get_request_usage_audit),
|
||||||
)
|
)
|
||||||
|
.route_layer(middleware::from_fn_with_state(
|
||||||
|
state,
|
||||||
|
authorize_operational_request,
|
||||||
|
));
|
||||||
|
router.merge(operational)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn authorize_operational_request(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
request: Request,
|
||||||
|
next: Next,
|
||||||
|
) -> Response<Body> {
|
||||||
|
let Some(permission) = operational_permission(request.method(), request.uri().path()) else {
|
||||||
|
return operational_error_response(
|
||||||
|
StatusCode::FORBIDDEN,
|
||||||
|
"operational route permission is not configured",
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
};
|
||||||
|
let Some(remote_addr) = request
|
||||||
|
.extensions()
|
||||||
|
.get::<ConnectInfo<SocketAddr>>()
|
||||||
|
.map(|value| value.0)
|
||||||
|
else {
|
||||||
|
return operational_error_response(
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE,
|
||||||
|
"operational authentication unavailable",
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
};
|
||||||
|
let headers = request.headers().clone();
|
||||||
|
let uri = request.uri().clone();
|
||||||
|
if headers.get_all(http::header::AUTHORIZATION).iter().count() > 1 {
|
||||||
|
return operational_auth_required_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
match crate::control::resolve_local_admin_session_principal(&state, &headers, &uri).await {
|
||||||
|
Ok(Some(principal)) => {
|
||||||
|
if permission.requires_full_admin_role
|
||||||
|
&& !crate::roles::is_full_admin_role(&principal.user_role)
|
||||||
|
{
|
||||||
|
return operational_permission_denied_response(permission.required_permissions[0]);
|
||||||
|
}
|
||||||
|
if permission.write && !crate::roles::can_write_admin_console(&principal.user_role) {
|
||||||
|
return operational_permission_denied_response(permission.required_permissions[0]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(None) => {
|
||||||
|
let client_ip = crate::headers::effective_client_ip(&headers, &remote_addr);
|
||||||
|
let authenticated = match crate::management_token_auth::authenticate_management_token(
|
||||||
|
&state, &headers, client_ip,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(authenticated) => authenticated,
|
||||||
|
Err(
|
||||||
|
crate::management_token_auth::ManagementTokenAuthError::Missing
|
||||||
|
| crate::management_token_auth::ManagementTokenAuthError::Invalid,
|
||||||
|
) => return operational_auth_required_response(),
|
||||||
|
Err(crate::management_token_auth::ManagementTokenAuthError::Unavailable) => {
|
||||||
|
return operational_error_response(
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE,
|
||||||
|
"operational authentication unavailable",
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if permission.requires_full_admin_role
|
||||||
|
&& !crate::roles::is_full_admin_role(&authenticated.user.role)
|
||||||
|
{
|
||||||
|
return operational_permission_denied_response(permission.required_permissions[0]);
|
||||||
|
}
|
||||||
|
if permission.write && !crate::roles::can_write_admin_console(&authenticated.user.role)
|
||||||
|
{
|
||||||
|
return operational_permission_denied_response(permission.required_permissions[0]);
|
||||||
|
}
|
||||||
|
let missing_permission =
|
||||||
|
permission
|
||||||
|
.required_permissions
|
||||||
|
.iter()
|
||||||
|
.copied()
|
||||||
|
.find(|required| {
|
||||||
|
!management_token_has_operational_permission(
|
||||||
|
&authenticated.permissions,
|
||||||
|
required,
|
||||||
|
)
|
||||||
|
});
|
||||||
|
if let Some(required_permission) = missing_permission {
|
||||||
|
return operational_permission_denied_response(required_permission);
|
||||||
|
}
|
||||||
|
|
||||||
|
let client_ip = client_ip.to_string();
|
||||||
|
if let Err(err) = state
|
||||||
|
.record_management_token_usage(&authenticated.token.id, Some(client_ip.as_str()))
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
warn!(
|
||||||
|
token_id = %authenticated.token.id,
|
||||||
|
error = ?err,
|
||||||
|
"gateway failed to record operational management token usage"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
warn!(error = ?err, "operational admin session authentication failed");
|
||||||
|
return operational_error_response(
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE,
|
||||||
|
"operational authentication unavailable",
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut response = next.run(request).await;
|
||||||
|
response.headers_mut().insert(
|
||||||
|
http::header::CACHE_CONTROL,
|
||||||
|
HeaderValue::from_static("no-store"),
|
||||||
|
);
|
||||||
|
response
|
||||||
|
}
|
||||||
|
|
||||||
|
fn operational_permission(method: &http::Method, path: &str) -> Option<OperationalPermission> {
|
||||||
|
if path == "/_gateway/metrics" {
|
||||||
|
return Some(OperationalPermission {
|
||||||
|
required_permissions: &["admin:monitoring:read"],
|
||||||
|
write: false,
|
||||||
|
requires_full_admin_role: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if path.starts_with("/_gateway/async-tasks/video-tasks") {
|
||||||
|
let write = *method == http::Method::POST && path.ends_with("/cancel");
|
||||||
|
return Some(OperationalPermission {
|
||||||
|
required_permissions: if write {
|
||||||
|
&["admin:video_tasks:write"]
|
||||||
|
} else {
|
||||||
|
&["admin:video_tasks:read"]
|
||||||
|
},
|
||||||
|
write,
|
||||||
|
requires_full_admin_role: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if path.starts_with("/_gateway/audit/auth/users/") {
|
||||||
|
return Some(OperationalPermission {
|
||||||
|
required_permissions: &["admin:api_keys:read"],
|
||||||
|
write: false,
|
||||||
|
requires_full_admin_role: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if path.starts_with("/_gateway/audit/request-audit/") {
|
||||||
|
return Some(OperationalPermission {
|
||||||
|
required_permissions: &[
|
||||||
|
"admin:monitoring:admin",
|
||||||
|
"admin:usage:read",
|
||||||
|
"admin:api_keys:read",
|
||||||
|
],
|
||||||
|
write: false,
|
||||||
|
requires_full_admin_role: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if path.starts_with("/_gateway/audit/request-candidates/")
|
||||||
|
|| path.starts_with("/_gateway/audit/decision-trace/")
|
||||||
|
{
|
||||||
|
return Some(OperationalPermission {
|
||||||
|
required_permissions: &["admin:monitoring:admin"],
|
||||||
|
write: false,
|
||||||
|
requires_full_admin_role: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if path.starts_with("/_gateway/audit/") {
|
||||||
|
return Some(OperationalPermission {
|
||||||
|
required_permissions: &["admin:usage:read"],
|
||||||
|
write: false,
|
||||||
|
requires_full_admin_role: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
fn management_token_has_operational_permission(
|
||||||
|
permissions: &[String],
|
||||||
|
required_permission: &str,
|
||||||
|
) -> bool {
|
||||||
|
let scope = required_permission
|
||||||
|
.rsplit_once(':')
|
||||||
|
.map(|(scope, _)| scope)
|
||||||
|
.unwrap_or(required_permission);
|
||||||
|
let admin_permission = format!("{scope}:admin");
|
||||||
|
permissions
|
||||||
|
.iter()
|
||||||
|
.any(|permission| permission == required_permission || permission == &admin_permission)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn operational_auth_required_response() -> Response<Body> {
|
||||||
|
let mut response = operational_error_response(
|
||||||
|
StatusCode::UNAUTHORIZED,
|
||||||
|
"admin authentication required",
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
response.headers_mut().insert(
|
||||||
|
http::header::WWW_AUTHENTICATE,
|
||||||
|
HeaderValue::from_static("Bearer"),
|
||||||
|
);
|
||||||
|
response
|
||||||
|
}
|
||||||
|
|
||||||
|
fn operational_permission_denied_response(required_permission: &'static str) -> Response<Body> {
|
||||||
|
operational_error_response(
|
||||||
|
StatusCode::FORBIDDEN,
|
||||||
|
"operational permission denied",
|
||||||
|
Some(required_permission),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn operational_error_response(
|
||||||
|
status: StatusCode,
|
||||||
|
detail: &'static str,
|
||||||
|
required_permission: Option<&'static str>,
|
||||||
|
) -> Response<Body> {
|
||||||
|
let mut response = (
|
||||||
|
status,
|
||||||
|
Json(json!({
|
||||||
|
"detail": detail,
|
||||||
|
"required_permission": required_permission,
|
||||||
|
})),
|
||||||
|
)
|
||||||
|
.into_response();
|
||||||
|
response.headers_mut().insert(
|
||||||
|
http::header::CACHE_CONTROL,
|
||||||
|
HeaderValue::from_static("no-store"),
|
||||||
|
);
|
||||||
|
response
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ use crate::constants::*;
|
|||||||
use crate::control::GatewayControlDecision;
|
use crate::control::GatewayControlDecision;
|
||||||
use crate::control::GatewayLocalAuthRejection;
|
use crate::control::GatewayLocalAuthRejection;
|
||||||
use crate::headers::should_skip_response_header;
|
use crate::headers::should_skip_response_header;
|
||||||
|
use crate::plan_usage_policy::PlanUsagePolicyRejection;
|
||||||
use crate::rate_limit::FrontdoorUserRpmRejection;
|
use crate::rate_limit::FrontdoorUserRpmRejection;
|
||||||
use crate::{insert_header_if_missing, GatewayError};
|
use crate::{insert_header_if_missing, GatewayError};
|
||||||
|
|
||||||
@@ -52,22 +53,34 @@ pub(crate) fn apply_streaming_response_headers(headers: &mut http::HeaderMap) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn apply_gateway_browser_security_headers(headers: &mut http::HeaderMap) {
|
||||||
|
// Provider responses are API data, even when an untrusted provider labels
|
||||||
|
// them as HTML or SVG. Keep a direct navigation to a gateway API route
|
||||||
|
// from becoming same-origin active content, and prevent referrer leakage
|
||||||
|
// if a user follows a link rendered from such a response.
|
||||||
|
headers.insert(
|
||||||
|
http::header::X_CONTENT_TYPE_OPTIONS,
|
||||||
|
HeaderValue::from_static("nosniff"),
|
||||||
|
);
|
||||||
|
headers.insert(
|
||||||
|
HeaderName::from_static("content-security-policy"),
|
||||||
|
HeaderValue::from_static(
|
||||||
|
"default-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'; sandbox",
|
||||||
|
),
|
||||||
|
);
|
||||||
|
headers.insert(
|
||||||
|
HeaderName::from_static("referrer-policy"),
|
||||||
|
HeaderValue::from_static("no-referrer"),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn build_client_response(
|
pub(crate) fn build_client_response(
|
||||||
upstream_response: reqwest::Response,
|
upstream_response: reqwest::Response,
|
||||||
trace_id: &str,
|
trace_id: &str,
|
||||||
control_decision: Option<&GatewayControlDecision>,
|
control_decision: Option<&GatewayControlDecision>,
|
||||||
) -> Result<Response<Body>, GatewayError> {
|
) -> Result<Response<Body>, GatewayError> {
|
||||||
let status = upstream_response.status();
|
let status = upstream_response.status();
|
||||||
let upstream_headers = upstream_response
|
let upstream_headers = collect_safe_response_headers(upstream_response.headers());
|
||||||
.headers()
|
|
||||||
.iter()
|
|
||||||
.map(|(name, value)| {
|
|
||||||
(
|
|
||||||
name.as_str().to_string(),
|
|
||||||
value.to_str().unwrap_or_default().to_string(),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.collect::<BTreeMap<_, _>>();
|
|
||||||
let upstream_stream = upstream_response.bytes_stream();
|
let upstream_stream = upstream_response.bytes_stream();
|
||||||
build_client_response_from_parts(
|
build_client_response_from_parts(
|
||||||
status.as_u16(),
|
status.as_u16(),
|
||||||
@@ -78,6 +91,40 @@ pub(crate) fn build_client_response(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn collect_safe_response_headers(headers: &http::HeaderMap) -> BTreeMap<String, String> {
|
||||||
|
let connection_declared = aether_http::connection_declared_header_names(
|
||||||
|
headers
|
||||||
|
.get_all(http::header::CONNECTION)
|
||||||
|
.iter()
|
||||||
|
.filter_map(|value| value.to_str().ok()),
|
||||||
|
);
|
||||||
|
headers
|
||||||
|
.iter()
|
||||||
|
.filter_map(|(name, value)| {
|
||||||
|
let normalized = name.as_str().to_ascii_lowercase();
|
||||||
|
if should_skip_client_response_header(&normalized)
|
||||||
|
|| connection_declared.contains(&normalized)
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
value
|
||||||
|
.to_str()
|
||||||
|
.ok()
|
||||||
|
.map(|value| (normalized, value.to_string()))
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn should_skip_client_response_header(name: &str) -> bool {
|
||||||
|
should_skip_response_header(name)
|
||||||
|
// A provider Location is relative to the provider, not to the gateway.
|
||||||
|
// Forwarding it lets redirect-following clients bypass the gateway and
|
||||||
|
// can disclose their gateway Authorization header to another origin.
|
||||||
|
// Keep Location available inside execution reports, but never expose
|
||||||
|
// it on the client-facing response boundary.
|
||||||
|
|| name.eq_ignore_ascii_case(http::header::LOCATION.as_str())
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn build_client_response_from_parts(
|
pub(crate) fn build_client_response_from_parts(
|
||||||
status_code: u16,
|
status_code: u16,
|
||||||
upstream_headers: &BTreeMap<String, String>,
|
upstream_headers: &BTreeMap<String, String>,
|
||||||
@@ -111,8 +158,17 @@ where
|
|||||||
.body(body)
|
.body(body)
|
||||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||||
|
|
||||||
|
let connection_declared = aether_http::connection_declared_header_names(
|
||||||
|
upstream_headers
|
||||||
|
.iter()
|
||||||
|
.filter(|(name, _)| name.eq_ignore_ascii_case(http::header::CONNECTION.as_str()))
|
||||||
|
.map(|(_, value)| value.as_str()),
|
||||||
|
);
|
||||||
|
|
||||||
for (name, value) in upstream_headers {
|
for (name, value) in upstream_headers {
|
||||||
if should_skip_response_header(name.as_str()) {
|
if should_skip_client_response_header(name.as_str())
|
||||||
|
|| connection_declared.contains(&name.to_ascii_lowercase())
|
||||||
|
{
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let header_name = HeaderName::from_bytes(name.as_bytes())
|
let header_name = HeaderName::from_bytes(name.as_bytes())
|
||||||
@@ -123,6 +179,7 @@ where
|
|||||||
}
|
}
|
||||||
mutate_headers(response.headers_mut())?;
|
mutate_headers(response.headers_mut())?;
|
||||||
apply_streaming_response_headers(response.headers_mut());
|
apply_streaming_response_headers(response.headers_mut());
|
||||||
|
apply_gateway_browser_security_headers(response.headers_mut());
|
||||||
insert_header_if_missing(response.headers_mut(), TRACE_ID_HEADER, trace_id)?;
|
insert_header_if_missing(response.headers_mut(), TRACE_ID_HEADER, trace_id)?;
|
||||||
insert_header_if_missing(response.headers_mut(), GATEWAY_HEADER, "rust-phase3b")?;
|
insert_header_if_missing(response.headers_mut(), GATEWAY_HEADER, "rust-phase3b")?;
|
||||||
if let Some(decision) = control_decision {
|
if let Some(decision) = control_decision {
|
||||||
@@ -258,6 +315,57 @@ pub(crate) fn build_local_user_rpm_limited_response(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn build_local_plan_usage_limited_response(
|
||||||
|
trace_id: &str,
|
||||||
|
control_decision: Option<&GatewayControlDecision>,
|
||||||
|
rejection: &PlanUsagePolicyRejection,
|
||||||
|
) -> Result<Response<Body>, GatewayError> {
|
||||||
|
let message = "套餐使用限制已达到上限,请稍后重试";
|
||||||
|
let fallback_payload = json!({
|
||||||
|
"error": {
|
||||||
|
"type": "plan_usage_limit_exceeded",
|
||||||
|
"message": message,
|
||||||
|
"details": {
|
||||||
|
"metric": rejection.metric,
|
||||||
|
"window": rejection.window,
|
||||||
|
"limit": rejection.limit,
|
||||||
|
"retry_after": rejection.retry_after,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
let payload = build_local_error_payload(
|
||||||
|
control_decision,
|
||||||
|
None,
|
||||||
|
message,
|
||||||
|
LocalCoreSyncErrorKind::RateLimit,
|
||||||
|
fallback_payload,
|
||||||
|
);
|
||||||
|
let body =
|
||||||
|
serde_json::to_vec(&payload).map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||||
|
let headers = BTreeMap::from([
|
||||||
|
("content-type".to_string(), "application/json".to_string()),
|
||||||
|
("Retry-After".to_string(), rejection.retry_after.to_string()),
|
||||||
|
("X-RateLimit-Limit".to_string(), rejection.limit.to_string()),
|
||||||
|
("X-RateLimit-Remaining".to_string(), "0".to_string()),
|
||||||
|
("X-RateLimit-Scope".to_string(), "plan".to_string()),
|
||||||
|
(
|
||||||
|
"X-RateLimit-Metric".to_string(),
|
||||||
|
rejection.metric.to_string(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"X-RateLimit-Window".to_string(),
|
||||||
|
rejection.window.to_string(),
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
build_client_response_from_parts(
|
||||||
|
StatusCode::TOO_MANY_REQUESTS.as_u16(),
|
||||||
|
&headers,
|
||||||
|
Body::from(body),
|
||||||
|
trace_id,
|
||||||
|
control_decision,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn build_local_http_error_response(
|
pub(crate) fn build_local_http_error_response(
|
||||||
trace_id: &str,
|
trace_id: &str,
|
||||||
control_decision: Option<&GatewayControlDecision>,
|
control_decision: Option<&GatewayControlDecision>,
|
||||||
@@ -454,11 +562,13 @@ fn local_error_kind_for_status(status: StatusCode) -> LocalCoreSyncErrorKind {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{
|
use super::{
|
||||||
build_client_response_from_parts, build_local_auth_rejection_response,
|
build_client_response, build_client_response_from_parts,
|
||||||
|
build_client_response_from_parts_with_mutator, build_local_auth_rejection_response,
|
||||||
build_local_http_error_response_with_request_path, build_local_overloaded_response,
|
build_local_http_error_response_with_request_path, build_local_overloaded_response,
|
||||||
build_local_user_rpm_limited_response,
|
build_local_plan_usage_limited_response, build_local_user_rpm_limited_response,
|
||||||
};
|
};
|
||||||
use crate::control::{GatewayControlDecision, GatewayLocalAuthRejection};
|
use crate::control::{GatewayControlDecision, GatewayLocalAuthRejection};
|
||||||
|
use crate::plan_usage_policy::PlanUsagePolicyRejection;
|
||||||
use crate::rate_limit::FrontdoorUserRpmRejection;
|
use crate::rate_limit::FrontdoorUserRpmRejection;
|
||||||
use axum::body::{to_bytes, Body};
|
use axum::body::{to_bytes, Body};
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
@@ -490,6 +600,163 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn upstream_security_headers_are_stripped_before_gateway_headers_are_added() {
|
||||||
|
let response = build_client_response_from_parts_with_mutator(
|
||||||
|
200,
|
||||||
|
&BTreeMap::from([
|
||||||
|
("set-cookie".to_string(), "session=attacker".to_string()),
|
||||||
|
(
|
||||||
|
"x-aether-gateway".to_string(),
|
||||||
|
"attacker-gateway".to_string(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"x-aether-control-action".to_string(),
|
||||||
|
"attacker-action".to_string(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"x-aether-future-control".to_string(),
|
||||||
|
"attacker-future".to_string(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"x-accel-redirect".to_string(),
|
||||||
|
"/internal/private-file".to_string(),
|
||||||
|
),
|
||||||
|
("x-sendfile".to_string(), "/etc/passwd".to_string()),
|
||||||
|
(
|
||||||
|
"x-reproxy-url".to_string(),
|
||||||
|
"http://127.0.0.1:9000/private".to_string(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"access-control-allow-origin".to_string(),
|
||||||
|
"https://attacker.example".to_string(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"access-control-allow-credentials".to_string(),
|
||||||
|
"true".to_string(),
|
||||||
|
),
|
||||||
|
("content-length".to_string(), "999999".to_string()),
|
||||||
|
(
|
||||||
|
"content-security-policy".to_string(),
|
||||||
|
"default-src * 'unsafe-inline' 'unsafe-eval'".to_string(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"content-security-policy-report-only".to_string(),
|
||||||
|
"default-src 'none'; report-uri https://attacker.example/csp".to_string(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"reporting-endpoints".to_string(),
|
||||||
|
"attacker=\"https://attacker.example/reports\"".to_string(),
|
||||||
|
),
|
||||||
|
("report-to".to_string(), "attacker".to_string()),
|
||||||
|
(
|
||||||
|
"nel".to_string(),
|
||||||
|
"{\"report_to\":\"attacker\"}".to_string(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"refresh".to_string(),
|
||||||
|
"0; url=https://attacker.example".to_string(),
|
||||||
|
),
|
||||||
|
("referrer-policy".to_string(), "unsafe-url".to_string()),
|
||||||
|
("x-content-type-options".to_string(), "invalid".to_string()),
|
||||||
|
(
|
||||||
|
"location".to_string(),
|
||||||
|
"https://provider.example/direct".to_string(),
|
||||||
|
),
|
||||||
|
("x-upstream-visible".to_string(), "ok".to_string()),
|
||||||
|
]),
|
||||||
|
Body::empty(),
|
||||||
|
"trace-upstream-header-filter",
|
||||||
|
None,
|
||||||
|
|headers| {
|
||||||
|
headers.insert(
|
||||||
|
http::HeaderName::from_static("x-aether-control-action"),
|
||||||
|
http::HeaderValue::from_static("gateway-action"),
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.expect("response should build");
|
||||||
|
|
||||||
|
assert!(response.headers().get(http::header::SET_COOKIE).is_none());
|
||||||
|
assert!(response.headers().get("x-aether-future-control").is_none());
|
||||||
|
assert!(response.headers().get("x-accel-redirect").is_none());
|
||||||
|
assert!(response.headers().get("x-sendfile").is_none());
|
||||||
|
assert!(response.headers().get("x-reproxy-url").is_none());
|
||||||
|
assert!(response
|
||||||
|
.headers()
|
||||||
|
.get("access-control-allow-origin")
|
||||||
|
.is_none());
|
||||||
|
assert!(response
|
||||||
|
.headers()
|
||||||
|
.get("access-control-allow-credentials")
|
||||||
|
.is_none());
|
||||||
|
assert!(response
|
||||||
|
.headers()
|
||||||
|
.get(http::header::CONTENT_LENGTH)
|
||||||
|
.is_none());
|
||||||
|
assert!(response
|
||||||
|
.headers()
|
||||||
|
.get("content-security-policy-report-only")
|
||||||
|
.is_none());
|
||||||
|
assert!(response.headers().get("reporting-endpoints").is_none());
|
||||||
|
assert!(response.headers().get("report-to").is_none());
|
||||||
|
assert!(response.headers().get("nel").is_none());
|
||||||
|
assert!(response.headers().get("refresh").is_none());
|
||||||
|
assert!(response.headers().get(http::header::LOCATION).is_none());
|
||||||
|
assert_eq!(
|
||||||
|
response.headers()["content-security-policy"],
|
||||||
|
"default-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'; sandbox"
|
||||||
|
);
|
||||||
|
assert_eq!(response.headers()["referrer-policy"], "no-referrer");
|
||||||
|
assert_eq!(
|
||||||
|
response.headers()[http::header::X_CONTENT_TYPE_OPTIONS],
|
||||||
|
"nosniff"
|
||||||
|
);
|
||||||
|
assert_eq!(response.headers()["x-aether-gateway"], "rust-phase3b");
|
||||||
|
assert_eq!(
|
||||||
|
response.headers()["x-aether-control-action"],
|
||||||
|
"gateway-action"
|
||||||
|
);
|
||||||
|
assert_eq!(response.headers()["x-upstream-visible"], "ok");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn raw_response_collector_honors_all_connection_header_lines() {
|
||||||
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||||
|
.await
|
||||||
|
.expect("listener");
|
||||||
|
let addr = listener.local_addr().expect("address");
|
||||||
|
let server = tokio::spawn(async move {
|
||||||
|
let (mut stream, _) = listener.accept().await.expect("connection");
|
||||||
|
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
|
||||||
|
let mut request = [0_u8; 1024];
|
||||||
|
let _ = stream.read(&mut request).await.expect("request read");
|
||||||
|
stream
|
||||||
|
.write_all(
|
||||||
|
b"HTTP/1.1 200 OK\r\nConnection: x-first-hop\r\nConnection: x-second-hop\r\nX-First-Hop: first-secret\r\nX-Second-Hop: second-secret\r\nContent-Length: 2\r\n\r\nok",
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("response write");
|
||||||
|
});
|
||||||
|
let upstream = reqwest::Client::builder()
|
||||||
|
.redirect(reqwest::redirect::Policy::none())
|
||||||
|
.build()
|
||||||
|
.expect("client")
|
||||||
|
.get(format!("http://{addr}/"))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.expect("upstream response");
|
||||||
|
|
||||||
|
let response = build_client_response(upstream, "trace-connection-lines", None)
|
||||||
|
.expect("client response");
|
||||||
|
server.await.expect("server");
|
||||||
|
|
||||||
|
assert!(response.headers().get("connection").is_none());
|
||||||
|
assert!(response.headers().get("x-first-hop").is_none());
|
||||||
|
assert!(response.headers().get("x-second-hop").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
fn claude_decision() -> GatewayControlDecision {
|
fn claude_decision() -> GatewayControlDecision {
|
||||||
GatewayControlDecision::synthetic(
|
GatewayControlDecision::synthetic(
|
||||||
"/v1/messages",
|
"/v1/messages",
|
||||||
@@ -581,4 +848,25 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn plan_usage_rejection_exposes_machine_readable_limit_headers() {
|
||||||
|
let response = build_local_plan_usage_limited_response(
|
||||||
|
"trace-plan-limit",
|
||||||
|
None,
|
||||||
|
&PlanUsagePolicyRejection {
|
||||||
|
metric: "request_count",
|
||||||
|
limit: 100.0,
|
||||||
|
retry_after: 42,
|
||||||
|
window: "calendar_week",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.expect("response");
|
||||||
|
assert_eq!(response.status(), http::StatusCode::TOO_MANY_REQUESTS);
|
||||||
|
assert_eq!(response.headers()["retry-after"], "42");
|
||||||
|
assert_eq!(response.headers()["x-ratelimit-scope"], "plan");
|
||||||
|
assert_eq!(response.headers()["x-ratelimit-window"], "calendar_week");
|
||||||
|
let payload = response_json(response).await;
|
||||||
|
assert_eq!(payload["error"]["type"], "plan_usage_limit_exceeded");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
use std::net::{IpAddr, SocketAddr};
|
||||||
use std::time::{SystemTime, UNIX_EPOCH};
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
use aether_contracts::ExecutionResult;
|
use aether_contracts::ExecutionResult;
|
||||||
@@ -21,7 +22,9 @@ use super::{
|
|||||||
};
|
};
|
||||||
use crate::{AppState, GatewayError};
|
use crate::{AppState, GatewayError};
|
||||||
|
|
||||||
pub(crate) use self::cancel::{cancel_video_task_record, CancelVideoTaskError};
|
pub(crate) use self::cancel::{
|
||||||
|
cancel_video_task_record, cancel_video_task_record_for_user, CancelVideoTaskError,
|
||||||
|
};
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
pub(crate) struct ListVideoTasksQuery {
|
pub(crate) struct ListVideoTasksQuery {
|
||||||
@@ -146,18 +149,21 @@ pub(crate) async fn get_video_task_video(
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn build_video_task_video_response(
|
pub(crate) async fn build_video_task_video_response(
|
||||||
state: &AppState,
|
_state: &AppState,
|
||||||
task_id: &str,
|
task_id: &str,
|
||||||
source: VideoTaskVideoSource,
|
source: VideoTaskVideoSource,
|
||||||
) -> Result<axum::response::Response, GatewayError> {
|
) -> Result<axum::response::Response, GatewayError> {
|
||||||
match source {
|
match source {
|
||||||
VideoTaskVideoSource::Redirect { url } => Ok(Redirect::temporary(&url).into_response()),
|
VideoTaskVideoSource::Redirect { url } => {
|
||||||
|
resolve_public_video_target(&url).await?;
|
||||||
|
Ok(Redirect::temporary(url.as_str()).into_response())
|
||||||
|
}
|
||||||
VideoTaskVideoSource::Proxy {
|
VideoTaskVideoSource::Proxy {
|
||||||
url,
|
url,
|
||||||
header_name,
|
header_name,
|
||||||
header_value,
|
header_value,
|
||||||
filename,
|
filename,
|
||||||
} => proxy_video_stream(state, task_id, &url, &header_name, &header_value, &filename).await,
|
} => proxy_video_stream(task_id, &url, &header_name, &header_value, &filename).await,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -209,25 +215,34 @@ fn video_task_status_name(status: VideoTaskStatus) -> &'static str {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn proxy_video_stream(
|
async fn proxy_video_stream(
|
||||||
state: &AppState,
|
|
||||||
task_id: &str,
|
task_id: &str,
|
||||||
url: &str,
|
url: &url::Url,
|
||||||
header_name: &str,
|
header_name: &str,
|
||||||
header_value: &str,
|
header_value: &str,
|
||||||
filename: &str,
|
filename: &str,
|
||||||
) -> Result<axum::response::Response, GatewayError> {
|
) -> Result<axum::response::Response, GatewayError> {
|
||||||
let response = state
|
let target = resolve_public_video_target(url).await?;
|
||||||
.client
|
let client = build_pinned_video_client(&target)?;
|
||||||
.get(url)
|
let response = client
|
||||||
|
.get(url.clone())
|
||||||
.header(header_name, header_value)
|
.header(header_name, header_value)
|
||||||
.send()
|
.send()
|
||||||
.await
|
.await
|
||||||
.map_err(|err| GatewayError::UpstreamUnavailable {
|
.map_err(|err| GatewayError::UpstreamUnavailable {
|
||||||
trace_id: task_id.to_string(),
|
trace_id: task_id.to_string(),
|
||||||
message: err.to_string(),
|
message: video_request_failure_message(&err).to_string(),
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
if response.status().is_client_error() || response.status().is_server_error() {
|
if response.status().is_redirection() {
|
||||||
|
return Err(GatewayError::UpstreamUnavailable {
|
||||||
|
trace_id: task_id.to_string(),
|
||||||
|
message: format!(
|
||||||
|
"video upstream redirect was rejected with HTTP {}",
|
||||||
|
response.status()
|
||||||
|
),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if !response.status().is_success() {
|
||||||
return Err(GatewayError::UpstreamUnavailable {
|
return Err(GatewayError::UpstreamUnavailable {
|
||||||
trace_id: task_id.to_string(),
|
trace_id: task_id.to_string(),
|
||||||
message: format!("video upstream returned HTTP {}", response.status()),
|
message: format!("video upstream returned HTTP {}", response.status()),
|
||||||
@@ -235,47 +250,321 @@ async fn proxy_video_stream(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let status = response.status();
|
let status = response.status();
|
||||||
let content_type = response
|
// Do not copy the provider's Content-Length onto a newly wrapped stream.
|
||||||
.headers()
|
// Reqwest may decode transfer/content encodings and the provider controls
|
||||||
.get(axum::http::header::CONTENT_TYPE)
|
// the declaration; forwarding a stale value would make the client-facing
|
||||||
.cloned()
|
// HTTP framing disagree with the bytes produced by this Body. Axum/Hyper
|
||||||
.unwrap_or_else(|| axum::http::HeaderValue::from_static("video/mp4"));
|
// will select safe framing for the actual stream.
|
||||||
let content_length = response
|
let upstream_headers = response.headers().clone();
|
||||||
.headers()
|
|
||||||
.get(axum::http::header::CONTENT_LENGTH)
|
|
||||||
.cloned();
|
|
||||||
let cache_control = response
|
|
||||||
.headers()
|
|
||||||
.get(axum::http::header::CACHE_CONTROL)
|
|
||||||
.cloned();
|
|
||||||
let body = Body::from_stream(response.bytes_stream());
|
let body = Body::from_stream(response.bytes_stream());
|
||||||
|
|
||||||
let mut outbound = axum::http::Response::builder()
|
let mut outbound = axum::http::Response::builder()
|
||||||
.status(status)
|
.status(status)
|
||||||
.body(body)
|
.body(body)
|
||||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||||
outbound
|
apply_safe_video_response_metadata(outbound.headers_mut(), &upstream_headers, filename)?;
|
||||||
.headers_mut()
|
|
||||||
.insert(axum::http::header::CONTENT_TYPE, content_type);
|
|
||||||
outbound.headers_mut().insert(
|
|
||||||
axum::http::header::CONTENT_DISPOSITION,
|
|
||||||
axum::http::HeaderValue::from_str(&format!("inline; filename=\"{filename}\""))
|
|
||||||
.map_err(|err| GatewayError::Internal(err.to_string()))?,
|
|
||||||
);
|
|
||||||
if let Some(content_length) = content_length {
|
|
||||||
outbound
|
|
||||||
.headers_mut()
|
|
||||||
.insert(axum::http::header::CONTENT_LENGTH, content_length);
|
|
||||||
}
|
|
||||||
if let Some(cache_control) = cache_control {
|
|
||||||
outbound
|
|
||||||
.headers_mut()
|
|
||||||
.insert(axum::http::header::CACHE_CONTROL, cache_control);
|
|
||||||
} else {
|
|
||||||
outbound.headers_mut().insert(
|
|
||||||
axum::http::header::CACHE_CONTROL,
|
|
||||||
axum::http::HeaderValue::from_static("private, max-age=3600"),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
Ok(outbound)
|
Ok(outbound)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn apply_safe_video_response_metadata(
|
||||||
|
outbound: &mut axum::http::HeaderMap,
|
||||||
|
upstream: &axum::http::HeaderMap,
|
||||||
|
filename: &str,
|
||||||
|
) -> Result<(), GatewayError> {
|
||||||
|
let content_type = upstream
|
||||||
|
.get(axum::http::header::CONTENT_TYPE)
|
||||||
|
.and_then(|value| value.to_str().ok())
|
||||||
|
.and_then(safe_video_content_type)
|
||||||
|
.unwrap_or_else(|| axum::http::HeaderValue::from_static("application/octet-stream"));
|
||||||
|
outbound.insert(axum::http::header::CONTENT_TYPE, content_type);
|
||||||
|
outbound.insert(
|
||||||
|
axum::http::header::CONTENT_DISPOSITION,
|
||||||
|
axum::http::HeaderValue::from_str(&format!(
|
||||||
|
"inline; filename=\"{}\"",
|
||||||
|
safe_video_filename(filename)
|
||||||
|
))
|
||||||
|
.map_err(|err| GatewayError::Internal(err.to_string()))?,
|
||||||
|
);
|
||||||
|
outbound.remove(axum::http::header::CONTENT_LENGTH);
|
||||||
|
outbound.insert(
|
||||||
|
axum::http::header::CACHE_CONTROL,
|
||||||
|
axum::http::HeaderValue::from_static("private, no-store"),
|
||||||
|
);
|
||||||
|
outbound.insert(
|
||||||
|
axum::http::header::X_CONTENT_TYPE_OPTIONS,
|
||||||
|
axum::http::HeaderValue::from_static("nosniff"),
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn safe_video_content_type(raw_value: &str) -> Option<axum::http::HeaderValue> {
|
||||||
|
let media_type = raw_value.split(';').next()?.trim().to_ascii_lowercase();
|
||||||
|
let subtype = media_type.strip_prefix("video/")?;
|
||||||
|
if subtype.is_empty()
|
||||||
|
|| !subtype.bytes().all(|byte| {
|
||||||
|
byte.is_ascii_alphanumeric()
|
||||||
|
|| matches!(
|
||||||
|
byte,
|
||||||
|
b'!' | b'#' | b'$' | b'&' | b'-' | b'^' | b'_' | b'.' | b'+'
|
||||||
|
)
|
||||||
|
})
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
axum::http::HeaderValue::from_str(raw_value).ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn safe_video_filename(filename: &str) -> String {
|
||||||
|
let filename = filename
|
||||||
|
.chars()
|
||||||
|
.take(255)
|
||||||
|
.map(|character| {
|
||||||
|
if character.is_ascii_alphanumeric() || matches!(character, '.' | '-' | '_') {
|
||||||
|
character
|
||||||
|
} else {
|
||||||
|
'_'
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect::<String>();
|
||||||
|
if filename.is_empty() {
|
||||||
|
"video.mp4".to_string()
|
||||||
|
} else {
|
||||||
|
filename
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ResolvedVideoTarget {
|
||||||
|
host: String,
|
||||||
|
addrs: Vec<SocketAddr>,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn resolve_public_video_target(url: &url::Url) -> Result<ResolvedVideoTarget, GatewayError> {
|
||||||
|
if !matches!(url.scheme(), "http" | "https")
|
||||||
|
|| !url.username().is_empty()
|
||||||
|
|| url.password().is_some()
|
||||||
|
{
|
||||||
|
return Err(video_target_rejected(
|
||||||
|
"video URL must be an absolute HTTP(S) URL without credentials",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let port = url
|
||||||
|
.port_or_known_default()
|
||||||
|
.ok_or_else(|| video_target_rejected("video URL is missing a port"))?;
|
||||||
|
let (host, addrs) = match url.host() {
|
||||||
|
Some(url::Host::Ipv4(ip)) => (ip.to_string(), vec![SocketAddr::new(IpAddr::V4(ip), port)]),
|
||||||
|
Some(url::Host::Ipv6(ip)) => (ip.to_string(), vec![SocketAddr::new(IpAddr::V6(ip), port)]),
|
||||||
|
Some(url::Host::Domain(host)) if !host.is_empty() => {
|
||||||
|
let addrs = aether_http::lookup_host_with_limits(
|
||||||
|
host,
|
||||||
|
port,
|
||||||
|
aether_http::DEFAULT_DNS_LOOKUP_TIMEOUT,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|_| video_target_rejected("video URL DNS resolution failed"))?;
|
||||||
|
(host.to_string(), addrs)
|
||||||
|
}
|
||||||
|
_ => return Err(video_target_rejected("video URL is missing a host")),
|
||||||
|
};
|
||||||
|
if addrs.is_empty()
|
||||||
|
|| addrs
|
||||||
|
.iter()
|
||||||
|
.any(|addr| aether_http::is_private_or_reserved_ip(addr.ip()))
|
||||||
|
{
|
||||||
|
return Err(video_target_rejected(
|
||||||
|
"video URL resolves to a private or reserved address",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(ResolvedVideoTarget { host, addrs })
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_pinned_video_client(
|
||||||
|
target: &ResolvedVideoTarget,
|
||||||
|
) -> Result<reqwest::Client, GatewayError> {
|
||||||
|
let mut builder = aether_http::apply_http_client_config(
|
||||||
|
reqwest::Client::builder()
|
||||||
|
.no_proxy()
|
||||||
|
.redirect(reqwest::redirect::Policy::none()),
|
||||||
|
&aether_http::HttpClientConfig {
|
||||||
|
connect_timeout_ms: Some(10_000),
|
||||||
|
request_timeout_ms: Some(300_000),
|
||||||
|
http2_adaptive_window: true,
|
||||||
|
..aether_http::HttpClientConfig::default()
|
||||||
|
},
|
||||||
|
);
|
||||||
|
if target.host.parse::<IpAddr>().is_err() {
|
||||||
|
builder = builder.resolve_to_addrs(&target.host, &target.addrs);
|
||||||
|
}
|
||||||
|
builder
|
||||||
|
.build()
|
||||||
|
.map_err(|_| GatewayError::Internal("video HTTP client initialization failed".to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn video_target_rejected(message: &str) -> GatewayError {
|
||||||
|
GatewayError::Client {
|
||||||
|
status: axum::http::StatusCode::BAD_GATEWAY,
|
||||||
|
message: message.to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn video_request_failure_message(error: &reqwest::Error) -> &'static str {
|
||||||
|
if error.is_timeout() {
|
||||||
|
"video upstream request timed out"
|
||||||
|
} else if error.is_connect() {
|
||||||
|
"video upstream connection failed"
|
||||||
|
} else if error.is_body() || error.is_decode() {
|
||||||
|
"video upstream response failed"
|
||||||
|
} else {
|
||||||
|
"video upstream request failed"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use axum::response::IntoResponse;
|
||||||
|
|
||||||
|
use super::{
|
||||||
|
apply_safe_video_response_metadata, build_video_task_video_response,
|
||||||
|
resolve_public_video_target, safe_video_content_type, safe_video_filename,
|
||||||
|
VideoTaskVideoSource,
|
||||||
|
};
|
||||||
|
use crate::AppState;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn video_redirect_response_accepts_public_target() {
|
||||||
|
let state = AppState::new().expect("gateway state should build");
|
||||||
|
let target = "https://8.8.8.8/video.mp4";
|
||||||
|
|
||||||
|
let response = build_video_task_video_response(
|
||||||
|
&state,
|
||||||
|
"task-public-redirect",
|
||||||
|
VideoTaskVideoSource::Redirect {
|
||||||
|
url: url::Url::parse(target).expect("public target should parse"),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("public redirect should build");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
response.status(),
|
||||||
|
axum::http::StatusCode::TEMPORARY_REDIRECT
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
response
|
||||||
|
.headers()
|
||||||
|
.get(axum::http::header::LOCATION)
|
||||||
|
.and_then(|value| value.to_str().ok()),
|
||||||
|
Some(target)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn video_redirect_response_rejects_private_and_reserved_targets() {
|
||||||
|
let state = AppState::new().expect("gateway state should build");
|
||||||
|
|
||||||
|
for raw_url in [
|
||||||
|
"http://127.0.0.1/video.mp4",
|
||||||
|
"http://169.254.169.254/latest/meta-data",
|
||||||
|
"http://10.0.0.1/video.mp4",
|
||||||
|
"http://[::1]/video.mp4",
|
||||||
|
] {
|
||||||
|
let error = build_video_task_video_response(
|
||||||
|
&state,
|
||||||
|
"task-rejected-redirect",
|
||||||
|
VideoTaskVideoSource::Redirect {
|
||||||
|
url: url::Url::parse(raw_url).expect("target should parse"),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect_err("private or reserved redirect target should be rejected");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
error.into_response().status(),
|
||||||
|
axum::http::StatusCode::BAD_GATEWAY,
|
||||||
|
"unexpected status for {raw_url}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn video_target_resolution_rejects_private_and_reserved_ip_literals() {
|
||||||
|
for raw_url in [
|
||||||
|
"http://127.0.0.1/video.mp4",
|
||||||
|
"http://169.254.169.254/latest/meta-data",
|
||||||
|
"http://10.0.0.1/video.mp4",
|
||||||
|
"http://[::1]/video.mp4",
|
||||||
|
"http://[::ffff:127.0.0.1]/video.mp4",
|
||||||
|
] {
|
||||||
|
let url = url::Url::parse(raw_url).unwrap();
|
||||||
|
assert!(
|
||||||
|
resolve_public_video_target(&url).await.is_err(),
|
||||||
|
"target should be rejected: {raw_url}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn video_target_resolution_accepts_public_ip_literals() {
|
||||||
|
for raw_url in [
|
||||||
|
"https://8.8.8.8/video.mp4",
|
||||||
|
"https://[2606:4700:4700::1111]/video.mp4",
|
||||||
|
] {
|
||||||
|
let url = url::Url::parse(raw_url).unwrap();
|
||||||
|
assert!(
|
||||||
|
resolve_public_video_target(&url).await.is_ok(),
|
||||||
|
"target should be accepted: {raw_url}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn video_response_metadata_rejects_active_content_and_sanitizes_filename() {
|
||||||
|
assert!(safe_video_content_type("video/mp4").is_some());
|
||||||
|
assert!(safe_video_content_type("video/webm; charset=binary").is_some());
|
||||||
|
assert!(safe_video_content_type("video/").is_none());
|
||||||
|
assert!(safe_video_content_type("video/; charset=binary").is_none());
|
||||||
|
assert!(safe_video_content_type("text/html").is_none());
|
||||||
|
assert!(safe_video_content_type("video/mp4\r\nx-test: injected").is_none());
|
||||||
|
assert_eq!(
|
||||||
|
safe_video_filename("video_123.mp4\"; filename=\"attack.html"),
|
||||||
|
"video_123.mp4___filename__attack.html"
|
||||||
|
);
|
||||||
|
assert_eq!(safe_video_filename(&"x".repeat(1024)).len(), 255);
|
||||||
|
|
||||||
|
let mut upstream = axum::http::HeaderMap::new();
|
||||||
|
upstream.insert(
|
||||||
|
axum::http::header::CONTENT_TYPE,
|
||||||
|
axum::http::HeaderValue::from_static("text/html"),
|
||||||
|
);
|
||||||
|
upstream.insert(
|
||||||
|
axum::http::header::CONTENT_LENGTH,
|
||||||
|
axum::http::HeaderValue::from_static("999999"),
|
||||||
|
);
|
||||||
|
let mut outbound = upstream.clone();
|
||||||
|
apply_safe_video_response_metadata(
|
||||||
|
&mut outbound,
|
||||||
|
&upstream,
|
||||||
|
"video.mp4\"; filename=\"attack.html",
|
||||||
|
)
|
||||||
|
.expect("video metadata should build");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
outbound
|
||||||
|
.get(axum::http::header::CONTENT_TYPE)
|
||||||
|
.and_then(|value| value.to_str().ok()),
|
||||||
|
Some("application/octet-stream")
|
||||||
|
);
|
||||||
|
assert!(outbound.get(axum::http::header::CONTENT_LENGTH).is_none());
|
||||||
|
assert_eq!(
|
||||||
|
outbound
|
||||||
|
.get(axum::http::header::X_CONTENT_TYPE_OPTIONS)
|
||||||
|
.and_then(|value| value.to_str().ok()),
|
||||||
|
Some("nosniff")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
outbound
|
||||||
|
.get(axum::http::header::CONTENT_DISPOSITION)
|
||||||
|
.and_then(|value| value.to_str().ok()),
|
||||||
|
Some("inline; filename=\"video.mp4___filename__attack.html\"")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,12 +3,13 @@ use aether_data_contracts::repository::video_tasks::{
|
|||||||
};
|
};
|
||||||
use axum::response::IntoResponse;
|
use axum::response::IntoResponse;
|
||||||
use axum::Json;
|
use axum::Json;
|
||||||
use serde_json::{json, Map, Value};
|
use serde_json::json;
|
||||||
|
|
||||||
|
use crate::state::VideoTaskRouteAccess;
|
||||||
use crate::{AppState, GatewayError};
|
use crate::{AppState, GatewayError};
|
||||||
|
|
||||||
use super::super::finalize_video_task_if_terminal;
|
use super::super::finalize_video_task_if_terminal;
|
||||||
use super::super::read_video_task_detail;
|
use super::super::{read_video_task_detail, read_video_task_detail_for_user};
|
||||||
use super::current_unix_secs;
|
use super::current_unix_secs;
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
@@ -29,7 +30,31 @@ pub(crate) async fn cancel_video_task_record(
|
|||||||
state: &AppState,
|
state: &AppState,
|
||||||
task_id: &str,
|
task_id: &str,
|
||||||
) -> Result<StoredVideoTask, CancelVideoTaskError> {
|
) -> Result<StoredVideoTask, CancelVideoTaskError> {
|
||||||
let Some(task) = read_video_task_detail(state, task_id).await? else {
|
cancel_video_task_record_inner(state, task_id, None).await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn cancel_video_task_record_for_user(
|
||||||
|
state: &AppState,
|
||||||
|
task_id: &str,
|
||||||
|
user_id: &str,
|
||||||
|
) -> Result<StoredVideoTask, CancelVideoTaskError> {
|
||||||
|
let user_id = user_id.trim();
|
||||||
|
if user_id.is_empty() {
|
||||||
|
return Err(CancelVideoTaskError::NotFound);
|
||||||
|
}
|
||||||
|
cancel_video_task_record_inner(state, task_id, Some(user_id)).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn cancel_video_task_record_inner(
|
||||||
|
state: &AppState,
|
||||||
|
task_id: &str,
|
||||||
|
expected_user_id: Option<&str>,
|
||||||
|
) -> Result<StoredVideoTask, CancelVideoTaskError> {
|
||||||
|
let task = match expected_user_id {
|
||||||
|
Some(user_id) => read_video_task_detail_for_user(state, task_id, user_id).await?,
|
||||||
|
None => read_video_task_detail(state, task_id).await?,
|
||||||
|
};
|
||||||
|
let Some(task) = task else {
|
||||||
return Err(CancelVideoTaskError::NotFound);
|
return Err(CancelVideoTaskError::NotFound);
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -45,39 +70,84 @@ pub(crate) async fn cancel_video_task_record(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let trace_id = format!("async-task-admin-cancel-{task_id}");
|
let trace_id = format!("async-task-admin-cancel-{task_id}");
|
||||||
|
let mut finalize_mutation = None;
|
||||||
if let Some(cancel_plan) = build_video_task_cancel_plan(&task) {
|
if let Some(cancel_plan) = build_video_task_cancel_plan(&task) {
|
||||||
state
|
|
||||||
.hydrate_video_task_for_route(Some(cancel_plan.route_family), &cancel_plan.request_path)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
let body_json = json!({});
|
let body_json = json!({});
|
||||||
let follow_up = state.video_tasks.prepare_follow_up_sync_plan(
|
let follow_up = if let Some(user_id) = expected_user_id {
|
||||||
cancel_plan.plan_kind,
|
if state
|
||||||
&cancel_plan.request_path,
|
.hydrate_video_task_for_route_for_user(
|
||||||
Some(&body_json),
|
Some(cancel_plan.route_family),
|
||||||
None,
|
&cancel_plan.request_path,
|
||||||
&trace_id,
|
user_id,
|
||||||
);
|
)
|
||||||
|
.await?
|
||||||
|
!= VideoTaskRouteAccess::Allowed
|
||||||
|
{
|
||||||
|
return Err(CancelVideoTaskError::NotFound);
|
||||||
|
}
|
||||||
|
state.video_tasks.prepare_follow_up_sync_plan_for_user_id(
|
||||||
|
cancel_plan.plan_kind,
|
||||||
|
&cancel_plan.request_path,
|
||||||
|
Some(&body_json),
|
||||||
|
user_id,
|
||||||
|
task.api_key_id.as_deref(),
|
||||||
|
&trace_id,
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
state
|
||||||
|
.hydrate_video_task_for_route(
|
||||||
|
Some(cancel_plan.route_family),
|
||||||
|
&cancel_plan.request_path,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
state.video_tasks.prepare_follow_up_sync_plan(
|
||||||
|
cancel_plan.plan_kind,
|
||||||
|
&cancel_plan.request_path,
|
||||||
|
Some(&body_json),
|
||||||
|
None,
|
||||||
|
&trace_id,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
|
||||||
if let Some(follow_up) = follow_up {
|
if let Some(follow_up) = follow_up {
|
||||||
execute_video_task_cancel_plan(state, &trace_id, follow_up.plan)
|
execute_video_task_cancel_plan(state, &trace_id, follow_up.plan)
|
||||||
.await
|
.await
|
||||||
.map_err(CancelVideoTaskError::Response)?;
|
.map_err(CancelVideoTaskError::Response)?;
|
||||||
|
finalize_mutation = Some((
|
||||||
|
cancel_plan.request_path,
|
||||||
|
cancel_plan.report_kind.to_string(),
|
||||||
|
));
|
||||||
|
} else if expected_user_id.is_none() {
|
||||||
|
finalize_mutation = Some((
|
||||||
|
cancel_plan.request_path,
|
||||||
|
cancel_plan.report_kind.to_string(),
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
state
|
|
||||||
.video_tasks
|
|
||||||
.apply_finalize_mutation(&cancel_plan.request_path, cancel_plan.report_kind);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let request_metadata = build_cancelled_request_metadata(state, &task).await?;
|
let stored = match persist_cancelled_video_task(state, &task).await? {
|
||||||
let stored = persist_cancelled_video_task(state, &task, request_metadata)
|
Some(stored) => stored,
|
||||||
.await?
|
None => {
|
||||||
.ok_or_else(|| {
|
let current = match expected_user_id {
|
||||||
CancelVideoTaskError::Gateway(GatewayError::Internal(
|
Some(user_id) => read_video_task_detail_for_user(state, task_id, user_id).await?,
|
||||||
|
None => read_video_task_detail(state, task_id).await?,
|
||||||
|
};
|
||||||
|
let Some(current) = current else {
|
||||||
|
return Err(CancelVideoTaskError::NotFound);
|
||||||
|
};
|
||||||
|
if !current.status.is_active() {
|
||||||
|
return Err(CancelVideoTaskError::InvalidStatus(current.status));
|
||||||
|
}
|
||||||
|
return Err(CancelVideoTaskError::Gateway(GatewayError::Internal(
|
||||||
"video task repository is unavailable".to_string(),
|
"video task repository is unavailable".to_string(),
|
||||||
))
|
)));
|
||||||
})?;
|
}
|
||||||
|
};
|
||||||
|
if let Some((request_path, report_kind)) = finalize_mutation {
|
||||||
|
state
|
||||||
|
.video_tasks
|
||||||
|
.apply_finalize_mutation(&request_path, &report_kind);
|
||||||
|
}
|
||||||
finalize_video_task_if_terminal(state, &stored).await;
|
finalize_video_task_if_terminal(state, &stored).await;
|
||||||
Ok(stored)
|
Ok(stored)
|
||||||
}
|
}
|
||||||
@@ -91,12 +161,7 @@ struct VideoTaskCancelPlan<'a> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn build_video_task_cancel_plan(task: &StoredVideoTask) -> Option<VideoTaskCancelPlan<'_>> {
|
fn build_video_task_cancel_plan(task: &StoredVideoTask) -> Option<VideoTaskCancelPlan<'_>> {
|
||||||
let provider_api_format = task
|
let provider_api_format = task.effective_api_format()?;
|
||||||
.provider_api_format
|
|
||||||
.as_deref()
|
|
||||||
.or(task.client_api_format.as_deref())
|
|
||||||
.map(str::trim)
|
|
||||||
.filter(|value| !value.is_empty())?;
|
|
||||||
|
|
||||||
match provider_api_format {
|
match provider_api_format {
|
||||||
"openai:video" => Some(VideoTaskCancelPlan {
|
"openai:video" => Some(VideoTaskCancelPlan {
|
||||||
@@ -131,99 +196,52 @@ async fn execute_video_task_cancel_plan(
|
|||||||
let result =
|
let result =
|
||||||
crate::execution_runtime::execute_execution_runtime_sync_plan(state, Some(trace_id), &plan)
|
crate::execution_runtime::execute_execution_runtime_sync_plan(state, Some(trace_id), &plan)
|
||||||
.await
|
.await
|
||||||
.map_err(|err| {
|
.map_err(|_| {
|
||||||
GatewayError::UpstreamUnavailable {
|
GatewayError::UpstreamUnavailable {
|
||||||
trace_id: trace_id.to_string(),
|
trace_id: trace_id.to_string(),
|
||||||
message: format!("{err:?}"),
|
message: "video cancellation request failed".to_string(),
|
||||||
}
|
}
|
||||||
.into_response()
|
.into_response()
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
if result.status_code >= 400 {
|
if result.status_code >= 400 {
|
||||||
let status = axum::http::StatusCode::from_u16(result.status_code)
|
return Err(build_video_task_cancel_upstream_error_response(&result));
|
||||||
.unwrap_or(axum::http::StatusCode::BAD_GATEWAY);
|
|
||||||
let body_json = result
|
|
||||||
.body
|
|
||||||
.and_then(|body| body.json_body)
|
|
||||||
.unwrap_or_else(|| {
|
|
||||||
json!({
|
|
||||||
"error": {
|
|
||||||
"message": result
|
|
||||||
.error
|
|
||||||
.as_ref()
|
|
||||||
.map(|error| error.message.clone())
|
|
||||||
.unwrap_or_else(|| {
|
|
||||||
format!("execution runtime returned {}", result.status_code)
|
|
||||||
}),
|
|
||||||
}
|
|
||||||
})
|
|
||||||
});
|
|
||||||
return Err((status, Json(body_json)).into_response());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn build_cancelled_request_metadata(
|
fn build_video_task_cancel_upstream_error_response(
|
||||||
state: &AppState,
|
result: &aether_contracts::ExecutionResult,
|
||||||
task: &StoredVideoTask,
|
) -> axum::response::Response {
|
||||||
) -> Result<Option<Value>, GatewayError> {
|
let status = axum::http::StatusCode::from_u16(result.status_code)
|
||||||
let mut metadata = match task.request_metadata.clone() {
|
.unwrap_or(axum::http::StatusCode::BAD_GATEWAY);
|
||||||
Some(Value::Object(object)) => object,
|
tracing::warn!(
|
||||||
_ => Map::new(),
|
event_name = "video_task_cancel_upstream_error",
|
||||||
};
|
upstream_status = result.status_code,
|
||||||
let mut snapshot_value = metadata.get("rust_local_snapshot").cloned();
|
"video cancellation upstream response body discarded"
|
||||||
if snapshot_value.is_none() {
|
);
|
||||||
snapshot_value = state
|
(
|
||||||
.reconstruct_video_task_snapshot(task)
|
status,
|
||||||
.await?
|
Json(json!({
|
||||||
.map(|snapshot| {
|
"error": {
|
||||||
serde_json::to_value(snapshot)
|
"message": format!(
|
||||||
.map_err(|err| GatewayError::Internal(err.to_string()))
|
"video cancellation upstream returned HTTP {}",
|
||||||
})
|
result.status_code
|
||||||
.transpose()?;
|
),
|
||||||
}
|
}
|
||||||
if let Some(snapshot_value_ref) = snapshot_value.as_mut() {
|
})),
|
||||||
mark_snapshot_value_cancelled(snapshot_value_ref);
|
)
|
||||||
metadata.insert(
|
.into_response()
|
||||||
"rust_owner".to_string(),
|
|
||||||
Value::String("async_task".to_string()),
|
|
||||||
);
|
|
||||||
metadata.insert(
|
|
||||||
"rust_local_snapshot".to_string(),
|
|
||||||
snapshot_value_ref.clone(),
|
|
||||||
);
|
|
||||||
return Ok(Some(Value::Object(metadata)));
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(task.request_metadata.clone())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn mark_snapshot_value_cancelled(snapshot_value: &mut Value) {
|
|
||||||
if let Some(object) = snapshot_value
|
|
||||||
.get_mut("OpenAi")
|
|
||||||
.and_then(Value::as_object_mut)
|
|
||||||
{
|
|
||||||
object.insert("status".to_string(), Value::String("Cancelled".to_string()));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if let Some(object) = snapshot_value
|
|
||||||
.get_mut("Gemini")
|
|
||||||
.and_then(Value::as_object_mut)
|
|
||||||
{
|
|
||||||
object.insert("status".to_string(), Value::String("Cancelled".to_string()));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn persist_cancelled_video_task(
|
async fn persist_cancelled_video_task(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
task: &StoredVideoTask,
|
task: &StoredVideoTask,
|
||||||
request_metadata: Option<Value>,
|
|
||||||
) -> Result<Option<StoredVideoTask>, GatewayError> {
|
) -> Result<Option<StoredVideoTask>, GatewayError> {
|
||||||
let now_unix_secs = current_unix_secs();
|
let now_unix_secs = current_unix_secs();
|
||||||
state
|
state
|
||||||
.data
|
.update_active_video_task(UpsertVideoTask {
|
||||||
.upsert_video_task(UpsertVideoTask {
|
|
||||||
id: task.id.clone(),
|
id: task.id.clone(),
|
||||||
short_id: task.short_id.clone(),
|
short_id: task.short_id.clone(),
|
||||||
request_id: task.request_id.clone(),
|
request_id: task.request_id.clone(),
|
||||||
@@ -240,14 +258,14 @@ async fn persist_cancelled_video_task(
|
|||||||
format_converted: task.format_converted,
|
format_converted: task.format_converted,
|
||||||
model: task.model.clone(),
|
model: task.model.clone(),
|
||||||
prompt: task.prompt.clone(),
|
prompt: task.prompt.clone(),
|
||||||
original_request_body: task.original_request_body.clone(),
|
original_request_body: None,
|
||||||
duration_seconds: task.duration_seconds,
|
duration_seconds: task.duration_seconds,
|
||||||
resolution: task.resolution.clone(),
|
resolution: task.resolution.clone(),
|
||||||
aspect_ratio: task.aspect_ratio.clone(),
|
aspect_ratio: task.aspect_ratio.clone(),
|
||||||
size: task.size.clone(),
|
size: task.size.clone(),
|
||||||
status: VideoTaskStatus::Cancelled,
|
status: VideoTaskStatus::Cancelled,
|
||||||
progress_percent: task.progress_percent,
|
progress_percent: task.progress_percent,
|
||||||
progress_message: task.progress_message.clone(),
|
progress_message: None,
|
||||||
retry_count: task.retry_count,
|
retry_count: task.retry_count,
|
||||||
poll_interval_seconds: task.poll_interval_seconds,
|
poll_interval_seconds: task.poll_interval_seconds,
|
||||||
next_poll_at_unix_secs: None,
|
next_poll_at_unix_secs: None,
|
||||||
@@ -258,10 +276,73 @@ async fn persist_cancelled_video_task(
|
|||||||
completed_at_unix_secs: Some(now_unix_secs),
|
completed_at_unix_secs: Some(now_unix_secs),
|
||||||
updated_at_unix_secs: now_unix_secs,
|
updated_at_unix_secs: now_unix_secs,
|
||||||
error_code: task.error_code.clone(),
|
error_code: task.error_code.clone(),
|
||||||
error_message: task.error_message.clone(),
|
error_message: None,
|
||||||
video_url: task.video_url.clone(),
|
video_url: task.video_url.clone(),
|
||||||
request_metadata,
|
request_metadata: None,
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.map_err(|err| GatewayError::Internal(err.to_string()))
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
|
use aether_contracts::{
|
||||||
|
ExecutionError, ExecutionErrorKind, ExecutionPhase, ExecutionResult, ResponseBody,
|
||||||
|
};
|
||||||
|
use axum::body::to_bytes;
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
use super::build_video_task_cancel_upstream_error_response;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cancellation_upstream_errors_do_not_expose_runtime_payloads() {
|
||||||
|
let result = ExecutionResult {
|
||||||
|
request_id: "cancel-secret-request-id".to_string(),
|
||||||
|
candidate_id: Some("cancel-secret-candidate-id".to_string()),
|
||||||
|
status_code: 502,
|
||||||
|
headers: BTreeMap::from([(
|
||||||
|
"x-internal-secret".to_string(),
|
||||||
|
"cancel-secret-header".to_string(),
|
||||||
|
)]),
|
||||||
|
response_observation: None,
|
||||||
|
body: Some(ResponseBody {
|
||||||
|
json_body: Some(json!({
|
||||||
|
"error": {
|
||||||
|
"message": "cancel-secret-upstream-body",
|
||||||
|
}
|
||||||
|
})),
|
||||||
|
body_bytes_b64: None,
|
||||||
|
}),
|
||||||
|
telemetry: None,
|
||||||
|
error: Some(ExecutionError {
|
||||||
|
kind: ExecutionErrorKind::Upstream5xx,
|
||||||
|
phase: ExecutionPhase::FirstByte,
|
||||||
|
message: "cancel-secret-runtime-error".to_string(),
|
||||||
|
upstream_status: Some(502),
|
||||||
|
retryable: true,
|
||||||
|
failover_recommended: false,
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
|
||||||
|
let response = build_video_task_cancel_upstream_error_response(&result);
|
||||||
|
assert_eq!(response.status(), axum::http::StatusCode::BAD_GATEWAY);
|
||||||
|
assert!(response.headers().get("x-internal-secret").is_none());
|
||||||
|
let body = to_bytes(response.into_body(), usize::MAX)
|
||||||
|
.await
|
||||||
|
.expect("response body should read");
|
||||||
|
let payload: serde_json::Value =
|
||||||
|
serde_json::from_slice(&body).expect("response body should parse");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
payload,
|
||||||
|
json!({
|
||||||
|
"error": {
|
||||||
|
"message": "video cancellation upstream returned HTTP 502",
|
||||||
|
}
|
||||||
|
})
|
||||||
|
);
|
||||||
|
let body = String::from_utf8(body.to_vec()).expect("response body should be utf-8");
|
||||||
|
assert!(!body.contains("cancel-secret"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,13 +6,14 @@ pub(crate) use crate::video_tasks::VideoTaskService;
|
|||||||
pub use crate::video_tasks::VideoTaskTruthSourceMode;
|
pub use crate::video_tasks::VideoTaskTruthSourceMode;
|
||||||
pub(crate) use http::{
|
pub(crate) use http::{
|
||||||
build_video_task_video_response, cancel_video_task, cancel_video_task_record,
|
build_video_task_video_response, cancel_video_task, cancel_video_task_record,
|
||||||
get_video_task_detail, get_video_task_stats, get_video_task_video, list_video_tasks,
|
cancel_video_task_record_for_user, get_video_task_detail, get_video_task_stats,
|
||||||
CancelVideoTaskError,
|
get_video_task_video, list_video_tasks, CancelVideoTaskError,
|
||||||
};
|
};
|
||||||
pub(crate) use query::{
|
pub(crate) use query::{
|
||||||
read_video_task_detail, read_video_task_page, read_video_task_page_summary,
|
read_video_task_detail, read_video_task_detail_for_user, read_video_task_page,
|
||||||
read_video_task_stats, read_video_task_video_source, VideoTaskPageResponse,
|
read_video_task_page_summary, read_video_task_stats, read_video_task_video_source,
|
||||||
VideoTaskStatsResponse, VideoTaskVideoSource,
|
video_task_video_source_from_task, VideoTaskPageResponse, VideoTaskStatsResponse,
|
||||||
|
VideoTaskVideoSource,
|
||||||
};
|
};
|
||||||
pub(crate) use runtime::{
|
pub(crate) use runtime::{
|
||||||
execute_video_task_refresh_plan, finalize_video_task_if_terminal, spawn_video_task_poller,
|
execute_video_task_refresh_plan, finalize_video_task_if_terminal, spawn_video_task_poller,
|
||||||
|
|||||||
@@ -26,13 +26,12 @@ pub(crate) struct VideoTaskStatsResponse {
|
|||||||
pub(crate) processing_count: u64,
|
pub(crate) processing_count: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
|
||||||
pub(crate) enum VideoTaskVideoSource {
|
pub(crate) enum VideoTaskVideoSource {
|
||||||
Redirect {
|
Redirect {
|
||||||
url: String,
|
url: url::Url,
|
||||||
},
|
},
|
||||||
Proxy {
|
Proxy {
|
||||||
url: String,
|
url: url::Url,
|
||||||
header_name: String,
|
header_name: String,
|
||||||
header_value: String,
|
header_value: String,
|
||||||
filename: String,
|
filename: String,
|
||||||
@@ -102,6 +101,14 @@ pub(crate) async fn read_video_task_detail(
|
|||||||
state.find_video_task_by_id(task_id).await
|
state.find_video_task_by_id(task_id).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn read_video_task_detail_for_user(
|
||||||
|
state: &AppState,
|
||||||
|
task_id: &str,
|
||||||
|
user_id: &str,
|
||||||
|
) -> Result<Option<StoredVideoTask>, GatewayError> {
|
||||||
|
state.find_video_task_by_id_for_user(task_id, user_id).await
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn read_video_task_video_source(
|
pub(crate) async fn read_video_task_video_source(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
task_id: &str,
|
task_id: &str,
|
||||||
@@ -109,6 +116,13 @@ pub(crate) async fn read_video_task_video_source(
|
|||||||
let Some(task) = read_video_task_detail(state, task_id).await? else {
|
let Some(task) = read_video_task_detail(state, task_id).await? else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
|
video_task_video_source_from_task(state, &task).await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn video_task_video_source_from_task(
|
||||||
|
state: &AppState,
|
||||||
|
task: &StoredVideoTask,
|
||||||
|
) -> Result<Option<VideoTaskVideoSource>, GatewayError> {
|
||||||
let Some(video_url) = task
|
let Some(video_url) = task
|
||||||
.video_url
|
.video_url
|
||||||
.as_deref()
|
.as_deref()
|
||||||
@@ -119,7 +133,9 @@ pub(crate) async fn read_video_task_video_source(
|
|||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
|
|
||||||
if !video_url.contains("generativelanguage.googleapis.com") {
|
let video_url = parse_video_url(&video_url)?;
|
||||||
|
|
||||||
|
if task.effective_api_format() != Some("gemini:video") {
|
||||||
return Ok(Some(VideoTaskVideoSource::Redirect { url: video_url }));
|
return Ok(Some(VideoTaskVideoSource::Redirect { url: video_url }));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -148,6 +164,15 @@ pub(crate) async fn read_video_task_video_source(
|
|||||||
));
|
));
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let endpoint_url = parse_video_url(transport.endpoint.base_url.trim()).map_err(|_| {
|
||||||
|
GatewayError::Internal("provider endpoint URL is invalid for proxied video".to_string())
|
||||||
|
})?;
|
||||||
|
if !video_urls_share_origin(&endpoint_url, &video_url) {
|
||||||
|
return Err(GatewayError::Client {
|
||||||
|
status: axum::http::StatusCode::BAD_GATEWAY,
|
||||||
|
message: "video URL origin does not match its provider endpoint".to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
let api_key = transport.key.decrypted_api_key.trim();
|
let api_key = transport.key.decrypted_api_key.trim();
|
||||||
if api_key.is_empty() {
|
if api_key.is_empty() {
|
||||||
return Err(GatewayError::Internal(
|
return Err(GatewayError::Internal(
|
||||||
@@ -159,10 +184,34 @@ pub(crate) async fn read_video_task_video_source(
|
|||||||
url: video_url,
|
url: video_url,
|
||||||
header_name: "x-goog-api-key".to_string(),
|
header_name: "x-goog-api-key".to_string(),
|
||||||
header_value: api_key.to_string(),
|
header_value: api_key.to_string(),
|
||||||
filename: format!("video_{task_id}.mp4"),
|
filename: format!("video_{}.mp4", task.id),
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn parse_video_url(raw_url: &str) -> Result<url::Url, GatewayError> {
|
||||||
|
let url = url::Url::parse(raw_url.trim()).map_err(|_| GatewayError::Client {
|
||||||
|
status: axum::http::StatusCode::BAD_GATEWAY,
|
||||||
|
message: "video URL is invalid".to_string(),
|
||||||
|
})?;
|
||||||
|
if !matches!(url.scheme(), "http" | "https")
|
||||||
|
|| url.host_str().is_none()
|
||||||
|
|| !url.username().is_empty()
|
||||||
|
|| url.password().is_some()
|
||||||
|
{
|
||||||
|
return Err(GatewayError::Client {
|
||||||
|
status: axum::http::StatusCode::BAD_GATEWAY,
|
||||||
|
message: "video URL must be an absolute HTTP(S) URL without credentials".to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Ok(url)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn video_urls_share_origin(left: &url::Url, right: &url::Url) -> bool {
|
||||||
|
left.scheme() == right.scheme()
|
||||||
|
&& left.host() == right.host()
|
||||||
|
&& left.port_or_known_default() == right.port_or_known_default()
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn read_video_task_stats(
|
pub(crate) async fn read_video_task_stats(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
filter: &VideoTaskQueryFilter,
|
filter: &VideoTaskQueryFilter,
|
||||||
@@ -226,3 +275,212 @@ fn status_key(status: VideoTaskStatus) -> String {
|
|||||||
fn start_of_utc_day(now_unix_secs: u64) -> u64 {
|
fn start_of_utc_day(now_unix_secs: u64) -> u64 {
|
||||||
now_unix_secs - (now_unix_secs % 86_400)
|
now_unix_secs - (now_unix_secs % 86_400)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||||
|
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
||||||
|
use aether_data::repository::video_tasks::InMemoryVideoTaskRepository;
|
||||||
|
use aether_data_contracts::repository::provider_catalog::{
|
||||||
|
ProviderCatalogReadRepository, StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
|
||||||
|
StoredProviderCatalogProvider,
|
||||||
|
};
|
||||||
|
use aether_data_contracts::repository::video_tasks::{UpsertVideoTask, VideoTaskStatus};
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
use super::{
|
||||||
|
parse_video_url, video_task_video_source_from_task, video_urls_share_origin,
|
||||||
|
VideoTaskVideoSource,
|
||||||
|
};
|
||||||
|
use crate::{data::GatewayDataState, AppState};
|
||||||
|
|
||||||
|
fn legacy_gemini_video_task() -> aether_data_contracts::repository::video_tasks::StoredVideoTask
|
||||||
|
{
|
||||||
|
UpsertVideoTask {
|
||||||
|
id: "legacy-gemini-task".to_string(),
|
||||||
|
short_id: Some("legacy-short".to_string()),
|
||||||
|
request_id: "legacy-request".to_string(),
|
||||||
|
user_id: Some("user-1".to_string()),
|
||||||
|
api_key_id: Some("client-key-1".to_string()),
|
||||||
|
username: None,
|
||||||
|
api_key_name: None,
|
||||||
|
external_task_id: Some("operations/upstream-1".to_string()),
|
||||||
|
provider_id: Some("provider-1".to_string()),
|
||||||
|
endpoint_id: Some("endpoint-1".to_string()),
|
||||||
|
key_id: Some("provider-key-1".to_string()),
|
||||||
|
client_api_format: Some("gemini:video".to_string()),
|
||||||
|
provider_api_format: None,
|
||||||
|
format_converted: false,
|
||||||
|
model: Some("veo-3".to_string()),
|
||||||
|
prompt: None,
|
||||||
|
original_request_body: None,
|
||||||
|
duration_seconds: Some(8),
|
||||||
|
resolution: Some("720p".to_string()),
|
||||||
|
aspect_ratio: Some("16:9".to_string()),
|
||||||
|
size: Some("1280x720".to_string()),
|
||||||
|
status: VideoTaskStatus::Completed,
|
||||||
|
progress_percent: 100,
|
||||||
|
progress_message: None,
|
||||||
|
retry_count: 0,
|
||||||
|
poll_interval_seconds: 10,
|
||||||
|
next_poll_at_unix_secs: None,
|
||||||
|
poll_count: 1,
|
||||||
|
max_poll_count: 360,
|
||||||
|
created_at_unix_ms: 1,
|
||||||
|
submitted_at_unix_secs: Some(1),
|
||||||
|
completed_at_unix_secs: Some(2),
|
||||||
|
updated_at_unix_secs: 2,
|
||||||
|
error_code: None,
|
||||||
|
error_message: None,
|
||||||
|
video_url: Some(
|
||||||
|
"https://generativelanguage.googleapis.com/v1beta/files/video-1:download?alt=media"
|
||||||
|
.to_string(),
|
||||||
|
),
|
||||||
|
request_metadata: None,
|
||||||
|
}
|
||||||
|
.into_stored()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn state_with_gemini_transport() -> AppState {
|
||||||
|
let state = AppState::new().expect("gateway state should build");
|
||||||
|
let provider = StoredProviderCatalogProvider::new(
|
||||||
|
"provider-1".to_string(),
|
||||||
|
"Gemini".to_string(),
|
||||||
|
Some("https://ai.google.dev".to_string()),
|
||||||
|
"gemini".to_string(),
|
||||||
|
)
|
||||||
|
.expect("provider should build");
|
||||||
|
let endpoint = StoredProviderCatalogEndpoint::new(
|
||||||
|
"endpoint-1".to_string(),
|
||||||
|
"provider-1".to_string(),
|
||||||
|
"gemini:video".to_string(),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
true,
|
||||||
|
)
|
||||||
|
.expect("endpoint should build")
|
||||||
|
.with_transport_fields(
|
||||||
|
"https://generativelanguage.googleapis.com".to_string(),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.expect("endpoint transport should build");
|
||||||
|
let encrypted_api_key = state
|
||||||
|
.seal_provider_catalog_key_api_key(
|
||||||
|
"provider-1",
|
||||||
|
"provider-key-1",
|
||||||
|
"gemini-provider-secret",
|
||||||
|
)
|
||||||
|
.expect("provider key should encrypt");
|
||||||
|
let key = StoredProviderCatalogKey::new(
|
||||||
|
"provider-key-1".to_string(),
|
||||||
|
"provider-1".to_string(),
|
||||||
|
"default".to_string(),
|
||||||
|
"api_key".to_string(),
|
||||||
|
None,
|
||||||
|
true,
|
||||||
|
)
|
||||||
|
.expect("provider key should build")
|
||||||
|
.with_transport_fields(
|
||||||
|
Some(json!(["gemini:video"])),
|
||||||
|
encrypted_api_key,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.expect("provider key transport should build");
|
||||||
|
let provider_catalog: Arc<dyn ProviderCatalogReadRepository> = Arc::new(
|
||||||
|
InMemoryProviderCatalogReadRepository::seed(vec![provider], vec![endpoint], vec![key]),
|
||||||
|
);
|
||||||
|
let video_tasks = Arc::new(InMemoryVideoTaskRepository::default());
|
||||||
|
let data = GatewayDataState::with_video_task_repository_and_provider_transport_for_tests(
|
||||||
|
video_tasks,
|
||||||
|
provider_catalog,
|
||||||
|
DEVELOPMENT_ENCRYPTION_KEY,
|
||||||
|
);
|
||||||
|
state.with_data_state_for_tests(data)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn video_url_parser_rejects_non_http_and_embedded_credentials() {
|
||||||
|
for raw_url in [
|
||||||
|
"file:///etc/passwd",
|
||||||
|
"data:video/mp4;base64,AAAA",
|
||||||
|
"https://[email protected]/video.mp4",
|
||||||
|
"https://user:[email protected]/video.mp4",
|
||||||
|
"/relative/video.mp4",
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
parse_video_url(raw_url).is_err(),
|
||||||
|
"URL should be rejected: {raw_url}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn video_origin_comparison_uses_scheme_host_and_effective_port() {
|
||||||
|
let base = parse_video_url("https://generativelanguage.googleapis.com/v1beta").unwrap();
|
||||||
|
for same_origin in [
|
||||||
|
"https://generativelanguage.googleapis.com/file",
|
||||||
|
"https://generativelanguage.googleapis.com:443/file",
|
||||||
|
] {
|
||||||
|
assert!(video_urls_share_origin(
|
||||||
|
&base,
|
||||||
|
&parse_video_url(same_origin).unwrap()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
for different_origin in [
|
||||||
|
"http://generativelanguage.googleapis.com/file",
|
||||||
|
"https://generativelanguage.googleapis.com:444/file",
|
||||||
|
"https://generativelanguage.googleapis.com.evil.test/file",
|
||||||
|
"https://evil.test/generativelanguage.googleapis.com/file",
|
||||||
|
] {
|
||||||
|
assert!(!video_urls_share_origin(
|
||||||
|
&base,
|
||||||
|
&parse_video_url(different_origin).unwrap()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn legacy_gemini_client_format_uses_authenticated_proxy_source() {
|
||||||
|
let source = video_task_video_source_from_task(
|
||||||
|
&state_with_gemini_transport(),
|
||||||
|
&legacy_gemini_video_task(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("video source should resolve")
|
||||||
|
.expect("video source should exist");
|
||||||
|
|
||||||
|
match source {
|
||||||
|
VideoTaskVideoSource::Proxy {
|
||||||
|
url,
|
||||||
|
header_name,
|
||||||
|
header_value,
|
||||||
|
filename,
|
||||||
|
} => {
|
||||||
|
assert_eq!(
|
||||||
|
url.as_str(),
|
||||||
|
"https://generativelanguage.googleapis.com/v1beta/files/video-1:download?alt=media"
|
||||||
|
);
|
||||||
|
assert_eq!(header_name, "x-goog-api-key");
|
||||||
|
assert_eq!(header_value, "gemini-provider-secret");
|
||||||
|
assert_eq!(filename, "video_legacy-gemini-task.mp4");
|
||||||
|
}
|
||||||
|
VideoTaskVideoSource::Redirect { .. } => {
|
||||||
|
panic!("legacy Gemini video must not bypass the authenticated proxy")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ const VIDEO_TASK_POLL_CLAIM_SECONDS: u64 = 30;
|
|||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
struct VideoTaskRefreshError {
|
struct VideoTaskRefreshError {
|
||||||
message: String,
|
category: &'static str,
|
||||||
permanent: bool,
|
permanent: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -55,7 +55,7 @@ pub(crate) async fn execute_video_task_refresh_plan(
|
|||||||
warn!(
|
warn!(
|
||||||
event_name = "video_task_refresh_failed",
|
event_name = "video_task_refresh_failed",
|
||||||
log_type = "event",
|
log_type = "event",
|
||||||
error = %err.message,
|
error_category = err.category,
|
||||||
permanent = err.permanent,
|
permanent = err.permanent,
|
||||||
"gateway video task refresh failed"
|
"gateway video task refresh failed"
|
||||||
);
|
);
|
||||||
@@ -79,23 +79,32 @@ async fn poll_video_tasks_once(state: &AppState, batch_size: usize) -> Result<us
|
|||||||
let mut refreshed = 0usize;
|
let mut refreshed = 0usize;
|
||||||
for (index, task) in tasks.into_iter().enumerate() {
|
for (index, task) in tasks.into_iter().enumerate() {
|
||||||
let trace_id = format!("video-task-poller-{index}");
|
let trace_id = format!("video-task-poller-{index}");
|
||||||
|
let Some(snapshot) = state.reconstruct_video_task_snapshot(&task).await? else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
let Some(refresh_plan) = state
|
let Some(refresh_plan) = state
|
||||||
.video_tasks
|
.video_tasks
|
||||||
.prepare_poll_refresh_plan_for_stored_task(&task, &trace_id)
|
.prepare_poll_refresh_plan_for_snapshot(snapshot.clone(), &trace_id)
|
||||||
else {
|
else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
|
|
||||||
match fetch_video_task_refresh_attempt(state, &refresh_plan).await? {
|
match fetch_video_task_refresh_attempt(state, &refresh_plan).await? {
|
||||||
VideoTaskRefreshAttempt::Success { provider_body } => {
|
VideoTaskRefreshAttempt::Success { provider_body } => {
|
||||||
let Some(updated) =
|
let Some(updated) = build_successful_poll_update(
|
||||||
build_successful_poll_update(&task, &provider_body, now_unix_secs)?
|
&task,
|
||||||
|
snapshot.clone(),
|
||||||
|
&provider_body,
|
||||||
|
now_unix_secs,
|
||||||
|
)?
|
||||||
else {
|
else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
match state.update_active_video_task(updated).await? {
|
match state.update_active_video_task(updated).await? {
|
||||||
Some(stored) => {
|
Some(stored) => {
|
||||||
if let Some(snapshot) = LocalVideoTaskSnapshot::from_stored_task(&stored) {
|
if let Some(snapshot) =
|
||||||
|
state.reconstruct_video_task_snapshot(&stored).await?
|
||||||
|
{
|
||||||
state.video_tasks.record_snapshot(snapshot);
|
state.video_tasks.record_snapshot(snapshot);
|
||||||
}
|
}
|
||||||
info!(
|
info!(
|
||||||
@@ -116,7 +125,9 @@ async fn poll_video_tasks_once(state: &AppState, batch_size: usize) -> Result<us
|
|||||||
let updated = build_failed_poll_update(&task, &err, now_unix_secs);
|
let updated = build_failed_poll_update(&task, &err, now_unix_secs);
|
||||||
match state.update_active_video_task(updated).await? {
|
match state.update_active_video_task(updated).await? {
|
||||||
Some(stored) => {
|
Some(stored) => {
|
||||||
if let Some(snapshot) = LocalVideoTaskSnapshot::from_stored_task(&stored) {
|
if let Some(snapshot) =
|
||||||
|
state.reconstruct_video_task_snapshot(&stored).await?
|
||||||
|
{
|
||||||
state.video_tasks.record_snapshot(snapshot);
|
state.video_tasks.record_snapshot(snapshot);
|
||||||
}
|
}
|
||||||
info!(
|
info!(
|
||||||
@@ -190,9 +201,9 @@ async fn fetch_video_task_refresh_attempt(
|
|||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(result) => result,
|
Ok(result) => result,
|
||||||
Err(err) => {
|
Err(_) => {
|
||||||
return Ok(VideoTaskRefreshAttempt::Error(VideoTaskRefreshError {
|
return Ok(VideoTaskRefreshAttempt::Error(VideoTaskRefreshError {
|
||||||
message: format!("{err:?}"),
|
category: "transport_error",
|
||||||
permanent: false,
|
permanent: false,
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
@@ -209,7 +220,7 @@ async fn fetch_video_task_refresh_attempt(
|
|||||||
.and_then(|body| body.as_object().cloned())
|
.and_then(|body| body.as_object().cloned())
|
||||||
else {
|
else {
|
||||||
return Ok(VideoTaskRefreshAttempt::Error(VideoTaskRefreshError {
|
return Ok(VideoTaskRefreshAttempt::Error(VideoTaskRefreshError {
|
||||||
message: "video task refresh missing json provider body".to_string(),
|
category: "invalid_provider_response",
|
||||||
permanent: false,
|
permanent: false,
|
||||||
}));
|
}));
|
||||||
};
|
};
|
||||||
@@ -223,20 +234,19 @@ fn classify_refresh_result_error(result: &ExecutionResult) -> VideoTaskRefreshEr
|
|||||||
.as_ref()
|
.as_ref()
|
||||||
.and_then(|error| error.upstream_status)
|
.and_then(|error| error.upstream_status)
|
||||||
.unwrap_or(result.status_code);
|
.unwrap_or(result.status_code);
|
||||||
let message = result
|
let category = if status_code == 401 {
|
||||||
.error
|
"authentication_error"
|
||||||
.as_ref()
|
} else if status_code == 403 {
|
||||||
.map(|error| error.message.clone())
|
"permission_denied"
|
||||||
.or_else(|| {
|
} else if status_code == 404 {
|
||||||
result
|
"not_found"
|
||||||
.body
|
} else if status_code == 429 {
|
||||||
.as_ref()
|
"rate_limit"
|
||||||
.and_then(|body| body.json_body.as_ref())
|
} else if status_code >= 500 {
|
||||||
.and_then(|value| value.get("error"))
|
"server_error"
|
||||||
.and_then(Value::as_str)
|
} else {
|
||||||
.map(str::to_string)
|
"provider_error"
|
||||||
})
|
};
|
||||||
.unwrap_or_else(|| format!("upstream returned {status_code}"));
|
|
||||||
let permanent = result.error.as_ref().map_or(
|
let permanent = result.error.as_ref().map_or(
|
||||||
matches!(status_code, 400 | 401 | 403 | 404 | 422),
|
matches!(status_code, 400 | 401 | 403 | 404 | 422),
|
||||||
|error| match error.kind {
|
|error| match error.kind {
|
||||||
@@ -253,17 +263,18 @@ fn classify_refresh_result_error(result: &ExecutionResult) -> VideoTaskRefreshEr
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
VideoTaskRefreshError { message, permanent }
|
VideoTaskRefreshError {
|
||||||
|
category,
|
||||||
|
permanent,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn build_successful_poll_update(
|
fn build_successful_poll_update(
|
||||||
task: &StoredVideoTask,
|
task: &StoredVideoTask,
|
||||||
|
mut snapshot: LocalVideoTaskSnapshot,
|
||||||
provider_body: &Map<String, Value>,
|
provider_body: &Map<String, Value>,
|
||||||
now_unix_secs: u64,
|
now_unix_secs: u64,
|
||||||
) -> Result<Option<UpsertVideoTask>, GatewayError> {
|
) -> Result<Option<UpsertVideoTask>, GatewayError> {
|
||||||
let Some(mut snapshot) = LocalVideoTaskSnapshot::from_stored_task(task) else {
|
|
||||||
return Ok(None);
|
|
||||||
};
|
|
||||||
snapshot.apply_provider_body(provider_body);
|
snapshot.apply_provider_body(provider_body);
|
||||||
|
|
||||||
let mut record = snapshot.to_upsert_record();
|
let mut record = snapshot.to_upsert_record();
|
||||||
@@ -283,10 +294,7 @@ fn build_successful_poll_update(
|
|||||||
record.format_converted = task.format_converted;
|
record.format_converted = task.format_converted;
|
||||||
record.model = task.model.clone().or(record.model);
|
record.model = task.model.clone().or(record.model);
|
||||||
record.prompt = task.prompt.clone().or(record.prompt);
|
record.prompt = task.prompt.clone().or(record.prompt);
|
||||||
record.original_request_body = task
|
record.original_request_body = None;
|
||||||
.original_request_body
|
|
||||||
.clone()
|
|
||||||
.or(record.original_request_body);
|
|
||||||
record.duration_seconds = task.duration_seconds.or(record.duration_seconds);
|
record.duration_seconds = task.duration_seconds.or(record.duration_seconds);
|
||||||
record.resolution = task.resolution.clone().or(record.resolution);
|
record.resolution = task.resolution.clone().or(record.resolution);
|
||||||
record.aspect_ratio = task.aspect_ratio.clone().or(record.aspect_ratio);
|
record.aspect_ratio = task.aspect_ratio.clone().or(record.aspect_ratio);
|
||||||
@@ -309,17 +317,11 @@ fn build_successful_poll_update(
|
|||||||
if record.status.is_active() && record.poll_count >= record.max_poll_count {
|
if record.status.is_active() && record.poll_count >= record.max_poll_count {
|
||||||
record.status = VideoTaskStatus::Failed;
|
record.status = VideoTaskStatus::Failed;
|
||||||
record.error_code = Some("poll_timeout".to_string());
|
record.error_code = Some("poll_timeout".to_string());
|
||||||
record.error_message = Some(format!("Task timed out after {} polls", record.poll_count));
|
record.error_message = None;
|
||||||
record.completed_at_unix_secs = Some(now_unix_secs);
|
record.completed_at_unix_secs = Some(now_unix_secs);
|
||||||
record.next_poll_at_unix_secs = None;
|
record.next_poll_at_unix_secs = None;
|
||||||
}
|
}
|
||||||
record.request_metadata = merge_video_task_request_metadata(
|
record.request_metadata = None;
|
||||||
task.request_metadata.clone(),
|
|
||||||
&snapshot,
|
|
||||||
Some(provider_body),
|
|
||||||
None,
|
|
||||||
)
|
|
||||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
|
||||||
|
|
||||||
Ok(Some(record))
|
Ok(Some(record))
|
||||||
}
|
}
|
||||||
@@ -332,11 +334,11 @@ fn build_failed_poll_update(
|
|||||||
let mut record = stored_task_to_upsert(task);
|
let mut record = stored_task_to_upsert(task);
|
||||||
record.updated_at_unix_secs = now_unix_secs;
|
record.updated_at_unix_secs = now_unix_secs;
|
||||||
record.poll_count = task.poll_count.saturating_add(1);
|
record.poll_count = task.poll_count.saturating_add(1);
|
||||||
record.progress_message = Some(format!("Poll error: {}", err.message));
|
record.progress_message = None;
|
||||||
if err.permanent {
|
if err.permanent {
|
||||||
record.status = VideoTaskStatus::Failed;
|
record.status = VideoTaskStatus::Failed;
|
||||||
record.error_code = Some("poll_permanent_error".to_string());
|
record.error_code = Some("poll_permanent_error".to_string());
|
||||||
record.error_message = Some(err.message.clone());
|
record.error_message = None;
|
||||||
record.completed_at_unix_secs = Some(now_unix_secs);
|
record.completed_at_unix_secs = Some(now_unix_secs);
|
||||||
record.next_poll_at_unix_secs = None;
|
record.next_poll_at_unix_secs = None;
|
||||||
} else {
|
} else {
|
||||||
@@ -348,28 +350,15 @@ fn build_failed_poll_update(
|
|||||||
if record.status.is_active() && record.poll_count >= record.max_poll_count {
|
if record.status.is_active() && record.poll_count >= record.max_poll_count {
|
||||||
record.status = VideoTaskStatus::Failed;
|
record.status = VideoTaskStatus::Failed;
|
||||||
record.error_code = Some("poll_timeout".to_string());
|
record.error_code = Some("poll_timeout".to_string());
|
||||||
record.error_message = Some(format!("Task timed out after {} polls", record.poll_count));
|
record.error_message = None;
|
||||||
record.completed_at_unix_secs = Some(now_unix_secs);
|
record.completed_at_unix_secs = Some(now_unix_secs);
|
||||||
record.next_poll_at_unix_secs = None;
|
record.next_poll_at_unix_secs = None;
|
||||||
}
|
}
|
||||||
record.request_metadata = LocalVideoTaskSnapshot::from_stored_task(task)
|
record.request_metadata = None;
|
||||||
.and_then(|snapshot| {
|
|
||||||
merge_video_task_request_metadata(
|
|
||||||
task.request_metadata.clone(),
|
|
||||||
&snapshot,
|
|
||||||
None,
|
|
||||||
Some(err),
|
|
||||||
)
|
|
||||||
.ok()
|
|
||||||
.flatten()
|
|
||||||
})
|
|
||||||
.or(task.request_metadata.clone());
|
|
||||||
record
|
record
|
||||||
}
|
}
|
||||||
|
|
||||||
fn stored_task_to_upsert(task: &StoredVideoTask) -> UpsertVideoTask {
|
fn stored_task_to_upsert(task: &StoredVideoTask) -> UpsertVideoTask {
|
||||||
let snapshot_record =
|
|
||||||
LocalVideoTaskSnapshot::from_stored_task(task).map(|snapshot| snapshot.to_upsert_record());
|
|
||||||
UpsertVideoTask {
|
UpsertVideoTask {
|
||||||
id: task.id.clone(),
|
id: task.id.clone(),
|
||||||
short_id: task.short_id.clone(),
|
short_id: task.short_id.clone(),
|
||||||
@@ -386,39 +375,15 @@ fn stored_task_to_upsert(task: &StoredVideoTask) -> UpsertVideoTask {
|
|||||||
provider_api_format: task.provider_api_format.clone(),
|
provider_api_format: task.provider_api_format.clone(),
|
||||||
format_converted: task.format_converted,
|
format_converted: task.format_converted,
|
||||||
model: task.model.clone(),
|
model: task.model.clone(),
|
||||||
prompt: task.prompt.clone().or_else(|| {
|
prompt: task.prompt.clone(),
|
||||||
snapshot_record
|
original_request_body: None,
|
||||||
.as_ref()
|
duration_seconds: task.duration_seconds,
|
||||||
.and_then(|record| record.prompt.clone())
|
resolution: task.resolution.clone(),
|
||||||
}),
|
aspect_ratio: task.aspect_ratio.clone(),
|
||||||
original_request_body: task.original_request_body.clone().or_else(|| {
|
size: task.size.clone(),
|
||||||
snapshot_record
|
|
||||||
.as_ref()
|
|
||||||
.and_then(|record| record.original_request_body.clone())
|
|
||||||
}),
|
|
||||||
duration_seconds: task.duration_seconds.or_else(|| {
|
|
||||||
snapshot_record
|
|
||||||
.as_ref()
|
|
||||||
.and_then(|record| record.duration_seconds)
|
|
||||||
}),
|
|
||||||
resolution: task.resolution.clone().or_else(|| {
|
|
||||||
snapshot_record
|
|
||||||
.as_ref()
|
|
||||||
.and_then(|record| record.resolution.clone())
|
|
||||||
}),
|
|
||||||
aspect_ratio: task.aspect_ratio.clone().or_else(|| {
|
|
||||||
snapshot_record
|
|
||||||
.as_ref()
|
|
||||||
.and_then(|record| record.aspect_ratio.clone())
|
|
||||||
}),
|
|
||||||
size: task.size.clone().or_else(|| {
|
|
||||||
snapshot_record
|
|
||||||
.as_ref()
|
|
||||||
.and_then(|record| record.size.clone())
|
|
||||||
}),
|
|
||||||
status: task.status,
|
status: task.status,
|
||||||
progress_percent: task.progress_percent,
|
progress_percent: task.progress_percent,
|
||||||
progress_message: task.progress_message.clone(),
|
progress_message: None,
|
||||||
retry_count: task.retry_count,
|
retry_count: task.retry_count,
|
||||||
poll_interval_seconds: task.poll_interval_seconds.max(1),
|
poll_interval_seconds: task.poll_interval_seconds.max(1),
|
||||||
next_poll_at_unix_secs: task.next_poll_at_unix_secs,
|
next_poll_at_unix_secs: task.next_poll_at_unix_secs,
|
||||||
@@ -429,9 +394,9 @@ fn stored_task_to_upsert(task: &StoredVideoTask) -> UpsertVideoTask {
|
|||||||
completed_at_unix_secs: task.completed_at_unix_secs,
|
completed_at_unix_secs: task.completed_at_unix_secs,
|
||||||
updated_at_unix_secs: task.updated_at_unix_secs,
|
updated_at_unix_secs: task.updated_at_unix_secs,
|
||||||
error_code: task.error_code.clone(),
|
error_code: task.error_code.clone(),
|
||||||
error_message: task.error_message.clone(),
|
error_message: None,
|
||||||
video_url: task.video_url.clone(),
|
video_url: task.video_url.clone(),
|
||||||
request_metadata: task.request_metadata.clone(),
|
request_metadata: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -443,44 +408,6 @@ fn compute_poll_backoff_seconds(poll_interval_seconds: u32, retry_count: u32) ->
|
|||||||
.min(MAX_VIDEO_TASK_POLL_BACKOFF_SECONDS)
|
.min(MAX_VIDEO_TASK_POLL_BACKOFF_SECONDS)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn merge_video_task_request_metadata(
|
|
||||||
existing: Option<Value>,
|
|
||||||
snapshot: &LocalVideoTaskSnapshot,
|
|
||||||
provider_body: Option<&Map<String, Value>>,
|
|
||||||
poll_error: Option<&VideoTaskRefreshError>,
|
|
||||||
) -> Result<Option<Value>, serde_json::Error> {
|
|
||||||
let mut metadata = match existing {
|
|
||||||
Some(Value::Object(object)) => object,
|
|
||||||
_ => Map::new(),
|
|
||||||
};
|
|
||||||
metadata.insert(
|
|
||||||
"rust_owner".to_string(),
|
|
||||||
Value::String("async_task".to_string()),
|
|
||||||
);
|
|
||||||
metadata.insert(
|
|
||||||
"rust_local_snapshot".to_string(),
|
|
||||||
serde_json::to_value(snapshot)?,
|
|
||||||
);
|
|
||||||
if let Some(provider_body) = provider_body {
|
|
||||||
metadata.insert(
|
|
||||||
"poll_raw_response".to_string(),
|
|
||||||
Value::Object(provider_body.clone()),
|
|
||||||
);
|
|
||||||
metadata.remove("poll_error");
|
|
||||||
}
|
|
||||||
if let Some(poll_error) = poll_error {
|
|
||||||
metadata.insert(
|
|
||||||
"poll_error".to_string(),
|
|
||||||
serde_json::json!({
|
|
||||||
"message": poll_error.message,
|
|
||||||
"permanent": poll_error.permanent,
|
|
||||||
"observed_at_unix_secs": now_unix_secs(),
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
Ok(Some(Value::Object(metadata)))
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) async fn finalize_video_task_if_terminal(state: &AppState, task: &StoredVideoTask) {
|
pub(crate) async fn finalize_video_task_if_terminal(state: &AppState, task: &StoredVideoTask) {
|
||||||
let Some(event) = build_video_task_terminal_usage_event(task) else {
|
let Some(event) = build_video_task_terminal_usage_event(task) else {
|
||||||
return;
|
return;
|
||||||
@@ -543,9 +470,9 @@ fn build_video_task_terminal_usage_event(task: &StoredVideoTask) -> Option<Usage
|
|||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let provider_name = LocalVideoTaskSnapshot::from_stored_task(task)
|
let provider_name = task
|
||||||
.and_then(|snapshot| snapshot.provider_name().map(str::to_string))
|
.provider_id
|
||||||
.or_else(|| task.provider_id.clone())
|
.clone()
|
||||||
.unwrap_or_else(|| "unknown".to_string());
|
.unwrap_or_else(|| "unknown".to_string());
|
||||||
let response_time_ms = task
|
let response_time_ms = task
|
||||||
.submitted_at_unix_secs
|
.submitted_at_unix_secs
|
||||||
@@ -580,10 +507,10 @@ fn build_video_task_terminal_usage_event(task: &StoredVideoTask) -> Option<Usage
|
|||||||
has_format_conversion: Some(task.format_converted),
|
has_format_conversion: Some(task.format_converted),
|
||||||
is_stream: Some(false),
|
is_stream: Some(false),
|
||||||
status_code,
|
status_code,
|
||||||
error_message: task.error_message.clone().or(task.error_code.clone()),
|
error_message: task.error_code.clone(),
|
||||||
response_time_ms,
|
response_time_ms,
|
||||||
request_body: task.original_request_body.clone(),
|
request_body: None,
|
||||||
request_metadata: task.request_metadata.clone(),
|
request_metadata: None,
|
||||||
..UsageEventData::default()
|
..UsageEventData::default()
|
||||||
},
|
},
|
||||||
))
|
))
|
||||||
@@ -701,48 +628,36 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn stored_task_to_upsert_restores_sparse_fields_from_snapshot() {
|
fn stored_task_to_upsert_does_not_restore_sensitive_legacy_snapshot_fields() {
|
||||||
let record = stored_task_to_upsert(&sample_sparse_stored_task());
|
let record = stored_task_to_upsert(&sample_sparse_stored_task());
|
||||||
|
|
||||||
assert_eq!(record.prompt.as_deref(), Some("hello"));
|
assert!(record.prompt.is_none());
|
||||||
assert_eq!(
|
assert!(record.original_request_body.is_none());
|
||||||
record.original_request_body,
|
assert!(record.duration_seconds.is_none());
|
||||||
Some(json!({
|
assert!(record.resolution.is_none());
|
||||||
"prompt": "hello",
|
assert!(record.aspect_ratio.is_none());
|
||||||
"seconds": "4",
|
assert!(record.size.is_none());
|
||||||
"resolution": "720p",
|
assert!(record.progress_message.is_none());
|
||||||
"aspect_ratio": "16:9",
|
assert!(record.error_message.is_none());
|
||||||
"size": "1280x720"
|
assert!(record.request_metadata.is_none());
|
||||||
}))
|
|
||||||
);
|
|
||||||
assert_eq!(record.duration_seconds, Some(4));
|
|
||||||
assert_eq!(record.resolution.as_deref(), Some("720p"));
|
|
||||||
assert_eq!(record.aspect_ratio.as_deref(), Some("16:9"));
|
|
||||||
assert_eq!(record.size.as_deref(), Some("1280x720"));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn failed_poll_update_keeps_snapshot_backed_request_body() {
|
fn failed_poll_update_drops_snapshot_backed_sensitive_fields() {
|
||||||
let record = build_failed_poll_update(
|
let record = build_failed_poll_update(
|
||||||
&sample_sparse_stored_task(),
|
&sample_sparse_stored_task(),
|
||||||
&VideoTaskRefreshError {
|
&VideoTaskRefreshError {
|
||||||
message: "temporary failure".to_string(),
|
category: "transport_error",
|
||||||
permanent: false,
|
permanent: false,
|
||||||
},
|
},
|
||||||
100,
|
100,
|
||||||
);
|
);
|
||||||
|
|
||||||
assert_eq!(
|
assert!(record.original_request_body.is_none());
|
||||||
record.original_request_body,
|
assert!(record.prompt.is_none());
|
||||||
Some(json!({
|
assert!(record.resolution.is_none());
|
||||||
"prompt": "hello",
|
assert!(record.progress_message.is_none());
|
||||||
"seconds": "4",
|
assert!(record.error_message.is_none());
|
||||||
"resolution": "720p",
|
assert!(record.request_metadata.is_none());
|
||||||
"aspect_ratio": "16:9",
|
|
||||||
"size": "1280x720"
|
|
||||||
}))
|
|
||||||
);
|
|
||||||
assert_eq!(record.prompt.as_deref(), Some("hello"));
|
|
||||||
assert_eq!(record.resolution.as_deref(), Some("720p"));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -34,6 +34,7 @@ pub(crate) fn emit_admin_audit(
|
|||||||
path_and_query: &str,
|
path_and_query: &str,
|
||||||
control_decision: Option<&GatewayControlDecision>,
|
control_decision: Option<&GatewayControlDecision>,
|
||||||
) {
|
) {
|
||||||
|
let sanitized_path_and_query = sanitize_admin_audit_path(path_and_query);
|
||||||
let Some(decision) = control_decision else {
|
let Some(decision) = control_decision else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
@@ -64,9 +65,10 @@ pub(crate) fn emit_admin_audit(
|
|||||||
},
|
},
|
||||||
route_kind,
|
route_kind,
|
||||||
default_target_type(route_family),
|
default_target_type(route_family),
|
||||||
path_and_query.to_string(),
|
sanitized_path_and_query.clone(),
|
||||||
)
|
)
|
||||||
};
|
};
|
||||||
|
let target_id = sanitize_admin_audit_target_id(target_id);
|
||||||
|
|
||||||
let (audit_status, log_level) = classify_admin_audit_response(method, response.status());
|
let (audit_status, log_level) = classify_admin_audit_response(method, response.status());
|
||||||
if log_level == AdminAuditLogLevel::Info {
|
if log_level == AdminAuditLogLevel::Info {
|
||||||
@@ -83,7 +85,7 @@ pub(crate) fn emit_admin_audit(
|
|||||||
route_family,
|
route_family,
|
||||||
route_kind,
|
route_kind,
|
||||||
method = %method,
|
method = %method,
|
||||||
path = %path_and_query,
|
path = %sanitized_path_and_query,
|
||||||
action,
|
action,
|
||||||
target_type,
|
target_type,
|
||||||
target_id = %target_id,
|
target_id = %target_id,
|
||||||
@@ -103,7 +105,7 @@ pub(crate) fn emit_admin_audit(
|
|||||||
route_family,
|
route_family,
|
||||||
route_kind,
|
route_kind,
|
||||||
method = %method,
|
method = %method,
|
||||||
path = %path_and_query,
|
path = %sanitized_path_and_query,
|
||||||
action,
|
action,
|
||||||
target_type,
|
target_type,
|
||||||
target_id = %target_id,
|
target_id = %target_id,
|
||||||
@@ -112,6 +114,17 @@ pub(crate) fn emit_admin_audit(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn sanitize_admin_audit_path(path_and_query: &str) -> String {
|
||||||
|
crate::middleware::sanitize_access_log_path(path_and_query)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sanitize_admin_audit_target_id(target_id: String) -> String {
|
||||||
|
if target_id.trim_start().starts_with('/') {
|
||||||
|
return sanitize_admin_audit_path(&target_id);
|
||||||
|
}
|
||||||
|
target_id
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
enum AdminAuditLogLevel {
|
enum AdminAuditLogLevel {
|
||||||
Info,
|
Info,
|
||||||
@@ -151,7 +164,10 @@ fn is_admin_read_method(method: &http::Method) -> bool {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{classify_admin_audit_response, AdminAuditLogLevel};
|
use super::{
|
||||||
|
classify_admin_audit_response, sanitize_admin_audit_path, sanitize_admin_audit_target_id,
|
||||||
|
AdminAuditLogLevel,
|
||||||
|
};
|
||||||
use axum::http::{Method, StatusCode};
|
use axum::http::{Method, StatusCode};
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -169,4 +185,32 @@ mod tests {
|
|||||||
("failed", AdminAuditLogLevel::Warn)
|
("failed", AdminAuditLogLevel::Warn)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn audit_paths_drop_sensitive_query_values() {
|
||||||
|
assert_eq!(
|
||||||
|
sanitize_admin_audit_path(
|
||||||
|
"/api/admin/providers?token=secret&api_key=live-key&limit=25"
|
||||||
|
),
|
||||||
|
"/api/admin/providers?limit=25"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
sanitize_admin_audit_path("/install/one-time-secret?view=raw"),
|
||||||
|
"/install/[redacted]?view=raw"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn path_shaped_audit_targets_drop_sensitive_query_values() {
|
||||||
|
assert_eq!(
|
||||||
|
sanitize_admin_audit_target_id(
|
||||||
|
"/api/admin/monitoring/trace/request-1?token=secret&limit=25".to_string(),
|
||||||
|
),
|
||||||
|
"/api/admin/monitoring/trace/request-1?limit=25"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
sanitize_admin_audit_target_id("resource-id?literal".to_string()),
|
||||||
|
"resource-id?literal"
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -26,7 +26,10 @@ pub(crate) async fn get_request_candidate_trace(
|
|||||||
.map_err(|err| GatewayError::Internal(err.to_string()).into_response())?;
|
.map_err(|err| GatewayError::Internal(err.to_string()).into_response())?;
|
||||||
|
|
||||||
match trace {
|
match trace {
|
||||||
Some(trace) => Ok(Json(trace)),
|
Some(mut trace) => {
|
||||||
|
trace.sanitize_sensitive_diagnostics();
|
||||||
|
Ok(Json(trace))
|
||||||
|
}
|
||||||
None => Err((
|
None => Err((
|
||||||
axum::http::StatusCode::NOT_FOUND,
|
axum::http::StatusCode::NOT_FOUND,
|
||||||
Json(json!({
|
Json(json!({
|
||||||
@@ -52,7 +55,10 @@ pub(crate) async fn get_decision_trace(
|
|||||||
.map_err(|err| GatewayError::Internal(err.to_string()).into_response())?;
|
.map_err(|err| GatewayError::Internal(err.to_string()).into_response())?;
|
||||||
|
|
||||||
match trace {
|
match trace {
|
||||||
Some(trace) => Ok(Json(trace)),
|
Some(mut trace) => {
|
||||||
|
trace.sanitize_sensitive_diagnostics();
|
||||||
|
Ok(Json(trace))
|
||||||
|
}
|
||||||
None => Err((
|
None => Err((
|
||||||
axum::http::StatusCode::NOT_FOUND,
|
axum::http::StatusCode::NOT_FOUND,
|
||||||
Json(json!({
|
Json(json!({
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ use serde_json::{Map, Value};
|
|||||||
use super::schedule::{BackupSchedule, BackupScheduleUnit};
|
use super::schedule::{BackupSchedule, BackupScheduleUnit};
|
||||||
use super::scopes::BackupScope;
|
use super::scopes::BackupScope;
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Clone, PartialEq, Eq)]
|
||||||
pub(crate) struct S3BackupConfig {
|
pub(crate) struct S3BackupConfig {
|
||||||
pub(crate) enabled: bool,
|
pub(crate) enabled: bool,
|
||||||
pub(crate) scope: BackupScope,
|
pub(crate) scope: BackupScope,
|
||||||
@@ -22,6 +22,28 @@ pub(crate) struct S3BackupConfig {
|
|||||||
pub(crate) retention_count: u32,
|
pub(crate) retention_count: u32,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl fmt::Debug for S3BackupConfig {
|
||||||
|
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
|
let endpoint_origin = sanitized_endpoint_origin(&self.endpoint);
|
||||||
|
formatter
|
||||||
|
.debug_struct("S3BackupConfig")
|
||||||
|
.field("enabled", &self.enabled)
|
||||||
|
.field("scope", &self.scope)
|
||||||
|
.field("endpoint_origin", &endpoint_origin)
|
||||||
|
.field("region", &self.region)
|
||||||
|
.field("user_agent", &self.user_agent)
|
||||||
|
.field("bucket", &self.bucket)
|
||||||
|
.field("prefix", &self.prefix)
|
||||||
|
.field("has_access_key_id", &!self.access_key_id.is_empty())
|
||||||
|
.field("has_secret_access_key", &!self.secret_access_key.is_empty())
|
||||||
|
.field("path_style", &self.path_style)
|
||||||
|
.field("compression", &self.compression)
|
||||||
|
.field("schedule", &self.schedule)
|
||||||
|
.field("retention_count", &self.retention_count)
|
||||||
|
.finish()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
pub(crate) struct BackupConfigError {
|
pub(crate) struct BackupConfigError {
|
||||||
message: String,
|
message: String,
|
||||||
@@ -84,6 +106,9 @@ impl S3BackupConfig {
|
|||||||
"Endpoint(S3 地址)",
|
"Endpoint(S3 地址)",
|
||||||
enabled,
|
enabled,
|
||||||
)?;
|
)?;
|
||||||
|
if enabled {
|
||||||
|
validate_s3_endpoint(&endpoint)?;
|
||||||
|
}
|
||||||
let bucket =
|
let bucket =
|
||||||
required_or_disabled_string(entries, "backup_s3_bucket", "Bucket(存储桶)", enabled)?;
|
required_or_disabled_string(entries, "backup_s3_bucket", "Bucket(存储桶)", enabled)?;
|
||||||
let access_key_id = required_or_disabled_string(
|
let access_key_id = required_or_disabled_string(
|
||||||
@@ -99,6 +124,11 @@ impl S3BackupConfig {
|
|||||||
enabled,
|
enabled,
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
|
let prefix = normalize_s3_prefix(
|
||||||
|
&optional_string(entries, "backup_s3_prefix")?
|
||||||
|
.unwrap_or_else(|| "aether/backups/".to_string()),
|
||||||
|
)?;
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
enabled,
|
enabled,
|
||||||
scope,
|
scope,
|
||||||
@@ -108,8 +138,7 @@ impl S3BackupConfig {
|
|||||||
user_agent: optional_string(entries, "backup_s3_user_agent")?
|
user_agent: optional_string(entries, "backup_s3_user_agent")?
|
||||||
.unwrap_or_else(|| "rclone/v1.68.0".to_string()),
|
.unwrap_or_else(|| "rclone/v1.68.0".to_string()),
|
||||||
bucket,
|
bucket,
|
||||||
prefix: optional_string(entries, "backup_s3_prefix")?
|
prefix,
|
||||||
.unwrap_or_else(|| "aether/backups/".to_string()),
|
|
||||||
access_key_id,
|
access_key_id,
|
||||||
secret_access_key,
|
secret_access_key,
|
||||||
path_style: optional_bool(entries, "backup_s3_path_style")?.unwrap_or(true),
|
path_style: optional_bool(entries, "backup_s3_path_style")?.unwrap_or(true),
|
||||||
@@ -121,6 +150,48 @@ impl S3BackupConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn normalize_s3_prefix(prefix: &str) -> Result<String, BackupConfigError> {
|
||||||
|
let prefix = prefix.trim().trim_matches('/');
|
||||||
|
if prefix.is_empty() {
|
||||||
|
return Ok(String::new());
|
||||||
|
}
|
||||||
|
if prefix
|
||||||
|
.split('/')
|
||||||
|
.any(|segment| segment.is_empty() || segment == "." || segment == "..")
|
||||||
|
|| prefix.contains('\\')
|
||||||
|
{
|
||||||
|
return Err(BackupConfigError::new(
|
||||||
|
"Prefix(备份前缀)不能包含空路径段、相对路径段或反斜杠",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(format!("{prefix}/"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_s3_endpoint(endpoint: &str) -> Result<(), BackupConfigError> {
|
||||||
|
let parsed = url::Url::parse(endpoint)
|
||||||
|
.map_err(|_| BackupConfigError::new("Endpoint(S3 地址)必须是有效的 HTTPS URL"))?;
|
||||||
|
if parsed.scheme() != "https"
|
||||||
|
|| parsed.host_str().is_none()
|
||||||
|
|| !parsed.username().is_empty()
|
||||||
|
|| parsed.password().is_some()
|
||||||
|
|| parsed.query().is_some()
|
||||||
|
|| parsed.fragment().is_some()
|
||||||
|
{
|
||||||
|
return Err(BackupConfigError::new(
|
||||||
|
"Endpoint(S3 地址)必须使用 HTTPS,且不能包含用户凭据、查询参数或片段",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sanitized_endpoint_origin(endpoint: &str) -> String {
|
||||||
|
url::Url::parse(endpoint)
|
||||||
|
.ok()
|
||||||
|
.map(|parsed| parsed.origin().ascii_serialization())
|
||||||
|
.unwrap_or_else(|| "<invalid>".to_string())
|
||||||
|
}
|
||||||
|
|
||||||
fn validate_range(label: &str, value: u32, min: u32, max: u32) -> Result<(), BackupConfigError> {
|
fn validate_range(label: &str, value: u32, min: u32, max: u32) -> Result<(), BackupConfigError> {
|
||||||
if (min..=max).contains(&value) {
|
if (min..=max).contains(&value) {
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -374,6 +445,69 @@ mod tests {
|
|||||||
assert!(err.to_string().contains("Endpoint"));
|
assert!(err.to_string().contains("Endpoint"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rejects_insecure_or_credential_bearing_endpoints() {
|
||||||
|
for endpoint in [
|
||||||
|
"http://s3.example.com",
|
||||||
|
"https://user:[email protected]",
|
||||||
|
"https://s3.example.com?token=secret",
|
||||||
|
"https://s3.example.com/#fragment",
|
||||||
|
] {
|
||||||
|
let entries = serde_json::json!({
|
||||||
|
"backup_s3_enabled": true,
|
||||||
|
"backup_s3_endpoint": endpoint,
|
||||||
|
"backup_s3_bucket": "aether-backups",
|
||||||
|
"backup_s3_access_key_id": "access",
|
||||||
|
"backup_s3_secret_access_key": "secret"
|
||||||
|
});
|
||||||
|
|
||||||
|
let error = S3BackupConfig::from_json_map(entries.as_object().unwrap())
|
||||||
|
.expect_err("unsafe endpoint should fail closed");
|
||||||
|
assert!(error.to_string().contains("Endpoint"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn debug_output_does_not_expose_s3_credentials() {
|
||||||
|
let entries = serde_json::json!({
|
||||||
|
"backup_s3_enabled": true,
|
||||||
|
"backup_s3_endpoint": "https://s3.example.com/path",
|
||||||
|
"backup_s3_bucket": "aether-backups",
|
||||||
|
"backup_s3_access_key_id": "access-key-value",
|
||||||
|
"backup_s3_secret_access_key": "secret-key-value"
|
||||||
|
});
|
||||||
|
let config = S3BackupConfig::from_json_map(entries.as_object().unwrap())
|
||||||
|
.expect("config should parse");
|
||||||
|
|
||||||
|
let debug = format!("{config:?}");
|
||||||
|
assert!(debug.contains("https://s3.example.com"));
|
||||||
|
assert!(!debug.contains("/path"));
|
||||||
|
assert!(!debug.contains("access-key-value"));
|
||||||
|
assert!(!debug.contains("secret-key-value"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn canonicalizes_s3_backup_prefix_once() {
|
||||||
|
let entries = serde_json::json!({
|
||||||
|
"backup_s3_enabled": true,
|
||||||
|
"backup_s3_endpoint": "https://s3.example.com",
|
||||||
|
"backup_s3_bucket": "aether-backups",
|
||||||
|
"backup_s3_prefix": "/prod/backups//",
|
||||||
|
"backup_s3_access_key_id": "access",
|
||||||
|
"backup_s3_secret_access_key": "secret"
|
||||||
|
});
|
||||||
|
let config = S3BackupConfig::from_json_map(entries.as_object().unwrap())
|
||||||
|
.expect("prefix should be canonicalized");
|
||||||
|
|
||||||
|
assert_eq!(config.prefix, "prod/backups/");
|
||||||
|
|
||||||
|
for invalid_prefix in ["prod//backups", "prod/../backups", "prod\\backups"] {
|
||||||
|
let mut entries = entries.clone();
|
||||||
|
entries["backup_s3_prefix"] = serde_json::json!(invalid_prefix);
|
||||||
|
assert!(S3BackupConfig::from_json_map(entries.as_object().unwrap()).is_err());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn applies_default_values_from_system_config_contract() {
|
fn applies_default_values_from_system_config_contract() {
|
||||||
let entries = serde_json::json!({
|
let entries = serde_json::json!({
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -6,5 +6,167 @@ pub(crate) mod store;
|
|||||||
pub(crate) mod task;
|
pub(crate) mod task;
|
||||||
pub(crate) mod worker;
|
pub(crate) mod worker;
|
||||||
|
|
||||||
|
pub use executor::{
|
||||||
|
restore_backup_json, BackupDecryptionKey, BackupRestoreError, BackupRestoreLimits,
|
||||||
|
RestoredBackupJson, DEFAULT_BACKUP_MAX_ENCRYPTED_BYTES, DEFAULT_BACKUP_MAX_JSON_BYTES,
|
||||||
|
};
|
||||||
|
|
||||||
|
use axum::body::Bytes;
|
||||||
|
use serde_json::Value;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum BackupRestoreScope {
|
||||||
|
Config,
|
||||||
|
Users,
|
||||||
|
Data,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl BackupRestoreScope {
|
||||||
|
pub const fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Config => "config",
|
||||||
|
Self::Users => "users",
|
||||||
|
Self::Data => "data",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, thiserror::Error)]
|
||||||
|
#[error("backup database apply failed: {0}")]
|
||||||
|
pub struct BackupApplyError(String);
|
||||||
|
|
||||||
|
pub async fn apply_restored_backup(
|
||||||
|
app: &crate::AppState,
|
||||||
|
restored: RestoredBackupJson,
|
||||||
|
scope: BackupRestoreScope,
|
||||||
|
operator_id: Option<&str>,
|
||||||
|
) -> Result<Result<Value, (http::StatusCode, Value)>, BackupApplyError> {
|
||||||
|
let (json_bytes, authority) = restored.into_authenticated_parts();
|
||||||
|
if authority.scope() != scope {
|
||||||
|
return Err(BackupApplyError(format!(
|
||||||
|
"authenticated {} backup cannot be applied to {} scope",
|
||||||
|
authority.scope().as_str(),
|
||||||
|
scope.as_str(),
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
let request_body = Bytes::from(json_bytes);
|
||||||
|
let state = crate::admin_api::AdminAppState::new(app);
|
||||||
|
let result = crate::admin_api::execute_admin_system_import_exclusively(app, async {
|
||||||
|
match scope {
|
||||||
|
BackupRestoreScope::Config => {
|
||||||
|
state
|
||||||
|
.restore_admin_system_config_backup(&request_body, authority)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
BackupRestoreScope::Users => {
|
||||||
|
state
|
||||||
|
.restore_admin_system_users_backup(&request_body, operator_id, authority)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
BackupRestoreScope::Data => {
|
||||||
|
state
|
||||||
|
.restore_admin_system_data_backup(&request_body, operator_id, authority)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|error| {
|
||||||
|
let message = match error {
|
||||||
|
crate::admin_api::AdminSystemImportLockError::Conflict => {
|
||||||
|
"another system import or restore is already running"
|
||||||
|
}
|
||||||
|
crate::admin_api::AdminSystemImportLockError::Unavailable => {
|
||||||
|
"system import coordination is unavailable"
|
||||||
|
}
|
||||||
|
crate::admin_api::AdminSystemImportLockError::Lost => {
|
||||||
|
"system import coordination lease was lost; restore was cancelled and may have partially applied changes"
|
||||||
|
}
|
||||||
|
};
|
||||||
|
BackupApplyError(message.to_string())
|
||||||
|
})?;
|
||||||
|
result.map_err(|error| BackupApplyError(error.into_message()))
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) const S3_BACKUP_ENABLED_KEY: &str = "backup_s3_enabled";
|
pub(crate) const S3_BACKUP_ENABLED_KEY: &str = "backup_s3_enabled";
|
||||||
pub(crate) const S3_BACKUP_LAST_SLOT_KEY: &str = "backup_s3_last_slot";
|
pub(crate) const S3_BACKUP_LAST_SLOT_KEY: &str = "backup_s3_last_slot";
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::{
|
||||||
|
apply_restored_backup, BackupDecryptionKey, BackupRestoreLimits, BackupRestoreScope,
|
||||||
|
RestoredBackupJson,
|
||||||
|
};
|
||||||
|
use crate::backup::executor::encrypt_backup_bytes;
|
||||||
|
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
fn authenticated_users_backup() -> RestoredBackupJson {
|
||||||
|
let object_key = "prod/aether-users-backup-20260830-120000.json.zst.aes256gcm";
|
||||||
|
let compressed = zstd::stream::encode_all(
|
||||||
|
serde_json::to_vec(&json!({
|
||||||
|
"version": "1.5",
|
||||||
|
"exported_at": "2026-08-30T12:00:00Z",
|
||||||
|
"users": [],
|
||||||
|
"standalone_keys": [],
|
||||||
|
}))
|
||||||
|
.expect("test backup should serialize")
|
||||||
|
.as_slice(),
|
||||||
|
0,
|
||||||
|
)
|
||||||
|
.expect("test backup should compress");
|
||||||
|
let (envelope, _) =
|
||||||
|
encrypt_backup_bytes(DEVELOPMENT_ENCRYPTION_KEY, object_key, &compressed)
|
||||||
|
.expect("test backup should encrypt");
|
||||||
|
super::restore_backup_json(
|
||||||
|
object_key,
|
||||||
|
&envelope,
|
||||||
|
&[BackupDecryptionKey::current(DEVELOPMENT_ENCRYPTION_KEY)
|
||||||
|
.expect("test restore key should build")],
|
||||||
|
BackupRestoreLimits::default(),
|
||||||
|
)
|
||||||
|
.expect("test backup should authenticate")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn authenticated_backup_cannot_be_applied_to_a_different_scope() {
|
||||||
|
let restored = authenticated_users_backup();
|
||||||
|
|
||||||
|
let error = apply_restored_backup(
|
||||||
|
&crate::AppState::new().expect("test state should build"),
|
||||||
|
restored,
|
||||||
|
BackupRestoreScope::Config,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect_err("scope mismatch must fail before database access");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
error.to_string(),
|
||||||
|
"backup database apply failed: authenticated users backup cannot be applied to config scope"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn authenticated_backup_apply_uses_the_shared_system_import_lock() {
|
||||||
|
let app = crate::AppState::new().expect("test state should build");
|
||||||
|
let lock = crate::admin_api::try_acquire_admin_system_import_lease(&app)
|
||||||
|
.await
|
||||||
|
.expect("test should acquire the shared import lease");
|
||||||
|
|
||||||
|
let error = apply_restored_backup(
|
||||||
|
&app,
|
||||||
|
authenticated_users_backup(),
|
||||||
|
BackupRestoreScope::Users,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect_err("restore must not interleave with another system import");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
error.to_string(),
|
||||||
|
"backup database apply failed: another system import or restore is already running"
|
||||||
|
);
|
||||||
|
crate::admin_api::release_admin_system_import_lease(&app, &lock).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,5 +1,8 @@
|
|||||||
use std::fmt;
|
use std::fmt;
|
||||||
|
|
||||||
|
const ENCRYPTED_BACKUP_FILE_SUFFIX: &str = ".json.zst.aes256gcm";
|
||||||
|
const LEGACY_PLAINTEXT_BACKUP_FILE_SUFFIX: &str = ".json.zst";
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
pub(crate) enum BackupScope {
|
pub(crate) enum BackupScope {
|
||||||
Config,
|
Config,
|
||||||
@@ -52,10 +55,81 @@ impl BackupScope {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn from_encrypted_object_key(object_key: &str) -> Option<Self> {
|
||||||
|
if object_key.is_empty()
|
||||||
|
|| object_key.starts_with('/')
|
||||||
|
|| object_key.contains('\0')
|
||||||
|
|| object_key.contains('\\')
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let mut segments = object_key.split('/').peekable();
|
||||||
|
let mut file_name = None;
|
||||||
|
while let Some(segment) = segments.next() {
|
||||||
|
if segment.is_empty()
|
||||||
|
|| segment == "."
|
||||||
|
|| segment == ".."
|
||||||
|
|| segment.chars().any(char::is_control)
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
if segments.peek().is_none() {
|
||||||
|
file_name = Some(segment);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let file_name = file_name?;
|
||||||
|
|
||||||
|
[Self::Config, Self::Users, Self::Data]
|
||||||
|
.into_iter()
|
||||||
|
.find(|scope| {
|
||||||
|
file_name
|
||||||
|
.strip_prefix(&format!("{}-", scope.file_stem()))
|
||||||
|
.and_then(|rest| rest.strip_suffix(ENCRYPTED_BACKUP_FILE_SUFFIX))
|
||||||
|
.is_some_and(is_aether_backup_object_id)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
pub(crate) fn matching_backup_keys(
|
pub(crate) fn matching_backup_keys(
|
||||||
self,
|
self,
|
||||||
prefix: &str,
|
prefix: &str,
|
||||||
keys: impl IntoIterator<Item = String>,
|
keys: impl IntoIterator<Item = String>,
|
||||||
|
) -> Vec<String> {
|
||||||
|
self.matching_backup_keys_with_suffixes(
|
||||||
|
prefix,
|
||||||
|
keys,
|
||||||
|
&[
|
||||||
|
ENCRYPTED_BACKUP_FILE_SUFFIX,
|
||||||
|
LEGACY_PLAINTEXT_BACKUP_FILE_SUFFIX,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn matching_encrypted_backup_keys(
|
||||||
|
self,
|
||||||
|
prefix: &str,
|
||||||
|
keys: impl IntoIterator<Item = String>,
|
||||||
|
) -> Vec<String> {
|
||||||
|
self.matching_backup_keys_with_suffixes(prefix, keys, &[ENCRYPTED_BACKUP_FILE_SUFFIX])
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn matching_legacy_plaintext_backup_keys(
|
||||||
|
self,
|
||||||
|
prefix: &str,
|
||||||
|
keys: impl IntoIterator<Item = String>,
|
||||||
|
) -> Vec<String> {
|
||||||
|
self.matching_backup_keys_with_suffixes(
|
||||||
|
prefix,
|
||||||
|
keys,
|
||||||
|
&[LEGACY_PLAINTEXT_BACKUP_FILE_SUFFIX],
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn matching_backup_keys_with_suffixes(
|
||||||
|
self,
|
||||||
|
prefix: &str,
|
||||||
|
keys: impl IntoIterator<Item = String>,
|
||||||
|
file_suffixes: &[&str],
|
||||||
) -> Vec<String> {
|
) -> Vec<String> {
|
||||||
let normalized_prefix = normalized_prefix(prefix);
|
let normalized_prefix = normalized_prefix(prefix);
|
||||||
let expected_prefix = if normalized_prefix.is_empty() {
|
let expected_prefix = if normalized_prefix.is_empty() {
|
||||||
@@ -64,7 +138,6 @@ impl BackupScope {
|
|||||||
format!("{normalized_prefix}/")
|
format!("{normalized_prefix}/")
|
||||||
};
|
};
|
||||||
let file_prefix = format!("{}-", self.file_stem());
|
let file_prefix = format!("{}-", self.file_stem());
|
||||||
let file_suffix = ".json.zst";
|
|
||||||
|
|
||||||
keys.into_iter()
|
keys.into_iter()
|
||||||
.filter(|key| {
|
.filter(|key| {
|
||||||
@@ -74,20 +147,24 @@ impl BackupScope {
|
|||||||
if file_name.contains('/') {
|
if file_name.contains('/') {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
let Some(timestamp) = file_name
|
let Some(timestamp) = file_name.strip_prefix(&file_prefix).and_then(|rest| {
|
||||||
.strip_prefix(&file_prefix)
|
file_suffixes
|
||||||
.and_then(|rest| rest.strip_suffix(file_suffix))
|
.iter()
|
||||||
else {
|
.find_map(|suffix| rest.strip_suffix(suffix))
|
||||||
|
}) else {
|
||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
|
|
||||||
is_aether_backup_timestamp(timestamp)
|
is_aether_backup_object_id(timestamp)
|
||||||
})
|
})
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn file_name(self, timestamp: &str) -> String {
|
fn file_name(self, timestamp: &str) -> String {
|
||||||
format!("{}-{timestamp}.json.zst", self.file_stem())
|
format!(
|
||||||
|
"{}-{timestamp}{ENCRYPTED_BACKUP_FILE_SUFFIX}",
|
||||||
|
self.file_stem()
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -110,6 +187,25 @@ fn is_aether_backup_timestamp(timestamp: &str) -> bool {
|
|||||||
&& bytes[9..].iter().all(|byte| byte.is_ascii_digit())
|
&& bytes[9..].iter().all(|byte| byte.is_ascii_digit())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn is_aether_backup_object_id(value: &str) -> bool {
|
||||||
|
if is_aether_backup_timestamp(value) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
let Some((timestamp, collision_digest)) = value.split_once('-').and_then(|(date, rest)| {
|
||||||
|
let (time, digest) = rest.split_once('-')?;
|
||||||
|
Some((format!("{date}-{time}"), digest))
|
||||||
|
}) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
|
||||||
|
is_aether_backup_timestamp(×tamp)
|
||||||
|
&& collision_digest.len() == 64
|
||||||
|
&& collision_digest
|
||||||
|
.bytes()
|
||||||
|
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::BackupScope;
|
use super::BackupScope;
|
||||||
@@ -130,15 +226,15 @@ mod tests {
|
|||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
BackupScope::Config.object_key("prod/", "20260524-031500"),
|
BackupScope::Config.object_key("prod/", "20260524-031500"),
|
||||||
"prod/aether-config-backup-20260524-031500.json.zst"
|
"prod/aether-config-backup-20260524-031500.json.zst.aes256gcm"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
BackupScope::Users.object_key("prod/", "20260524-031500"),
|
BackupScope::Users.object_key("prod/", "20260524-031500"),
|
||||||
"prod/aether-users-backup-20260524-031500.json.zst"
|
"prod/aether-users-backup-20260524-031500.json.zst.aes256gcm"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
BackupScope::Data.object_key("prod/", "20260524-031500"),
|
BackupScope::Data.object_key("prod/", "20260524-031500"),
|
||||||
"prod/aether-data-backup-20260524-031500.json.zst"
|
"prod/aether-data-backup-20260524-031500.json.zst.aes256gcm"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -146,7 +242,7 @@ mod tests {
|
|||||||
fn retention_filter_only_matches_same_scope() {
|
fn retention_filter_only_matches_same_scope() {
|
||||||
let keys = vec![
|
let keys = vec![
|
||||||
"prod/aether-config-backup-20260524-010000.json.zst".to_string(),
|
"prod/aether-config-backup-20260524-010000.json.zst".to_string(),
|
||||||
"prod/aether-users-backup-20260524-010000.json.zst".to_string(),
|
"prod/aether-users-backup-20260524-010000.json.zst.aes256gcm".to_string(),
|
||||||
"prod/aether-data-backup-20260524-010000.json.zst".to_string(),
|
"prod/aether-data-backup-20260524-010000.json.zst".to_string(),
|
||||||
"prod/random.json.zst".to_string(),
|
"prod/random.json.zst".to_string(),
|
||||||
];
|
];
|
||||||
@@ -155,14 +251,18 @@ mod tests {
|
|||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
matched,
|
matched,
|
||||||
vec!["prod/aether-users-backup-20260524-010000.json.zst"]
|
vec!["prod/aether-users-backup-20260524-010000.json.zst.aes256gcm"]
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn retention_filter_requires_aether_timestamp_format() {
|
fn retention_filter_requires_aether_timestamp_format() {
|
||||||
|
let collision_digest = "a".repeat(64);
|
||||||
let keys = vec![
|
let keys = vec![
|
||||||
"prod/aether-users-backup-20260524-010000.json.zst".to_string(),
|
"prod/aether-users-backup-20260524-010000.json.zst".to_string(),
|
||||||
|
format!(
|
||||||
|
"prod/aether-users-backup-20260524-010000-{collision_digest}.json.zst.aes256gcm"
|
||||||
|
),
|
||||||
"prod/aether-users-backup-foo.json.zst".to_string(),
|
"prod/aether-users-backup-foo.json.zst".to_string(),
|
||||||
"prod/aether-users-backup-2026052-010000.json.zst".to_string(),
|
"prod/aether-users-backup-2026052-010000.json.zst".to_string(),
|
||||||
"prod/aether-users-backup-202605240-010000.json.zst".to_string(),
|
"prod/aether-users-backup-202605240-010000.json.zst".to_string(),
|
||||||
@@ -171,13 +271,19 @@ mod tests {
|
|||||||
"prod/aether-users-backup-20260524010000.json.zst".to_string(),
|
"prod/aether-users-backup-20260524010000.json.zst".to_string(),
|
||||||
"prod/aether-users-backup-2026052a-010000.json.zst".to_string(),
|
"prod/aether-users-backup-2026052a-010000.json.zst".to_string(),
|
||||||
"prod/aether-users-backup-20260524-01000x.json.zst".to_string(),
|
"prod/aether-users-backup-20260524-01000x.json.zst".to_string(),
|
||||||
|
"prod/aether-users-backup-20260524-010000-short.json.zst.aes256gcm".to_string(),
|
||||||
];
|
];
|
||||||
|
|
||||||
let matched = BackupScope::Users.matching_backup_keys("prod/", keys);
|
let matched = BackupScope::Users.matching_backup_keys("prod/", keys);
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
matched,
|
matched,
|
||||||
vec!["prod/aether-users-backup-20260524-010000.json.zst"]
|
vec![
|
||||||
|
"prod/aether-users-backup-20260524-010000.json.zst".to_string(),
|
||||||
|
format!(
|
||||||
|
"prod/aether-users-backup-20260524-010000-{collision_digest}.json.zst.aes256gcm"
|
||||||
|
),
|
||||||
|
]
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -185,11 +291,11 @@ mod tests {
|
|||||||
fn backup_key_prefix_boundaries_are_exact() {
|
fn backup_key_prefix_boundaries_are_exact() {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
BackupScope::Config.object_key("", "20260524-031500"),
|
BackupScope::Config.object_key("", "20260524-031500"),
|
||||||
"aether-config-backup-20260524-031500.json.zst"
|
"aether-config-backup-20260524-031500.json.zst.aes256gcm"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
BackupScope::Config.object_key("prod", "20260524-031500"),
|
BackupScope::Config.object_key("prod", "20260524-031500"),
|
||||||
"prod/aether-config-backup-20260524-031500.json.zst"
|
"prod/aether-config-backup-20260524-031500.json.zst.aes256gcm"
|
||||||
);
|
);
|
||||||
|
|
||||||
let keys = vec![
|
let keys = vec![
|
||||||
@@ -208,4 +314,36 @@ mod tests {
|
|||||||
vec!["prod/aether-config-backup-20260524-010000.json.zst"]
|
vec!["prod/aether-config-backup-20260524-010000.json.zst"]
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn encrypted_object_key_parser_binds_scope_and_rejects_path_traversal() {
|
||||||
|
let collision_digest = "a".repeat(64);
|
||||||
|
assert_eq!(
|
||||||
|
BackupScope::from_encrypted_object_key(
|
||||||
|
"prod/aether-config-backup-20260524-010000.json.zst.aes256gcm"
|
||||||
|
),
|
||||||
|
Some(BackupScope::Config)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
BackupScope::from_encrypted_object_key(&format!(
|
||||||
|
"prod/aether-users-backup-20260524-010000-{collision_digest}.json.zst.aes256gcm"
|
||||||
|
)),
|
||||||
|
Some(BackupScope::Users)
|
||||||
|
);
|
||||||
|
for key in [
|
||||||
|
"../aether-data-backup-20260524-010000.json.zst.aes256gcm",
|
||||||
|
"/aether-data-backup-20260524-010000.json.zst.aes256gcm",
|
||||||
|
"prod//aether-data-backup-20260524-010000.json.zst.aes256gcm",
|
||||||
|
"prod/./aether-data-backup-20260524-010000.json.zst.aes256gcm",
|
||||||
|
"prod\\aether-data-backup-20260524-010000.json.zst.aes256gcm",
|
||||||
|
"prod/aether-data-backup-invalid.json.zst.aes256gcm",
|
||||||
|
"prod/unrelated-20260524-010000.json.zst.aes256gcm",
|
||||||
|
] {
|
||||||
|
assert_eq!(
|
||||||
|
BackupScope::from_encrypted_object_key(key),
|
||||||
|
None,
|
||||||
|
"unsafe or unrelated key: {key}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,23 +2,45 @@ use std::collections::BTreeMap;
|
|||||||
use std::fmt;
|
use std::fmt;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use bytes::Bytes;
|
use bytes::{Bytes, BytesMut};
|
||||||
use futures_util::TryStreamExt;
|
use futures_util::TryStreamExt;
|
||||||
use object_store::aws::AmazonS3Builder;
|
use object_store::aws::AmazonS3Builder;
|
||||||
use object_store::path::Path;
|
use object_store::path::Path;
|
||||||
use object_store::{ClientOptions, ObjectStore};
|
use object_store::{ClientOptions, ObjectStore, ObjectStoreExt, PutMode, PutOptions};
|
||||||
use reqwest::header::HeaderValue;
|
use reqwest::header::HeaderValue;
|
||||||
use tokio::sync::RwLock;
|
use tokio::sync::RwLock;
|
||||||
|
|
||||||
use super::config::S3BackupConfig;
|
use super::config::S3BackupConfig;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub(crate) enum BackupObjectCreateResult {
|
||||||
|
Created,
|
||||||
|
AlreadyExists,
|
||||||
|
}
|
||||||
|
|
||||||
#[async_trait::async_trait]
|
#[async_trait::async_trait]
|
||||||
pub(crate) trait BackupObjectStore: Send + Sync {
|
pub(crate) trait BackupObjectStore: Send + Sync {
|
||||||
async fn put_object(&self, key: &str, bytes: Bytes) -> Result<(), BackupStoreError>;
|
async fn put_object(&self, key: &str, bytes: Bytes) -> Result<(), BackupStoreError>;
|
||||||
|
|
||||||
async fn list_keys(&self, prefix: &str) -> Result<Vec<String>, BackupStoreError>;
|
async fn put_object_if_absent(
|
||||||
|
&self,
|
||||||
|
key: &str,
|
||||||
|
bytes: Bytes,
|
||||||
|
) -> Result<BackupObjectCreateResult, BackupStoreError>;
|
||||||
|
|
||||||
|
async fn get_object_limited(
|
||||||
|
&self,
|
||||||
|
key: &str,
|
||||||
|
max_bytes: usize,
|
||||||
|
) -> Result<Bytes, BackupStoreError>;
|
||||||
|
|
||||||
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError>;
|
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError>;
|
||||||
|
|
||||||
|
async fn list_keys_limited(
|
||||||
|
&self,
|
||||||
|
prefix: &str,
|
||||||
|
max_objects: usize,
|
||||||
|
) -> Result<Vec<String>, BackupStoreError>;
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
@@ -60,21 +82,72 @@ impl BackupObjectStore for FakeBackupObjectStore {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn list_keys(&self, prefix: &str) -> Result<Vec<String>, BackupStoreError> {
|
async fn put_object_if_absent(
|
||||||
|
&self,
|
||||||
|
key: &str,
|
||||||
|
bytes: Bytes,
|
||||||
|
) -> Result<BackupObjectCreateResult, BackupStoreError> {
|
||||||
|
let mut objects = self.objects.write().await;
|
||||||
|
if objects.contains_key(key) {
|
||||||
|
Ok(BackupObjectCreateResult::AlreadyExists)
|
||||||
|
} else {
|
||||||
|
objects.insert(key.to_string(), bytes);
|
||||||
|
Ok(BackupObjectCreateResult::Created)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_object_limited(
|
||||||
|
&self,
|
||||||
|
key: &str,
|
||||||
|
max_bytes: usize,
|
||||||
|
) -> Result<Bytes, BackupStoreError> {
|
||||||
|
let bytes = self
|
||||||
|
.objects
|
||||||
|
.read()
|
||||||
|
.await
|
||||||
|
.get(key)
|
||||||
|
.cloned()
|
||||||
|
.ok_or_else(|| BackupStoreError::new(format!("backup object `{key}` not found")))?;
|
||||||
|
if bytes.len() > max_bytes {
|
||||||
|
return Err(BackupStoreError::new(format!(
|
||||||
|
"backup object `{key}` exceeds the configured {max_bytes} byte read limit"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(bytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError> {
|
||||||
|
self.objects.write().await.remove(key);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list_keys_limited(
|
||||||
|
&self,
|
||||||
|
prefix: &str,
|
||||||
|
max_objects: usize,
|
||||||
|
) -> Result<Vec<String>, BackupStoreError> {
|
||||||
let prefix = directory_list_prefix(prefix);
|
let prefix = directory_list_prefix(prefix);
|
||||||
Ok(self
|
let keys: Vec<_> = self
|
||||||
.objects
|
.objects
|
||||||
.read()
|
.read()
|
||||||
.await
|
.await
|
||||||
.keys()
|
.keys()
|
||||||
.filter(|key| key.starts_with(&prefix))
|
.filter(|key| key.starts_with(&prefix))
|
||||||
.cloned()
|
.cloned()
|
||||||
.collect())
|
.collect();
|
||||||
|
if keys.len() > max_objects {
|
||||||
|
return Err(BackupStoreError::new(format!(
|
||||||
|
"backup object listing exceeds the configured {max_objects} object limit"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(keys)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError> {
|
#[cfg(test)]
|
||||||
self.objects.write().await.remove(key);
|
impl FakeBackupObjectStore {
|
||||||
Ok(())
|
pub(crate) async fn object_bytes(&self, key: &str) -> Option<Bytes> {
|
||||||
|
self.objects.read().await.get(key).cloned()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -125,17 +198,68 @@ impl BackupObjectStore for ObjectStoreS3BackupStore {
|
|||||||
.map_err(|error| BackupStoreError::object_store("put", key, error))
|
.map_err(|error| BackupStoreError::object_store("put", key, error))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn list_keys(&self, prefix: &str) -> Result<Vec<String>, BackupStoreError> {
|
async fn put_object_if_absent(
|
||||||
let prefix_path = list_prefix_path(prefix);
|
&self,
|
||||||
let mut keys = self
|
key: &str,
|
||||||
|
bytes: Bytes,
|
||||||
|
) -> Result<BackupObjectCreateResult, BackupStoreError> {
|
||||||
|
let options = PutOptions {
|
||||||
|
mode: PutMode::Create,
|
||||||
|
..PutOptions::default()
|
||||||
|
};
|
||||||
|
match self
|
||||||
.store
|
.store
|
||||||
.list(prefix_path.as_ref())
|
.put_opts(&Path::from(key), bytes.into(), options)
|
||||||
.map_ok(|meta| meta.location.to_string())
|
|
||||||
.try_collect::<Vec<_>>()
|
|
||||||
.await
|
.await
|
||||||
.map_err(|error| BackupStoreError::object_store("list", prefix, error))?;
|
{
|
||||||
keys.sort();
|
Ok(_) => Ok(BackupObjectCreateResult::Created),
|
||||||
Ok(keys)
|
Err(object_store::Error::AlreadyExists { .. }) => {
|
||||||
|
Ok(BackupObjectCreateResult::AlreadyExists)
|
||||||
|
}
|
||||||
|
Err(error) => Err(BackupStoreError::object_store(
|
||||||
|
"conditional put",
|
||||||
|
key,
|
||||||
|
error,
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_object_limited(
|
||||||
|
&self,
|
||||||
|
key: &str,
|
||||||
|
max_bytes: usize,
|
||||||
|
) -> Result<Bytes, BackupStoreError> {
|
||||||
|
let result = self
|
||||||
|
.store
|
||||||
|
.get(&Path::from(key))
|
||||||
|
.await
|
||||||
|
.map_err(|error| BackupStoreError::object_store("get", key, error))?;
|
||||||
|
if result.meta.size > u64::try_from(max_bytes).unwrap_or(u64::MAX) {
|
||||||
|
return Err(BackupStoreError::new(format!(
|
||||||
|
"backup object `{key}` exceeds the configured {max_bytes} byte read limit"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
let object_size = result.meta.size;
|
||||||
|
let mut stream = result.into_stream();
|
||||||
|
let mut bytes = BytesMut::with_capacity(
|
||||||
|
usize::try_from(object_size)
|
||||||
|
.unwrap_or(max_bytes)
|
||||||
|
.min(max_bytes)
|
||||||
|
.min(8 * 1024 * 1024),
|
||||||
|
);
|
||||||
|
while let Some(chunk) = stream
|
||||||
|
.try_next()
|
||||||
|
.await
|
||||||
|
.map_err(|error| BackupStoreError::object_store("read", key, error))?
|
||||||
|
{
|
||||||
|
if bytes.len().saturating_add(chunk.len()) > max_bytes {
|
||||||
|
return Err(BackupStoreError::new(format!(
|
||||||
|
"backup object `{key}` exceeds the configured {max_bytes} byte read limit"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
bytes.extend_from_slice(&chunk);
|
||||||
|
}
|
||||||
|
Ok(bytes.freeze())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError> {
|
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError> {
|
||||||
@@ -144,6 +268,30 @@ impl BackupObjectStore for ObjectStoreS3BackupStore {
|
|||||||
.await
|
.await
|
||||||
.map_err(|error| BackupStoreError::object_store("delete", key, error))
|
.map_err(|error| BackupStoreError::object_store("delete", key, error))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn list_keys_limited(
|
||||||
|
&self,
|
||||||
|
prefix: &str,
|
||||||
|
max_objects: usize,
|
||||||
|
) -> Result<Vec<String>, BackupStoreError> {
|
||||||
|
let prefix_path = list_prefix_path(prefix);
|
||||||
|
let mut objects = self.store.list(prefix_path.as_ref());
|
||||||
|
let mut keys = Vec::new();
|
||||||
|
while let Some(meta) = objects
|
||||||
|
.try_next()
|
||||||
|
.await
|
||||||
|
.map_err(|error| BackupStoreError::object_store("list", prefix, error))?
|
||||||
|
{
|
||||||
|
if keys.len() >= max_objects {
|
||||||
|
return Err(BackupStoreError::new(format!(
|
||||||
|
"backup object listing exceeds the configured {max_objects} object limit"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
keys.push(meta.location.to_string());
|
||||||
|
}
|
||||||
|
keys.sort();
|
||||||
|
Ok(keys)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn directory_list_prefix(prefix: &str) -> String {
|
fn directory_list_prefix(prefix: &str) -> String {
|
||||||
@@ -166,10 +314,12 @@ fn list_prefix_path(prefix: &str) -> Option<Path> {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{list_prefix_path, BackupObjectStore, FakeBackupObjectStore};
|
use super::{
|
||||||
|
list_prefix_path, BackupObjectCreateResult, BackupObjectStore, FakeBackupObjectStore,
|
||||||
|
};
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn fake_backup_object_store_puts_lists_and_deletes() {
|
async fn fake_backup_object_store_puts_and_lists() {
|
||||||
let store = FakeBackupObjectStore::default();
|
let store = FakeBackupObjectStore::default();
|
||||||
store
|
store
|
||||||
.put_object(
|
.put_object(
|
||||||
@@ -186,17 +336,59 @@ mod tests {
|
|||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let keys = store.list_keys("prod/").await.unwrap();
|
let keys = store.list_keys_limited("prod/", 2).await.unwrap();
|
||||||
assert_eq!(keys.len(), 2);
|
|
||||||
|
|
||||||
store
|
|
||||||
.delete_object("prod/aether-data-backup-20260524-010000.json.zst")
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
let keys = store.list_keys("prod/").await.unwrap();
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
keys,
|
keys,
|
||||||
vec!["prod/aether-data-backup-20260524-020000.json.zst"]
|
vec![
|
||||||
|
"prod/aether-data-backup-20260524-010000.json.zst",
|
||||||
|
"prod/aether-data-backup-20260524-020000.json.zst",
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn fake_backup_object_store_enforces_read_and_listing_limits() {
|
||||||
|
let store = FakeBackupObjectStore::default();
|
||||||
|
store
|
||||||
|
.put_object("prod/one", bytes::Bytes::from_static(b"1234"))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
store
|
||||||
|
.put_object("prod/two", bytes::Bytes::from_static(b"5678"))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert!(store.get_object_limited("prod/one", 3).await.is_err());
|
||||||
|
assert_eq!(
|
||||||
|
store.get_object_limited("prod/one", 4).await.unwrap(),
|
||||||
|
bytes::Bytes::from_static(b"1234")
|
||||||
|
);
|
||||||
|
assert!(store.list_keys_limited("prod/", 1).await.is_err());
|
||||||
|
assert_eq!(store.list_keys_limited("prod/", 2).await.unwrap().len(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn fake_backup_object_store_conditional_put_never_overwrites() {
|
||||||
|
let store = FakeBackupObjectStore::default();
|
||||||
|
let key = "prod/aether-data-backup-20260524-010000.json.zst.aes256gcm";
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.put_object_if_absent(key, bytes::Bytes::from_static(b"first"))
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
BackupObjectCreateResult::Created
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.put_object_if_absent(key, bytes::Bytes::from_static(b"second"))
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
BackupObjectCreateResult::AlreadyExists
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
store.object_bytes(key).await.as_deref(),
|
||||||
|
Some(b"first".as_slice())
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -218,7 +410,7 @@ mod tests {
|
|||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let keys = store.list_keys("prod").await.unwrap();
|
let keys = store.list_keys_limited("prod", 10).await.unwrap();
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
keys,
|
keys,
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
use std::fmt;
|
use std::fmt;
|
||||||
|
use std::future::Future;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use aether_admin::system::admin_system_config_default_value;
|
use aether_admin::system::admin_system_config_default_value;
|
||||||
@@ -12,14 +13,15 @@ use chrono::Utc;
|
|||||||
use futures_util::FutureExt;
|
use futures_util::FutureExt;
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
use serde_json::{json, Map, Value};
|
use serde_json::{json, Map, Value};
|
||||||
|
use tokio::task::{JoinError, JoinHandle};
|
||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
|
|
||||||
use super::config::S3BackupConfig;
|
use super::config::S3BackupConfig;
|
||||||
use super::executor::{run_backup_with_store, BackupRunResult};
|
use super::executor::{run_backup_with_store, BackupRunResult};
|
||||||
use super::scopes::BackupScope;
|
use super::scopes::BackupScope;
|
||||||
use super::store::ObjectStoreS3BackupStore;
|
use super::store::ObjectStoreS3BackupStore;
|
||||||
use crate::admin_api::AdminAppState;
|
use crate::admin_api::{AdminAppState, SystemExportMode};
|
||||||
use crate::handlers::shared::decrypt_catalog_secret_with_fallbacks;
|
use crate::handlers::shared::decrypt_or_migrate_system_config_secret;
|
||||||
use crate::task_runtime::{
|
use crate::task_runtime::{
|
||||||
append_event_with_logging, build_task_run_id, now_unix_secs, spawn_fire_and_forget,
|
append_event_with_logging, build_task_run_id, now_unix_secs, spawn_fire_and_forget,
|
||||||
task_definition, update_run_status, upsert_run_with_logging, TASK_KEY_SYSTEM_S3_BACKUP,
|
task_definition, update_run_status, upsert_run_with_logging, TASK_KEY_SYSTEM_S3_BACKUP,
|
||||||
@@ -48,6 +50,9 @@ const S3_BACKUP_CONFIG_KEYS: &[&str] = &[
|
|||||||
];
|
];
|
||||||
|
|
||||||
const S3_BACKUP_QUEUED_MESSAGE: &str = "S3 备份任务已提交";
|
const S3_BACKUP_QUEUED_MESSAGE: &str = "S3 备份任务已提交";
|
||||||
|
const S3_BACKUP_INTERNAL_ERROR_DETAIL: &str = "S3 备份服务暂时不可用";
|
||||||
|
const S3_BACKUP_TASK_FAILURE_CODE: &str = "s3_backup_failed";
|
||||||
|
const S3_BACKUP_SLOT_RECORD_FAILURE_CODE: &str = "s3_backup_slot_record_failed";
|
||||||
const S3_BACKUP_TASK_LOCK_KEY: &str = "task_runtime:lock:system.s3.backup";
|
const S3_BACKUP_TASK_LOCK_KEY: &str = "task_runtime:lock:system.s3.backup";
|
||||||
const S3_BACKUP_TASK_LOCK_TTL: Duration = Duration::from_secs(60 * 60 * 6);
|
const S3_BACKUP_TASK_LOCK_TTL: Duration = Duration::from_secs(60 * 60 * 6);
|
||||||
const S3_BACKUP_TASK_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(60 * 5);
|
const S3_BACKUP_TASK_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(60 * 5);
|
||||||
@@ -67,6 +72,16 @@ pub(crate) struct S3BackupTaskError {
|
|||||||
detail: String,
|
detail: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
enum BackupLockRenewalFailure<E> {
|
||||||
|
Lost,
|
||||||
|
Backend(E),
|
||||||
|
}
|
||||||
|
|
||||||
|
enum BackupLockRaceOutcome<T> {
|
||||||
|
BackupCompleted(T),
|
||||||
|
LeaseLost(Result<(), JoinError>),
|
||||||
|
}
|
||||||
|
|
||||||
impl S3BackupTaskError {
|
impl S3BackupTaskError {
|
||||||
fn bad_request(detail: impl Into<String>) -> Self {
|
fn bad_request(detail: impl Into<String>) -> Self {
|
||||||
Self {
|
Self {
|
||||||
@@ -114,8 +129,12 @@ impl fmt::Display for S3BackupTaskError {
|
|||||||
impl std::error::Error for S3BackupTaskError {}
|
impl std::error::Error for S3BackupTaskError {}
|
||||||
|
|
||||||
impl From<GatewayError> for S3BackupTaskError {
|
impl From<GatewayError> for S3BackupTaskError {
|
||||||
fn from(error: GatewayError) -> Self {
|
fn from(_error: GatewayError) -> Self {
|
||||||
Self::internal(format!("{error:?}"))
|
warn!(
|
||||||
|
error_category = "dependency_failed",
|
||||||
|
"S3 backup dependency failed"
|
||||||
|
);
|
||||||
|
Self::internal(S3_BACKUP_INTERNAL_ERROR_DETAIL)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -220,8 +239,6 @@ fn s3_backup_task_payload_json(
|
|||||||
) -> Value {
|
) -> Value {
|
||||||
let mut payload = json!({
|
let mut payload = json!({
|
||||||
"scope": config.scope.as_config_value(),
|
"scope": config.scope.as_config_value(),
|
||||||
"bucket": config.bucket.clone(),
|
|
||||||
"prefix": config.prefix.clone(),
|
|
||||||
"compression": config.compression.clone(),
|
"compression": config.compression.clone(),
|
||||||
"trigger": trigger,
|
"trigger": trigger,
|
||||||
});
|
});
|
||||||
@@ -259,7 +276,7 @@ fn spawn_s3_backup_worker(
|
|||||||
Some(100),
|
Some(100),
|
||||||
Some("S3 备份任务异常退出".to_string()),
|
Some("S3 备份任务异常退出".to_string()),
|
||||||
None,
|
None,
|
||||||
Some("S3 backup task panicked".to_string()),
|
Some("background_task_panicked".to_string()),
|
||||||
None,
|
None,
|
||||||
Some(now_unix_secs()),
|
Some(now_unix_secs()),
|
||||||
)
|
)
|
||||||
@@ -294,16 +311,67 @@ async fn run_s3_backup_worker_inner(
|
|||||||
.await;
|
.await;
|
||||||
append_event_with_logging(&app, &run_id, "running", "S3 backup task started", None).await;
|
append_event_with_logging(&app, &run_id, "running", "S3 backup task started", None).await;
|
||||||
|
|
||||||
let heartbeat = spawn_s3_backup_task_heartbeat(app.clone(), run_id.clone(), lock);
|
let heartbeat = spawn_s3_backup_task_heartbeat(app.clone(), run_id.clone(), lock.clone());
|
||||||
let result = run_s3_backup_once(&app, &config).await;
|
let result = match race_backup_with_lock_heartbeat(run_s3_backup_once(&app, &config), heartbeat)
|
||||||
heartbeat.abort();
|
.await
|
||||||
let _ = heartbeat.await;
|
{
|
||||||
|
BackupLockRaceOutcome::BackupCompleted(result) => {
|
||||||
|
match require_successful_backup_lock_renewal(
|
||||||
|
app.runtime_state
|
||||||
|
.lock_renew(&lock, S3_BACKUP_TASK_LOCK_TTL)
|
||||||
|
.await,
|
||||||
|
) {
|
||||||
|
Ok(()) => result,
|
||||||
|
Err(BackupLockRenewalFailure::Lost) => {
|
||||||
|
warn!(
|
||||||
|
run_id = %run_id,
|
||||||
|
lock_key = %lock.key,
|
||||||
|
"S3 backup task lost its distributed lock before publishing completion"
|
||||||
|
);
|
||||||
|
Err(S3BackupTaskError::service_unavailable(
|
||||||
|
"S3 备份任务锁已失效,任务完成状态未发布",
|
||||||
|
))
|
||||||
|
}
|
||||||
|
Err(BackupLockRenewalFailure::Backend(error)) => {
|
||||||
|
warn!(
|
||||||
|
run_id = %run_id,
|
||||||
|
lock_key = %lock.key,
|
||||||
|
error = %error,
|
||||||
|
"S3 backup task could not verify its distributed lock before publishing completion"
|
||||||
|
);
|
||||||
|
Err(S3BackupTaskError::service_unavailable(
|
||||||
|
"无法确认 S3 备份任务锁所有权,任务完成状态未发布",
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
BackupLockRaceOutcome::LeaseLost(heartbeat_result) => {
|
||||||
|
match heartbeat_result {
|
||||||
|
Ok(()) => warn!(
|
||||||
|
run_id = %run_id,
|
||||||
|
"S3 backup task stopped after losing its distributed lock"
|
||||||
|
),
|
||||||
|
Err(error) => warn!(
|
||||||
|
run_id = %run_id,
|
||||||
|
error = %error,
|
||||||
|
"S3 backup lock heartbeat task failed"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
Err(S3BackupTaskError::service_unavailable(
|
||||||
|
"S3 备份任务锁已失效,任务已停止",
|
||||||
|
))
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
match result {
|
match result {
|
||||||
Ok(result) => {
|
Ok(result) => {
|
||||||
if let Some(slot) = scheduled_backup_slot_to_record(scheduled_slot.as_deref(), true) {
|
if let Some(slot) = scheduled_backup_slot_to_record(scheduled_slot.as_deref(), true) {
|
||||||
if let Err(error) = record_scheduled_backup_slot(&app, &slot).await {
|
if record_scheduled_backup_slot(&app, &slot).await.is_err() {
|
||||||
warn!(error = ?error, run_id = %run_id, "S3 backup slot record failed");
|
warn!(
|
||||||
|
error_category = "slot_record_failed",
|
||||||
|
run_id = %run_id,
|
||||||
|
"S3 backup slot record failed"
|
||||||
|
);
|
||||||
let _ = update_run_status(
|
let _ = update_run_status(
|
||||||
&app,
|
&app,
|
||||||
&run_id,
|
&run_id,
|
||||||
@@ -311,7 +379,7 @@ async fn run_s3_backup_worker_inner(
|
|||||||
Some(100),
|
Some(100),
|
||||||
Some("S3 备份任务完成,但记录调度时间失败".to_string()),
|
Some("S3 备份任务完成,但记录调度时间失败".to_string()),
|
||||||
None,
|
None,
|
||||||
Some(format!("S3 backup slot record failed: {error:?}")),
|
Some(S3_BACKUP_SLOT_RECORD_FAILURE_CODE.to_string()),
|
||||||
None,
|
None,
|
||||||
Some(now_unix_secs()),
|
Some(now_unix_secs()),
|
||||||
)
|
)
|
||||||
@@ -321,7 +389,7 @@ async fn run_s3_backup_worker_inner(
|
|||||||
&run_id,
|
&run_id,
|
||||||
"failed",
|
"failed",
|
||||||
"S3 backup slot record failed",
|
"S3 backup slot record failed",
|
||||||
Some(json!({ "error": format!("{error:?}") })),
|
Some(json!({ "error_code": S3_BACKUP_SLOT_RECORD_FAILURE_CODE })),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
return;
|
return;
|
||||||
@@ -349,8 +417,12 @@ async fn run_s3_backup_worker_inner(
|
|||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
Err(error) => {
|
Err(_) => {
|
||||||
warn!(error = %error, run_id = %run_id, "S3 backup task failed");
|
warn!(
|
||||||
|
error_category = "backup_execution_failed",
|
||||||
|
run_id = %run_id,
|
||||||
|
"S3 backup task failed"
|
||||||
|
);
|
||||||
let _ = update_run_status(
|
let _ = update_run_status(
|
||||||
&app,
|
&app,
|
||||||
&run_id,
|
&run_id,
|
||||||
@@ -358,7 +430,7 @@ async fn run_s3_backup_worker_inner(
|
|||||||
Some(100),
|
Some(100),
|
||||||
Some("S3 备份任务失败".to_string()),
|
Some("S3 备份任务失败".to_string()),
|
||||||
None,
|
None,
|
||||||
Some(error.to_string()),
|
Some(S3_BACKUP_TASK_FAILURE_CODE.to_string()),
|
||||||
None,
|
None,
|
||||||
Some(now_unix_secs()),
|
Some(now_unix_secs()),
|
||||||
)
|
)
|
||||||
@@ -368,13 +440,44 @@ async fn run_s3_backup_worker_inner(
|
|||||||
&run_id,
|
&run_id,
|
||||||
"failed",
|
"failed",
|
||||||
"S3 backup task failed",
|
"S3 backup task failed",
|
||||||
Some(json!({ "error": error.to_string() })),
|
Some(json!({ "error_code": S3_BACKUP_TASK_FAILURE_CODE })),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn race_backup_with_lock_heartbeat<F, T>(
|
||||||
|
backup: F,
|
||||||
|
mut heartbeat: JoinHandle<()>,
|
||||||
|
) -> BackupLockRaceOutcome<T>
|
||||||
|
where
|
||||||
|
F: Future<Output = T>,
|
||||||
|
{
|
||||||
|
tokio::pin!(backup);
|
||||||
|
tokio::select! {
|
||||||
|
biased;
|
||||||
|
heartbeat_result = &mut heartbeat => {
|
||||||
|
BackupLockRaceOutcome::LeaseLost(heartbeat_result)
|
||||||
|
}
|
||||||
|
result = &mut backup => {
|
||||||
|
heartbeat.abort();
|
||||||
|
let _ = heartbeat.await;
|
||||||
|
BackupLockRaceOutcome::BackupCompleted(result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn require_successful_backup_lock_renewal<E>(
|
||||||
|
result: Result<bool, E>,
|
||||||
|
) -> Result<(), BackupLockRenewalFailure<E>> {
|
||||||
|
match result {
|
||||||
|
Ok(true) => Ok(()),
|
||||||
|
Ok(false) => Err(BackupLockRenewalFailure::Lost),
|
||||||
|
Err(error) => Err(BackupLockRenewalFailure::Backend(error)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn spawn_s3_backup_task_heartbeat(
|
fn spawn_s3_backup_task_heartbeat(
|
||||||
app: AppState,
|
app: AppState,
|
||||||
run_id: String,
|
run_id: String,
|
||||||
@@ -386,10 +489,30 @@ fn spawn_s3_backup_task_heartbeat(
|
|||||||
interval.tick().await;
|
interval.tick().await;
|
||||||
loop {
|
loop {
|
||||||
interval.tick().await;
|
interval.tick().await;
|
||||||
let _ = app
|
match require_successful_backup_lock_renewal(
|
||||||
.runtime_state
|
app.runtime_state
|
||||||
.lock_renew(&lock, S3_BACKUP_TASK_LOCK_TTL)
|
.lock_renew(&lock, S3_BACKUP_TASK_LOCK_TTL)
|
||||||
.await;
|
.await,
|
||||||
|
) {
|
||||||
|
Ok(()) => {}
|
||||||
|
Err(BackupLockRenewalFailure::Lost) => {
|
||||||
|
warn!(
|
||||||
|
run_id = %run_id,
|
||||||
|
lock_key = %lock.key,
|
||||||
|
"S3 backup task distributed lock is no longer owned"
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Err(BackupLockRenewalFailure::Backend(error)) => {
|
||||||
|
warn!(
|
||||||
|
run_id = %run_id,
|
||||||
|
lock_key = %lock.key,
|
||||||
|
error = %error,
|
||||||
|
"S3 backup task distributed lock renewal failed"
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
let _ = update_run_status(
|
let _ = update_run_status(
|
||||||
&app,
|
&app,
|
||||||
&run_id,
|
&run_id,
|
||||||
@@ -458,9 +581,15 @@ async fn acquire_s3_backup_task_lock(
|
|||||||
Ok(None) => Err(S3BackupTaskError::conflict(
|
Ok(None) => Err(S3BackupTaskError::conflict(
|
||||||
"已有 S3 备份任务正在执行,请等待当前任务完成后再试",
|
"已有 S3 备份任务正在执行,请等待当前任务完成后再试",
|
||||||
)),
|
)),
|
||||||
Err(error) => Err(S3BackupTaskError::service_unavailable(format!(
|
Err(_) => {
|
||||||
"无法获取 S3 备份任务锁:{error}"
|
warn!(
|
||||||
))),
|
error_category = "lock_acquisition_failed",
|
||||||
|
"S3 backup task lock acquisition failed"
|
||||||
|
);
|
||||||
|
Err(S3BackupTaskError::service_unavailable(
|
||||||
|
"无法获取 S3 备份任务锁,请稍后重试",
|
||||||
|
))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -509,25 +638,77 @@ async fn run_s3_backup_once(
|
|||||||
app: &AppState,
|
app: &AppState,
|
||||||
config: &S3BackupConfig,
|
config: &S3BackupConfig,
|
||||||
) -> Result<BackupRunResult, S3BackupTaskError> {
|
) -> Result<BackupRunResult, S3BackupTaskError> {
|
||||||
let admin_state = AdminAppState::new(app);
|
let Some(encryption_secret) = effective_backup_encryption_secret(app) else {
|
||||||
let payload = match config.scope {
|
return Err(S3BackupTaskError::service_unavailable(
|
||||||
BackupScope::Config => {
|
"S3 备份需要 AETHER_BACKUP_ENCRYPTION_KEY 或可用的数据加密密钥",
|
||||||
admin_state
|
));
|
||||||
.build_admin_system_config_export_payload()
|
|
||||||
.await?
|
|
||||||
}
|
|
||||||
BackupScope::Users => {
|
|
||||||
admin_state
|
|
||||||
.build_admin_system_users_export_payload()
|
|
||||||
.await?
|
|
||||||
}
|
|
||||||
BackupScope::Data => admin_state.build_admin_system_data_export_payload().await?,
|
|
||||||
};
|
};
|
||||||
let store = ObjectStoreS3BackupStore::from_config(config)
|
let payload = build_s3_backup_payload_exclusively(app, config.scope).await?;
|
||||||
.map_err(|error| S3BackupTaskError::internal(error.to_string()))?;
|
let store = ObjectStoreS3BackupStore::from_config(config).map_err(|_| {
|
||||||
run_backup_with_store(config, &store, payload, Utc::now())
|
warn!(
|
||||||
|
error_category = "object_store_initialization_failed",
|
||||||
|
"S3 backup object store initialization failed"
|
||||||
|
);
|
||||||
|
S3BackupTaskError::internal(S3_BACKUP_INTERNAL_ERROR_DETAIL)
|
||||||
|
})?;
|
||||||
|
run_backup_with_store(config, &store, payload, Utc::now(), &encryption_secret)
|
||||||
.await
|
.await
|
||||||
.map_err(|error| S3BackupTaskError::internal(error.to_string()))
|
.map_err(|_| {
|
||||||
|
warn!(
|
||||||
|
error_category = "backup_execution_failed",
|
||||||
|
"S3 backup execution failed"
|
||||||
|
);
|
||||||
|
S3BackupTaskError::internal(S3_BACKUP_INTERNAL_ERROR_DETAIL)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn build_s3_backup_payload_exclusively(
|
||||||
|
app: &AppState,
|
||||||
|
scope: BackupScope,
|
||||||
|
) -> Result<Value, S3BackupTaskError> {
|
||||||
|
let admin_state = AdminAppState::new(app);
|
||||||
|
crate::admin_api::execute_admin_system_import_exclusively(app, async {
|
||||||
|
match scope {
|
||||||
|
BackupScope::Config => {
|
||||||
|
admin_state
|
||||||
|
.build_admin_system_config_export_payload(SystemExportMode::RecoveryBackup)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
BackupScope::Users => {
|
||||||
|
admin_state
|
||||||
|
.build_admin_system_users_export_payload(SystemExportMode::RecoveryBackup)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
BackupScope::Data => {
|
||||||
|
admin_state
|
||||||
|
.build_admin_system_data_export_payload(SystemExportMode::RecoveryBackup)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|error| {
|
||||||
|
warn!(
|
||||||
|
error_category = "system_import_coordination_failed",
|
||||||
|
lock_error = ?error,
|
||||||
|
"S3 backup snapshot could not acquire or retain the system import lock"
|
||||||
|
);
|
||||||
|
S3BackupTaskError::service_unavailable(S3_BACKUP_INTERNAL_ERROR_DETAIL)
|
||||||
|
})?
|
||||||
|
.map_err(S3BackupTaskError::from)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn effective_backup_encryption_secret(app: &AppState) -> Option<String> {
|
||||||
|
std::env::var("AETHER_BACKUP_ENCRYPTION_KEY")
|
||||||
|
.ok()
|
||||||
|
.map(|value| value.trim().to_string())
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.or_else(|| {
|
||||||
|
app.encryption_key()
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.map(ToOwned::to_owned)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn load_s3_backup_config_for_run(
|
async fn load_s3_backup_config_for_run(
|
||||||
@@ -551,7 +732,7 @@ pub(crate) async fn load_s3_backup_config_values(
|
|||||||
.or_else(|| admin_system_config_default_value(key));
|
.or_else(|| admin_system_config_default_value(key));
|
||||||
if let Some(value) = value {
|
if let Some(value) = value {
|
||||||
let value = if *key == "backup_s3_secret_access_key" {
|
let value = if *key == "backup_s3_secret_access_key" {
|
||||||
decrypt_s3_secret_access_key(app, value)?
|
decrypt_s3_secret_access_key(app, value).await?
|
||||||
} else {
|
} else {
|
||||||
value
|
value
|
||||||
};
|
};
|
||||||
@@ -561,39 +742,56 @@ pub(crate) async fn load_s3_backup_config_values(
|
|||||||
Ok(values)
|
Ok(values)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn decrypt_s3_secret_access_key(app: &AppState, value: Value) -> Result<Value, S3BackupTaskError> {
|
async fn decrypt_s3_secret_access_key(
|
||||||
let Some(ciphertext) = value
|
app: &AppState,
|
||||||
|
value: Value,
|
||||||
|
) -> Result<Value, S3BackupTaskError> {
|
||||||
|
let Some(stored_value) = value
|
||||||
.as_str()
|
.as_str()
|
||||||
.map(str::trim)
|
.map(str::trim)
|
||||||
.filter(|value| !value.is_empty())
|
.filter(|value| !value.is_empty())
|
||||||
else {
|
else {
|
||||||
return Ok(value);
|
return Ok(value);
|
||||||
};
|
};
|
||||||
let Some(plaintext) = decrypt_catalog_secret_with_fallbacks(app.encryption_key(), ciphertext)
|
let plaintext = decrypt_or_migrate_system_config_secret(
|
||||||
else {
|
app,
|
||||||
return Err(S3BackupTaskError::bad_request(
|
"backup_s3_secret_access_key",
|
||||||
|
stored_value.to_string(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|_| {
|
||||||
|
S3BackupTaskError::bad_request(
|
||||||
"S3 备份配置无效:Secret Access Key(访问密钥)无法解密,请重新填写",
|
"S3 备份配置无效:Secret Access Key(访问密钥)无法解密,请重新填写",
|
||||||
));
|
)
|
||||||
};
|
})?;
|
||||||
Ok(Value::String(plaintext))
|
Ok(Value::String(plaintext))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn backup_run_result_json(result: &BackupRunResult) -> Value {
|
fn backup_run_result_json(result: &BackupRunResult) -> Value {
|
||||||
json!({
|
json!({
|
||||||
"scope": result.scope.as_config_value(),
|
"scope": result.scope.as_config_value(),
|
||||||
"bucket": result.bucket,
|
|
||||||
"object_key": result.object_key,
|
|
||||||
"bytes": result.bytes,
|
"bytes": result.bytes,
|
||||||
"sha256": result.sha256,
|
"sha256": result.sha256,
|
||||||
"export_version": result.export_version,
|
"export_version": result.export_version,
|
||||||
"exported_at": result.exported_at,
|
"exported_at": result.exported_at,
|
||||||
"compression": result.compression,
|
"compression": result.compression,
|
||||||
"deleted_old_objects": result.deleted_old_objects,
|
"encryption": result.encryption,
|
||||||
|
"legacy_encrypted_copies_created": result.legacy_encrypted_copies_created,
|
||||||
|
"legacy_encrypted_copies_verified": result.legacy_encrypted_copies_verified,
|
||||||
|
"legacy_plaintext_objects_deleted": result.legacy_plaintext_objects_deleted,
|
||||||
|
"legacy_plaintext_objects_retained": result.legacy_plaintext_objects_retained,
|
||||||
|
"retention_cleanup_candidates": result.retention_cleanup_candidates,
|
||||||
|
"automatic_deletions": result.legacy_plaintext_objects_deleted,
|
||||||
|
"object_cleanup_mode": "legacy_plaintext_deleted_after_verified_encryption",
|
||||||
|
"versioned_storage_cleanup_required": result.versioned_storage_cleanup_required,
|
||||||
|
"versioned_storage_cleanup_notice": "legacy_plaintext_versions_require_external_cleanup",
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
use std::convert::Infallible;
|
||||||
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
|
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
|
||||||
@@ -603,9 +801,78 @@ mod tests {
|
|||||||
};
|
};
|
||||||
|
|
||||||
use crate::data::GatewayDataState;
|
use crate::data::GatewayDataState;
|
||||||
|
use crate::handlers::shared::decrypt_system_config_secret;
|
||||||
use crate::state::AppState;
|
use crate::state::AppState;
|
||||||
use crate::task_runtime::{now_unix_secs, TASK_KEY_SYSTEM_S3_BACKUP};
|
use crate::task_runtime::{now_unix_secs, TASK_KEY_SYSTEM_S3_BACKUP};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn backup_lock_renewal_requires_ownership_and_preserves_backend_errors() {
|
||||||
|
assert!(matches!(
|
||||||
|
super::require_successful_backup_lock_renewal::<Infallible>(Ok(true)),
|
||||||
|
Ok(())
|
||||||
|
));
|
||||||
|
assert!(matches!(
|
||||||
|
super::require_successful_backup_lock_renewal::<Infallible>(Ok(false)),
|
||||||
|
Err(super::BackupLockRenewalFailure::Lost)
|
||||||
|
));
|
||||||
|
assert!(matches!(
|
||||||
|
super::require_successful_backup_lock_renewal(Err("redis unavailable")),
|
||||||
|
Err(super::BackupLockRenewalFailure::Backend(
|
||||||
|
"redis unavailable"
|
||||||
|
))
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn lost_backup_lock_stops_race_without_publishing_backup_result() {
|
||||||
|
let destructive_stage_reached = Arc::new(AtomicBool::new(false));
|
||||||
|
let destructive_stage_for_backup = Arc::clone(&destructive_stage_reached);
|
||||||
|
let backup = async move {
|
||||||
|
std::future::pending::<()>().await;
|
||||||
|
destructive_stage_for_backup.store(true, Ordering::Release);
|
||||||
|
Ok::<(), super::S3BackupTaskError>(())
|
||||||
|
};
|
||||||
|
let heartbeat = tokio::spawn(async {});
|
||||||
|
let outcome = super::race_backup_with_lock_heartbeat(backup, heartbeat).await;
|
||||||
|
assert!(matches!(
|
||||||
|
outcome,
|
||||||
|
super::BackupLockRaceOutcome::LeaseLost(Ok(()))
|
||||||
|
));
|
||||||
|
assert!(!destructive_stage_reached.load(Ordering::Acquire));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn completed_heartbeat_wins_when_backup_completion_is_also_ready() {
|
||||||
|
let heartbeat = tokio::spawn(async {});
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
|
||||||
|
let outcome = super::race_backup_with_lock_heartbeat(async { 42_u8 }, heartbeat).await;
|
||||||
|
|
||||||
|
assert!(matches!(
|
||||||
|
outcome,
|
||||||
|
super::BackupLockRaceOutcome::LeaseLost(Ok(()))
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn s3_backup_snapshot_refuses_to_overlap_system_import() {
|
||||||
|
let app = AppState::new().expect("app state should build");
|
||||||
|
let lease = crate::admin_api::try_acquire_admin_system_import_lease(&app)
|
||||||
|
.await
|
||||||
|
.expect("test should acquire the system import lease");
|
||||||
|
|
||||||
|
let error = super::build_s3_backup_payload_exclusively(
|
||||||
|
&app,
|
||||||
|
crate::backup::scopes::BackupScope::Config,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect_err("backup snapshot must not overlap a system import");
|
||||||
|
|
||||||
|
crate::admin_api::release_admin_system_import_lease(&app, &lease).await;
|
||||||
|
assert_eq!(error.status(), axum::http::StatusCode::SERVICE_UNAVAILABLE);
|
||||||
|
assert_eq!(error.detail(), super::S3_BACKUP_INTERNAL_ERROR_DETAIL);
|
||||||
|
}
|
||||||
|
|
||||||
fn valid_s3_backup_config_values() -> Vec<(String, serde_json::Value)> {
|
fn valid_s3_backup_config_values() -> Vec<(String, serde_json::Value)> {
|
||||||
vec![
|
vec![
|
||||||
(
|
(
|
||||||
@@ -632,6 +899,92 @@ mod tests {
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn legacy_plaintext_s3_secret_is_migrated_when_config_loads() {
|
||||||
|
let plaintext = "legacy-s3-secret-access-key";
|
||||||
|
let mut entries = valid_s3_backup_config_values();
|
||||||
|
entries
|
||||||
|
.iter_mut()
|
||||||
|
.find(|(key, _)| key == "backup_s3_secret_access_key")
|
||||||
|
.expect("secret config fixture should exist")
|
||||||
|
.1 = serde_json::json!(plaintext);
|
||||||
|
let app = AppState::new()
|
||||||
|
.expect("app state should build")
|
||||||
|
.with_data_state_for_tests(
|
||||||
|
GatewayDataState::disabled()
|
||||||
|
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY)
|
||||||
|
.with_system_config_values_for_tests(entries),
|
||||||
|
);
|
||||||
|
|
||||||
|
let values = super::load_s3_backup_config_values(&app)
|
||||||
|
.await
|
||||||
|
.expect("legacy S3 config should load");
|
||||||
|
assert_eq!(
|
||||||
|
values.get("backup_s3_secret_access_key"),
|
||||||
|
Some(&serde_json::json!(plaintext))
|
||||||
|
);
|
||||||
|
|
||||||
|
let stored = app
|
||||||
|
.read_system_config_json_value_strong("backup_s3_secret_access_key")
|
||||||
|
.await
|
||||||
|
.expect("stored S3 secret should read")
|
||||||
|
.and_then(|value| value.as_str().map(ToOwned::to_owned))
|
||||||
|
.expect("stored S3 secret should remain a string");
|
||||||
|
assert_ne!(stored, plaintext);
|
||||||
|
assert_eq!(
|
||||||
|
decrypt_system_config_secret(&app, "backup_s3_secret_access_key", &stored)
|
||||||
|
.expect("migrated S3 secret should decrypt"),
|
||||||
|
plaintext
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn undecryptable_s3_fernet_secret_fails_closed() {
|
||||||
|
let plaintext = "s3-secret-from-unavailable-key";
|
||||||
|
let ciphertext = encrypt_python_fernet_plaintext("unavailable-s3-key", plaintext)
|
||||||
|
.expect("unknown-key fixture should encrypt");
|
||||||
|
let mut entries = valid_s3_backup_config_values();
|
||||||
|
entries
|
||||||
|
.iter_mut()
|
||||||
|
.find(|(key, _)| key == "backup_s3_secret_access_key")
|
||||||
|
.expect("secret config fixture should exist")
|
||||||
|
.1 = serde_json::json!(ciphertext.clone());
|
||||||
|
let app = AppState::new()
|
||||||
|
.expect("app state should build")
|
||||||
|
.with_data_state_for_tests(
|
||||||
|
GatewayDataState::disabled()
|
||||||
|
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY)
|
||||||
|
.with_system_config_values_for_tests(entries),
|
||||||
|
);
|
||||||
|
|
||||||
|
let error = super::load_s3_backup_config_values(&app)
|
||||||
|
.await
|
||||||
|
.expect_err("unknown-key S3 ciphertext must fail closed");
|
||||||
|
let error_text = error.to_string();
|
||||||
|
assert!(!error_text.contains(plaintext));
|
||||||
|
assert!(!error_text.contains(&ciphertext));
|
||||||
|
assert_eq!(
|
||||||
|
app.read_system_config_json_value_strong("backup_s3_secret_access_key")
|
||||||
|
.await
|
||||||
|
.expect("stored S3 secret should read"),
|
||||||
|
Some(serde_json::json!(ciphertext))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn gateway_dependency_errors_are_not_exposed_to_backup_clients() {
|
||||||
|
let error = super::S3BackupTaskError::from(crate::GatewayError::Internal(
|
||||||
|
"postgresql://admin:[email protected]/aether".to_string(),
|
||||||
|
));
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
error.status(),
|
||||||
|
axum::http::StatusCode::INTERNAL_SERVER_ERROR
|
||||||
|
);
|
||||||
|
assert_eq!(error.detail(), super::S3_BACKUP_INTERNAL_ERROR_DETAIL);
|
||||||
|
assert!(!error.detail().contains("database-secret"));
|
||||||
|
}
|
||||||
|
|
||||||
fn stored_s3_backup_run(status: BackgroundTaskStatus) -> StoredBackgroundTaskRun {
|
fn stored_s3_backup_run(status: BackgroundTaskStatus) -> StoredBackgroundTaskRun {
|
||||||
let now = now_unix_secs();
|
let now = now_unix_secs();
|
||||||
StoredBackgroundTaskRun {
|
StoredBackgroundTaskRun {
|
||||||
@@ -774,7 +1127,9 @@ mod tests {
|
|||||||
|
|
||||||
let payload = super::s3_backup_task_payload_json(&config, "manual", None);
|
let payload = super::s3_backup_task_payload_json(&config, "manual", None);
|
||||||
|
|
||||||
assert!(payload["bucket"].is_string());
|
assert!(payload.get("bucket").is_none());
|
||||||
|
assert!(payload.get("prefix").is_none());
|
||||||
|
assert_eq!(payload["scope"], serde_json::json!("data"));
|
||||||
assert_eq!(payload["trigger"], serde_json::json!("manual"));
|
assert_eq!(payload["trigger"], serde_json::json!("manual"));
|
||||||
assert!(!payload.to_string().contains("secret"));
|
assert!(!payload.to_string().contains("secret"));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -29,8 +29,11 @@ pub(crate) fn spawn_s3_backup_worker(app: AppState) -> Option<JoinHandle<()>> {
|
|||||||
interval.tick().await;
|
interval.tick().await;
|
||||||
loop {
|
loop {
|
||||||
interval.tick().await;
|
interval.tick().await;
|
||||||
if let Err(error) = run_s3_backup_schedule_tick(&app, Utc::now()).await {
|
if run_s3_backup_schedule_tick(&app, Utc::now()).await.is_err() {
|
||||||
warn!(error = ?error, "S3 backup schedule tick failed");
|
warn!(
|
||||||
|
error_category = "schedule_tick_failed",
|
||||||
|
"S3 backup schedule tick failed"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -43,15 +46,21 @@ async fn run_s3_backup_schedule_tick(
|
|||||||
) -> Result<(), GatewayError> {
|
) -> Result<(), GatewayError> {
|
||||||
let values = match super::task::load_s3_backup_config_values(app).await {
|
let values = match super::task::load_s3_backup_config_values(app).await {
|
||||||
Ok(values) => values,
|
Ok(values) => values,
|
||||||
Err(error) => {
|
Err(_) => {
|
||||||
warn!(error = %error, "S3 backup schedule config load failed");
|
warn!(
|
||||||
|
error_category = "config_load_failed",
|
||||||
|
"S3 backup schedule config load failed"
|
||||||
|
);
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let config = match S3BackupConfig::from_json_map(&values) {
|
let config = match S3BackupConfig::from_json_map(&values) {
|
||||||
Ok(config) => config,
|
Ok(config) => config,
|
||||||
Err(error) => {
|
Err(_) => {
|
||||||
warn!(error = %error, "S3 backup schedule config is invalid");
|
warn!(
|
||||||
|
error_category = "config_invalid",
|
||||||
|
"S3 backup schedule config is invalid"
|
||||||
|
);
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -68,8 +77,11 @@ async fn run_s3_backup_schedule_tick(
|
|||||||
|
|
||||||
match super::task::start_s3_backup_task_for_schedule(app.clone(), slot).await {
|
match super::task::start_s3_backup_task_for_schedule(app.clone(), slot).await {
|
||||||
Ok(_) => {}
|
Ok(_) => {}
|
||||||
Err(error) => {
|
Err(_) => {
|
||||||
warn!(error = %error, "S3 backup scheduled task submission failed");
|
warn!(
|
||||||
|
error_category = "task_submission_failed",
|
||||||
|
"S3 backup scheduled task submission failed"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
|
|||||||
@@ -1,8 +1,10 @@
|
|||||||
use crate::handlers::shared::{
|
use crate::handlers::shared::{
|
||||||
decrypt_catalog_secret_with_fallbacks, system_config_bool, system_config_string,
|
bark_device_key_binding, canonical_bark_server_url, decrypt_or_migrate_bark_device_key,
|
||||||
|
system_config_bool, system_config_string,
|
||||||
};
|
};
|
||||||
use crate::{AppState, GatewayError};
|
use crate::{AppState, GatewayError};
|
||||||
use serde_json::{json, Value};
|
use serde_json::{json, Value};
|
||||||
|
use std::net::{IpAddr, SocketAddr};
|
||||||
|
|
||||||
pub(crate) const BARK_PUSH_ENABLED_KEY: &str = "module.bark_push.enabled";
|
pub(crate) const BARK_PUSH_ENABLED_KEY: &str = "module.bark_push.enabled";
|
||||||
pub(crate) const BARK_PUSH_DEVICE_KEY_KEY: &str = "module.bark_push.device_key";
|
pub(crate) const BARK_PUSH_DEVICE_KEY_KEY: &str = "module.bark_push.device_key";
|
||||||
@@ -10,8 +12,20 @@ pub(crate) const BARK_PUSH_SERVER_URL_KEY: &str = "module.bark_push.server_url";
|
|||||||
pub(crate) const BARK_PUSH_TEMPLATE_KEY: &str = "module.bark_push.template";
|
pub(crate) const BARK_PUSH_TEMPLATE_KEY: &str = "module.bark_push.template";
|
||||||
|
|
||||||
const DEFAULT_BARK_API_BASE: &str = "https://api.day.app";
|
const DEFAULT_BARK_API_BASE: &str = "https://api.day.app";
|
||||||
|
const BARK_ALLOW_HTTP_ENV: &str = "AETHER_BARK_ALLOW_HTTP";
|
||||||
|
const BARK_ALLOW_PRIVATE_TARGETS_ENV: &str = "AETHER_BARK_ALLOW_PRIVATE_TARGETS";
|
||||||
|
const MAX_BARK_RESPONSE_BYTES: usize = 64 * 1024;
|
||||||
|
const BARK_CONNECT_TIMEOUT_MS: u64 = 10_000;
|
||||||
|
const BARK_REQUEST_TIMEOUT_MS: u64 = 300_000;
|
||||||
|
const MAX_BARK_SERVER_URL_BYTES: usize = 2 * 1024;
|
||||||
|
const MAX_BARK_DEVICE_KEY_BYTES: usize = 512;
|
||||||
|
const MAX_BARK_TEMPLATE_BYTES: usize = 256 * 1024;
|
||||||
|
const MAX_BARK_TITLE_BYTES: usize = 512;
|
||||||
|
const MAX_BARK_BODY_BYTES: usize = 2 * 1024 * 1024;
|
||||||
|
const MAX_BARK_RENDERED_BODY_BYTES: usize = 2 * 1024 * 1024;
|
||||||
|
const MAX_BARK_RESOLVED_ADDRESSES: usize = 32;
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Clone)]
|
||||||
pub(crate) struct BarkPushConfig {
|
pub(crate) struct BarkPushConfig {
|
||||||
pub(crate) enabled: bool,
|
pub(crate) enabled: bool,
|
||||||
pub(crate) device_key: Option<String>,
|
pub(crate) device_key: Option<String>,
|
||||||
@@ -19,6 +33,21 @@ pub(crate) struct BarkPushConfig {
|
|||||||
pub(crate) template: Option<String>,
|
pub(crate) template: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Debug for BarkPushConfig {
|
||||||
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
formatter
|
||||||
|
.debug_struct("BarkPushConfig")
|
||||||
|
.field("enabled", &self.enabled)
|
||||||
|
.field(
|
||||||
|
"device_key",
|
||||||
|
&self.device_key.as_ref().map(|_| "[REDACTED]"),
|
||||||
|
)
|
||||||
|
.field("server_url", &self.server_url)
|
||||||
|
.field("template", &self.template.as_ref().map(|_| "[REDACTED]"))
|
||||||
|
.finish()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn bark_push_module_enabled(state: &AppState) -> Result<bool, GatewayError> {
|
pub(crate) async fn bark_push_module_enabled(state: &AppState) -> Result<bool, GatewayError> {
|
||||||
let value = state
|
let value = state
|
||||||
.read_system_config_json_value(BARK_PUSH_ENABLED_KEY)
|
.read_system_config_json_value(BARK_PUSH_ENABLED_KEY)
|
||||||
@@ -35,23 +64,34 @@ pub(crate) async fn read_bark_push_config(
|
|||||||
state: &AppState,
|
state: &AppState,
|
||||||
) -> Result<BarkPushConfig, GatewayError> {
|
) -> Result<BarkPushConfig, GatewayError> {
|
||||||
let enabled = bark_push_module_enabled(state).await?;
|
let enabled = bark_push_module_enabled(state).await?;
|
||||||
let device_key = state
|
|
||||||
.read_system_config_json_value(BARK_PUSH_DEVICE_KEY_KEY)
|
|
||||||
.await?
|
|
||||||
.and_then(|value| system_config_string(Some(&value)))
|
|
||||||
.map(|value| {
|
|
||||||
decrypt_catalog_secret_with_fallbacks(state.encryption_key(), &value).unwrap_or(value)
|
|
||||||
});
|
|
||||||
let server_url = state
|
let server_url = state
|
||||||
.read_system_config_json_value(BARK_PUSH_SERVER_URL_KEY)
|
.read_system_config_json_value(BARK_PUSH_SERVER_URL_KEY)
|
||||||
.await?
|
.await?
|
||||||
.and_then(|value| system_config_string(Some(&value)))
|
.and_then(|value| system_config_string(Some(&value)))
|
||||||
.filter(|value| !value.trim().is_empty())
|
.filter(|value| !value.trim().is_empty())
|
||||||
.unwrap_or_else(|| DEFAULT_BARK_API_BASE.to_string());
|
.unwrap_or_else(|| DEFAULT_BARK_API_BASE.to_string());
|
||||||
|
validate_bark_config_field("server_url", &server_url, MAX_BARK_SERVER_URL_BYTES)?;
|
||||||
|
let server_url = normalized_bark_server_url(&server_url)?;
|
||||||
|
let binding = bark_device_key_binding(&server_url)
|
||||||
|
.ok_or_else(|| GatewayError::Internal("Bark 服务器地址不是合法 URL".to_string()))?;
|
||||||
|
let device_key = state
|
||||||
|
.read_system_config_json_value(BARK_PUSH_DEVICE_KEY_KEY)
|
||||||
|
.await?
|
||||||
|
.and_then(|value| system_config_string(Some(&value)));
|
||||||
|
let device_key = match device_key {
|
||||||
|
Some(value) => Some(decrypt_or_migrate_bark_device_key(state, &binding, value).await?),
|
||||||
|
None => None,
|
||||||
|
};
|
||||||
|
if let Some(device_key) = device_key.as_deref() {
|
||||||
|
validate_bark_config_field("device_key", device_key, MAX_BARK_DEVICE_KEY_BYTES)?;
|
||||||
|
}
|
||||||
let template = state
|
let template = state
|
||||||
.read_system_config_json_value(BARK_PUSH_TEMPLATE_KEY)
|
.read_system_config_json_value(BARK_PUSH_TEMPLATE_KEY)
|
||||||
.await?
|
.await?
|
||||||
.and_then(|value| system_config_string(Some(&value)));
|
.and_then(|value| system_config_string(Some(&value)));
|
||||||
|
if let Some(template) = template.as_deref() {
|
||||||
|
validate_bark_config_field("template", template, MAX_BARK_TEMPLATE_BYTES)?;
|
||||||
|
}
|
||||||
|
|
||||||
Ok(BarkPushConfig {
|
Ok(BarkPushConfig {
|
||||||
enabled,
|
enabled,
|
||||||
@@ -62,7 +102,7 @@ pub(crate) async fn read_bark_push_config(
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn send_bark_push(
|
pub(crate) async fn send_bark_push(
|
||||||
state: &AppState,
|
_state: &AppState,
|
||||||
config: &BarkPushConfig,
|
config: &BarkPushConfig,
|
||||||
title: &str,
|
title: &str,
|
||||||
markdown_body: &str,
|
markdown_body: &str,
|
||||||
@@ -76,11 +116,13 @@ pub(crate) async fn send_bark_push(
|
|||||||
"Bark Device Key 不能为空".to_string(),
|
"Bark Device Key 不能为空".to_string(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
let server_url = normalized_bark_server_url(&config.server_url)?;
|
validate_bark_config_field("device_key", device_key, MAX_BARK_DEVICE_KEY_BYTES)?;
|
||||||
let body = render_bark_body(config.template.as_deref(), title, markdown_body);
|
validate_bark_content_field("title", title, MAX_BARK_TITLE_BYTES)?;
|
||||||
let response = state
|
validate_bark_content_field("body", markdown_body, MAX_BARK_BODY_BYTES)?;
|
||||||
.client
|
let (client, push_url) = build_bark_push_client_and_url(&config.server_url).await?;
|
||||||
.post(format!("{server_url}/push"))
|
let body = render_bark_body(config.template.as_deref(), title, markdown_body)?;
|
||||||
|
let response = client
|
||||||
|
.post(push_url)
|
||||||
.json(&json!({
|
.json(&json!({
|
||||||
"device_key": device_key,
|
"device_key": device_key,
|
||||||
"title": title,
|
"title": title,
|
||||||
@@ -88,16 +130,14 @@ pub(crate) async fn send_bark_push(
|
|||||||
}))
|
}))
|
||||||
.send()
|
.send()
|
||||||
.await
|
.await
|
||||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
.map_err(|err| GatewayError::Internal(bark_request_error_message(&err)))?;
|
||||||
let status = response.status();
|
let status = response.status();
|
||||||
let text = response
|
let body = aether_http::read_response_bytes_with_limit(response, MAX_BARK_RESPONSE_BYTES)
|
||||||
.text()
|
|
||||||
.await
|
.await
|
||||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
.map_err(|err| GatewayError::Internal(bark_response_body_error_message(&err)))?;
|
||||||
|
let text = String::from_utf8_lossy(&body);
|
||||||
if !status.is_success() {
|
if !status.is_success() {
|
||||||
return Err(GatewayError::Internal(format!(
|
return Err(GatewayError::Internal(format!("Bark 返回 HTTP {status}")));
|
||||||
"Bark 返回 HTTP {status}: {text}"
|
|
||||||
)));
|
|
||||||
}
|
}
|
||||||
if let Ok(payload) = serde_json::from_str::<Value>(&text) {
|
if let Ok(payload) = serde_json::from_str::<Value>(&text) {
|
||||||
let code_is_ok = payload
|
let code_is_ok = payload
|
||||||
@@ -114,53 +154,259 @@ pub(crate) async fn send_bark_push(
|
|||||||
})
|
})
|
||||||
.unwrap_or(true);
|
.unwrap_or(true);
|
||||||
if !code_is_ok {
|
if !code_is_ok {
|
||||||
return Err(GatewayError::Internal(format!("Bark 返回失败: {payload}")));
|
return Err(GatewayError::Internal("Bark 返回失败".to_string()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn normalized_bark_server_url(server_url: &str) -> Result<String, GatewayError> {
|
fn bark_request_error_message(error: &reqwest::Error) -> String {
|
||||||
let server_url = server_url.trim().trim_end_matches('/');
|
format!("Bark 请求失败 ({})", bark_reqwest_error_kind(error))
|
||||||
if server_url.is_empty() {
|
|
||||||
return Err(GatewayError::Internal(
|
|
||||||
"Bark 服务器地址不能为空".to_string(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
if !server_url.starts_with("https://") && !server_url.starts_with("http://") {
|
|
||||||
return Err(GatewayError::Internal(
|
|
||||||
"Bark 服务器地址必须以 http:// 或 https:// 开头".to_string(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
Ok(server_url.to_string())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn render_bark_body(template: Option<&str>, title: &str, markdown_body: &str) -> String {
|
fn bark_response_body_error_message(error: &aether_http::ResponseBodyReadError) -> String {
|
||||||
match template {
|
match error {
|
||||||
Some(template) if !template.trim().is_empty() => template
|
aether_http::ResponseBodyReadError::TooLarge { max_bytes } => {
|
||||||
.replace("{title}", title)
|
format!("Bark 响应超过 {max_bytes} 字节")
|
||||||
.replace("{body}", markdown_body),
|
}
|
||||||
_ => markdown_body.to_string(),
|
aether_http::ResponseBodyReadError::Read(error) => {
|
||||||
|
format!("Bark 响应读取失败 ({})", bark_reqwest_error_kind(error))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn bark_reqwest_error_kind(error: &reqwest::Error) -> &'static str {
|
||||||
|
if error.is_timeout() {
|
||||||
|
"timeout"
|
||||||
|
} else if error.is_connect() {
|
||||||
|
"connect"
|
||||||
|
} else if error.is_request() {
|
||||||
|
"request"
|
||||||
|
} else {
|
||||||
|
"transport"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn normalized_bark_server_url(server_url: &str) -> Result<String, GatewayError> {
|
||||||
|
canonical_bark_server_url(server_url)
|
||||||
|
.ok_or_else(|| GatewayError::Internal("Bark 服务器地址不是合法 URL".to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn build_bark_push_client_and_url(
|
||||||
|
server_url: &str,
|
||||||
|
) -> Result<(reqwest::Client, url::Url), GatewayError> {
|
||||||
|
validate_bark_config_field("server_url", server_url, MAX_BARK_SERVER_URL_BYTES)?;
|
||||||
|
let normalized = normalized_bark_server_url(server_url)?;
|
||||||
|
let mut push_url = url::Url::parse(&normalized)
|
||||||
|
.map_err(|_| GatewayError::Internal("Bark 服务器地址不是合法 URL".to_string()))?;
|
||||||
|
validate_bark_transport_policy(&push_url, env_flag_enabled(BARK_ALLOW_HTTP_ENV))?;
|
||||||
|
|
||||||
|
let host = push_url
|
||||||
|
.host_str()
|
||||||
|
.ok_or_else(|| GatewayError::Internal("Bark 服务器地址缺少主机名".to_string()))?
|
||||||
|
.to_string();
|
||||||
|
let port = push_url
|
||||||
|
.port_or_known_default()
|
||||||
|
.ok_or_else(|| GatewayError::Internal("Bark 服务器地址缺少端口".to_string()))?;
|
||||||
|
let addresses = if let Ok(ip) = host.parse::<IpAddr>() {
|
||||||
|
vec![SocketAddr::new(ip, port)]
|
||||||
|
} else {
|
||||||
|
tokio::time::timeout(
|
||||||
|
std::time::Duration::from_millis(BARK_CONNECT_TIMEOUT_MS),
|
||||||
|
tokio::net::lookup_host((host.as_str(), port)),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|_| GatewayError::Internal("Bark 服务器 DNS 解析超时".to_string()))?
|
||||||
|
.map_err(|_| GatewayError::Internal("Bark 服务器 DNS 解析失败".to_string()))?
|
||||||
|
.take(MAX_BARK_RESOLVED_ADDRESSES)
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
};
|
||||||
|
let allow_benchmarking_ip = push_url.scheme() == "https"
|
||||||
|
&& push_url.port_or_known_default() == Some(443)
|
||||||
|
&& host.eq_ignore_ascii_case("api.day.app");
|
||||||
|
validate_bark_resolved_addresses(
|
||||||
|
&addresses,
|
||||||
|
env_flag_enabled(BARK_ALLOW_PRIVATE_TARGETS_ENV),
|
||||||
|
allow_benchmarking_ip,
|
||||||
|
)?;
|
||||||
|
|
||||||
|
push_url
|
||||||
|
.path_segments_mut()
|
||||||
|
.map_err(|_| GatewayError::Internal("Bark 服务器地址不能作为基础 URL".to_string()))?
|
||||||
|
.pop_if_empty()
|
||||||
|
.push("push");
|
||||||
|
|
||||||
|
let mut builder = aether_http::apply_http_client_config(
|
||||||
|
reqwest::Client::builder()
|
||||||
|
.no_proxy()
|
||||||
|
.redirect(reqwest::redirect::Policy::none()),
|
||||||
|
&aether_http::HttpClientConfig {
|
||||||
|
connect_timeout_ms: Some(BARK_CONNECT_TIMEOUT_MS),
|
||||||
|
request_timeout_ms: Some(BARK_REQUEST_TIMEOUT_MS),
|
||||||
|
http2_adaptive_window: true,
|
||||||
|
..aether_http::HttpClientConfig::default()
|
||||||
|
},
|
||||||
|
);
|
||||||
|
if host.parse::<IpAddr>().is_err() {
|
||||||
|
builder = builder.resolve_to_addrs(&host, &addresses);
|
||||||
|
}
|
||||||
|
let client = builder
|
||||||
|
.build()
|
||||||
|
.map_err(|_| GatewayError::Internal("Bark HTTP 客户端初始化失败".to_string()))?;
|
||||||
|
Ok((client, push_url))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_bark_transport_policy(url: &url::Url, allow_http: bool) -> Result<(), GatewayError> {
|
||||||
|
if url.scheme() == "http" && !allow_http {
|
||||||
|
return Err(GatewayError::Internal(format!(
|
||||||
|
"Bark 服务器必须使用 HTTPS;如确需明文 HTTP,请显式设置 {BARK_ALLOW_HTTP_ENV}=true"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_bark_resolved_addresses(
|
||||||
|
addresses: &[SocketAddr],
|
||||||
|
allow_private: bool,
|
||||||
|
allow_benchmarking_ip: bool,
|
||||||
|
) -> Result<(), GatewayError> {
|
||||||
|
if addresses.is_empty() {
|
||||||
|
return Err(GatewayError::Internal(
|
||||||
|
"Bark 服务器 DNS 解析未返回地址".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if !allow_private
|
||||||
|
&& addresses.iter().any(|address| {
|
||||||
|
aether_http::is_private_or_reserved_ip(address.ip())
|
||||||
|
&& !(allow_benchmarking_ip
|
||||||
|
&& aether_http::is_ipv4_benchmarking_fake_ip(address.ip()))
|
||||||
|
})
|
||||||
|
{
|
||||||
|
return Err(GatewayError::Internal(format!(
|
||||||
|
"Bark 服务器解析到私有或保留地址;如确需内网自建服务,请显式设置 {BARK_ALLOW_PRIVATE_TARGETS_ENV}=true"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn env_flag_enabled(key: &str) -> bool {
|
||||||
|
std::env::var(key).ok().is_some_and(|value| {
|
||||||
|
matches!(
|
||||||
|
value.trim().to_ascii_lowercase().as_str(),
|
||||||
|
"1" | "true" | "yes" | "on"
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_bark_config_field(
|
||||||
|
field: &str,
|
||||||
|
value: &str,
|
||||||
|
max_bytes: usize,
|
||||||
|
) -> Result<(), GatewayError> {
|
||||||
|
if value.len() > max_bytes || value.bytes().any(|byte| byte == 0) {
|
||||||
|
return Err(GatewayError::Internal(format!(
|
||||||
|
"Bark {field} exceeds the allowed size or contains a NUL byte"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_bark_content_field(
|
||||||
|
field: &str,
|
||||||
|
value: &str,
|
||||||
|
max_bytes: usize,
|
||||||
|
) -> Result<(), GatewayError> {
|
||||||
|
validate_bark_config_field(field, value, max_bytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn render_bark_body(
|
||||||
|
template: Option<&str>,
|
||||||
|
title: &str,
|
||||||
|
markdown_body: &str,
|
||||||
|
) -> Result<String, GatewayError> {
|
||||||
|
validate_bark_content_field("title", title, MAX_BARK_TITLE_BYTES)?;
|
||||||
|
validate_bark_content_field("body", markdown_body, MAX_BARK_BODY_BYTES)?;
|
||||||
|
let template = template
|
||||||
|
.filter(|value| !value.trim().is_empty())
|
||||||
|
.unwrap_or("{body}");
|
||||||
|
validate_bark_config_field("template", template, MAX_BARK_TEMPLATE_BYTES)?;
|
||||||
|
|
||||||
|
let mut rendered = String::with_capacity(template.len().min(MAX_BARK_RENDERED_BODY_BYTES));
|
||||||
|
let mut cursor = 0usize;
|
||||||
|
while cursor < template.len() {
|
||||||
|
let remaining = &template[cursor..];
|
||||||
|
let title_match = remaining.find("{title}");
|
||||||
|
let body_match = remaining.find("{body}");
|
||||||
|
let next = match (title_match, body_match) {
|
||||||
|
(None, None) => {
|
||||||
|
append_bark_rendered_part(&mut rendered, remaining)?;
|
||||||
|
cursor = template.len();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
(Some(index), None) => (index, "{title}", title),
|
||||||
|
(None, Some(index)) => (index, "{body}", markdown_body),
|
||||||
|
(Some(title_index), Some(body_index)) if title_index <= body_index => {
|
||||||
|
(title_index, "{title}", title)
|
||||||
|
}
|
||||||
|
(Some(_), Some(body_index)) => (body_index, "{body}", markdown_body),
|
||||||
|
};
|
||||||
|
append_bark_rendered_part(&mut rendered, &remaining[..next.0])?;
|
||||||
|
append_bark_rendered_part(&mut rendered, next.2)?;
|
||||||
|
cursor += next.0 + next.1.len();
|
||||||
|
}
|
||||||
|
if rendered.is_empty() && template.is_empty() {
|
||||||
|
return Ok(String::new());
|
||||||
|
}
|
||||||
|
Ok(rendered)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn append_bark_rendered_part(output: &mut String, part: &str) -> Result<(), GatewayError> {
|
||||||
|
let next_len = output
|
||||||
|
.len()
|
||||||
|
.checked_add(part.len())
|
||||||
|
.ok_or_else(|| GatewayError::Internal("Bark rendered body is too large".to_string()))?;
|
||||||
|
if next_len > MAX_BARK_RENDERED_BODY_BYTES {
|
||||||
|
return Err(GatewayError::Internal(
|
||||||
|
"Bark rendered body exceeds the allowed size".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
output.push_str(part);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{normalized_bark_server_url, render_bark_body};
|
use super::{
|
||||||
|
bark_request_error_message, bark_response_body_error_message, normalized_bark_server_url,
|
||||||
|
render_bark_body, validate_bark_resolved_addresses, validate_bark_transport_policy,
|
||||||
|
};
|
||||||
|
use std::net::SocketAddr;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn bark_body_uses_template_when_provided() {
|
fn bark_body_uses_template_when_provided() {
|
||||||
let rendered = render_bark_body(Some("{title}\n\n{body}"), "告警", "原始正文");
|
let rendered = render_bark_body(Some("{title}\n\n{body}"), "告警", "原始正文")
|
||||||
|
.expect("template should render");
|
||||||
assert_eq!(rendered, "告警\n\n原始正文");
|
assert_eq!(rendered, "告警\n\n原始正文");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn bark_body_falls_back_to_markdown_body_for_empty_template() {
|
fn bark_body_falls_back_to_markdown_body_for_empty_template() {
|
||||||
assert_eq!(render_bark_body(None, "告警", "原始正文"), "原始正文");
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
render_bark_body(Some(" "), "告警", "原始正文"),
|
render_bark_body(None, "告警", "原始正文").expect("fallback should render"),
|
||||||
"原始正文"
|
"原始正文"
|
||||||
);
|
);
|
||||||
|
assert_eq!(
|
||||||
|
render_bark_body(Some(" "), "告警", "原始正文").expect("fallback should render"),
|
||||||
|
"原始正文"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn bark_body_rejects_template_expansion_bombs_and_oversized_content() {
|
||||||
|
let template = "x".repeat(super::MAX_BARK_TEMPLATE_BYTES + 1);
|
||||||
|
assert!(render_bark_body(Some(&template), "告警", "正文").is_err());
|
||||||
|
let body = "x".repeat(super::MAX_BARK_BODY_BYTES + 1);
|
||||||
|
assert!(render_bark_body(None, "告警", &body).is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -170,4 +416,61 @@ mod tests {
|
|||||||
"https://api.day.app"
|
"https://api.day.app"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn bark_server_url_rejects_credentials_query_and_fragments() {
|
||||||
|
for invalid in [
|
||||||
|
"https://[email protected]",
|
||||||
|
"https://example.com?target=internal",
|
||||||
|
"https://example.com/#fragment",
|
||||||
|
] {
|
||||||
|
assert!(normalized_bark_server_url(invalid).is_err(), "{invalid}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn bark_http_transport_requires_explicit_opt_in() {
|
||||||
|
let url = url::Url::parse("http://bark.example.com").unwrap();
|
||||||
|
assert!(validate_bark_transport_policy(&url, false).is_err());
|
||||||
|
assert!(validate_bark_transport_policy(&url, true).is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn bark_private_targets_require_explicit_opt_in() {
|
||||||
|
let private = [SocketAddr::from(([127, 0, 0, 1], 443))];
|
||||||
|
assert!(validate_bark_resolved_addresses(&private, false, false).is_err());
|
||||||
|
assert!(validate_bark_resolved_addresses(&private, true, false).is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn bark_builtin_server_allows_benchmarking_ip_only_with_https_default_port() {
|
||||||
|
let fake = [SocketAddr::from(([198, 18, 75, 234], 443))];
|
||||||
|
assert!(validate_bark_resolved_addresses(&fake, false, true).is_ok());
|
||||||
|
assert!(validate_bark_resolved_addresses(
|
||||||
|
&[fake[0], SocketAddr::from(([127, 0, 0, 1], 443))],
|
||||||
|
false,
|
||||||
|
true,
|
||||||
|
)
|
||||||
|
.is_err());
|
||||||
|
assert!(validate_bark_resolved_addresses(&fake, false, false).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn bark_transport_errors_do_not_expose_server_url_or_response_body() {
|
||||||
|
let secret = "bark-secret-query";
|
||||||
|
let error = reqwest::Client::new()
|
||||||
|
.post(format!("ftp://bark.example.test/push?token={secret}"))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.expect_err("unsupported URL scheme should fail before network I/O");
|
||||||
|
|
||||||
|
let message = bark_request_error_message(&error);
|
||||||
|
assert!(!message.contains(secret));
|
||||||
|
assert!(!message.contains("bark.example.test"));
|
||||||
|
|
||||||
|
let body_error = aether_http::ResponseBodyReadError::Read(error);
|
||||||
|
let message = bark_response_body_error_message(&body_error);
|
||||||
|
assert!(!message.contains(secret));
|
||||||
|
assert!(!message.contains("bark.example.test"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -193,6 +193,7 @@ pub(crate) fn parse_probe_url(raw: &str) -> Result<Url, ProbeFailure> {
|
|||||||
|| url.password().is_some()
|
|| url.password().is_some()
|
||||||
|| url.query().is_some()
|
|| url.query().is_some()
|
||||||
|| url.fragment().is_some()
|
|| url.fragment().is_some()
|
||||||
|
|| (url.scheme() == "ws" && !aether_http::url_has_literal_loopback_host(&url))
|
||||||
{
|
{
|
||||||
return Err(ProbeFailure::InvalidEndpoint);
|
return Err(ProbeFailure::InvalidEndpoint);
|
||||||
}
|
}
|
||||||
@@ -210,6 +211,7 @@ pub(crate) const fn turn_timeout(args: &ProbeArgs) -> Duration {
|
|||||||
|
|
||||||
async fn run_probe(config: &ProbeConfig, started_at: Instant) -> Result<ProbeReport, ProbeFailure> {
|
async fn run_probe(config: &ProbeConfig, started_at: Instant) -> Result<ProbeReport, ProbeFailure> {
|
||||||
let client = wreq::Client::builder()
|
let client = wreq::Client::builder()
|
||||||
|
.no_proxy()
|
||||||
.connect_timeout(config.turn_timeout)
|
.connect_timeout(config.turn_timeout)
|
||||||
.timeout(config.turn_timeout)
|
.timeout(config.turn_timeout)
|
||||||
.build()
|
.build()
|
||||||
@@ -434,7 +436,14 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn probe_url_rejects_credentials_and_query_strings() {
|
fn probe_url_rejects_credentials_and_query_strings() {
|
||||||
assert!(parse_probe_url("wss://example.test/v1/responses").is_ok());
|
assert!(parse_probe_url("wss://example.test/v1/responses").is_ok());
|
||||||
|
assert!(parse_probe_url("ws://localhost:8080/v1/responses").is_ok());
|
||||||
|
assert!(parse_probe_url("ws://127.42.0.1:8080/v1/responses").is_ok());
|
||||||
|
assert!(parse_probe_url("ws://[::1]:8080/v1/responses").is_ok());
|
||||||
assert!(parse_probe_url("https://example.test/v1/responses").is_err());
|
assert!(parse_probe_url("https://example.test/v1/responses").is_err());
|
||||||
|
assert!(parse_probe_url("ws://example.test/v1/responses").is_err());
|
||||||
|
assert!(parse_probe_url("ws://10.0.0.1/v1/responses").is_err());
|
||||||
|
assert!(parse_probe_url("ws://0.0.0.0:8080/v1/responses").is_err());
|
||||||
|
assert!(parse_probe_url("ws://[::ffff:127.0.0.1]:8080/v1/responses").is_err());
|
||||||
assert!(parse_probe_url("wss://[email protected]/v1/responses").is_err());
|
assert!(parse_probe_url("wss://[email protected]/v1/responses").is_err());
|
||||||
assert!(parse_probe_url("wss://example.test/v1/responses?token=secret").is_err());
|
assert!(parse_probe_url("wss://example.test/v1/responses?token=secret").is_err());
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -422,6 +422,7 @@ mod tests {
|
|||||||
local_rejection: None,
|
local_rejection: None,
|
||||||
allowed_models: None,
|
allowed_models: None,
|
||||||
ip_rules: None,
|
ip_rules: None,
|
||||||
|
verified_api_key_hash: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -173,6 +173,15 @@ impl SystemConfigCache {
|
|||||||
self.detach_all_loads();
|
self.detach_all_loads();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn invalidate(&self, key: &str) {
|
||||||
|
let Ok(_mutation) = self.mutation.lock() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
self.generation.fetch_add(1, Ordering::AcqRel);
|
||||||
|
self.entries.remove(&key.to_string());
|
||||||
|
self.detach_all_loads();
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn insert_if_generation(
|
pub(crate) fn insert_if_generation(
|
||||||
&self,
|
&self,
|
||||||
key: String,
|
key: String,
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ pub(crate) const TUNNEL_AFFINITY_FORWARDED_BY_HEADER: &str =
|
|||||||
"x-aether-tunnel-affinity-forwarded-by";
|
"x-aether-tunnel-affinity-forwarded-by";
|
||||||
pub(crate) const TUNNEL_AFFINITY_OWNER_INSTANCE_HEADER: &str =
|
pub(crate) const TUNNEL_AFFINITY_OWNER_INSTANCE_HEADER: &str =
|
||||||
"x-aether-tunnel-affinity-owner-instance-id";
|
"x-aether-tunnel-affinity-owner-instance-id";
|
||||||
|
pub(crate) const TUNNEL_AFFINITY_NODE_ID_HEADER: &str = "x-aether-tunnel-affinity-node-id";
|
||||||
pub(crate) const EXECUTION_PATH_PUBLIC_PROXY_PASSTHROUGH: &str = "public_proxy_passthrough";
|
pub(crate) const EXECUTION_PATH_PUBLIC_PROXY_PASSTHROUGH: &str = "public_proxy_passthrough";
|
||||||
pub(crate) const EXECUTION_PATH_LOCAL_PROXY_PASSTHROUGH_REMOVED: &str =
|
pub(crate) const EXECUTION_PATH_LOCAL_PROXY_PASSTHROUGH_REMOVED: &str =
|
||||||
"local_proxy_passthrough_removed";
|
"local_proxy_passthrough_removed";
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ pub(crate) fn extract_requested_model(
|
|||||||
body: &Bytes,
|
body: &Bytes,
|
||||||
) -> Option<String> {
|
) -> Option<String> {
|
||||||
if decision.route_family.as_deref() == Some("gemini") {
|
if decision.route_family.as_deref() == Some("gemini") {
|
||||||
if let Some(model) = extract_gemini_model_from_path(uri.path()) {
|
if let Some(model) = extract_gemini_requested_model_from_path(uri.path()) {
|
||||||
return Some(model);
|
return Some(model);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -43,23 +43,46 @@ pub(crate) fn extract_requested_model(
|
|||||||
.filter(|value| !value.is_empty())
|
.filter(|value| !value.is_empty())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn extract_gemini_requested_model_from_path(path: &str) -> Option<String> {
|
||||||
|
let model = extract_gemini_model_from_path(path)?;
|
||||||
|
Some(
|
||||||
|
model
|
||||||
|
.split_once("/operations/")
|
||||||
|
.map(|(model, _)| model)
|
||||||
|
.unwrap_or(model.as_str())
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) fn extract_request_credentials(
|
pub(super) fn extract_request_credentials(
|
||||||
headers: &http::HeaderMap,
|
headers: &http::HeaderMap,
|
||||||
uri: &Uri,
|
uri: &Uri,
|
||||||
auth_endpoint_signature: &str,
|
auth_endpoint_signature: &str,
|
||||||
|
) -> GatewayExtractedCredentials {
|
||||||
|
extract_request_credentials_with_trusted_auth(headers, uri, auth_endpoint_signature, cfg!(test))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn extract_request_credentials_with_trusted_auth(
|
||||||
|
headers: &http::HeaderMap,
|
||||||
|
uri: &Uri,
|
||||||
|
auth_endpoint_signature: &str,
|
||||||
|
trusted_auth_verified: bool,
|
||||||
) -> GatewayExtractedCredentials {
|
) -> GatewayExtractedCredentials {
|
||||||
let bundle = GatewayCredentialBundle {
|
let bundle = GatewayCredentialBundle {
|
||||||
authorization_bearer: header_value_str(headers, http::header::AUTHORIZATION.as_str())
|
authorization_bearer: unique_header_value_str(
|
||||||
.as_deref()
|
headers,
|
||||||
.and_then(extract_bearer_token)
|
http::header::AUTHORIZATION.as_str(),
|
||||||
.map(ToOwned::to_owned),
|
)
|
||||||
|
.as_deref()
|
||||||
|
.and_then(extract_bearer_token)
|
||||||
|
.map(ToOwned::to_owned),
|
||||||
x_api_key: header_value_str(headers, "x-api-key"),
|
x_api_key: header_value_str(headers, "x-api-key"),
|
||||||
api_key: header_value_str(headers, "api-key"),
|
api_key: header_value_str(headers, "api-key"),
|
||||||
x_goog_api_key: header_value_str(headers, "x-goog-api-key"),
|
x_goog_api_key: header_value_str(headers, "x-goog-api-key"),
|
||||||
query_key: extract_query_api_key(uri),
|
query_key: extract_query_api_key(uri),
|
||||||
cookie_header: header_value_str(headers, http::header::COOKIE.as_str()),
|
cookie_header: header_value_str(headers, http::header::COOKIE.as_str()),
|
||||||
};
|
};
|
||||||
let trusted_headers = extract_trusted_auth_headers(headers);
|
let trusted_headers = extract_trusted_auth_headers(headers, trusted_auth_verified);
|
||||||
let trusted_admin_headers = extract_trusted_admin_headers(headers);
|
let trusted_admin_headers = extract_trusted_admin_headers(headers);
|
||||||
let primary = select_primary_credential(auth_endpoint_signature, &bundle);
|
let primary = select_primary_credential(auth_endpoint_signature, &bundle);
|
||||||
|
|
||||||
@@ -71,6 +94,20 @@ pub(super) fn extract_request_credentials(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn unique_header_value_str(headers: &http::HeaderMap, key: &str) -> Option<String> {
|
||||||
|
let mut values = headers.get_all(key).iter();
|
||||||
|
let value = values.next()?;
|
||||||
|
if values.next().is_some() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
value
|
||||||
|
.to_str()
|
||||||
|
.ok()
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.map(ToOwned::to_owned)
|
||||||
|
}
|
||||||
|
|
||||||
pub(in crate::control) fn resolve_gateway_credential_carrier(
|
pub(in crate::control) fn resolve_gateway_credential_carrier(
|
||||||
headers: &http::HeaderMap,
|
headers: &http::HeaderMap,
|
||||||
uri: &Uri,
|
uri: &Uri,
|
||||||
@@ -85,6 +122,7 @@ pub(in crate::control) fn resolve_gateway_credential_carrier(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
fn has_trusted_gateway_marker(headers: &http::HeaderMap) -> bool {
|
fn has_trusted_gateway_marker(headers: &http::HeaderMap) -> bool {
|
||||||
header_value_str(headers, crate::constants::GATEWAY_HEADER)
|
header_value_str(headers, crate::constants::GATEWAY_HEADER)
|
||||||
.unwrap_or_default()
|
.unwrap_or_default()
|
||||||
@@ -97,13 +135,32 @@ pub(super) fn build_auth_context_cache_key(
|
|||||||
headers: &http::HeaderMap,
|
headers: &http::HeaderMap,
|
||||||
uri: &Uri,
|
uri: &Uri,
|
||||||
auth_endpoint_signature: &str,
|
auth_endpoint_signature: &str,
|
||||||
|
) -> Option<String> {
|
||||||
|
build_auth_context_cache_key_with_trusted_auth(
|
||||||
|
headers,
|
||||||
|
uri,
|
||||||
|
auth_endpoint_signature,
|
||||||
|
cfg!(test),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn build_auth_context_cache_key_with_trusted_auth(
|
||||||
|
headers: &http::HeaderMap,
|
||||||
|
uri: &Uri,
|
||||||
|
auth_endpoint_signature: &str,
|
||||||
|
trusted_auth_verified: bool,
|
||||||
) -> Option<String> {
|
) -> Option<String> {
|
||||||
let signature = auth_endpoint_signature.trim();
|
let signature = auth_endpoint_signature.trim();
|
||||||
if signature.is_empty() {
|
if signature.is_empty() {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
|
||||||
let extracted = extract_request_credentials(headers, uri, signature);
|
let extracted = extract_request_credentials_with_trusted_auth(
|
||||||
|
headers,
|
||||||
|
uri,
|
||||||
|
signature,
|
||||||
|
trusted_auth_verified,
|
||||||
|
);
|
||||||
let trusted_headers = extracted.trusted_headers;
|
let trusted_headers = extracted.trusted_headers;
|
||||||
let bundle = extracted.bundle;
|
let bundle = extracted.bundle;
|
||||||
if bundle.authorization_bearer.is_none()
|
if bundle.authorization_bearer.is_none()
|
||||||
@@ -135,7 +192,7 @@ pub(super) fn build_auth_context_cache_key(
|
|||||||
})
|
})
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
|
||||||
Some(format!(
|
let raw_cache_identity = format!(
|
||||||
"{signature}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}",
|
"{signature}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}",
|
||||||
bundle.authorization_bearer.unwrap_or_default(),
|
bundle.authorization_bearer.unwrap_or_default(),
|
||||||
bundle.x_api_key.unwrap_or_default(),
|
bundle.x_api_key.unwrap_or_default(),
|
||||||
@@ -147,11 +204,26 @@ pub(super) fn build_auth_context_cache_key(
|
|||||||
trusted_api_key_id,
|
trusted_api_key_id,
|
||||||
trusted_balance_remaining,
|
trusted_balance_remaining,
|
||||||
trusted_access_allowed,
|
trusted_access_allowed,
|
||||||
))
|
);
|
||||||
|
let mut hasher = Sha256::new();
|
||||||
|
hasher.update(raw_cache_identity.as_bytes());
|
||||||
|
Some(format!("auth-context:sha256:{:x}", hasher.finalize()))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn extract_trusted_auth_headers(headers: &http::HeaderMap) -> Option<GatewayTrustedAuthHeaders> {
|
fn extract_trusted_auth_headers(
|
||||||
if !has_trusted_gateway_marker(headers) {
|
headers: &http::HeaderMap,
|
||||||
|
trusted_auth_verified: bool,
|
||||||
|
) -> Option<GatewayTrustedAuthHeaders> {
|
||||||
|
if !trusted_auth_verified {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
#[cfg(test)]
|
||||||
|
if !header_value_str(headers, crate::constants::GATEWAY_HEADER)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.trim()
|
||||||
|
.to_ascii_lowercase()
|
||||||
|
.starts_with("rust-phase3")
|
||||||
|
{
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
let user_id = header_value_str(headers, crate::constants::TRUSTED_AUTH_USER_ID_HEADER)
|
let user_id = header_value_str(headers, crate::constants::TRUSTED_AUTH_USER_ID_HEADER)
|
||||||
@@ -387,7 +459,7 @@ fn extract_bearer_token(value: &str) -> Option<&str> {
|
|||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
let token = token.trim();
|
let token = token.trim();
|
||||||
if token.is_empty() {
|
if token.is_empty() || token.chars().any(char::is_whitespace) {
|
||||||
None
|
None
|
||||||
} else {
|
} else {
|
||||||
Some(token)
|
Some(token)
|
||||||
@@ -472,6 +544,46 @@ mod tests {
|
|||||||
assert_eq!(requested_model.as_deref(), Some("gpt-5.4"));
|
assert_eq!(requested_model.as_deref(), Some("gpt-5.4"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extract_requested_model_handles_gemini_generation_and_operation_paths() {
|
||||||
|
let generation_decision = GatewayControlDecision::synthetic(
|
||||||
|
"/v1beta/models/gemini-2.5-pro:generateContent",
|
||||||
|
Some("ai_public".to_string()),
|
||||||
|
Some("gemini".to_string()),
|
||||||
|
Some("generate_content".to_string()),
|
||||||
|
Some("gemini:generate_content".to_string()),
|
||||||
|
);
|
||||||
|
let operation_decision = GatewayControlDecision::synthetic(
|
||||||
|
"/v1beta/models/veo-3/operations/task-123:cancel",
|
||||||
|
Some("ai_public".to_string()),
|
||||||
|
Some("gemini".to_string()),
|
||||||
|
Some("video".to_string()),
|
||||||
|
Some("gemini:video".to_string()),
|
||||||
|
);
|
||||||
|
let headers = http::HeaderMap::new();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
extract_requested_model(
|
||||||
|
&generation_decision,
|
||||||
|
&uri("/v1beta/models/gemini-2.5-pro:generateContent"),
|
||||||
|
&headers,
|
||||||
|
&Bytes::new(),
|
||||||
|
)
|
||||||
|
.as_deref(),
|
||||||
|
Some("gemini-2.5-pro")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
extract_requested_model(
|
||||||
|
&operation_decision,
|
||||||
|
&uri("/v1beta/models/veo-3/operations/task-123:cancel"),
|
||||||
|
&headers,
|
||||||
|
&Bytes::new(),
|
||||||
|
)
|
||||||
|
.as_deref(),
|
||||||
|
Some("veo-3")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn selects_openai_bearer_as_provider_api_key() {
|
fn selects_openai_bearer_as_provider_api_key() {
|
||||||
let mut headers = http::HeaderMap::new();
|
let mut headers = http::HeaderMap::new();
|
||||||
@@ -491,6 +603,33 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rejects_duplicate_or_combined_authorization_credentials() {
|
||||||
|
let mut duplicate = http::HeaderMap::new();
|
||||||
|
duplicate.append(
|
||||||
|
http::header::AUTHORIZATION,
|
||||||
|
"Bearer first-token".parse().unwrap(),
|
||||||
|
);
|
||||||
|
duplicate.append(
|
||||||
|
http::header::AUTHORIZATION,
|
||||||
|
"Bearer second-token".parse().unwrap(),
|
||||||
|
);
|
||||||
|
let extracted =
|
||||||
|
extract_request_credentials(&duplicate, &uri("/api/admin/system"), "admin:operational");
|
||||||
|
assert!(extracted.bundle.authorization_bearer.is_none());
|
||||||
|
assert!(extracted.primary.is_none());
|
||||||
|
|
||||||
|
let mut combined = http::HeaderMap::new();
|
||||||
|
combined.insert(
|
||||||
|
http::header::AUTHORIZATION,
|
||||||
|
"Bearer first-token, Bearer second-token".parse().unwrap(),
|
||||||
|
);
|
||||||
|
let extracted =
|
||||||
|
extract_request_credentials(&combined, &uri("/api/admin/system"), "admin:operational");
|
||||||
|
assert!(extracted.bundle.authorization_bearer.is_none());
|
||||||
|
assert!(extracted.primary.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn selects_codex_live_bearer_as_provider_api_key() {
|
fn selects_codex_live_bearer_as_provider_api_key() {
|
||||||
let mut headers = http::HeaderMap::new();
|
let mut headers = http::HeaderMap::new();
|
||||||
@@ -608,7 +747,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn cache_key_includes_cookie_header() {
|
fn cache_key_hashes_cookie_header_instead_of_retaining_session_secret() {
|
||||||
let mut headers = http::HeaderMap::new();
|
let mut headers = http::HeaderMap::new();
|
||||||
headers.insert(http::header::COOKIE, "session=abc123".parse().unwrap());
|
headers.insert(http::header::COOKIE, "session=abc123".parse().unwrap());
|
||||||
|
|
||||||
@@ -618,7 +757,8 @@ mod tests {
|
|||||||
"internal:session",
|
"internal:session",
|
||||||
)
|
)
|
||||||
.expect("cache key should exist");
|
.expect("cache key should exist");
|
||||||
assert!(cache_key.contains("session=abc123"));
|
assert!(cache_key.starts_with("auth-context:sha256:"));
|
||||||
|
assert!(!cache_key.contains("session=abc123"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -669,12 +809,10 @@ mod tests {
|
|||||||
.expect("trusted cache key should exist");
|
.expect("trusted cache key should exist");
|
||||||
|
|
||||||
assert_ne!(first, second);
|
assert_ne!(first, second);
|
||||||
assert!(first.contains("user-1"));
|
for raw_identity in ["user-1", "key-1", "1.5", "user-2"] {
|
||||||
assert!(first.contains("key-1"));
|
assert!(!first.contains(raw_identity));
|
||||||
assert!(first.contains("1.5"));
|
assert!(!second.contains(raw_identity));
|
||||||
assert!(first.contains("true"));
|
}
|
||||||
assert!(second.contains("user-2"));
|
|
||||||
assert!(second.contains("false"));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -224,7 +224,7 @@ fn wallet_finite_available_usd(
|
|||||||
Some(wallet.balance.max(0.0) + wallet.gift_balance.max(0.0))
|
Some(wallet.balance.max(0.0) + wallet.gift_balance.max(0.0))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn estimate_execution_plan_cost_upper_bound_usd(
|
pub(crate) async fn estimate_execution_plan_cost_upper_bound_usd(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
plan: &aether_contracts::ExecutionPlan,
|
plan: &aether_contracts::ExecutionPlan,
|
||||||
report_context: Option<&serde_json::Value>,
|
report_context: Option<&serde_json::Value>,
|
||||||
@@ -925,6 +925,7 @@ mod tests {
|
|||||||
local_rejection: None,
|
local_rejection: None,
|
||||||
allowed_models: Some(allowed_models),
|
allowed_models: Some(allowed_models),
|
||||||
ip_rules: None,
|
ip_rules: None,
|
||||||
|
verified_api_key_hash: None,
|
||||||
});
|
});
|
||||||
decision
|
decision
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,13 +7,16 @@ mod types;
|
|||||||
pub(crate) use credentials::extract_requested_model;
|
pub(crate) use credentials::extract_requested_model;
|
||||||
pub(super) use credentials::resolve_gateway_credential_carrier;
|
pub(super) use credentials::resolve_gateway_credential_carrier;
|
||||||
pub(crate) use gate::{
|
pub(crate) use gate::{
|
||||||
execution_plan_balance_capacity_rejection, request_model_local_rejection,
|
estimate_execution_plan_cost_upper_bound_usd, execution_plan_balance_capacity_rejection,
|
||||||
should_buffer_request_for_local_auth, trusted_auth_local_rejection, GatewayLocalAuthRejection,
|
request_model_local_rejection, should_buffer_request_for_local_auth,
|
||||||
|
trusted_auth_local_rejection, GatewayLocalAuthRejection,
|
||||||
};
|
};
|
||||||
pub(crate) use resolution::{
|
pub(crate) use resolution::{
|
||||||
refresh_execution_runtime_auth_context, refresh_execution_runtime_auth_context_with_snapshot,
|
refresh_execution_runtime_auth_context, refresh_execution_runtime_auth_context_with_snapshot,
|
||||||
resolve_execution_runtime_auth_context, GatewayAdminPrincipalContext,
|
resolve_execution_runtime_auth_context, resolve_local_admin_session_principal,
|
||||||
GatewayControlAuthContext,
|
GatewayAdminPrincipalContext, GatewayControlAuthContext,
|
||||||
|
};
|
||||||
|
pub(super) use resolution::{
|
||||||
|
resolve_control_decision_auth_with_trusted_auth, ControlDecisionAuthResolution,
|
||||||
};
|
};
|
||||||
pub(super) use resolution::{resolve_control_decision_auth, ControlDecisionAuthResolution};
|
|
||||||
pub(crate) use types::GatewayCredentialCarrier;
|
pub(crate) use types::GatewayCredentialCarrier;
|
||||||
|
|||||||
@@ -4,11 +4,9 @@ use aether_data_contracts::repository::provider_catalog::{
|
|||||||
StoredProviderCatalogEndpoint, StoredProviderCatalogProvider,
|
StoredProviderCatalogEndpoint, StoredProviderCatalogProvider,
|
||||||
};
|
};
|
||||||
use axum::http::Uri;
|
use axum::http::Uri;
|
||||||
use base64::Engine as _;
|
|
||||||
use hmac::Mac;
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use tracing::{debug, info};
|
use tracing::{debug, info, warn};
|
||||||
|
|
||||||
use crate::wallet_runtime::{
|
use crate::wallet_runtime::{
|
||||||
local_rejection_from_wallet_access, resolve_wallet_auth_gate_uncached,
|
local_rejection_from_wallet_access, resolve_wallet_auth_gate_uncached,
|
||||||
@@ -17,7 +15,8 @@ use crate::{AppState, GatewayError};
|
|||||||
|
|
||||||
use super::super::GatewayControlDecision;
|
use super::super::GatewayControlDecision;
|
||||||
use super::credentials::{
|
use super::credentials::{
|
||||||
build_auth_context_cache_key, current_unix_secs, extract_request_credentials,
|
build_auth_context_cache_key, build_auth_context_cache_key_with_trusted_auth,
|
||||||
|
current_unix_secs, extract_request_credentials, extract_request_credentials_with_trusted_auth,
|
||||||
extract_trusted_admin_headers, hash_api_key,
|
extract_trusted_admin_headers, hash_api_key,
|
||||||
};
|
};
|
||||||
use super::gate::GatewayLocalAuthRejection;
|
use super::gate::GatewayLocalAuthRejection;
|
||||||
@@ -27,6 +26,9 @@ use super::types::{
|
|||||||
};
|
};
|
||||||
use crate::cache::{AuthContextCacheGeneration, AuthContextInflightRegistration};
|
use crate::cache::{AuthContextCacheGeneration, AuthContextInflightRegistration};
|
||||||
use crate::headers::header_value_str;
|
use crate::headers::header_value_str;
|
||||||
|
use crate::local_auth_token::{
|
||||||
|
decode_local_auth_token, local_auth_token_identity_matches_user, LocalAuthTokenType,
|
||||||
|
};
|
||||||
|
|
||||||
const AUTH_CONTEXT_CACHE_TTL: Duration = Duration::from_secs(60);
|
const AUTH_CONTEXT_CACHE_TTL: Duration = Duration::from_secs(60);
|
||||||
const AUTH_CONTEXT_CACHE_REFRESH_INTERVAL: Duration = Duration::from_secs(10);
|
const AUTH_CONTEXT_CACHE_REFRESH_INTERVAL: Duration = Duration::from_secs(10);
|
||||||
@@ -93,6 +95,30 @@ pub(crate) struct GatewayControlAuthContext {
|
|||||||
pub(crate) allowed_models: Option<Vec<String>>,
|
pub(crate) allowed_models: Option<Vec<String>>,
|
||||||
#[serde(skip)]
|
#[serde(skip)]
|
||||||
pub(crate) ip_rules: Option<Vec<String>>,
|
pub(crate) ip_rules: Option<Vec<String>>,
|
||||||
|
/// Credential verifier that established this API-key identity. Long-lived
|
||||||
|
/// executions use it to prove that a later row with the same IDs is still
|
||||||
|
/// the record authenticated by the original request.
|
||||||
|
#[serde(skip)]
|
||||||
|
pub(crate) verified_api_key_hash: Option<VerifiedApiKeyHash>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub(crate) struct VerifiedApiKeyHash(String);
|
||||||
|
|
||||||
|
impl VerifiedApiKeyHash {
|
||||||
|
fn new(value: String) -> Self {
|
||||||
|
Self(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn as_str(&self) -> &str {
|
||||||
|
self.0.as_str()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Debug for VerifiedApiKeyHash {
|
||||||
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
formatter.write_str("VerifiedApiKeyHash([REDACTED])")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
@@ -109,11 +135,30 @@ pub(in super::super) enum ControlDecisionAuthResolution {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(in super::super) async fn resolve_control_decision_auth(
|
pub(in super::super) async fn resolve_control_decision_auth(
|
||||||
|
state: &AppState,
|
||||||
|
headers: &http::HeaderMap,
|
||||||
|
uri: &Uri,
|
||||||
|
trace_id: &str,
|
||||||
|
decision: GatewayControlDecision,
|
||||||
|
) -> Result<ControlDecisionAuthResolution, GatewayError> {
|
||||||
|
resolve_control_decision_auth_with_trusted_auth(
|
||||||
|
state,
|
||||||
|
headers,
|
||||||
|
uri,
|
||||||
|
trace_id,
|
||||||
|
decision,
|
||||||
|
cfg!(test),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(in super::super) async fn resolve_control_decision_auth_with_trusted_auth(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
headers: &http::HeaderMap,
|
headers: &http::HeaderMap,
|
||||||
uri: &Uri,
|
uri: &Uri,
|
||||||
trace_id: &str,
|
trace_id: &str,
|
||||||
mut decision: GatewayControlDecision,
|
mut decision: GatewayControlDecision,
|
||||||
|
trusted_auth_verified: bool,
|
||||||
) -> Result<ControlDecisionAuthResolution, GatewayError> {
|
) -> Result<ControlDecisionAuthResolution, GatewayError> {
|
||||||
if let Some(admin_principal) =
|
if let Some(admin_principal) =
|
||||||
resolve_trusted_admin_principal(headers, decision.auth_endpoint_signature.as_deref())
|
resolve_trusted_admin_principal(headers, decision.auth_endpoint_signature.as_deref())
|
||||||
@@ -132,10 +177,18 @@ pub(in super::super) async fn resolve_control_decision_auth(
|
|||||||
decision.admin_principal = Some(admin_principal);
|
decision.admin_principal = Some(admin_principal);
|
||||||
}
|
}
|
||||||
|
|
||||||
let auth_context_cache_key = decision
|
let auth_context_cache_key =
|
||||||
.auth_endpoint_signature
|
decision
|
||||||
.as_deref()
|
.auth_endpoint_signature
|
||||||
.and_then(|signature| build_auth_context_cache_key(headers, uri, signature));
|
.as_deref()
|
||||||
|
.and_then(|signature| {
|
||||||
|
build_auth_context_cache_key_with_trusted_auth(
|
||||||
|
headers,
|
||||||
|
uri,
|
||||||
|
signature,
|
||||||
|
trusted_auth_verified,
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
let mut resolved_auth_context = None;
|
let mut resolved_auth_context = None;
|
||||||
if let Some(cache_key) = auth_context_cache_key.as_deref() {
|
if let Some(cache_key) = auth_context_cache_key.as_deref() {
|
||||||
@@ -149,6 +202,7 @@ pub(in super::super) async fn resolve_control_decision_auth(
|
|||||||
decision.auth_endpoint_signature.as_deref(),
|
decision.auth_endpoint_signature.as_deref(),
|
||||||
headers,
|
headers,
|
||||||
uri,
|
uri,
|
||||||
|
trusted_auth_verified,
|
||||||
)
|
)
|
||||||
.await?,
|
.await?,
|
||||||
);
|
);
|
||||||
@@ -168,6 +222,7 @@ pub(in super::super) async fn resolve_control_decision_auth(
|
|||||||
uri,
|
uri,
|
||||||
decision.auth_endpoint_signature.as_deref(),
|
decision.auth_endpoint_signature.as_deref(),
|
||||||
true,
|
true,
|
||||||
|
trusted_auth_verified,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
}
|
}
|
||||||
@@ -336,7 +391,7 @@ async fn resolve_local_admin_principal(
|
|||||||
let Some(access_token) = extracted.bundle.authorization_bearer.as_deref() else {
|
let Some(access_token) = extracted.bundle.authorization_bearer.as_deref() else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
let claims = match decode_local_auth_token(access_token, "access") {
|
let claims = match decode_local_auth_token(access_token, LocalAuthTokenType::Access) {
|
||||||
Ok(claims) => claims,
|
Ok(claims) => claims,
|
||||||
Err(_) => return Ok(None),
|
Err(_) => return Ok(None),
|
||||||
};
|
};
|
||||||
@@ -351,6 +406,14 @@ async fn resolve_local_admin_principal(
|
|||||||
resolve_local_admin_principal_from_claims(state, headers, uri, &claims).await
|
resolve_local_admin_principal_from_claims(state, headers, uri, &claims).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn resolve_local_admin_session_principal(
|
||||||
|
state: &AppState,
|
||||||
|
headers: &http::HeaderMap,
|
||||||
|
uri: &Uri,
|
||||||
|
) -> Result<Option<GatewayAdminPrincipalContext>, GatewayError> {
|
||||||
|
resolve_local_admin_principal(state, headers, uri, Some("admin:operational")).await
|
||||||
|
}
|
||||||
|
|
||||||
async fn resolve_local_admin_principal_from_claims(
|
async fn resolve_local_admin_principal_from_claims(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
headers: &http::HeaderMap,
|
headers: &http::HeaderMap,
|
||||||
@@ -373,6 +436,9 @@ async fn resolve_local_admin_principal_from_claims(
|
|||||||
if !user.is_active || user.is_deleted || !crate::roles::can_access_admin_console(&user.role) {
|
if !user.is_active || user.is_deleted || !crate::roles::can_access_admin_console(&user.role) {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
|
if !local_auth_token_identity_matches_user(claims, &user) {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
|
||||||
let now = chrono::Utc::now();
|
let now = chrono::Utc::now();
|
||||||
let Some(session) = state.find_user_session(user_id, session_id).await? else {
|
let Some(session) = state.find_user_session(user_id, session_id).await? else {
|
||||||
@@ -380,6 +446,7 @@ async fn resolve_local_admin_principal_from_claims(
|
|||||||
};
|
};
|
||||||
if session.is_revoked()
|
if session.is_revoked()
|
||||||
|| session.is_expired(now)
|
|| session.is_expired(now)
|
||||||
|
|| session.security_version != user.security_version
|
||||||
|| session.client_device_id != client_device_id
|
|| session.client_device_id != client_device_id
|
||||||
{
|
{
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
@@ -431,68 +498,6 @@ fn local_admin_user_agent(headers: &http::HeaderMap) -> Option<String> {
|
|||||||
.map(|value| value.chars().take(1000).collect())
|
.map(|value| value.chars().take(1000).collect())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn local_auth_secret() -> String {
|
|
||||||
std::env::var("JWT_SECRET_KEY")
|
|
||||||
.ok()
|
|
||||||
.map(|value| value.trim().to_string())
|
|
||||||
.filter(|value| !value.is_empty())
|
|
||||||
.unwrap_or_else(|| "aether-rust-dev-jwt-secret".to_string())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn decode_local_auth_token(
|
|
||||||
token: &str,
|
|
||||||
expected_type: &str,
|
|
||||||
) -> Result<serde_json::Map<String, Value>, String> {
|
|
||||||
let mut parts = token.split('.');
|
|
||||||
let Some(header_segment) = parts.next() else {
|
|
||||||
return Err("invalid token".to_string());
|
|
||||||
};
|
|
||||||
let Some(payload_segment) = parts.next() else {
|
|
||||||
return Err("invalid token".to_string());
|
|
||||||
};
|
|
||||||
let Some(signature_segment) = parts.next() else {
|
|
||||||
return Err("invalid token".to_string());
|
|
||||||
};
|
|
||||||
if parts.next().is_some() {
|
|
||||||
return Err("invalid token".to_string());
|
|
||||||
}
|
|
||||||
|
|
||||||
let signing_input = format!("{header_segment}.{payload_segment}");
|
|
||||||
let signature = base64::engine::general_purpose::URL_SAFE_NO_PAD
|
|
||||||
.decode(signature_segment)
|
|
||||||
.map_err(|_| "invalid token".to_string())?;
|
|
||||||
let mut mac = hmac::Hmac::<sha2::Sha256>::new_from_slice(local_auth_secret().as_bytes())
|
|
||||||
.map_err(|_| "invalid token".to_string())?;
|
|
||||||
mac.update(signing_input.as_bytes());
|
|
||||||
mac.verify_slice(&signature)
|
|
||||||
.map_err(|_| "invalid token".to_string())?;
|
|
||||||
|
|
||||||
let payload_bytes = base64::engine::general_purpose::URL_SAFE_NO_PAD
|
|
||||||
.decode(payload_segment)
|
|
||||||
.map_err(|_| "invalid token".to_string())?;
|
|
||||||
let payload =
|
|
||||||
serde_json::from_slice::<Value>(&payload_bytes).map_err(|_| "invalid token".to_string())?;
|
|
||||||
let payload = payload
|
|
||||||
.as_object()
|
|
||||||
.cloned()
|
|
||||||
.ok_or_else(|| "invalid token".to_string())?;
|
|
||||||
let actual_type = payload
|
|
||||||
.get("type")
|
|
||||||
.and_then(Value::as_str)
|
|
||||||
.unwrap_or_default();
|
|
||||||
if actual_type != expected_type {
|
|
||||||
return Err("invalid token".to_string());
|
|
||||||
}
|
|
||||||
let exp = payload
|
|
||||||
.get("exp")
|
|
||||||
.and_then(Value::as_i64)
|
|
||||||
.ok_or_else(|| "invalid token".to_string())?;
|
|
||||||
if exp <= chrono::Utc::now().timestamp() {
|
|
||||||
return Err("expired token".to_string());
|
|
||||||
}
|
|
||||||
Ok(payload)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) async fn resolve_execution_runtime_auth_context(
|
pub(crate) async fn resolve_execution_runtime_auth_context(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
decision: &GatewayControlDecision,
|
decision: &GatewayControlDecision,
|
||||||
@@ -525,6 +530,7 @@ pub(crate) async fn resolve_execution_runtime_auth_context(
|
|||||||
Some(auth_endpoint_signature),
|
Some(auth_endpoint_signature),
|
||||||
headers,
|
headers,
|
||||||
uri,
|
uri,
|
||||||
|
cfg!(test),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map(Some);
|
.map(Some);
|
||||||
@@ -539,6 +545,7 @@ pub(crate) async fn resolve_execution_runtime_auth_context(
|
|||||||
uri,
|
uri,
|
||||||
Some(auth_endpoint_signature),
|
Some(auth_endpoint_signature),
|
||||||
true,
|
true,
|
||||||
|
cfg!(test),
|
||||||
)
|
)
|
||||||
.await?
|
.await?
|
||||||
{
|
{
|
||||||
@@ -558,6 +565,7 @@ async fn revalidate_cached_auth_context(
|
|||||||
auth_endpoint_signature: Option<&str>,
|
auth_endpoint_signature: Option<&str>,
|
||||||
headers: &http::HeaderMap,
|
headers: &http::HeaderMap,
|
||||||
uri: &Uri,
|
uri: &Uri,
|
||||||
|
trusted_auth_verified: bool,
|
||||||
) -> Result<GatewayControlAuthContext, GatewayError> {
|
) -> Result<GatewayControlAuthContext, GatewayError> {
|
||||||
if is_negative_auth_context(&auth_context)
|
if is_negative_auth_context(&auth_context)
|
||||||
|| !auth_context.access_allowed
|
|| !auth_context.access_allowed
|
||||||
@@ -581,6 +589,7 @@ async fn revalidate_cached_auth_context(
|
|||||||
uri,
|
uri,
|
||||||
auth_context.clone(),
|
auth_context.clone(),
|
||||||
auth_endpoint_signature,
|
auth_endpoint_signature,
|
||||||
|
trusted_auth_verified,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
@@ -622,6 +631,7 @@ async fn revalidate_cached_auth_context(
|
|||||||
uri,
|
uri,
|
||||||
auth_context,
|
auth_context,
|
||||||
auth_endpoint_signature,
|
auth_endpoint_signature,
|
||||||
|
trusted_auth_verified,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
if refreshed.is_err() {
|
if refreshed.is_err() {
|
||||||
@@ -639,9 +649,16 @@ async fn resolve_security_fresh_auth_context(
|
|||||||
uri: &Uri,
|
uri: &Uri,
|
||||||
stale: GatewayControlAuthContext,
|
stale: GatewayControlAuthContext,
|
||||||
auth_endpoint_signature: Option<&str>,
|
auth_endpoint_signature: Option<&str>,
|
||||||
|
trusted_auth_verified: bool,
|
||||||
) -> Result<GatewayControlAuthContext, GatewayError> {
|
) -> Result<GatewayControlAuthContext, GatewayError> {
|
||||||
if let Some(refreshed) =
|
if let Some(refreshed) = resolve_data_backed_auth_context_with_trusted_auth(
|
||||||
resolve_data_backed_auth_context(state, headers, uri, auth_endpoint_signature).await?
|
state,
|
||||||
|
headers,
|
||||||
|
uri,
|
||||||
|
auth_endpoint_signature,
|
||||||
|
trusted_auth_verified,
|
||||||
|
)
|
||||||
|
.await?
|
||||||
{
|
{
|
||||||
return Ok(refreshed);
|
return Ok(refreshed);
|
||||||
}
|
}
|
||||||
@@ -660,19 +677,27 @@ async fn resolve_data_backed_auth_context_cached(
|
|||||||
uri: &Uri,
|
uri: &Uri,
|
||||||
auth_endpoint_signature: Option<&str>,
|
auth_endpoint_signature: Option<&str>,
|
||||||
cache_negative: bool,
|
cache_negative: bool,
|
||||||
|
trusted_auth_verified: bool,
|
||||||
) -> Result<Option<GatewayControlAuthContext>, GatewayError> {
|
) -> Result<Option<GatewayControlAuthContext>, GatewayError> {
|
||||||
let Some(cache_key) = cache_key else {
|
let Some(cache_key) = cache_key else {
|
||||||
return resolve_data_backed_auth_context(state, headers, uri, auth_endpoint_signature)
|
return resolve_data_backed_auth_context_with_trusted_auth(
|
||||||
.await;
|
state,
|
||||||
|
headers,
|
||||||
|
uri,
|
||||||
|
auth_endpoint_signature,
|
||||||
|
trusted_auth_verified,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
};
|
};
|
||||||
loop {
|
loop {
|
||||||
match state.auth_context_cache.register_inflight(cache_key) {
|
match state.auth_context_cache.register_inflight(cache_key) {
|
||||||
AuthContextInflightRegistration::Leader(guard) => {
|
AuthContextInflightRegistration::Leader(guard) => {
|
||||||
let resolved = match resolve_data_backed_auth_context(
|
let resolved = match resolve_data_backed_auth_context_with_trusted_auth(
|
||||||
state,
|
state,
|
||||||
headers,
|
headers,
|
||||||
uri,
|
uri,
|
||||||
auth_endpoint_signature,
|
auth_endpoint_signature,
|
||||||
|
trusted_auth_verified,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
@@ -708,11 +733,12 @@ async fn resolve_data_backed_auth_context_cached(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
AuthContextInflightRegistration::Bypass => {
|
AuthContextInflightRegistration::Bypass => {
|
||||||
return resolve_data_backed_auth_context(
|
return resolve_data_backed_auth_context_with_trusted_auth(
|
||||||
state,
|
state,
|
||||||
headers,
|
headers,
|
||||||
uri,
|
uri,
|
||||||
auth_endpoint_signature,
|
auth_endpoint_signature,
|
||||||
|
trusted_auth_verified,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
@@ -768,28 +794,39 @@ pub(crate) async fn refresh_execution_runtime_auth_context_with_snapshot(
|
|||||||
return Ok((auth_context, None));
|
return Ok((auth_context, None));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let verified_api_key_hash = auth_context.verified_api_key_hash.clone();
|
||||||
let snapshot = {
|
let snapshot = {
|
||||||
let _permit = state.acquire_auth_snapshot_load_gate().await?;
|
let _permit = state.acquire_auth_snapshot_load_gate().await?;
|
||||||
state
|
if let Some(key_hash) = verified_api_key_hash.as_ref() {
|
||||||
.data
|
state
|
||||||
.read_auth_api_key_snapshot_strong(
|
.data
|
||||||
&auth_context.user_id,
|
.read_auth_api_key_snapshot_by_key_hash_strong(
|
||||||
&auth_context.api_key_id,
|
key_hash.as_str(),
|
||||||
current_unix_secs(),
|
current_unix_secs(),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||||
|
} else {
|
||||||
|
state
|
||||||
|
.data
|
||||||
|
.read_auth_api_key_snapshot_strong(
|
||||||
|
&auth_context.user_id,
|
||||||
|
&auth_context.api_key_id,
|
||||||
|
current_unix_secs(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|err| GatewayError::Internal(err.to_string()))?
|
||||||
|
}
|
||||||
};
|
};
|
||||||
let Some(snapshot) = snapshot else {
|
let Some(snapshot) = snapshot else {
|
||||||
let mut denied = auth_context;
|
return Ok((deny_refreshed_auth_context(auth_context), None));
|
||||||
denied.access_allowed = false;
|
};
|
||||||
denied.local_rejection = Some(GatewayLocalAuthRejection::InvalidApiKey);
|
if snapshot.user_id != auth_context.user_id || snapshot.api_key_id != auth_context.api_key_id {
|
||||||
denied.balance_remaining = None;
|
return Ok((deny_refreshed_auth_context(auth_context), None));
|
||||||
return Ok((denied, None));
|
|
||||||
};
|
};
|
||||||
|
|
||||||
let wallet_access = resolve_wallet_auth_gate_uncached(state, &snapshot).await?;
|
let wallet_access = resolve_wallet_auth_gate_uncached(state, &snapshot).await?;
|
||||||
let refreshed = build_data_backed_auth_context(
|
let mut refreshed = build_data_backed_auth_context(
|
||||||
state,
|
state,
|
||||||
snapshot.clone(),
|
snapshot.clone(),
|
||||||
auth_endpoint_signature,
|
auth_endpoint_signature,
|
||||||
@@ -798,9 +835,19 @@ pub(crate) async fn refresh_execution_runtime_auth_context_with_snapshot(
|
|||||||
wallet_access,
|
wallet_access,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
refreshed.verified_api_key_hash = verified_api_key_hash;
|
||||||
Ok((refreshed, Some(snapshot)))
|
Ok((refreshed, Some(snapshot)))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn deny_refreshed_auth_context(
|
||||||
|
mut auth_context: GatewayControlAuthContext,
|
||||||
|
) -> GatewayControlAuthContext {
|
||||||
|
auth_context.access_allowed = false;
|
||||||
|
auth_context.local_rejection = Some(GatewayLocalAuthRejection::InvalidApiKey);
|
||||||
|
auth_context.balance_remaining = None;
|
||||||
|
auth_context
|
||||||
|
}
|
||||||
|
|
||||||
fn put_cached_auth_context(
|
fn put_cached_auth_context(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
cache_key: String,
|
cache_key: String,
|
||||||
@@ -913,6 +960,23 @@ pub(super) async fn resolve_data_backed_auth_context(
|
|||||||
headers: &http::HeaderMap,
|
headers: &http::HeaderMap,
|
||||||
uri: &Uri,
|
uri: &Uri,
|
||||||
auth_endpoint_signature: Option<&str>,
|
auth_endpoint_signature: Option<&str>,
|
||||||
|
) -> Result<Option<GatewayControlAuthContext>, GatewayError> {
|
||||||
|
resolve_data_backed_auth_context_with_trusted_auth(
|
||||||
|
state,
|
||||||
|
headers,
|
||||||
|
uri,
|
||||||
|
auth_endpoint_signature,
|
||||||
|
cfg!(test),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn resolve_data_backed_auth_context_with_trusted_auth(
|
||||||
|
state: &AppState,
|
||||||
|
headers: &http::HeaderMap,
|
||||||
|
uri: &Uri,
|
||||||
|
auth_endpoint_signature: Option<&str>,
|
||||||
|
trusted_auth_verified: bool,
|
||||||
) -> Result<Option<GatewayControlAuthContext>, GatewayError> {
|
) -> Result<Option<GatewayControlAuthContext>, GatewayError> {
|
||||||
let Some(signature) = auth_endpoint_signature
|
let Some(signature) = auth_endpoint_signature
|
||||||
.map(str::trim)
|
.map(str::trim)
|
||||||
@@ -923,7 +987,12 @@ pub(super) async fn resolve_data_backed_auth_context(
|
|||||||
if !state.has_auth_api_key_reader() {
|
if !state.has_auth_api_key_reader() {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
let extracted = extract_request_credentials(headers, uri, signature);
|
let extracted = extract_request_credentials_with_trusted_auth(
|
||||||
|
headers,
|
||||||
|
uri,
|
||||||
|
signature,
|
||||||
|
trusted_auth_verified,
|
||||||
|
);
|
||||||
let principal = derive_principal_candidate(&extracted);
|
let principal = derive_principal_candidate(&extracted);
|
||||||
let now_unix_secs = current_unix_secs();
|
let now_unix_secs = current_unix_secs();
|
||||||
|
|
||||||
@@ -955,6 +1024,7 @@ pub(super) async fn resolve_data_backed_auth_context(
|
|||||||
local_rejection: Some(GatewayLocalAuthRejection::InvalidApiKey),
|
local_rejection: Some(GatewayLocalAuthRejection::InvalidApiKey),
|
||||||
allowed_models: None,
|
allowed_models: None,
|
||||||
ip_rules: None,
|
ip_rules: None,
|
||||||
|
verified_api_key_hash: None,
|
||||||
}));
|
}));
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -963,17 +1033,17 @@ pub(super) async fn resolve_data_backed_auth_context(
|
|||||||
.await;
|
.await;
|
||||||
|
|
||||||
let wallet_access = resolve_wallet_auth_gate_uncached(state, &snapshot).await?;
|
let wallet_access = resolve_wallet_auth_gate_uncached(state, &snapshot).await?;
|
||||||
Ok(Some(
|
let mut auth_context = build_data_backed_auth_context(
|
||||||
build_data_backed_auth_context(
|
state,
|
||||||
state,
|
snapshot,
|
||||||
snapshot,
|
signature,
|
||||||
signature,
|
None,
|
||||||
None,
|
None,
|
||||||
None,
|
wallet_access,
|
||||||
wallet_access,
|
)
|
||||||
)
|
.await;
|
||||||
.await,
|
auth_context.verified_api_key_hash = Some(VerifiedApiKeyHash::new(key_hash));
|
||||||
))
|
Ok(Some(auth_context))
|
||||||
}
|
}
|
||||||
Some(GatewayPrincipalCandidate::DeferredBearerToken { raw, carrier }) => {
|
Some(GatewayPrincipalCandidate::DeferredBearerToken { raw, carrier }) => {
|
||||||
if let Some(auth_context) = resolve_antigravity_bearer_bridge_auth_context(
|
if let Some(auth_context) = resolve_antigravity_bearer_bridge_auth_context(
|
||||||
@@ -1068,6 +1138,7 @@ async fn resolve_antigravity_bearer_bridge_auth_context(
|
|||||||
local_rejection: Some(GatewayLocalAuthRejection::InvalidApiKey),
|
local_rejection: Some(GatewayLocalAuthRejection::InvalidApiKey),
|
||||||
allowed_models: None,
|
allowed_models: None,
|
||||||
ip_rules: None,
|
ip_rules: None,
|
||||||
|
verified_api_key_hash: None,
|
||||||
}));
|
}));
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -1127,6 +1198,7 @@ async fn resolve_trusted_auth_context(
|
|||||||
local_rejection: Some(GatewayLocalAuthRejection::InvalidApiKey),
|
local_rejection: Some(GatewayLocalAuthRejection::InvalidApiKey),
|
||||||
allowed_models: None,
|
allowed_models: None,
|
||||||
ip_rules: None,
|
ip_rules: None,
|
||||||
|
verified_api_key_hash: None,
|
||||||
}));
|
}));
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -1158,9 +1230,7 @@ async fn build_data_backed_auth_context(
|
|||||||
let invalid_api_key = !snapshot.user_is_active
|
let invalid_api_key = !snapshot.user_is_active
|
||||||
|| snapshot.user_is_deleted
|
|| snapshot.user_is_deleted
|
||||||
|| !snapshot.api_key_is_active
|
|| !snapshot.api_key_is_active
|
||||||
|| snapshot
|
|| api_key_is_expired(snapshot.api_key_expires_at_unix_secs, current_unix_secs());
|
||||||
.api_key_expires_at_unix_secs
|
|
||||||
.is_some_and(|expires_at| expires_at < current_unix_secs());
|
|
||||||
let locked_api_key = snapshot.api_key_is_locked && !snapshot.api_key_is_standalone;
|
let locked_api_key = snapshot.api_key_is_locked && !snapshot.api_key_is_standalone;
|
||||||
let key_access_allowed = header_access_allowed
|
let key_access_allowed = header_access_allowed
|
||||||
.map(|value| value && snapshot.currently_usable)
|
.map(|value| value && snapshot.currently_usable)
|
||||||
@@ -1225,9 +1295,14 @@ async fn build_data_backed_auth_context(
|
|||||||
local_rejection,
|
local_rejection,
|
||||||
allowed_models,
|
allowed_models,
|
||||||
ip_rules: snapshot.api_key_ip_rules,
|
ip_rules: snapshot.api_key_ip_rules,
|
||||||
|
verified_api_key_hash: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn api_key_is_expired(expires_at_unix_secs: Option<u64>, now_unix_secs: u64) -> bool {
|
||||||
|
expires_at_unix_secs.is_some_and(|expires_at| expires_at <= now_unix_secs)
|
||||||
|
}
|
||||||
|
|
||||||
fn contains_api_format_or_alias(items: &[String], target: &str) -> bool {
|
fn contains_api_format_or_alias(items: &[String], target: &str) -> bool {
|
||||||
items.iter().any(|item| api_format_matches(item, target))
|
items.iter().any(|item| api_format_matches(item, target))
|
||||||
}
|
}
|
||||||
@@ -1282,18 +1357,21 @@ async fn auth_snapshot_allows_requested_provider(
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
if !state.has_provider_catalog_data_reader() {
|
if !state.has_provider_catalog_data_reader() {
|
||||||
return true;
|
debug!(
|
||||||
|
"deny requested provider {}: provider catalog is unavailable for allowlist resolution",
|
||||||
|
requested_provider
|
||||||
|
);
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
let providers = match state.list_provider_catalog_providers(true).await {
|
let providers = match state.list_provider_catalog_providers(true).await {
|
||||||
Ok(value) => value,
|
Ok(value) => value,
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
debug!(
|
warn!(
|
||||||
"skip local provider auth gate for requested provider {}: provider catalog lookup failed: {:?}",
|
"deny requested provider {}: provider catalog lookup failed: {:?}",
|
||||||
requested_provider,
|
requested_provider, err
|
||||||
err
|
|
||||||
);
|
);
|
||||||
return true;
|
return false;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -1331,11 +1409,11 @@ async fn auth_snapshot_allows_requested_provider(
|
|||||||
{
|
{
|
||||||
Ok(value) => value,
|
Ok(value) => value,
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
debug!(
|
warn!(
|
||||||
"skip local provider auth gate for requested provider {}: provider endpoint lookup failed: {:?}",
|
"deny requested provider {}: provider endpoint lookup failed: {:?}",
|
||||||
requested_provider, err
|
requested_provider, err
|
||||||
);
|
);
|
||||||
return true;
|
return false;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -1421,12 +1499,12 @@ fn get_cached_auth_context_with_age(
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use std::path::PathBuf;
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use aether_data::repository::auth::{
|
use aether_data::repository::auth::{
|
||||||
AuthApiKeyWriteRepository, InMemoryAuthApiKeySnapshotRepository, StoredAuthApiKeySnapshot,
|
AuthApiKeyWriteRepository, CreateUserApiKeyRecord, InMemoryAuthApiKeySnapshotRepository,
|
||||||
|
StoredAuthApiKeySnapshot,
|
||||||
};
|
};
|
||||||
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
||||||
use aether_data::repository::wallet::{
|
use aether_data::repository::wallet::{
|
||||||
@@ -1441,9 +1519,10 @@ mod tests {
|
|||||||
use futures_util::future::join_all;
|
use futures_util::future::join_all;
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
get_cached_auth_context, resolve_control_decision_auth, resolve_data_backed_auth_context,
|
api_key_is_expired, get_cached_auth_context,
|
||||||
resolve_execution_runtime_auth_context, ControlDecisionAuthResolution,
|
refresh_execution_runtime_auth_context_with_snapshot, resolve_control_decision_auth,
|
||||||
GatewayLocalAuthRejection,
|
resolve_data_backed_auth_context, resolve_execution_runtime_auth_context,
|
||||||
|
ControlDecisionAuthResolution, GatewayLocalAuthRejection,
|
||||||
};
|
};
|
||||||
use crate::control::auth::credentials::{build_auth_context_cache_key, hash_api_key};
|
use crate::control::auth::credentials::{build_auth_context_cache_key, hash_api_key};
|
||||||
use crate::control::GatewayControlDecision;
|
use crate::control::GatewayControlDecision;
|
||||||
@@ -1481,6 +1560,14 @@ mod tests {
|
|||||||
path.parse().expect("uri should parse")
|
path.parse().expect("uri should parse")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn api_key_expiry_is_inclusive_at_the_declared_second() {
|
||||||
|
assert!(!api_key_is_expired(None, 100));
|
||||||
|
assert!(!api_key_is_expired(Some(101), 100));
|
||||||
|
assert!(api_key_is_expired(Some(100), 100));
|
||||||
|
assert!(api_key_is_expired(Some(99), 100));
|
||||||
|
}
|
||||||
|
|
||||||
fn sample_provider(id: &str, name: &str, provider_type: &str) -> StoredProviderCatalogProvider {
|
fn sample_provider(id: &str, name: &str, provider_type: &str) -> StoredProviderCatalogProvider {
|
||||||
StoredProviderCatalogProvider::new(
|
StoredProviderCatalogProvider::new(
|
||||||
id.to_string(),
|
id.to_string(),
|
||||||
@@ -1507,40 +1594,24 @@ mod tests {
|
|||||||
.expect("endpoint should build")
|
.expect("endpoint should build")
|
||||||
}
|
}
|
||||||
|
|
||||||
struct SqliteAuthConfigNodes {
|
struct PostgresAuthConfigNodes {
|
||||||
first: AppState,
|
first: AppState,
|
||||||
second: AppState,
|
second: AppState,
|
||||||
_database: TemporarySqliteDatabase,
|
_database: aether_testkit::ManagedPostgresServer,
|
||||||
}
|
}
|
||||||
|
|
||||||
struct TemporarySqliteDatabase(PathBuf);
|
async fn postgres_auth_config_nodes(
|
||||||
|
|
||||||
impl Drop for TemporarySqliteDatabase {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
for suffix in ["", "-shm", "-wal"] {
|
|
||||||
let mut path = self.0.as_os_str().to_os_string();
|
|
||||||
path.push(suffix);
|
|
||||||
let _ = std::fs::remove_file(PathBuf::from(path));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn sqlite_auth_config_nodes(
|
|
||||||
auth_repository: Arc<InMemoryAuthApiKeySnapshotRepository>,
|
auth_repository: Arc<InMemoryAuthApiKeySnapshotRepository>,
|
||||||
) -> SqliteAuthConfigNodes {
|
) -> PostgresAuthConfigNodes {
|
||||||
let database_path = std::env::temp_dir().join(format!(
|
let server = aether_testkit::ManagedPostgresServer::start()
|
||||||
"aether-auth-config-{}.sqlite",
|
.await
|
||||||
uuid::Uuid::new_v4()
|
.expect("temporary PostgreSQL should start");
|
||||||
));
|
|
||||||
let mut pool = SqlPoolConfig::default();
|
let mut pool = SqlPoolConfig::default();
|
||||||
pool.min_connections = 0;
|
pool.min_connections = 0;
|
||||||
pool.max_connections = 4;
|
pool.max_connections = 4;
|
||||||
let database = SqlDatabaseConfig::new(
|
let database =
|
||||||
DatabaseDriver::Sqlite,
|
SqlDatabaseConfig::new(DatabaseDriver::Postgres, server.database_url(), pool)
|
||||||
format!("sqlite://{}", database_path.display()),
|
.expect("postgres config should build");
|
||||||
pool,
|
|
||||||
)
|
|
||||||
.expect("sqlite config should build");
|
|
||||||
let config = GatewayDataConfig::from_database_config(database);
|
let config = GatewayDataConfig::from_database_config(database);
|
||||||
let first_data = GatewayDataState::from_config(config.clone())
|
let first_data = GatewayDataState::from_config(config.clone())
|
||||||
.expect("first data state should build")
|
.expect("first data state should build")
|
||||||
@@ -1549,27 +1620,27 @@ mod tests {
|
|||||||
assert!(first_data
|
assert!(first_data
|
||||||
.run_database_migrations()
|
.run_database_migrations()
|
||||||
.await
|
.await
|
||||||
.expect("sqlite migrations should run"));
|
.expect("postgres migrations should run"));
|
||||||
let second_data = GatewayDataState::from_config(config)
|
let second_data = GatewayDataState::from_config(config)
|
||||||
.expect("second data state should build")
|
.expect("second data state should build")
|
||||||
.with_auth_api_key_reader(auth_repository)
|
.with_auth_api_key_reader(auth_repository)
|
||||||
.without_wallet_reader_for_tests();
|
.without_wallet_reader_for_tests();
|
||||||
|
|
||||||
SqliteAuthConfigNodes {
|
PostgresAuthConfigNodes {
|
||||||
first: AppState::new()
|
first: AppState::new()
|
||||||
.expect("first app state should build")
|
.expect("first app state should build")
|
||||||
.with_data_state_for_tests(first_data),
|
.with_data_state_for_tests(first_data),
|
||||||
second: AppState::new()
|
second: AppState::new()
|
||||||
.expect("second app state should build")
|
.expect("second app state should build")
|
||||||
.with_data_state_for_tests(second_data),
|
.with_data_state_for_tests(second_data),
|
||||||
_database: TemporarySqliteDatabase(database_path),
|
_database: server,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn strong_system_config_read_bypasses_app_and_data_caches() {
|
async fn strong_system_config_read_bypasses_app_and_data_caches() {
|
||||||
let nodes =
|
let nodes =
|
||||||
sqlite_auth_config_nodes(Arc::new(InMemoryAuthApiKeySnapshotRepository::seed([])))
|
postgres_auth_config_nodes(Arc::new(InMemoryAuthApiKeySnapshotRepository::seed([])))
|
||||||
.await;
|
.await;
|
||||||
let key = format!("test.auth.strong-read.{}", uuid::Uuid::new_v4());
|
let key = format!("test.auth.strong-read.{}", uuid::Uuid::new_v4());
|
||||||
let old_value = serde_json::json!({"version": "old"});
|
let old_value = serde_json::json!({"version": "old"});
|
||||||
@@ -1769,6 +1840,97 @@ mod tests {
|
|||||||
assert_eq!(repository.touch_count("key-1"), 1);
|
assert_eq!(repository.touch_count("key-1"), 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn long_lived_refresh_rejects_same_ids_recreated_with_a_different_credential() {
|
||||||
|
let old_api_key = "sk-old-websocket-credential";
|
||||||
|
let new_api_key = "sk-new-websocket-credential";
|
||||||
|
let old_key_hash = hash_api_key(old_api_key);
|
||||||
|
let new_key_hash = hash_api_key(new_api_key);
|
||||||
|
let mut old_snapshot = sample_snapshot("key-stable-id", "user-stable-id");
|
||||||
|
old_snapshot.user_allowed_api_formats = Some(vec!["openai:responses".to_string()]);
|
||||||
|
old_snapshot.api_key_allowed_api_formats = Some(vec!["openai:responses".to_string()]);
|
||||||
|
let repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
|
||||||
|
Some(old_key_hash.clone()),
|
||||||
|
old_snapshot,
|
||||||
|
)]));
|
||||||
|
let data = GatewayDataState::with_auth_api_key_repository_for_tests(repository.clone());
|
||||||
|
let state = AppState::new()
|
||||||
|
.expect("state should build")
|
||||||
|
.with_data_state_for_tests(data);
|
||||||
|
let mut headers = HeaderMap::new();
|
||||||
|
headers.insert(
|
||||||
|
http::header::AUTHORIZATION,
|
||||||
|
format!("Bearer {old_api_key}").parse().unwrap(),
|
||||||
|
);
|
||||||
|
|
||||||
|
let original = resolve_data_backed_auth_context(
|
||||||
|
&state,
|
||||||
|
&headers,
|
||||||
|
&uri("/v1/responses"),
|
||||||
|
Some("openai:responses"),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("initial auth resolution should succeed")
|
||||||
|
.expect("the old API key should authenticate");
|
||||||
|
assert!(original.access_allowed);
|
||||||
|
assert!(original.verified_api_key_hash.is_some());
|
||||||
|
assert!(
|
||||||
|
!format!("{original:?}").contains(&old_key_hash),
|
||||||
|
"the credential verifier must stay redacted from Debug output"
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(repository
|
||||||
|
.delete_user_api_key("user-stable-id", "key-stable-id")
|
||||||
|
.await
|
||||||
|
.expect("old API key deletion should succeed"));
|
||||||
|
repository
|
||||||
|
.create_user_api_key(CreateUserApiKeyRecord {
|
||||||
|
user_id: "user-stable-id".to_string(),
|
||||||
|
api_key_id: "key-stable-id".to_string(),
|
||||||
|
key_hash: new_key_hash,
|
||||||
|
key_encrypted: None,
|
||||||
|
name: Some("restored-with-new-secret".to_string()),
|
||||||
|
allowed_providers: Some(vec!["openai".to_string()]),
|
||||||
|
allowed_api_formats: Some(vec!["openai:responses".to_string()]),
|
||||||
|
allowed_models: Some(vec!["gpt-4.1".to_string()]),
|
||||||
|
ip_rules: None,
|
||||||
|
rate_limit: 60,
|
||||||
|
concurrent_limit: Some(5),
|
||||||
|
force_capabilities: None,
|
||||||
|
feature_settings: None,
|
||||||
|
is_active: true,
|
||||||
|
expires_at_unix_secs: Some(4_102_444_800),
|
||||||
|
auto_delete_on_expiry: false,
|
||||||
|
total_requests: 0,
|
||||||
|
total_tokens: 0,
|
||||||
|
total_cost_usd: 0.0,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("same-ID API key recreation should resolve")
|
||||||
|
.expect("same-ID API key recreation should persist");
|
||||||
|
|
||||||
|
let (refreshed, snapshot) = refresh_execution_runtime_auth_context_with_snapshot(
|
||||||
|
&state,
|
||||||
|
original,
|
||||||
|
Some("openai:responses"),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("long-lived auth refresh should resolve");
|
||||||
|
|
||||||
|
assert!(!refreshed.access_allowed);
|
||||||
|
assert_eq!(
|
||||||
|
refreshed.local_rejection,
|
||||||
|
Some(GatewayLocalAuthRejection::InvalidApiKey)
|
||||||
|
);
|
||||||
|
assert!(snapshot.is_none());
|
||||||
|
assert_eq!(repository.key_hash_lookup_count(&old_key_hash), 1);
|
||||||
|
assert_eq!(
|
||||||
|
repository.snapshot_lookup_count("key-stable-id"),
|
||||||
|
0,
|
||||||
|
"a bound long-lived credential must not fall back to identity-only lookup"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn control_auth_context_singleflights_concurrent_cache_misses() {
|
async fn control_auth_context_singleflights_concurrent_cache_misses() {
|
||||||
let api_key = "sk-test-concurrent-auth-miss";
|
let api_key = "sk-test-concurrent-auth-miss";
|
||||||
@@ -2396,6 +2558,44 @@ mod tests {
|
|||||||
assert_eq!(auth_context.local_rejection, None);
|
assert_eq!(auth_context.local_rejection, None);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn data_backed_auth_context_denies_unresolved_provider_id_without_catalog_reader() {
|
||||||
|
let api_key = "sk-test-provider-no-catalog";
|
||||||
|
let mut snapshot = sample_snapshot("key-no-catalog", "user-no-catalog");
|
||||||
|
snapshot.user_allowed_providers = Some(vec!["provider-custom-claude".to_string()]);
|
||||||
|
snapshot.api_key_allowed_providers = Some(vec!["provider-custom-claude".to_string()]);
|
||||||
|
snapshot.user_allowed_api_formats = None;
|
||||||
|
snapshot.api_key_allowed_api_formats = None;
|
||||||
|
let repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
|
||||||
|
Some(hash_api_key(api_key)),
|
||||||
|
snapshot,
|
||||||
|
)]));
|
||||||
|
let data = GatewayDataState::with_auth_api_key_reader_for_tests(repository);
|
||||||
|
let state = AppState::new()
|
||||||
|
.expect("state should build")
|
||||||
|
.with_data_state_for_tests(data);
|
||||||
|
let mut headers = HeaderMap::new();
|
||||||
|
headers.insert("x-api-key", api_key.parse().unwrap());
|
||||||
|
|
||||||
|
let auth_context = resolve_data_backed_auth_context(
|
||||||
|
&state,
|
||||||
|
&headers,
|
||||||
|
&uri("/v1/messages"),
|
||||||
|
Some("claude:messages"),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("resolution should succeed")
|
||||||
|
.expect("auth context should exist");
|
||||||
|
|
||||||
|
assert!(!auth_context.access_allowed);
|
||||||
|
assert_eq!(
|
||||||
|
auth_context.local_rejection,
|
||||||
|
Some(GatewayLocalAuthRejection::ProviderNotAllowed {
|
||||||
|
provider: "claude".to_string(),
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn due_antigravity_bearer_refresh_observes_cross_node_allowlist_revocation() {
|
async fn due_antigravity_bearer_refresh_observes_cross_node_allowlist_revocation() {
|
||||||
let raw_bearer = "google-oauth-access-token-revoked-cross-node";
|
let raw_bearer = "google-oauth-access-token-revoked-cross-node";
|
||||||
@@ -2410,7 +2610,7 @@ mod tests {
|
|||||||
let auth_repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
|
let auth_repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(vec![(
|
||||||
None, snapshot,
|
None, snapshot,
|
||||||
)]));
|
)]));
|
||||||
let nodes = sqlite_auth_config_nodes(auth_repository.clone()).await;
|
let nodes = postgres_auth_config_nodes(auth_repository.clone()).await;
|
||||||
nodes
|
nodes
|
||||||
.first
|
.first
|
||||||
.auth_context_cache
|
.auth_context_cache
|
||||||
|
|||||||
@@ -44,7 +44,7 @@ pub(super) struct GatewayTrustedAdminHeaders {
|
|||||||
pub(super) management_token_id: Option<String>,
|
pub(super) management_token_id: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
#[derive(Clone, Default, PartialEq, Eq)]
|
||||||
pub(super) struct GatewayCredentialBundle {
|
pub(super) struct GatewayCredentialBundle {
|
||||||
pub(super) authorization_bearer: Option<String>,
|
pub(super) authorization_bearer: Option<String>,
|
||||||
pub(super) x_api_key: Option<String>,
|
pub(super) x_api_key: Option<String>,
|
||||||
@@ -54,7 +54,25 @@ pub(super) struct GatewayCredentialBundle {
|
|||||||
pub(super) cookie_header: Option<String>,
|
pub(super) cookie_header: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
impl std::fmt::Debug for GatewayCredentialBundle {
|
||||||
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
let redacted = |value: &Option<String>| value.as_ref().map(|_| "[REDACTED]");
|
||||||
|
formatter
|
||||||
|
.debug_struct("GatewayCredentialBundle")
|
||||||
|
.field(
|
||||||
|
"authorization_bearer",
|
||||||
|
&redacted(&self.authorization_bearer),
|
||||||
|
)
|
||||||
|
.field("x_api_key", &redacted(&self.x_api_key))
|
||||||
|
.field("api_key", &redacted(&self.api_key))
|
||||||
|
.field("x_goog_api_key", &redacted(&self.x_goog_api_key))
|
||||||
|
.field("query_key", &redacted(&self.query_key))
|
||||||
|
.field("cookie_header", &redacted(&self.cookie_header))
|
||||||
|
.finish()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, PartialEq, Eq)]
|
||||||
pub(super) enum GatewayPrimaryCredential {
|
pub(super) enum GatewayPrimaryCredential {
|
||||||
ProviderApiKey {
|
ProviderApiKey {
|
||||||
raw: String,
|
raw: String,
|
||||||
@@ -70,6 +88,21 @@ pub(super) enum GatewayPrimaryCredential {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Debug for GatewayPrimaryCredential {
|
||||||
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
let (variant, carrier) = match self {
|
||||||
|
Self::ProviderApiKey { carrier, .. } => ("ProviderApiKey", carrier),
|
||||||
|
Self::BearerToken { carrier, .. } => ("BearerToken", carrier),
|
||||||
|
Self::CookieHeader { carrier, .. } => ("CookieHeader", carrier),
|
||||||
|
};
|
||||||
|
formatter
|
||||||
|
.debug_struct(variant)
|
||||||
|
.field("raw", &"[REDACTED]")
|
||||||
|
.field("carrier", carrier)
|
||||||
|
.finish()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq)]
|
#[derive(Debug, Clone, PartialEq)]
|
||||||
pub(super) struct GatewayExtractedCredentials {
|
pub(super) struct GatewayExtractedCredentials {
|
||||||
pub(super) trusted_headers: Option<GatewayTrustedAuthHeaders>,
|
pub(super) trusted_headers: Option<GatewayTrustedAuthHeaders>,
|
||||||
@@ -78,7 +111,7 @@ pub(super) struct GatewayExtractedCredentials {
|
|||||||
pub(super) primary: Option<GatewayPrimaryCredential>,
|
pub(super) primary: Option<GatewayPrimaryCredential>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq)]
|
#[derive(Clone, PartialEq)]
|
||||||
pub(super) enum GatewayPrincipalCandidate {
|
pub(super) enum GatewayPrincipalCandidate {
|
||||||
TrustedHeaders(GatewayTrustedAuthHeaders),
|
TrustedHeaders(GatewayTrustedAuthHeaders),
|
||||||
ApiKeyHash {
|
ApiKeyHash {
|
||||||
@@ -94,3 +127,58 @@ pub(super) enum GatewayPrincipalCandidate {
|
|||||||
carrier: GatewayCredentialCarrier,
|
carrier: GatewayCredentialCarrier,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Debug for GatewayPrincipalCandidate {
|
||||||
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
match self {
|
||||||
|
Self::TrustedHeaders(headers) => formatter
|
||||||
|
.debug_tuple("TrustedHeaders")
|
||||||
|
.field(headers)
|
||||||
|
.finish(),
|
||||||
|
Self::ApiKeyHash { carrier, .. } => formatter
|
||||||
|
.debug_struct("ApiKeyHash")
|
||||||
|
.field("key_hash", &"[REDACTED]")
|
||||||
|
.field("carrier", carrier)
|
||||||
|
.finish(),
|
||||||
|
Self::DeferredBearerToken { carrier, .. } => formatter
|
||||||
|
.debug_struct("DeferredBearerToken")
|
||||||
|
.field("raw", &"[REDACTED]")
|
||||||
|
.field("carrier", carrier)
|
||||||
|
.finish(),
|
||||||
|
Self::DeferredCookieHeader { carrier, .. } => formatter
|
||||||
|
.debug_struct("DeferredCookieHeader")
|
||||||
|
.field("raw", &"[REDACTED]")
|
||||||
|
.field("carrier", carrier)
|
||||||
|
.finish(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod debug_redaction_tests {
|
||||||
|
use super::{GatewayCredentialBundle, GatewayCredentialCarrier, GatewayPrimaryCredential};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn gateway_credential_debug_output_redacts_raw_authorization_values() {
|
||||||
|
let bundle = GatewayCredentialBundle {
|
||||||
|
authorization_bearer: Some("bundle-bearer-canary".to_string()),
|
||||||
|
api_key: Some("bundle-api-key-canary".to_string()),
|
||||||
|
cookie_header: Some("bundle-cookie-canary".to_string()),
|
||||||
|
..GatewayCredentialBundle::default()
|
||||||
|
};
|
||||||
|
let primary = GatewayPrimaryCredential::ProviderApiKey {
|
||||||
|
raw: "primary-api-key-canary".to_string(),
|
||||||
|
carrier: GatewayCredentialCarrier::ApiKey,
|
||||||
|
};
|
||||||
|
let debug = format!("{bundle:?} {primary:?}");
|
||||||
|
assert!(debug.contains("[REDACTED]"));
|
||||||
|
for secret in [
|
||||||
|
"bundle-bearer-canary",
|
||||||
|
"bundle-api-key-canary",
|
||||||
|
"bundle-cookie-canary",
|
||||||
|
"primary-api-key-canary",
|
||||||
|
] {
|
||||||
|
assert!(!debug.contains(secret), "debug output leaked {secret}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -8,9 +8,10 @@ mod public;
|
|||||||
mod route;
|
mod route;
|
||||||
|
|
||||||
pub(crate) use auth::{
|
pub(crate) use auth::{
|
||||||
execution_plan_balance_capacity_rejection, extract_requested_model,
|
estimate_execution_plan_cost_upper_bound_usd, execution_plan_balance_capacity_rejection,
|
||||||
refresh_execution_runtime_auth_context, refresh_execution_runtime_auth_context_with_snapshot,
|
extract_requested_model, refresh_execution_runtime_auth_context,
|
||||||
request_model_local_rejection, resolve_execution_runtime_auth_context,
|
refresh_execution_runtime_auth_context_with_snapshot, request_model_local_rejection,
|
||||||
|
resolve_execution_runtime_auth_context, resolve_local_admin_session_principal,
|
||||||
should_buffer_request_for_local_auth, trusted_auth_local_rejection,
|
should_buffer_request_for_local_auth, trusted_auth_local_rejection,
|
||||||
GatewayAdminPrincipalContext, GatewayControlAuthContext, GatewayCredentialCarrier,
|
GatewayAdminPrincipalContext, GatewayControlAuthContext, GatewayCredentialCarrier,
|
||||||
GatewayLocalAuthRejection,
|
GatewayLocalAuthRejection,
|
||||||
@@ -18,14 +19,16 @@ pub(crate) use auth::{
|
|||||||
pub(crate) use execute::{allows_control_execute_emergency, maybe_execute_via_control};
|
pub(crate) use execute::{allows_control_execute_emergency, maybe_execute_via_control};
|
||||||
pub(crate) use management_token_permissions::{
|
pub(crate) use management_token_permissions::{
|
||||||
all_assignable_management_token_permissions,
|
all_assignable_management_token_permissions,
|
||||||
audit_admin_read_only_management_token_permissions,
|
audit_admin_read_only_management_token_permissions, legacy_full_management_token_permissions,
|
||||||
management_token_permission_catalog_payload, management_token_permission_keys_from_value,
|
management_token_permission_catalog_payload, management_token_permission_keys_from_value,
|
||||||
management_token_permission_mode_and_summary,
|
management_token_permission_mode_and_summary, management_token_principal_has_permission,
|
||||||
management_token_permissions_cover_all_assignable_permissions,
|
|
||||||
management_token_required_permission, normalize_assignable_management_token_permissions,
|
management_token_required_permission, normalize_assignable_management_token_permissions,
|
||||||
read_only_management_token_permissions, validate_management_token_admin_route_permission,
|
read_only_management_token_permissions, validate_management_token_admin_route_permission,
|
||||||
};
|
};
|
||||||
pub(crate) use public::{resolve_public_request_context, GatewayPublicRequestContext};
|
pub(crate) use public::{
|
||||||
|
resolve_public_request_context, resolve_public_request_context_with_trusted_auth,
|
||||||
|
resolve_public_request_context_without_trusted_auth, GatewayPublicRequestContext,
|
||||||
|
};
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use route::classify_control_route;
|
pub(crate) use route::classify_control_route;
|
||||||
pub(crate) use route::{resolve_control_route, GatewayControlDecision};
|
pub(crate) use route::{resolve_control_route, GatewayControlDecision};
|
||||||
|
|||||||
@@ -2,7 +2,9 @@ use axum::http::Uri;
|
|||||||
|
|
||||||
use crate::{AppState, GatewayError};
|
use crate::{AppState, GatewayError};
|
||||||
|
|
||||||
use super::{resolve_control_route, GatewayControlDecision};
|
use super::{
|
||||||
|
resolve_control_route, route::resolve_control_route_with_trusted_auth, GatewayControlDecision,
|
||||||
|
};
|
||||||
|
|
||||||
pub(crate) type GatewayPublicRequestContext =
|
pub(crate) type GatewayPublicRequestContext =
|
||||||
aether_gateway_control::PublicRequestContext<GatewayControlDecision>;
|
aether_gateway_control::PublicRequestContext<GatewayControlDecision>;
|
||||||
@@ -23,3 +25,41 @@ pub(crate) async fn resolve_public_request_context(
|
|||||||
control_decision,
|
control_decision,
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn resolve_public_request_context_with_trusted_auth(
|
||||||
|
state: &AppState,
|
||||||
|
method: &http::Method,
|
||||||
|
uri: &Uri,
|
||||||
|
headers: &http::HeaderMap,
|
||||||
|
trace_id: &str,
|
||||||
|
) -> Result<GatewayPublicRequestContext, GatewayError> {
|
||||||
|
let control_decision =
|
||||||
|
resolve_control_route_with_trusted_auth(state, method, uri, headers, trace_id, true)
|
||||||
|
.await?;
|
||||||
|
Ok(GatewayPublicRequestContext::from_request_parts(
|
||||||
|
trace_id,
|
||||||
|
method,
|
||||||
|
uri,
|
||||||
|
headers,
|
||||||
|
control_decision,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn resolve_public_request_context_without_trusted_auth(
|
||||||
|
state: &AppState,
|
||||||
|
method: &http::Method,
|
||||||
|
uri: &Uri,
|
||||||
|
headers: &http::HeaderMap,
|
||||||
|
trace_id: &str,
|
||||||
|
) -> Result<GatewayPublicRequestContext, GatewayError> {
|
||||||
|
let control_decision =
|
||||||
|
resolve_control_route_with_trusted_auth(state, method, uri, headers, trace_id, false)
|
||||||
|
.await?;
|
||||||
|
Ok(GatewayPublicRequestContext::from_request_parts(
|
||||||
|
trace_id,
|
||||||
|
method,
|
||||||
|
uri,
|
||||||
|
headers,
|
||||||
|
control_decision,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|||||||
@@ -302,6 +302,19 @@ pub(super) fn classify_admin_endpoints_family_route(
|
|||||||
"admin:endpoints_manage",
|
"admin:endpoints_manage",
|
||||||
false,
|
false,
|
||||||
))
|
))
|
||||||
|
} else if method == http::Method::GET
|
||||||
|
&& normalized_path
|
||||||
|
.strip_prefix("/api/admin/endpoints/")
|
||||||
|
.and_then(|path| path.strip_suffix("/rules/reveal"))
|
||||||
|
.is_some_and(|endpoint_id| !endpoint_id.is_empty() && !endpoint_id.contains('/'))
|
||||||
|
{
|
||||||
|
Some(classified(
|
||||||
|
"admin_proxy",
|
||||||
|
"endpoints_manage",
|
||||||
|
"reveal_endpoint_rules",
|
||||||
|
"admin:endpoints_manage",
|
||||||
|
false,
|
||||||
|
))
|
||||||
} else if method == http::Method::GET
|
} else if method == http::Method::GET
|
||||||
&& normalized_path.starts_with("/api/admin/endpoints/")
|
&& normalized_path.starts_with("/api/admin/endpoints/")
|
||||||
&& !normalized_path.starts_with("/api/admin/endpoints/health/")
|
&& !normalized_path.starts_with("/api/admin/endpoints/health/")
|
||||||
|
|||||||
@@ -5,19 +5,21 @@ pub(super) fn classify_internal_route(
|
|||||||
method: &http::Method,
|
method: &http::Method,
|
||||||
normalized_path: &str,
|
normalized_path: &str,
|
||||||
) -> Option<ClassifiedRoute> {
|
) -> Option<ClassifiedRoute> {
|
||||||
if method == http::Method::POST && normalized_path.starts_with("/api/internal/gateway/") {
|
if normalized_path == "/api/internal/gateway"
|
||||||
let route_kind = match normalized_path {
|
|| normalized_path.starts_with("/api/internal/gateway/")
|
||||||
"/api/internal/gateway/resolve" => "resolve",
|
{
|
||||||
"/api/internal/gateway/auth-context" => "auth_context",
|
let route_kind = match (method, normalized_path) {
|
||||||
"/api/internal/gateway/decision-sync" => "decision_sync",
|
(&http::Method::POST, "/api/internal/gateway/resolve") => "resolve",
|
||||||
"/api/internal/gateway/decision-stream" => "decision_stream",
|
(&http::Method::POST, "/api/internal/gateway/auth-context") => "auth_context",
|
||||||
"/api/internal/gateway/plan-sync" => "plan_sync",
|
(&http::Method::POST, "/api/internal/gateway/decision-sync") => "decision_sync",
|
||||||
"/api/internal/gateway/plan-stream" => "plan_stream",
|
(&http::Method::POST, "/api/internal/gateway/decision-stream") => "decision_stream",
|
||||||
"/api/internal/gateway/report-sync" => "report_sync",
|
(&http::Method::POST, "/api/internal/gateway/plan-sync") => "plan_sync",
|
||||||
"/api/internal/gateway/report-stream" => "report_stream",
|
(&http::Method::POST, "/api/internal/gateway/plan-stream") => "plan_stream",
|
||||||
"/api/internal/gateway/finalize-sync" => "finalize_sync",
|
(&http::Method::POST, "/api/internal/gateway/report-sync") => "report_sync",
|
||||||
"/api/internal/gateway/execute-sync" => "execute_sync",
|
(&http::Method::POST, "/api/internal/gateway/report-stream") => "report_stream",
|
||||||
"/api/internal/gateway/execute-stream" => "execute_stream",
|
(&http::Method::POST, "/api/internal/gateway/finalize-sync") => "finalize_sync",
|
||||||
|
(&http::Method::POST, "/api/internal/gateway/execute-sync") => "execute_sync",
|
||||||
|
(&http::Method::POST, "/api/internal/gateway/execute-stream") => "execute_stream",
|
||||||
_ => "unhandled",
|
_ => "unhandled",
|
||||||
};
|
};
|
||||||
Some(classified(
|
Some(classified(
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ mod oauth;
|
|||||||
mod public_support;
|
mod public_support;
|
||||||
|
|
||||||
use super::auth::{
|
use super::auth::{
|
||||||
resolve_control_decision_auth, resolve_gateway_credential_carrier,
|
resolve_control_decision_auth_with_trusted_auth, resolve_gateway_credential_carrier,
|
||||||
ControlDecisionAuthResolution, GatewayCredentialCarrier,
|
ControlDecisionAuthResolution, GatewayCredentialCarrier,
|
||||||
};
|
};
|
||||||
use super::{GatewayAdminPrincipalContext, GatewayControlAuthContext, GatewayLocalAuthRejection};
|
use super::{GatewayAdminPrincipalContext, GatewayControlAuthContext, GatewayLocalAuthRejection};
|
||||||
@@ -175,6 +175,17 @@ pub(crate) async fn resolve_control_route(
|
|||||||
uri: &Uri,
|
uri: &Uri,
|
||||||
headers: &http::HeaderMap,
|
headers: &http::HeaderMap,
|
||||||
trace_id: &str,
|
trace_id: &str,
|
||||||
|
) -> Result<Option<GatewayControlDecision>, GatewayError> {
|
||||||
|
resolve_control_route_with_trusted_auth(state, method, uri, headers, trace_id, cfg!(test)).await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn resolve_control_route_with_trusted_auth(
|
||||||
|
state: &AppState,
|
||||||
|
method: &http::Method,
|
||||||
|
uri: &Uri,
|
||||||
|
headers: &http::HeaderMap,
|
||||||
|
trace_id: &str,
|
||||||
|
trusted_auth_verified: bool,
|
||||||
) -> Result<Option<GatewayControlDecision>, GatewayError> {
|
) -> Result<Option<GatewayControlDecision>, GatewayError> {
|
||||||
let Some(mut decision) = classify_control_route(method, uri, headers) else {
|
let Some(mut decision) = classify_control_route(method, uri, headers) else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
@@ -185,7 +196,16 @@ pub(crate) async fn resolve_control_route(
|
|||||||
crate::system_features::ModelDirectivePolicySnapshot::load(state).await;
|
crate::system_features::ModelDirectivePolicySnapshot::load(state).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
match resolve_control_decision_auth(state, headers, uri, trace_id, decision).await? {
|
match resolve_control_decision_auth_with_trusted_auth(
|
||||||
|
state,
|
||||||
|
headers,
|
||||||
|
uri,
|
||||||
|
trace_id,
|
||||||
|
decision,
|
||||||
|
trusted_auth_verified,
|
||||||
|
)
|
||||||
|
.await?
|
||||||
|
{
|
||||||
ControlDecisionAuthResolution::Resolved(decision) => Ok(Some(decision)),
|
ControlDecisionAuthResolution::Resolved(decision) => Ok(Some(decision)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -285,7 +285,7 @@ pub(super) fn classify_oauth_route(
|
|||||||
"admin_proxy",
|
"admin_proxy",
|
||||||
"provider_oauth_manage",
|
"provider_oauth_manage",
|
||||||
"batch_import_oauth",
|
"batch_import_oauth",
|
||||||
"admin:pool",
|
"admin:provider_oauth",
|
||||||
false,
|
false,
|
||||||
))
|
))
|
||||||
} else if method == http::Method::POST
|
} else if method == http::Method::POST
|
||||||
@@ -296,7 +296,7 @@ pub(super) fn classify_oauth_route(
|
|||||||
"admin_proxy",
|
"admin_proxy",
|
||||||
"provider_oauth_manage",
|
"provider_oauth_manage",
|
||||||
"start_batch_import_oauth_task",
|
"start_batch_import_oauth_task",
|
||||||
"admin:pool",
|
"admin:provider_oauth",
|
||||||
false,
|
false,
|
||||||
))
|
))
|
||||||
} else if method == http::Method::GET
|
} else if method == http::Method::GET
|
||||||
@@ -307,7 +307,7 @@ pub(super) fn classify_oauth_route(
|
|||||||
"admin_proxy",
|
"admin_proxy",
|
||||||
"provider_oauth_manage",
|
"provider_oauth_manage",
|
||||||
"get_batch_import_task_status",
|
"get_batch_import_task_status",
|
||||||
"admin:pool",
|
"admin:provider_oauth",
|
||||||
false,
|
false,
|
||||||
))
|
))
|
||||||
} else if method == http::Method::POST
|
} else if method == http::Method::POST
|
||||||
|
|||||||
@@ -197,18 +197,22 @@ pub(super) fn classify_public_support_route(
|
|||||||
"public:auth",
|
"public:auth",
|
||||||
false,
|
false,
|
||||||
))
|
))
|
||||||
} else if matches!(method, &http::Method::GET | &http::Method::POST)
|
// Authentication state-changing endpoints must never be dispatched for GET.
|
||||||
|
// Besides violating HTTP method semantics, accepting GET here would allow
|
||||||
|
// browser prefetches/cross-site requests to trigger login, refresh, logout,
|
||||||
|
// registration, or verification side effects. `/me` is the sole read route.
|
||||||
|
} else if (method == http::Method::POST
|
||||||
&& matches!(
|
&& matches!(
|
||||||
normalized_path,
|
normalized_path,
|
||||||
"/api/auth/login"
|
"/api/auth/login"
|
||||||
| "/api/auth/refresh"
|
| "/api/auth/refresh"
|
||||||
| "/api/auth/register"
|
| "/api/auth/register"
|
||||||
| "/api/auth/me"
|
|
||||||
| "/api/auth/logout"
|
| "/api/auth/logout"
|
||||||
| "/api/auth/send-verification-code"
|
| "/api/auth/send-verification-code"
|
||||||
| "/api/auth/verify-email"
|
| "/api/auth/verify-email"
|
||||||
| "/api/auth/verification-status"
|
| "/api/auth/verification-status"
|
||||||
)
|
))
|
||||||
|
|| (method == http::Method::GET && normalized_path == "/api/auth/me")
|
||||||
{
|
{
|
||||||
let route_kind = match normalized_path {
|
let route_kind = match normalized_path {
|
||||||
"/api/auth/login" => "login",
|
"/api/auth/login" => "login",
|
||||||
|
|||||||
@@ -628,7 +628,7 @@ fn classifies_admin_management_token_write_routes_and_permission_catalog() {
|
|||||||
http::Method::POST,
|
http::Method::POST,
|
||||||
"/api/admin/management-tokens",
|
"/api/admin/management-tokens",
|
||||||
"create_token",
|
"create_token",
|
||||||
"admin:management_tokens:write",
|
"admin:management_tokens:admin",
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
http::Method::PUT,
|
http::Method::PUT,
|
||||||
@@ -640,7 +640,7 @@ fn classifies_admin_management_token_write_routes_and_permission_catalog() {
|
|||||||
http::Method::POST,
|
http::Method::POST,
|
||||||
"/api/admin/management-tokens/token-123/regenerate",
|
"/api/admin/management-tokens/token-123/regenerate",
|
||||||
"regenerate_token",
|
"regenerate_token",
|
||||||
"admin:management_tokens:write",
|
"admin:management_tokens:admin",
|
||||||
),
|
),
|
||||||
];
|
];
|
||||||
|
|
||||||
|
|||||||
@@ -381,6 +381,28 @@ fn classifies_admin_get_endpoint_as_admin_proxy_route() {
|
|||||||
assert!(!decision.is_execution_runtime_candidate());
|
assert!(!decision.is_execution_runtime_candidate());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn classifies_admin_reveal_endpoint_rules_as_admin_proxy_route() {
|
||||||
|
let headers = headers(&[]);
|
||||||
|
let uri: Uri = "/api/admin/endpoints/endpoint-1/rules/reveal"
|
||||||
|
.parse()
|
||||||
|
.expect("uri should parse");
|
||||||
|
let decision =
|
||||||
|
classify_control_route(&http::Method::GET, &uri, &headers).expect("route should classify");
|
||||||
|
|
||||||
|
assert_eq!(decision.route_class.as_deref(), Some("admin_proxy"));
|
||||||
|
assert_eq!(decision.route_family.as_deref(), Some("endpoints_manage"));
|
||||||
|
assert_eq!(
|
||||||
|
decision.route_kind.as_deref(),
|
||||||
|
Some("reveal_endpoint_rules")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
decision.auth_endpoint_signature.as_deref(),
|
||||||
|
Some("admin:endpoints_manage")
|
||||||
|
);
|
||||||
|
assert!(!decision.is_execution_runtime_candidate());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn classifies_admin_create_endpoint_as_admin_proxy_route() {
|
fn classifies_admin_create_endpoint_as_admin_proxy_route() {
|
||||||
let headers = http::HeaderMap::new();
|
let headers = http::HeaderMap::new();
|
||||||
|
|||||||
@@ -68,11 +68,11 @@ fn classifies_admin_provider_oauth_batch_import_task_status_as_admin_proxy_route
|
|||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
decision.auth_endpoint_signature.as_deref(),
|
decision.auth_endpoint_signature.as_deref(),
|
||||||
Some("admin:pool")
|
Some("admin:provider_oauth")
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
management_token_required_permission(&http::Method::GET, &decision).as_deref(),
|
management_token_required_permission(&http::Method::GET, &decision).as_deref(),
|
||||||
Some("admin:pool:read")
|
Some("admin:provider_oauth:read")
|
||||||
);
|
);
|
||||||
assert!(!decision.is_execution_runtime_candidate());
|
assert!(!decision.is_execution_runtime_candidate());
|
||||||
}
|
}
|
||||||
@@ -86,42 +86,42 @@ fn classifies_admin_provider_oauth_maintenance_routes_as_admin_proxy_route() {
|
|||||||
"/api/admin/provider-oauth/keys/key-123/complete",
|
"/api/admin/provider-oauth/keys/key-123/complete",
|
||||||
"complete_key_oauth",
|
"complete_key_oauth",
|
||||||
"admin:provider_oauth",
|
"admin:provider_oauth",
|
||||||
"admin:provider_oauth:write",
|
"admin:provider_oauth:admin",
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
http::Method::POST,
|
http::Method::POST,
|
||||||
"/api/admin/provider-oauth/keys/key-123/refresh",
|
"/api/admin/provider-oauth/keys/key-123/refresh",
|
||||||
"refresh_key_oauth",
|
"refresh_key_oauth",
|
||||||
"admin:provider_oauth",
|
"admin:provider_oauth",
|
||||||
"admin:provider_oauth:write",
|
"admin:provider_oauth:admin",
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
http::Method::POST,
|
http::Method::POST,
|
||||||
"/api/admin/provider-oauth/providers/provider-123/complete",
|
"/api/admin/provider-oauth/providers/provider-123/complete",
|
||||||
"complete_provider_oauth",
|
"complete_provider_oauth",
|
||||||
"admin:provider_oauth",
|
"admin:provider_oauth",
|
||||||
"admin:provider_oauth:write",
|
"admin:provider_oauth:admin",
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
http::Method::POST,
|
http::Method::POST,
|
||||||
"/api/admin/provider-oauth/providers/provider-123/import-refresh-token",
|
"/api/admin/provider-oauth/providers/provider-123/import-refresh-token",
|
||||||
"import_refresh_token",
|
"import_refresh_token",
|
||||||
"admin:provider_oauth",
|
"admin:provider_oauth",
|
||||||
"admin:provider_oauth:write",
|
"admin:provider_oauth:admin",
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
http::Method::POST,
|
http::Method::POST,
|
||||||
"/api/admin/provider-oauth/providers/provider-123/cookie-authorize",
|
"/api/admin/provider-oauth/providers/provider-123/cookie-authorize",
|
||||||
"cookie_authorize",
|
"cookie_authorize",
|
||||||
"admin:provider_oauth",
|
"admin:provider_oauth",
|
||||||
"admin:provider_oauth:write",
|
"admin:provider_oauth:admin",
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
http::Method::POST,
|
http::Method::POST,
|
||||||
"/api/admin/provider-oauth/providers/provider-123/cookie-authorize/tasks",
|
"/api/admin/provider-oauth/providers/provider-123/cookie-authorize/tasks",
|
||||||
"start_cookie_authorize_task",
|
"start_cookie_authorize_task",
|
||||||
"admin:provider_oauth",
|
"admin:provider_oauth",
|
||||||
"admin:provider_oauth:write",
|
"admin:provider_oauth:admin",
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
http::Method::GET,
|
http::Method::GET,
|
||||||
@@ -135,7 +135,7 @@ fn classifies_admin_provider_oauth_maintenance_routes_as_admin_proxy_route() {
|
|||||||
"/api/admin/provider-oauth/providers/provider-123/agent-identity-import/tasks",
|
"/api/admin/provider-oauth/providers/provider-123/agent-identity-import/tasks",
|
||||||
"start_agent_identity_import_task",
|
"start_agent_identity_import_task",
|
||||||
"admin:provider_oauth",
|
"admin:provider_oauth",
|
||||||
"admin:provider_oauth:write",
|
"admin:provider_oauth:admin",
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
http::Method::GET,
|
http::Method::GET,
|
||||||
@@ -148,22 +148,22 @@ fn classifies_admin_provider_oauth_maintenance_routes_as_admin_proxy_route() {
|
|||||||
http::Method::POST,
|
http::Method::POST,
|
||||||
"/api/admin/provider-oauth/providers/provider-123/batch-import",
|
"/api/admin/provider-oauth/providers/provider-123/batch-import",
|
||||||
"batch_import_oauth",
|
"batch_import_oauth",
|
||||||
"admin:pool",
|
"admin:provider_oauth",
|
||||||
"admin:pool:write",
|
"admin:provider_oauth:admin",
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
http::Method::POST,
|
http::Method::POST,
|
||||||
"/api/admin/provider-oauth/providers/provider-123/batch-import/tasks",
|
"/api/admin/provider-oauth/providers/provider-123/batch-import/tasks",
|
||||||
"start_batch_import_oauth_task",
|
"start_batch_import_oauth_task",
|
||||||
"admin:pool",
|
"admin:provider_oauth",
|
||||||
"admin:pool:write",
|
"admin:provider_oauth:admin",
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
http::Method::GET,
|
http::Method::GET,
|
||||||
"/api/admin/provider-oauth/providers/provider-123/batch-import/tasks/task-123",
|
"/api/admin/provider-oauth/providers/provider-123/batch-import/tasks/task-123",
|
||||||
"get_batch_import_task_status",
|
"get_batch_import_task_status",
|
||||||
"admin:pool",
|
"admin:provider_oauth",
|
||||||
"admin:pool:read",
|
"admin:provider_oauth:read",
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
http::Method::POST,
|
http::Method::POST,
|
||||||
@@ -177,7 +177,7 @@ fn classifies_admin_provider_oauth_maintenance_routes_as_admin_proxy_route() {
|
|||||||
"/api/admin/provider-oauth/providers/provider-123/device-poll",
|
"/api/admin/provider-oauth/providers/provider-123/device-poll",
|
||||||
"device_poll",
|
"device_poll",
|
||||||
"admin:provider_oauth",
|
"admin:provider_oauth",
|
||||||
"admin:provider_oauth:write",
|
"admin:provider_oauth:admin",
|
||||||
),
|
),
|
||||||
] {
|
] {
|
||||||
let uri: Uri = path.parse().expect("uri should parse");
|
let uri: Uri = path.parse().expect("uri should parse");
|
||||||
|
|||||||
@@ -947,6 +947,34 @@ fn classifies_auth_routes_as_public_support_route() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn does_not_classify_state_changing_auth_routes_for_get() {
|
||||||
|
for path in [
|
||||||
|
"/api/auth/login",
|
||||||
|
"/api/auth/refresh",
|
||||||
|
"/api/auth/register",
|
||||||
|
"/api/auth/logout",
|
||||||
|
"/api/auth/send-verification-code",
|
||||||
|
"/api/auth/verify-email",
|
||||||
|
"/api/auth/verification-status",
|
||||||
|
] {
|
||||||
|
let headers = headers(&[]);
|
||||||
|
let uri: Uri = path.parse().expect("uri should parse");
|
||||||
|
assert!(
|
||||||
|
classify_control_route(&http::Method::GET, &uri, &headers).is_none(),
|
||||||
|
"state-changing auth route {path} must not accept GET"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let headers = headers(&[]);
|
||||||
|
let uri: Uri = "/api/auth/me".parse().expect("uri should parse");
|
||||||
|
assert_eq!(
|
||||||
|
classify_control_route(&http::Method::GET, &uri, &headers)
|
||||||
|
.and_then(|decision| decision.route_kind),
|
||||||
|
Some("me".to_string())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn classifies_oauth_public_providers_route() {
|
fn classifies_oauth_public_providers_route() {
|
||||||
let headers = headers(&[]);
|
let headers = headers(&[]);
|
||||||
|
|||||||
@@ -113,10 +113,7 @@ impl GatewayDataConfig {
|
|||||||
return (self.clone(), None);
|
return (self.clone(), None);
|
||||||
};
|
};
|
||||||
let total_max = database.pool.max_connections;
|
let total_max = database.pool.max_connections;
|
||||||
if total_max < 2
|
if total_max < 2 || configured_background_max == Some(0) {
|
||||||
|| configured_background_max == Some(0)
|
|
||||||
|| is_private_sqlite_memory_database(database)
|
|
||||||
{
|
|
||||||
return (self.clone(), None);
|
return (self.clone(), None);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -152,14 +149,9 @@ impl GatewayDataConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn is_private_sqlite_memory_database(database: &aether_data::SqlDatabaseConfig) -> bool {
|
|
||||||
database.driver == aether_data::DatabaseDriver::Sqlite
|
|
||||||
&& matches!(database.url.trim(), "sqlite::memory:" | "sqlite://:memory:")
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::GatewayDataConfig;
|
use super::{GatewayDataConfig, PostgresPoolConfig};
|
||||||
use aether_data::{DatabaseDriver, SqlDatabaseConfig, SqlPoolConfig};
|
use aether_data::{DatabaseDriver, SqlDatabaseConfig, SqlPoolConfig};
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -226,7 +218,10 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn runtime_pool_split_can_be_disabled_or_degrade_for_single_connection() {
|
fn runtime_pool_split_can_be_disabled_or_degrade_for_single_connection() {
|
||||||
let mut database = SqlDatabaseConfig::sqlite_default();
|
let mut database = SqlDatabaseConfig::from_postgres_config(PostgresPoolConfig {
|
||||||
|
database_url: "postgres://localhost/aether".to_string(),
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
database.pool.max_connections = 1;
|
database.pool.max_connections = 1;
|
||||||
let config = GatewayDataConfig::from_database_config(database);
|
let config = GatewayDataConfig::from_database_config(database);
|
||||||
assert!(config
|
assert!(config
|
||||||
@@ -234,7 +229,10 @@ mod tests {
|
|||||||
.1
|
.1
|
||||||
.is_none());
|
.is_none());
|
||||||
|
|
||||||
let mut database = SqlDatabaseConfig::sqlite_default();
|
let mut database = SqlDatabaseConfig::from_postgres_config(PostgresPoolConfig {
|
||||||
|
database_url: "postgres://localhost/aether".to_string(),
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
database.pool.max_connections = 8;
|
database.pool.max_connections = 8;
|
||||||
let config = GatewayDataConfig::from_database_config(database);
|
let config = GatewayDataConfig::from_database_config(database);
|
||||||
assert!(config
|
assert!(config
|
||||||
@@ -242,34 +240,4 @@ mod tests {
|
|||||||
.1
|
.1
|
||||||
.is_none());
|
.is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn runtime_pool_split_keeps_private_sqlite_memory_database_in_one_pool() {
|
|
||||||
for url in ["sqlite::memory:", "sqlite://:memory:"] {
|
|
||||||
let config = GatewayDataConfig::from_database_config(
|
|
||||||
SqlDatabaseConfig::new(
|
|
||||||
DatabaseDriver::Sqlite,
|
|
||||||
url,
|
|
||||||
SqlPoolConfig {
|
|
||||||
min_connections: 1,
|
|
||||||
max_connections: 8,
|
|
||||||
..SqlPoolConfig::default()
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.expect("sqlite memory database config should be valid"),
|
|
||||||
);
|
|
||||||
|
|
||||||
let (foreground, background) = config.split_runtime_pools_with_background_max(Some(2));
|
|
||||||
|
|
||||||
assert!(background.is_none(), "private SQLite URL {url} was split");
|
|
||||||
assert_eq!(
|
|
||||||
foreground
|
|
||||||
.database()
|
|
||||||
.expect("foreground database")
|
|
||||||
.pool
|
|
||||||
.max_connections,
|
|
||||||
8
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -172,7 +172,7 @@ mod tests {
|
|||||||
candidates: vec![DecisionTraceCandidate {
|
candidates: vec![DecisionTraceCandidate {
|
||||||
candidate: sample_candidate("req-1"),
|
candidate: sample_candidate("req-1"),
|
||||||
provider_name: Some("OpenAI".to_string()),
|
provider_name: Some("OpenAI".to_string()),
|
||||||
provider_website: Some("https://openai.com".to_string()),
|
provider_website: Some("https://openai.com/".to_string()),
|
||||||
provider_type: Some("custom".to_string()),
|
provider_type: Some("custom".to_string()),
|
||||||
provider_priority: Some(0),
|
provider_priority: Some(0),
|
||||||
provider_keep_priority_on_conversion: Some(false),
|
provider_keep_priority_on_conversion: Some(false),
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -1,23 +1,42 @@
|
|||||||
use super::{
|
use super::{
|
||||||
ApiKeyLastUsedDelta, DataLayerError, GatewayDataState, GeminiFileMappingListQuery,
|
ApiKeyLastUsedDelta, DataLayerError, GatewayDataState, GeminiFileMappingListQuery,
|
||||||
GeminiFileMappingStats, ProviderCatalogKeyAdaptiveStateUpdate,
|
GeminiFileMappingStats, ProviderCatalogKeyAdaptiveStateUpdate,
|
||||||
ProviderCatalogKeyAdminCasUpdate, ProviderCatalogKeyHealthStateUpdate,
|
ProviderCatalogKeyAdminCasUpdate, ProviderCatalogKeyCredentialsCasUpdate,
|
||||||
ProviderCatalogKeyListQuery, ProviderCatalogKeyOAuthCredentialCasDelete,
|
ProviderCatalogKeyHealthStateUpdate, ProviderCatalogKeyListQuery,
|
||||||
ProviderCatalogKeyOAuthRuntimeStateCasUpdate, ProviderCatalogKeyRuntimeMetadataUpdate,
|
ProviderCatalogKeyOAuthCredentialCasDelete, ProviderCatalogKeyOAuthRuntimeStateCasUpdate,
|
||||||
ProviderCatalogKeyStatusSnapshotUpdate, PublicHealthStatusCount, PublicHealthTimelineBucket,
|
ProviderCatalogKeyRuntimeMetadataUpdate, ProviderCatalogKeyStatusSnapshotUpdate,
|
||||||
StoredGeminiFileMapping, StoredGeminiFileMappingListPage, StoredProviderCatalogEndpoint,
|
ProviderCatalogProviderConfigCasUpdate, ProviderCatalogProxyCasUpdate, PublicHealthStatusCount,
|
||||||
StoredProviderCatalogKey, StoredProviderCatalogKeyMaintenanceSummary,
|
PublicHealthTimelineBucket, StoredGeminiFileMapping, StoredGeminiFileMappingListPage,
|
||||||
StoredProviderCatalogKeyPage, StoredProviderCatalogKeyStats, StoredProviderCatalogProvider,
|
StoredProviderCatalogEndpoint, StoredProviderCatalogKey,
|
||||||
StoredRequestCandidate, UpsertGeminiFileMappingRecord, UpsertRequestCandidateRecord,
|
StoredProviderCatalogKeyMaintenanceSummary, StoredProviderCatalogKeyPage,
|
||||||
|
StoredProviderCatalogKeyStats, StoredProviderCatalogProvider, StoredRequestCandidate,
|
||||||
|
UpsertGeminiFileMappingRecord, UpsertRequestCandidateRecord,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
fn sanitize_request_candidate_rows(
|
||||||
|
mut candidates: Vec<StoredRequestCandidate>,
|
||||||
|
) -> Vec<StoredRequestCandidate> {
|
||||||
|
for candidate in &mut candidates {
|
||||||
|
candidate.sanitize_for_persistence();
|
||||||
|
}
|
||||||
|
candidates
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sanitize_request_candidate_row(mut candidate: StoredRequestCandidate) -> StoredRequestCandidate {
|
||||||
|
candidate.sanitize_for_persistence();
|
||||||
|
candidate
|
||||||
|
}
|
||||||
|
|
||||||
impl GatewayDataState {
|
impl GatewayDataState {
|
||||||
pub(crate) async fn list_request_candidates_by_request_id(
|
pub(crate) async fn list_request_candidates_by_request_id(
|
||||||
&self,
|
&self,
|
||||||
request_id: &str,
|
request_id: &str,
|
||||||
) -> Result<Vec<StoredRequestCandidate>, DataLayerError> {
|
) -> Result<Vec<StoredRequestCandidate>, DataLayerError> {
|
||||||
match &self.request_candidate_reader {
|
match &self.request_candidate_reader {
|
||||||
Some(repository) => repository.list_by_request_id(request_id).await,
|
Some(repository) => repository
|
||||||
|
.list_by_request_id(request_id)
|
||||||
|
.await
|
||||||
|
.map(sanitize_request_candidate_rows),
|
||||||
None => Ok(Vec::new()),
|
None => Ok(Vec::new()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -27,7 +46,10 @@ impl GatewayDataState {
|
|||||||
request_id: &str,
|
request_id: &str,
|
||||||
) -> Result<Vec<StoredRequestCandidate>, DataLayerError> {
|
) -> Result<Vec<StoredRequestCandidate>, DataLayerError> {
|
||||||
match &self.request_candidate_reader {
|
match &self.request_candidate_reader {
|
||||||
Some(repository) => repository.list_attempted_by_request_id(request_id).await,
|
Some(repository) => repository
|
||||||
|
.list_attempted_by_request_id(request_id)
|
||||||
|
.await
|
||||||
|
.map(sanitize_request_candidate_rows),
|
||||||
None => Ok(Vec::new()),
|
None => Ok(Vec::new()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -38,7 +60,10 @@ impl GatewayDataState {
|
|||||||
limit: usize,
|
limit: usize,
|
||||||
) -> Result<Vec<StoredRequestCandidate>, DataLayerError> {
|
) -> Result<Vec<StoredRequestCandidate>, DataLayerError> {
|
||||||
match &self.request_candidate_reader {
|
match &self.request_candidate_reader {
|
||||||
Some(repository) => repository.list_by_provider_id(provider_id, limit).await,
|
Some(repository) => repository
|
||||||
|
.list_by_provider_id(provider_id, limit)
|
||||||
|
.await
|
||||||
|
.map(sanitize_request_candidate_rows),
|
||||||
None => Ok(Vec::new()),
|
None => Ok(Vec::new()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -48,7 +73,10 @@ impl GatewayDataState {
|
|||||||
limit: usize,
|
limit: usize,
|
||||||
) -> Result<Vec<StoredRequestCandidate>, DataLayerError> {
|
) -> Result<Vec<StoredRequestCandidate>, DataLayerError> {
|
||||||
match &self.request_candidate_reader {
|
match &self.request_candidate_reader {
|
||||||
Some(repository) => repository.list_recent(limit).await,
|
Some(repository) => repository
|
||||||
|
.list_recent(limit)
|
||||||
|
.await
|
||||||
|
.map(sanitize_request_candidate_rows),
|
||||||
None => Ok(Vec::new()),
|
None => Ok(Vec::new()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -60,11 +88,10 @@ impl GatewayDataState {
|
|||||||
limit: usize,
|
limit: usize,
|
||||||
) -> Result<Vec<StoredRequestCandidate>, DataLayerError> {
|
) -> Result<Vec<StoredRequestCandidate>, DataLayerError> {
|
||||||
match &self.request_candidate_reader {
|
match &self.request_candidate_reader {
|
||||||
Some(repository) => {
|
Some(repository) => repository
|
||||||
repository
|
.list_finalized_by_endpoint_ids_since(endpoint_ids, since_unix_secs, limit)
|
||||||
.list_finalized_by_endpoint_ids_since(endpoint_ids, since_unix_secs, limit)
|
.await
|
||||||
.await
|
.map(sanitize_request_candidate_rows),
|
||||||
}
|
|
||||||
None => Ok(Vec::new()),
|
None => Ok(Vec::new()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -108,14 +135,19 @@ impl GatewayDataState {
|
|||||||
|
|
||||||
pub(crate) async fn upsert_request_candidate(
|
pub(crate) async fn upsert_request_candidate(
|
||||||
&self,
|
&self,
|
||||||
candidate: UpsertRequestCandidateRecord,
|
mut candidate: UpsertRequestCandidateRecord,
|
||||||
) -> Result<Option<StoredRequestCandidate>, DataLayerError> {
|
) -> Result<Option<StoredRequestCandidate>, DataLayerError> {
|
||||||
|
candidate.sanitize_for_persistence();
|
||||||
crate::request_diagnostics::observe_db_operation(
|
crate::request_diagnostics::observe_db_operation(
|
||||||
"request_candidate_upsert",
|
"request_candidate_upsert",
|
||||||
self.database_pool_summary(),
|
self.database_pool_summary(),
|
||||||
async {
|
async {
|
||||||
match &self.request_candidate_writer {
|
match &self.request_candidate_writer {
|
||||||
Some(repository) => repository.upsert(candidate).await.map(Some),
|
Some(repository) => repository
|
||||||
|
.upsert(candidate)
|
||||||
|
.await
|
||||||
|
.map(sanitize_request_candidate_row)
|
||||||
|
.map(Some),
|
||||||
None => Ok(None),
|
None => Ok(None),
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -170,6 +202,16 @@ impl GatewayDataState {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn upsert_gemini_file_mapping_if_owner_matches(
|
||||||
|
&self,
|
||||||
|
record: UpsertGeminiFileMappingRecord,
|
||||||
|
) -> Result<Option<StoredGeminiFileMapping>, DataLayerError> {
|
||||||
|
match &self.gemini_file_mapping_writer {
|
||||||
|
Some(repository) => repository.upsert_if_owner_matches(record).await,
|
||||||
|
None => Ok(None),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn list_gemini_file_mappings(
|
pub(crate) async fn list_gemini_file_mappings(
|
||||||
&self,
|
&self,
|
||||||
query: &GeminiFileMappingListQuery,
|
query: &GeminiFileMappingListQuery,
|
||||||
@@ -183,6 +225,49 @@ impl GatewayDataState {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn find_gemini_file_mapping_by_file_name(
|
||||||
|
&self,
|
||||||
|
file_name: &str,
|
||||||
|
) -> Result<Option<StoredGeminiFileMapping>, DataLayerError> {
|
||||||
|
match &self.gemini_file_mapping_reader {
|
||||||
|
Some(repository) => repository.find_by_file_name(file_name).await,
|
||||||
|
None => Ok(None),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn find_active_gemini_file_mapping_for_user(
|
||||||
|
&self,
|
||||||
|
file_name: &str,
|
||||||
|
user_id: &str,
|
||||||
|
now_unix_secs: u64,
|
||||||
|
) -> Result<Option<StoredGeminiFileMapping>, DataLayerError> {
|
||||||
|
match &self.gemini_file_mapping_reader {
|
||||||
|
Some(repository) => {
|
||||||
|
repository
|
||||||
|
.find_active_by_file_name_for_user(file_name, user_id, now_unix_secs)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
None => Ok(None),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn find_active_gemini_file_mapping_for_owner(
|
||||||
|
&self,
|
||||||
|
file_name: &str,
|
||||||
|
key_id: &str,
|
||||||
|
user_id: &str,
|
||||||
|
now_unix_secs: u64,
|
||||||
|
) -> Result<Option<StoredGeminiFileMapping>, DataLayerError> {
|
||||||
|
match &self.gemini_file_mapping_reader {
|
||||||
|
Some(repository) => {
|
||||||
|
repository
|
||||||
|
.find_active_by_file_name_for_owner(file_name, key_id, user_id, now_unix_secs)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
None => Ok(None),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn summarize_gemini_file_mappings(
|
pub(crate) async fn summarize_gemini_file_mappings(
|
||||||
&self,
|
&self,
|
||||||
now_unix_secs: u64,
|
now_unix_secs: u64,
|
||||||
@@ -208,6 +293,37 @@ impl GatewayDataState {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn delete_gemini_file_mapping_by_file_name_for_user(
|
||||||
|
&self,
|
||||||
|
file_name: &str,
|
||||||
|
user_id: &str,
|
||||||
|
) -> Result<bool, DataLayerError> {
|
||||||
|
match &self.gemini_file_mapping_writer {
|
||||||
|
Some(repository) => {
|
||||||
|
repository
|
||||||
|
.delete_by_file_name_for_user(file_name, user_id)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
None => Ok(false),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn delete_gemini_file_mapping_by_file_name_for_owner(
|
||||||
|
&self,
|
||||||
|
file_name: &str,
|
||||||
|
key_id: &str,
|
||||||
|
user_id: &str,
|
||||||
|
) -> Result<bool, DataLayerError> {
|
||||||
|
match &self.gemini_file_mapping_writer {
|
||||||
|
Some(repository) => {
|
||||||
|
repository
|
||||||
|
.delete_by_file_name_for_owner(file_name, key_id, user_id)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
None => Ok(false),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn delete_gemini_file_mapping_by_id(
|
pub(crate) async fn delete_gemini_file_mapping_by_id(
|
||||||
&self,
|
&self,
|
||||||
mapping_id: &str,
|
mapping_id: &str,
|
||||||
@@ -357,38 +473,11 @@ impl GatewayDataState {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn update_provider_catalog_key_oauth_credentials(
|
|
||||||
&self,
|
|
||||||
key_id: &str,
|
|
||||||
encrypted_api_key: &str,
|
|
||||||
encrypted_auth_config: Option<&str>,
|
|
||||||
expires_at_unix_secs: Option<u64>,
|
|
||||||
) -> Result<bool, DataLayerError> {
|
|
||||||
let updated = match &self.provider_catalog_writer {
|
|
||||||
Some(repository) => {
|
|
||||||
repository
|
|
||||||
.update_key_oauth_credentials(
|
|
||||||
key_id,
|
|
||||||
encrypted_api_key,
|
|
||||||
encrypted_auth_config,
|
|
||||||
expires_at_unix_secs,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
}
|
|
||||||
None => Ok(false),
|
|
||||||
}?;
|
|
||||||
if updated {
|
|
||||||
self.clear_provider_catalog_cache();
|
|
||||||
}
|
|
||||||
Ok(updated)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) async fn update_provider_catalog_key_oauth_runtime_state(
|
pub(crate) async fn update_provider_catalog_key_oauth_runtime_state(
|
||||||
&self,
|
&self,
|
||||||
key_id: &str,
|
key_id: &str,
|
||||||
oauth_invalid_at_unix_secs: Option<u64>,
|
oauth_invalid_at_unix_secs: Option<u64>,
|
||||||
oauth_invalid_reason: Option<&str>,
|
oauth_invalid_reason: Option<&str>,
|
||||||
encrypted_auth_config_update: Option<&str>,
|
|
||||||
updated_at_unix_secs: Option<u64>,
|
updated_at_unix_secs: Option<u64>,
|
||||||
) -> Result<bool, DataLayerError> {
|
) -> Result<bool, DataLayerError> {
|
||||||
let updated = match &self.provider_catalog_writer {
|
let updated = match &self.provider_catalog_writer {
|
||||||
@@ -398,7 +487,6 @@ impl GatewayDataState {
|
|||||||
key_id,
|
key_id,
|
||||||
oauth_invalid_at_unix_secs,
|
oauth_invalid_at_unix_secs,
|
||||||
oauth_invalid_reason,
|
oauth_invalid_reason,
|
||||||
encrypted_auth_config_update,
|
|
||||||
updated_at_unix_secs,
|
updated_at_unix_secs,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
@@ -475,6 +563,30 @@ impl GatewayDataState {
|
|||||||
Ok(updated)
|
Ok(updated)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn compare_and_swap_provider_catalog_provider_config(
|
||||||
|
&self,
|
||||||
|
update: &ProviderCatalogProviderConfigCasUpdate,
|
||||||
|
) -> Result<bool, DataLayerError> {
|
||||||
|
let updated = match &self.provider_catalog_writer {
|
||||||
|
Some(repository) => repository.compare_and_swap_provider_config(update).await,
|
||||||
|
None => Ok(false),
|
||||||
|
}?;
|
||||||
|
self.clear_provider_catalog_cache();
|
||||||
|
Ok(updated)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn compare_and_swap_provider_catalog_provider_proxy(
|
||||||
|
&self,
|
||||||
|
update: &ProviderCatalogProxyCasUpdate,
|
||||||
|
) -> Result<bool, DataLayerError> {
|
||||||
|
let updated = match &self.provider_catalog_writer {
|
||||||
|
Some(repository) => repository.compare_and_swap_provider_proxy(update).await,
|
||||||
|
None => Ok(false),
|
||||||
|
}?;
|
||||||
|
self.clear_provider_catalog_cache();
|
||||||
|
Ok(updated)
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn delete_provider_catalog_provider(
|
pub(crate) async fn delete_provider_catalog_provider(
|
||||||
&self,
|
&self,
|
||||||
provider_id: &str,
|
provider_id: &str,
|
||||||
@@ -543,6 +655,18 @@ impl GatewayDataState {
|
|||||||
Ok(updated)
|
Ok(updated)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn compare_and_swap_provider_catalog_endpoint_proxy(
|
||||||
|
&self,
|
||||||
|
update: &ProviderCatalogProxyCasUpdate,
|
||||||
|
) -> Result<bool, DataLayerError> {
|
||||||
|
let updated = match &self.provider_catalog_writer {
|
||||||
|
Some(repository) => repository.compare_and_swap_endpoint_proxy(update).await,
|
||||||
|
None => Ok(false),
|
||||||
|
}?;
|
||||||
|
self.clear_provider_catalog_cache();
|
||||||
|
Ok(updated)
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn delete_provider_catalog_endpoint(
|
pub(crate) async fn delete_provider_catalog_endpoint(
|
||||||
&self,
|
&self,
|
||||||
endpoint_id: &str,
|
endpoint_id: &str,
|
||||||
@@ -571,6 +695,32 @@ impl GatewayDataState {
|
|||||||
Ok(updated)
|
Ok(updated)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn compare_and_swap_provider_catalog_key_proxy(
|
||||||
|
&self,
|
||||||
|
update: &ProviderCatalogProxyCasUpdate,
|
||||||
|
) -> Result<bool, DataLayerError> {
|
||||||
|
let updated = match &self.provider_catalog_writer {
|
||||||
|
Some(repository) => repository.compare_and_swap_key_proxy(update).await,
|
||||||
|
None => Ok(false),
|
||||||
|
}?;
|
||||||
|
self.clear_provider_catalog_cache();
|
||||||
|
Ok(updated)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn compare_and_swap_provider_catalog_key_credentials(
|
||||||
|
&self,
|
||||||
|
update: &ProviderCatalogKeyCredentialsCasUpdate,
|
||||||
|
) -> Result<bool, DataLayerError> {
|
||||||
|
let updated = match &self.provider_catalog_writer {
|
||||||
|
Some(repository) => repository.compare_and_swap_key_credentials(update).await,
|
||||||
|
None => Ok(false),
|
||||||
|
}?;
|
||||||
|
// Clear on both outcomes: a CAS miss proves the cached credential
|
||||||
|
// generation was stale and the retry must observe the winning record.
|
||||||
|
self.clear_provider_catalog_cache();
|
||||||
|
Ok(updated)
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn compare_and_update_provider_catalog_key_admin_state(
|
pub(crate) async fn compare_and_update_provider_catalog_key_admin_state(
|
||||||
&self,
|
&self,
|
||||||
update: &ProviderCatalogKeyAdminCasUpdate,
|
update: &ProviderCatalogKeyAdminCasUpdate,
|
||||||
@@ -843,3 +993,86 @@ impl GatewayDataState {
|
|||||||
Ok(updated)
|
Ok(updated)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod request_candidate_security_tests {
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
use super::{
|
||||||
|
sanitize_request_candidate_row, sanitize_request_candidate_rows, StoredRequestCandidate,
|
||||||
|
};
|
||||||
|
use aether_data_contracts::repository::candidates::RequestCandidateStatus;
|
||||||
|
|
||||||
|
fn untrusted_candidate() -> StoredRequestCandidate {
|
||||||
|
let mut candidate = StoredRequestCandidate::new(
|
||||||
|
"candidate-untrusted".to_string(),
|
||||||
|
"request-1".to_string(),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
Some("provider-1".to_string()),
|
||||||
|
Some("endpoint-1".to_string()),
|
||||||
|
Some("key-1".to_string()),
|
||||||
|
RequestCandidateStatus::Failed,
|
||||||
|
None,
|
||||||
|
false,
|
||||||
|
Some(500),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
1,
|
||||||
|
None,
|
||||||
|
Some(2),
|
||||||
|
)
|
||||||
|
.expect("candidate should build");
|
||||||
|
candidate.skip_reason = Some("Bearer candidate-secret".to_string());
|
||||||
|
candidate.error_type = Some("candidate-secret".to_string());
|
||||||
|
candidate.error_message = Some("Bearer candidate-secret".to_string());
|
||||||
|
candidate.extra_data = Some(json!({
|
||||||
|
"gateway_execution_runtime": true,
|
||||||
|
"request_body": {"token": "candidate-secret"}
|
||||||
|
}));
|
||||||
|
candidate.required_capabilities = Some(json!({
|
||||||
|
"vision": 1,
|
||||||
|
"tenant_secret": "candidate-secret"
|
||||||
|
}));
|
||||||
|
candidate
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_candidate_is_sanitized(candidate: &StoredRequestCandidate) {
|
||||||
|
assert_eq!(candidate.skip_reason.as_deref(), Some("unclassified_skip"));
|
||||||
|
assert_eq!(candidate.error_type.as_deref(), Some("unclassified_error"));
|
||||||
|
assert_eq!(
|
||||||
|
candidate.error_message.as_deref(),
|
||||||
|
Some("Bearer candidate-secret")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
candidate.extra_data,
|
||||||
|
Some(json!({"gateway_execution_runtime": true}))
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
candidate.required_capabilities,
|
||||||
|
Some(json!({"vision": true}))
|
||||||
|
);
|
||||||
|
let mut public_candidate = candidate.clone();
|
||||||
|
public_candidate.sanitize_sensitive_diagnostics();
|
||||||
|
assert!(!serde_json::to_string(&public_candidate)
|
||||||
|
.expect("candidate should serialize")
|
||||||
|
.contains("candidate-secret"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn gateway_candidate_boundary_preserves_admin_errors_and_removes_request_payloads() {
|
||||||
|
let candidate = sanitize_request_candidate_row(untrusted_candidate());
|
||||||
|
assert_candidate_is_sanitized(&candidate);
|
||||||
|
|
||||||
|
let candidates = sanitize_request_candidate_rows(vec![untrusted_candidate()]);
|
||||||
|
assert_candidate_is_sanitized(&candidates[0]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user