mirror of
https://github.com/fawney19/Aether.git
synced 2026-10-08 02:17:46 +08:00
Compare commits
162
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
911c7f8875 | ||
|
|
310098a853 | ||
|
|
466c7918a1 | ||
|
|
e7de935e61 | ||
|
|
b1943710b3 | ||
|
|
082348a5d1 | ||
|
|
ac7e3abab3 | ||
|
|
7acbaab82b | ||
|
|
716c35bf56 | ||
|
|
2539439656 | ||
|
|
e379bd791c | ||
|
|
84f0d5ce70 | ||
|
|
d4ed774423 | ||
|
|
0948f29da5 | ||
|
|
8a767c4309 | ||
|
|
bd35e885a7 | ||
|
|
8ffd8188e4 | ||
|
|
976c3feca6 | ||
|
|
5d5281b022 | ||
|
|
bda0d4e44d | ||
|
|
cb7b9c9ecd | ||
|
|
12d58327a1 | ||
|
|
2075cd95de | ||
|
|
625456ff24 | ||
|
|
e1dadf5b06 | ||
|
|
8f1e2decf2 | ||
|
|
9f829450d8 | ||
|
|
65f43a7aef | ||
|
|
2a63bafd20 | ||
|
|
ea6b739fd6 | ||
|
|
14befeda2c | ||
|
|
d13f52bdf8 | ||
|
|
2257c3959f | ||
|
|
066ea87d72 | ||
|
|
fae51a0ec8 | ||
|
|
db5d2dfbb5 | ||
|
|
54fbcc25a1 | ||
|
|
d068b62966 | ||
|
|
017d08d45e | ||
|
|
7f45617f79 | ||
|
|
9cc4018a37 | ||
|
|
b49f5c0fd7 | ||
|
|
125cd40aa5 | ||
|
|
491364867e | ||
|
|
8093899b5a | ||
|
|
fa06e9695d | ||
|
|
5d5880d75e | ||
|
|
9abbf268a1 | ||
|
|
e8ee7b4ecf | ||
|
|
c1aa5d618d | ||
|
|
309f507ef4 | ||
|
|
00a315e3bf | ||
|
|
f811e29f13 | ||
|
|
fb7e3fc224 | ||
|
|
cabca22d65 | ||
|
|
11989e278e | ||
|
|
532af797bf | ||
|
|
cafa05c4cb | ||
|
|
49ec53cbd9 | ||
|
|
73f1d79637 | ||
|
|
1b8f78a992 | ||
|
|
14ef19d9e1 | ||
|
|
bc0e9f94e2 | ||
|
|
74072e5007 | ||
|
|
c96f8272f4 | ||
|
|
4c07d9fcfb | ||
|
|
45678d9419 | ||
|
|
75bc32cfe9 | ||
|
|
d4bc058c2f | ||
|
|
2516e51b4e | ||
|
|
3541ccfe29 | ||
|
|
bd83cff58f | ||
|
|
85c04335c6 | ||
|
|
c08497c963 | ||
|
|
d30268f80f | ||
|
|
5679375f71 | ||
|
|
3465d23db3 | ||
|
|
e3333f1aef | ||
|
|
31d5e2d172 | ||
|
|
bb9f2eed8e | ||
|
|
926f5cc928 | ||
|
|
27ae884759 | ||
|
|
834eb9c308 | ||
|
|
390b73d4b6 | ||
|
|
bcd121d447 | ||
|
|
3394a51278 | ||
|
|
75471ae4a4 | ||
|
|
57f53903f5 | ||
|
|
81788d3a64 | ||
|
|
30bb0c3130 | ||
|
|
595b8e4e05 | ||
|
|
7e033d0571 | ||
|
|
5745442ed7 | ||
|
|
1a4eba1005 | ||
|
|
2a9d8d3b25 | ||
|
|
e25e240d16 | ||
|
|
cd765f2c2f | ||
|
|
ec95989e02 | ||
|
|
7f5e1a64fe | ||
|
|
f86dd10467 | ||
|
|
69930a6059 | ||
|
|
07cb401fd4 | ||
|
|
593327c803 | ||
|
|
a9a7c64e5d | ||
|
|
70d1a4ab74 | ||
|
|
0b7c7f94ac | ||
|
|
67d0414483 | ||
|
|
e3c01fb554 | ||
|
|
f960bbd2c8 | ||
|
|
906baae88e | ||
|
|
ba7c9f8b27 | ||
|
|
37e3a36680 | ||
|
|
166de33355 | ||
|
|
a95f0d2488 | ||
|
|
b296d46e97 | ||
|
|
0486435f16 | ||
|
|
fb25dde4c9 | ||
|
|
4124749a7d | ||
|
|
5a55116b62 | ||
|
|
681ce56c4f | ||
|
|
bcb2308000 | ||
|
|
6c92db2ba5 | ||
|
|
fdf55525f5 | ||
|
|
4ff4129034 | ||
|
|
364692da55 | ||
|
|
5842c7232e | ||
|
|
72a4bf3408 | ||
|
|
fe1723d87c | ||
|
|
03b198d5ab | ||
|
|
53562fd9de | ||
|
|
e66dd00b84 | ||
|
|
03496c46c5 | ||
|
|
c0ded116a9 | ||
|
|
e5ab73bf35 | ||
|
|
a5456cdc38 | ||
|
|
e9899200f8 | ||
|
|
5a6692ade0 | ||
|
|
6e431e2ff6 | ||
|
|
88df2a2ed5 | ||
|
|
e7864e5611 | ||
|
|
6e6407160a | ||
|
|
01acff0774 | ||
|
|
cc5050155f | ||
|
|
04c4a97766 | ||
|
|
e83399db2f | ||
|
|
f753f14fd2 | ||
|
|
7daf355e65 | ||
|
|
e4f89de90e | ||
|
|
dfe88e34e7 | ||
|
|
c5adcf0312 | ||
|
|
ea24d61910 | ||
|
|
23e0af7b12 | ||
|
|
60b89cc840 | ||
|
|
b748b5bfd2 | ||
|
|
30e36cd09a | ||
|
|
28cd77eb5e | ||
|
|
531f53b443 | ||
|
|
95e4d0149c | ||
|
|
72aea7898b | ||
|
|
8aedf87aad | ||
|
|
e9b64c3e9d | ||
|
|
33ea4ebf15 |
@@ -0,0 +1,6 @@
|
||||
# 不固定 test-threads:nextest 默认按 num-cpus 并发,固定值会在更大规格的
|
||||
# runner 或本地开发机上主动压低并发、反而变慢,且无法表达 min(4, num-cpus)。
|
||||
# 这里只保留卡死保护,避免单个挂起用例拖满整个 job。
|
||||
[profile.default]
|
||||
# 60 秒后标记慢测试,连续两轮仍未结束则终止;超时结果保持失败,不隐藏回归。
|
||||
slow-timeout = { period = "60s", terminate-after = 2, grace-period = "10s" }
|
||||
@@ -137,6 +137,10 @@ ADMIN_USERNAME=admin123456
|
||||
# 单请求解压后 Payload 上限(MiB),默认 256;显式 0 仍受 256 MiB 硬上限保护。
|
||||
# AETHER_MAX_REQUEST_BODY_MB=256
|
||||
# AETHER_GATEWAY_SECURITY_CACHE_TTL_MS=1000
|
||||
# 内部错误详情日志默认关闭;仅接受精确的小写 true(开启)和 false(关闭)。
|
||||
# 开启后 ERROR 日志输出不截断的错误详情,但仍移除 URL 凭据并遮盖常见密码、令牌等字段。
|
||||
# 脱敏无法覆盖任意敏感内容,排查完毕请关闭;未设置或无法识别的值均按关闭处理。
|
||||
# AETHER_GATEWAY_ERROR_DETAIL_LOGGING=false
|
||||
# AETHER_MAX_REDACTED_SYNC_RESPONSE_BODY_MB=64
|
||||
# AETHER_MAX_INTERNAL_BUFFERED_BODY_MB=64
|
||||
# AETHER_TUNNEL_NODE_STATUS_QUEUE_CAPACITY=1024
|
||||
|
||||
@@ -63,6 +63,8 @@ jobs:
|
||||
name: Rust CI
|
||||
needs: source
|
||||
uses: ./.github/workflows/rust-ci.yml
|
||||
with:
|
||||
full_scope: true
|
||||
|
||||
rust_extended:
|
||||
name: Rust extended checks
|
||||
@@ -332,7 +334,7 @@ jobs:
|
||||
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6 # v3
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
|
||||
+199
-30
@@ -2,6 +2,12 @@ name: Rust CI
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
full_scope:
|
||||
description: "Run all Rust and shell scopes, used by Nightly"
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
@@ -9,8 +15,11 @@ on:
|
||||
paths:
|
||||
- "Cargo.toml"
|
||||
- "Cargo.lock"
|
||||
- "rust-toolchain.toml"
|
||||
- ".cargo/**"
|
||||
- "crates/**"
|
||||
- "apps/**"
|
||||
- "*.sql"
|
||||
- "install.sh"
|
||||
- "deploy.sh"
|
||||
- "update.sh"
|
||||
@@ -28,17 +37,17 @@ on:
|
||||
- "tests/update_*_test.sh"
|
||||
- "tests/release_supply_chain_test.sh"
|
||||
- "tests/tunnel_installer_config_security_test.sh"
|
||||
- ".github/workflows/build-tunnel.yml"
|
||||
- ".github/workflows/deploy-pages.yml"
|
||||
- ".github/workflows/release.yml"
|
||||
- ".github/workflows/rust-ci.yml"
|
||||
- ".github/workflows/nightly.yml"
|
||||
- ".github/workflows/*.yml"
|
||||
- ".github/workflows/*.yaml"
|
||||
pull_request:
|
||||
paths:
|
||||
- "Cargo.toml"
|
||||
- "Cargo.lock"
|
||||
- "rust-toolchain.toml"
|
||||
- ".cargo/**"
|
||||
- "crates/**"
|
||||
- "apps/**"
|
||||
- "*.sql"
|
||||
- "install.sh"
|
||||
- "deploy.sh"
|
||||
- "update.sh"
|
||||
@@ -56,11 +65,8 @@ on:
|
||||
- "tests/update_*_test.sh"
|
||||
- "tests/release_supply_chain_test.sh"
|
||||
- "tests/tunnel_installer_config_security_test.sh"
|
||||
- ".github/workflows/build-tunnel.yml"
|
||||
- ".github/workflows/deploy-pages.yml"
|
||||
- ".github/workflows/release.yml"
|
||||
- ".github/workflows/rust-ci.yml"
|
||||
- ".github/workflows/nightly.yml"
|
||||
- ".github/workflows/*.yml"
|
||||
- ".github/workflows/*.yaml"
|
||||
|
||||
concurrency:
|
||||
group: rust-ci-${{ github.event_name }}-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
@@ -76,8 +82,77 @@ env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
jobs:
|
||||
changes:
|
||||
name: Detect Rust CI scope
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
rust: ${{ steps.scope.outputs.rust }}
|
||||
shell: ${{ steps.scope.outputs.shell }}
|
||||
steps:
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Classify changed paths
|
||||
id: scope
|
||||
shell: bash
|
||||
env:
|
||||
RUST_CI_FULL_SCOPE: ${{ inputs.full_scope || false }}
|
||||
run: |
|
||||
# 任何命令失败都必须让本 job 失败,否则 git fetch/diff 出错后仍会写出
|
||||
# rust=false/shell=false,下游会误判为“无需测试”而假绿放行。
|
||||
set -euo pipefail
|
||||
|
||||
# Nightly 通过 workflow_call 显式传入 full_scope;普通 push/PR 只按源码和构建
|
||||
# 指纹触发 Rust jobs,安装脚本、Compose、README 等由 shell scope 覆盖。
|
||||
if [ "$RUST_CI_FULL_SCOPE" = "true" ]; then
|
||||
echo "rust=true" >> "$GITHUB_OUTPUT"
|
||||
echo "shell=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$GITHUB_EVENT_NAME" = "pull_request" ] \
|
||||
&& [ -n "${GITHUB_BASE_REF:-}" ] \
|
||||
&& [ -n "${GITHUB_SHA:-}" ]; then
|
||||
git fetch --no-tags origin "$GITHUB_BASE_REF" --depth=1
|
||||
changed_paths=$(git diff --name-only "origin/$GITHUB_BASE_REF...$GITHUB_SHA")
|
||||
elif [ "$GITHUB_EVENT_NAME" = "push" ] \
|
||||
&& [ -n "${GITHUB_EVENT_BEFORE:-}" ] \
|
||||
&& [ "$GITHUB_EVENT_BEFORE" != "0000000000000000000000000000000000000000" ] \
|
||||
&& [ -n "${GITHUB_SHA:-}" ]; then
|
||||
changed_paths=$(git diff --name-only "$GITHUB_EVENT_BEFORE" "$GITHUB_SHA")
|
||||
else
|
||||
changed_paths=$(git ls-files)
|
||||
fi
|
||||
|
||||
# 防御性兜底:diff 结果为空(异常事件或比较失败)时按全量运行,
|
||||
# 宁可多跑也不能漏测。
|
||||
if [ -z "$changed_paths" ]; then
|
||||
echo "rust=true" >> "$GITHUB_OUTPUT"
|
||||
echo "shell=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
rust=false
|
||||
shell=false
|
||||
while IFS= read -r path; do
|
||||
case "$path" in
|
||||
Cargo.toml|Cargo.lock|rust-toolchain.toml|.cargo/*|*.rs|*/Cargo.toml|*/build.rs|*.sql|.github/workflows/*.yml|.github/workflows/*.yaml)
|
||||
rust=true
|
||||
;;
|
||||
*.sh|*.py|README.md|*/README.md|.env.example|Dockerfile*|docker-compose*.yml|docker-compose*.yaml)
|
||||
shell=true
|
||||
;;
|
||||
esac
|
||||
done <<< "$changed_paths"
|
||||
|
||||
echo "rust=$rust" >> "$GITHUB_OUTPUT"
|
||||
echo "shell=$shell" >> "$GITHUB_OUTPUT"
|
||||
|
||||
shell_security:
|
||||
name: Shell security fixtures
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.shell == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
@@ -99,6 +174,8 @@ jobs:
|
||||
|
||||
fmt:
|
||||
name: Format
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
@@ -114,6 +191,8 @@ jobs:
|
||||
|
||||
clippy_gateway:
|
||||
name: Clippy (Gateway)
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
@@ -127,7 +206,9 @@ jobs:
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
# Gateway lint 与 Gateway 测试都可能触发 mold/大型链接依赖,单独隔离缓存
|
||||
# 指纹,避免不同 job 的构建产物互相驱逐或复用错误的链接参数。
|
||||
shared-key: rust-ci-gateway-clippy-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
@@ -148,6 +229,8 @@ jobs:
|
||||
|
||||
clippy_data:
|
||||
name: Clippy (Data)
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
@@ -182,6 +265,8 @@ jobs:
|
||||
|
||||
clippy_rest:
|
||||
name: Clippy (Workspace Rest)
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
@@ -218,6 +303,7 @@ jobs:
|
||||
name: Clippy
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- changes
|
||||
- clippy_gateway
|
||||
- clippy_data
|
||||
- clippy_rest
|
||||
@@ -225,6 +311,10 @@ jobs:
|
||||
steps:
|
||||
- name: Verify clippy jobs
|
||||
run: |
|
||||
if [ "${{ needs.changes.outputs.rust }}" != "true" ]; then
|
||||
echo "Rust scope unchanged; clippy jobs skipped"
|
||||
exit 0
|
||||
fi
|
||||
if [ "${{ needs.clippy_gateway.result }}" != "success" ] || \
|
||||
[ "${{ needs.clippy_data.result }}" != "success" ] || \
|
||||
[ "${{ needs.clippy_rest.result }}" != "success" ]; then
|
||||
@@ -234,12 +324,24 @@ jobs:
|
||||
|
||||
test_gateway:
|
||||
name: Test (Gateway)
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
# 构建指纹提到 job 级:mold RUSTFLAGS / 栈 / sccache 对 lib、bins、integration 三步保持一致,
|
||||
# 避免 step 级 env 漂移导致同 job 内 rustc 指纹不一致。
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
RUST_MIN_STACK: "16777216"
|
||||
RUSTFLAGS: "-C link-arg=-fuse-ld=mold"
|
||||
steps:
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
# 与 rust-toolchain.toml、fmt/clippy 钉在同一版本,避免浮动 stable 换指纹导致全量重编
|
||||
toolchain: 1.95.0
|
||||
|
||||
- name: Show Rust toolchain
|
||||
run: rustup show active-toolchain
|
||||
@@ -247,7 +349,8 @@ jobs:
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
with:
|
||||
shared-key: rust-ci-${{ runner.os }}
|
||||
# mold RUSTFLAGS 只在本 job 生效:独立 cache key,避免与无 mold 的 job 互相污染指纹
|
||||
shared-key: rust-ci-gateway-test-${{ runner.os }}
|
||||
workspaces: . -> target
|
||||
|
||||
- name: Setup sccache
|
||||
@@ -263,36 +366,34 @@ jobs:
|
||||
run: pg_config --bindir >> "$GITHUB_PATH"
|
||||
|
||||
- name: Test lib
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
RUST_MIN_STACK: "16777216"
|
||||
RUSTFLAGS: "-C link-arg=-fuse-ld=mold"
|
||||
run: cargo nextest run -p aether-gateway --lib
|
||||
|
||||
- name: Test bins
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
RUST_MIN_STACK: "16777216"
|
||||
RUSTFLAGS: "-C link-arg=-fuse-ld=mold"
|
||||
run: cargo nextest run -p aether-gateway --bins
|
||||
|
||||
# 只运行独立 integration targets;显式列出目标,避免 --tests 再次执行 lib/bin 测试。
|
||||
- name: Test integration targets
|
||||
run: >-
|
||||
cargo nextest run -p aether-gateway
|
||||
--test admin_unsigned_identity_headers
|
||||
--test architecture_guard
|
||||
|
||||
- name: Show sccache stats
|
||||
if: always()
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
run: sccache --show-stats
|
||||
|
||||
test_data:
|
||||
name: Test (Data)
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
|
||||
- name: Show Rust toolchain
|
||||
run: rustup show active-toolchain
|
||||
@@ -328,6 +429,8 @@ jobs:
|
||||
|
||||
check_data_features:
|
||||
name: Check (Data Feature - ${{ matrix.feature }})
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -340,6 +443,8 @@ jobs:
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
@@ -365,12 +470,16 @@ jobs:
|
||||
|
||||
test_rest:
|
||||
name: Test (Workspace Rest)
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
|
||||
- name: Show Rust toolchain
|
||||
run: rustup show active-toolchain
|
||||
@@ -402,6 +511,8 @@ jobs:
|
||||
|
||||
test_data_adapters:
|
||||
name: Test (Data Adapter - ${{ matrix.package }})
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -413,6 +524,8 @@ jobs:
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
@@ -441,12 +554,16 @@ jobs:
|
||||
|
||||
check_integration_scenarios:
|
||||
name: Test (Integration Scenarios)
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@49a0bdc70d2e1b713ca9e2869b211fcce03d3c1c # v2
|
||||
@@ -477,6 +594,7 @@ jobs:
|
||||
name: Test
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- changes
|
||||
- test_gateway
|
||||
- test_data
|
||||
- check_data_features
|
||||
@@ -487,6 +605,10 @@ jobs:
|
||||
steps:
|
||||
- name: Verify test jobs
|
||||
run: |
|
||||
if [ "${{ needs.changes.outputs.rust }}" != "true" ]; then
|
||||
echo "Rust scope unchanged; test jobs skipped"
|
||||
exit 0
|
||||
fi
|
||||
if [ "${{ needs.test_gateway.result }}" != "success" ] || \
|
||||
[ "${{ needs.test_data.result }}" != "success" ] || \
|
||||
[ "${{ needs.check_data_features.result }}" != "success" ] || \
|
||||
@@ -499,6 +621,8 @@ jobs:
|
||||
|
||||
data_db_smoke_postgres:
|
||||
name: Data DB Smoke (Postgres)
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.rust == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
postgres:
|
||||
@@ -519,6 +643,8 @@ jobs:
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
|
||||
with:
|
||||
toolchain: 1.95.0
|
||||
|
||||
- name: Show Rust toolchain
|
||||
run: rustup show active-toolchain
|
||||
@@ -549,6 +675,20 @@ jobs:
|
||||
AETHER_TEST_POSTGRES_URL: postgres://aether:[email protected]:5432/aether_test
|
||||
run: cargo test -p aether-data --all-features postgres_provider_upstream_metadata_migration_preserves_json_when_url_is_set --lib -- --nocapture
|
||||
|
||||
- name: Run Postgres payment callback regression tests
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_DATABASE_URL: postgres://aether:[email protected]:5432/aether_test
|
||||
run: cargo test -p aether-data-postgres live_payment_callback --lib -- --ignored --nocapture
|
||||
|
||||
- name: Run Postgres batch wallet deduction regression
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: "true"
|
||||
AETHER_TEST_DATABASE_URL: postgres://aether:[email protected]:5432/aether_test
|
||||
run: cargo test -p aether-data-postgres live_bulk_wallet_adjustment_persists_actual_delta_and_skips_zero_ledger --lib -- --ignored --nocapture
|
||||
|
||||
- name: Run Postgres API key lifecycle tests
|
||||
env:
|
||||
RUSTC_WRAPPER: sccache
|
||||
@@ -579,11 +719,20 @@ jobs:
|
||||
name: Data DB Smoke
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- changes
|
||||
- data_db_smoke_postgres
|
||||
if: ${{ always() }}
|
||||
steps:
|
||||
- name: Verify database smoke jobs
|
||||
run: |
|
||||
if [ "${{ needs.changes.result }}" != "success" ]; then
|
||||
echo "Scope detection failed"
|
||||
exit 1
|
||||
fi
|
||||
if [ "${{ needs.changes.outputs.rust }}" != "true" ]; then
|
||||
echo "Rust scope unchanged; database smoke jobs skipped"
|
||||
exit 0
|
||||
fi
|
||||
if [ "${{ needs.data_db_smoke_postgres.result }}" != "success" ]; then
|
||||
echo "Data DB smoke failed"
|
||||
exit 1
|
||||
@@ -593,6 +742,7 @@ jobs:
|
||||
name: check
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- changes
|
||||
- fmt
|
||||
- clippy
|
||||
- test
|
||||
@@ -602,11 +752,30 @@ jobs:
|
||||
steps:
|
||||
- name: Verify required jobs
|
||||
run: |
|
||||
if [ "${{ needs.fmt.result }}" != "success" ] || \
|
||||
[ "${{ needs.clippy.result }}" != "success" ] || \
|
||||
[ "${{ needs.test.result }}" != "success" ] || \
|
||||
[ "${{ needs.data_db_smoke.result }}" != "success" ] || \
|
||||
[ "${{ needs.shell_security.result }}" != "success" ]; then
|
||||
# changes 失败或未产出 scope 时不允许直接放行,避免假绿。
|
||||
if [ "${{ needs.changes.result }}" != "success" ]; then
|
||||
echo "Scope detection failed"
|
||||
exit 1
|
||||
fi
|
||||
rust="${{ needs.changes.outputs.rust }}"
|
||||
shell="${{ needs.changes.outputs.shell }}"
|
||||
|
||||
if [ "$rust" != "true" ] && [ "$shell" != "true" ]; then
|
||||
echo "No Rust or shell scope changed"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$rust" = "true" ] && {
|
||||
[ "${{ needs.fmt.result }}" != "success" ] ||
|
||||
[ "${{ needs.clippy.result }}" != "success" ] ||
|
||||
[ "${{ needs.test.result }}" != "success" ] ||
|
||||
[ "${{ needs.data_db_smoke.result }}" != "success" ];
|
||||
}; then
|
||||
echo "Rust CI failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$shell" = "true" ] && [ "${{ needs.shell_security.result }}" != "success" ]; then
|
||||
echo "Rust CI failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
Generated
+86
-40
@@ -55,8 +55,9 @@ dependencies = [
|
||||
"aether-provider-pool",
|
||||
"aether-provider-transport",
|
||||
"axum",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"chrono",
|
||||
"chrono-tz",
|
||||
"http",
|
||||
"regex",
|
||||
"reqwest 0.12.28",
|
||||
@@ -82,8 +83,9 @@ name = "aether-ai-formats"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"aether-contracts",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"http",
|
||||
"os_info",
|
||||
"regex",
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -103,7 +105,7 @@ dependencies = [
|
||||
"aether-pool-core",
|
||||
"aether-scheduler-core",
|
||||
"async-trait",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"http",
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -134,7 +136,7 @@ name = "aether-contracts"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"flate2",
|
||||
"hmac",
|
||||
@@ -151,7 +153,7 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"aes",
|
||||
"aws-lc-rs",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"cbc",
|
||||
"hmac",
|
||||
"pbkdf2",
|
||||
@@ -193,7 +195,7 @@ dependencies = [
|
||||
"aether-contracts",
|
||||
"aether-routing-core",
|
||||
"async-trait",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bcrypt",
|
||||
"chrono",
|
||||
"chrono-tz",
|
||||
@@ -218,6 +220,7 @@ dependencies = [
|
||||
"chrono-tz",
|
||||
"flate2",
|
||||
"futures-util",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"sqlx",
|
||||
@@ -294,7 +297,7 @@ dependencies = [
|
||||
"async-trait",
|
||||
"aws-lc-rs",
|
||||
"axum",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bcrypt",
|
||||
"brotli",
|
||||
"bytes",
|
||||
@@ -389,7 +392,7 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"aether-admission-core",
|
||||
"aether-contracts",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"http",
|
||||
"serde",
|
||||
@@ -429,11 +432,14 @@ dependencies = [
|
||||
"aether-runtime",
|
||||
"aether-runtime-state",
|
||||
"aether-testkit",
|
||||
"aether-tunnel",
|
||||
"arc-swap",
|
||||
"async-stream",
|
||||
"axum",
|
||||
"futures-util",
|
||||
"http",
|
||||
"reqwest 0.12.28",
|
||||
"rustls",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
@@ -474,7 +480,7 @@ dependencies = [
|
||||
"aether-scheduler-core",
|
||||
"async-trait",
|
||||
"aws-lc-rs",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"regex",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
@@ -488,7 +494,7 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"aether-contracts",
|
||||
"async-trait",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"http",
|
||||
"reqwest 0.12.28",
|
||||
"serde",
|
||||
@@ -524,6 +530,7 @@ dependencies = [
|
||||
"aether-data-contracts",
|
||||
"aether-pool-core",
|
||||
"aether-provider-transport",
|
||||
"chrono",
|
||||
"serde_json",
|
||||
"url",
|
||||
"uuid",
|
||||
@@ -544,7 +551,7 @@ dependencies = [
|
||||
"async-trait",
|
||||
"aws-lc-rs",
|
||||
"axum",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"chrono",
|
||||
"crypto_box",
|
||||
"ed25519-dalek",
|
||||
@@ -671,7 +678,6 @@ name = "aether-tunnel"
|
||||
version = "0.3.17"
|
||||
dependencies = [
|
||||
"aether-contracts",
|
||||
"aether-gateway",
|
||||
"aether-gateway-tunnel",
|
||||
"aether-http",
|
||||
"aether-runtime",
|
||||
@@ -679,7 +685,7 @@ dependencies = [
|
||||
"anyhow",
|
||||
"arc-swap",
|
||||
"axum",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"clap",
|
||||
"crossterm 0.28.1",
|
||||
@@ -730,7 +736,7 @@ dependencies = [
|
||||
"aether-data-contracts",
|
||||
"aether-runtime-state",
|
||||
"async-trait",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"futures-util",
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -748,6 +754,7 @@ dependencies = [
|
||||
"async-trait",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"url",
|
||||
"uuid",
|
||||
]
|
||||
@@ -846,7 +853,7 @@ version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
||||
dependencies = [
|
||||
"windows-sys 0.60.2",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -857,7 +864,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"once_cell_polyfill",
|
||||
"windows-sys 0.60.2",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1027,7 +1034,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8"
|
||||
dependencies = [
|
||||
"axum-core",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"form_urlencoded",
|
||||
"futures-util",
|
||||
@@ -1090,6 +1097,12 @@ version = "0.22.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.23.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
|
||||
|
||||
[[package]]
|
||||
name = "base64ct"
|
||||
version = "1.8.3"
|
||||
@@ -1102,7 +1115,7 @@ version = "0.16.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2b1866ecef4f2d06a0bb77880015fdf2b89e25a1c2e5addacb87e459c86dc67e"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"blowfish",
|
||||
"getrandom 0.2.17",
|
||||
"subtle",
|
||||
@@ -1978,7 +1991,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2575,7 +2588,7 @@ version = "0.1.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"futures-channel",
|
||||
"futures-util",
|
||||
@@ -2586,7 +2599,7 @@ dependencies = [
|
||||
"libc",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"socket2 0.5.10",
|
||||
"socket2 0.6.3",
|
||||
"tokio",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
@@ -2733,12 +2746,12 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "2.13.0"
|
||||
version = "2.14.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017"
|
||||
checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855"
|
||||
dependencies = [
|
||||
"equivalent",
|
||||
"hashbrown 0.16.1",
|
||||
"hashbrown 0.17.1",
|
||||
"serde",
|
||||
"serde_core",
|
||||
]
|
||||
@@ -3221,7 +3234,7 @@ version = "0.50.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
|
||||
dependencies = [
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3314,7 +3327,7 @@ checksum = "d354792e39fa5f0009e47623cf8b15b099bf9a652fa55c6f817fe28ac84fea50"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"aws-lc-rs",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"crc-fast",
|
||||
@@ -3330,7 +3343,7 @@ dependencies = [
|
||||
"md-5 0.11.0",
|
||||
"parking_lot",
|
||||
"percent-encoding",
|
||||
"quick-xml",
|
||||
"quick-xml 0.41.0",
|
||||
"rand 0.10.2",
|
||||
"reqwest 0.13.4",
|
||||
"rustls-pki-types",
|
||||
@@ -3398,6 +3411,17 @@ dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "os_info"
|
||||
version = "3.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d0e1ac5fde8d43c34139135df8ea9ee9465394b2d8d20f032d38998f64afffc3"
|
||||
dependencies = [
|
||||
"log",
|
||||
"plist",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "palette"
|
||||
version = "0.7.7"
|
||||
@@ -3643,6 +3667,19 @@ version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6"
|
||||
|
||||
[[package]]
|
||||
name = "plist"
|
||||
version = "1.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2896bade328c13f7042a297ea5ac5b0951f6cf989dea5f32c2fd98da398195cb"
|
||||
dependencies = [
|
||||
"base64 0.23.1",
|
||||
"indexmap",
|
||||
"quick-xml 0.42.0",
|
||||
"serde",
|
||||
"time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "poly1305"
|
||||
version = "0.8.0"
|
||||
@@ -3725,6 +3762,15 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quick-xml"
|
||||
version = "0.42.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quinn"
|
||||
version = "0.11.9"
|
||||
@@ -3738,7 +3784,7 @@ dependencies = [
|
||||
"quinn-udp",
|
||||
"rustc-hash",
|
||||
"rustls",
|
||||
"socket2 0.5.10",
|
||||
"socket2 0.6.3",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
"tracing",
|
||||
@@ -3776,7 +3822,7 @@ dependencies = [
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
"once_cell",
|
||||
"socket2 0.5.10",
|
||||
"socket2 0.6.3",
|
||||
"tracing",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
@@ -4075,7 +4121,7 @@ version = "0.12.28"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
@@ -4117,7 +4163,7 @@ version = "0.13.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
@@ -4231,7 +4277,7 @@ dependencies = [
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys 0.12.1",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4290,7 +4336,7 @@ dependencies = [
|
||||
"security-framework",
|
||||
"security-framework-sys",
|
||||
"webpki-root-certs",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4616,7 +4662,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.60.2",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4663,7 +4709,7 @@ version = "0.8.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bigdecimal",
|
||||
"bytes",
|
||||
"chrono",
|
||||
@@ -4740,7 +4786,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526"
|
||||
dependencies = [
|
||||
"atoi",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bigdecimal",
|
||||
"bitflags 2.13.1",
|
||||
"byteorder",
|
||||
@@ -4784,7 +4830,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46"
|
||||
dependencies = [
|
||||
"atoi",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bigdecimal",
|
||||
"bitflags 2.13.1",
|
||||
"byteorder",
|
||||
@@ -4975,7 +5021,7 @@ dependencies = [
|
||||
"parking_lot",
|
||||
"rustix 1.1.4",
|
||||
"signal-hook",
|
||||
"windows-sys 0.60.2",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5006,7 +5052,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4676b37242ccbd1aabf56edb093a4827dc49086c0ffd764a5705899e0f35f8f7"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64",
|
||||
"base64 0.22.1",
|
||||
"bitflags 2.13.1",
|
||||
"fancy-regex",
|
||||
"filedescriptor",
|
||||
@@ -6003,7 +6049,7 @@ version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.48.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -92,6 +92,7 @@ aether-usage-core = { path = "crates/aether-usage/core" }
|
||||
aether-usage-runtime = { path = "crates/aether-usage/runtime" }
|
||||
aether-video-tasks-core = { path = "crates/aether-video-tasks-core" }
|
||||
aether-gateway = { path = "apps/aether-gateway" }
|
||||
aether-tunnel = { path = "apps/aether-tunnel" }
|
||||
aether-http = { path = "crates/aether-http" }
|
||||
aether-runtime = { path = "crates/aether-runtime/base" }
|
||||
aether-testkit = { path = "crates/aether-testing/testkit" }
|
||||
|
||||
@@ -123,7 +123,8 @@ Aether Tunnel 是配套的正向代理节点,部署在海外 VPS 上,为墙
|
||||
- `APP_PORT`:`aether-gateway` 唯一监听端口,固定绑定 `0.0.0.0:${APP_PORT}`
|
||||
- `DATABASE_URL`:PostgreSQL 连接串,例如 `postgresql://USER:PASSWORD@HOST:5432/aether`
|
||||
- `AETHER_GATEWAY_DATA_POSTGRES_MIN_CONNECTIONS` / `AETHER_GATEWAY_DATA_POSTGRES_MAX_CONNECTIONS`:数据库连接池手动覆盖值;未配置时 PostgreSQL 按每核 `4` 条自动推导,总池范围为 `32-100`。该预算按进程计算,多实例部署应按数据库连接上限显式分配
|
||||
- `AETHER_GATEWAY_DATA_POSTGRES_STATEMENT_TIMEOUT_MS` / `AETHER_GATEWAY_DATA_POSTGRES_LOCK_TIMEOUT_MS`:普通数据库连接的单条 SQL / 锁等待期限,默认 `30000` / `3000` 毫秒,显式 `0` 关闭;不是整个事务总期限。迁移与历史 backfill 使用独立连接放宽,事务可通过局部设置覆盖
|
||||
- `AETHER_GATEWAY_DATA_POSTGRES_STATEMENT_TIMEOUT_MS` / `AETHER_GATEWAY_DATA_POSTGRES_LOCK_TIMEOUT_MS`:普通数据库连接的单条 SQL / 锁等待期限,默认 `30000` / `3000` 毫秒,显式 `0` 关闭;不是整个事务总期限。schema 迁移使用独立超时配置,历史 backfill 使用独立连接放宽期限
|
||||
- `AETHER_POSTGRES_MIGRATION_LOCK_TIMEOUT_MS` / `AETHER_POSTGRES_MIGRATION_TIMEOUT_MS` / `AETHER_POSTGRES_MIGRATION_CONCURRENT_TIMEOUT_MS`:schema 迁移的锁等待、每个事务及并发索引迁移期限,默认 `1000` / `10000` / `900000` 毫秒,不接受 `0`。超时会中止当前迁移,已提交的迁移保留;空库 schema 初始化也受事务期限约束
|
||||
- `AETHER_USAGE_EVENT_CAPTURE_MEMORY_BUDGET_BYTES`:usage 诊断正文共享预算,默认 `134217728`(128 MiB),按 JSON 堆内存估算,覆盖进入终态队列的 seed、Redis 解码后的事件、数据库写入 DTO 及其正文副本。额度不足或显式 `0` 时先保留计费事实,再舍弃诊断正文;已有清空或禁用状态保持不变,其余标记截断。预算随正文保留到释放,后台构建或压缩不会因调用方取消而提前归还额度。该额度不覆盖原始 Redis 批次、解码临时分配、序列化及压缩结果、协议观察缓冲或进程总内存;可通过 `usage_runtime_event_capture_memory_*` 指标观察
|
||||
- `AETHER_GATEWAY_USAGE_QUEUE_PAYLOAD_MAX_BYTES`:新增 usage 队列消息的完整 JSON payload 上限,默认 `1048576`(1 MiB),按序列化后的 UTF-8 字节计算,显式 `0` 非法。超限先保留计费事实并舍弃诊断字段;仍超限或无法保留计费语义时拒绝入队,终态消息尝试受限数据库落库,失败则明确失败,不继续 Redis 重试。该限制不覆盖存量 Redis 消息、整个读取批次、DLQ 或进程总内存。`usage_runtime_queue_payload_*` 导出上限及进程级降级、拒绝编码尝试次数,包含入队和重试预校验,不代表唯一事件数;`usage_runtime_enqueue_retry_permanent_failure_total` 记录永久输入错误导致的重试拒绝或终止
|
||||
- `AETHER_USAGE_QUEUE_READ_PAYLOAD_BUDGET_BYTES` / `AETHER_USAGE_QUEUE_READ_BATCH_PAYLOAD_BYTES`:usage worker 读取和重领共用的进程级逻辑 payload 预留,默认总额 `134217728`(128 MiB)、单批目标 `8388608`(8 MiB)。按当前 `QUEUE_PAYLOAD_MAX_BYTES` 推导实际 COUNT,默认最多读取 8 条,自动扩容使用实际 COUNT 判断批次是否读满。预留覆盖读取、整批处理和确认,额度不足等待;取消/失败释放。单批目标至少允许一条,当前 payload 上限大于总额时读取报配置错误。`0` 或非法值回退默认,过大值收敛到约 4 GiB 的有效总额。收到消息后按全部字段值长度缩减多余预留;历史消息、其他生产者使用更高上限或额外字段可能超出估算,仍继续原计费流程并记录 `usage_runtime_queue_read_oversized_*`。`usage_runtime_queue_read_*` 同时导出预留、等待与累计字段字节;该预留不是 RESP 解码、连接缓冲容量、字段结构、诊断 JSON、DLQ 或进程 RSS 的硬上限,旧公开 Vec 读取接口不携带处理阶段预留
|
||||
|
||||
@@ -69,9 +69,14 @@ pub(crate) use aether_ai_formats::api::{
|
||||
OPENAI_VIDEO_REMIX_SYNC_PLAN_KIND,
|
||||
};
|
||||
pub(crate) use aether_ai_formats::protocol::stream::CanonicalUsage as StreamingCanonicalUsage;
|
||||
/// Codex client identity headers re-exported for out-of-crate probe binaries,
|
||||
/// which must reach `aether_ai_formats` through this seam.
|
||||
pub use aether_ai_formats::{CODEX_CLIENT_ORIGINATOR, CODEX_CLIENT_USER_AGENT};
|
||||
/// Codex client identity accessors re-exported for out-of-crate probe binaries,
|
||||
/// which must reach the runtime profile through this seam.
|
||||
pub use aether_ai_formats::{codex_client_originator, codex_client_user_agent};
|
||||
/// Codex 动态客户端画像 API 只允许经此根缝进入 gateway,避免其它模块直接依赖 formats crate。
|
||||
pub(crate) use aether_ai_formats::{
|
||||
codex_client_profile, codex_client_version, set_codex_cli_version, set_codex_client_profile,
|
||||
CodexClientProfile,
|
||||
};
|
||||
pub(crate) use aether_ai_formats::{CODEX_RESPONSES_LITE_HEADER, UPSTREAM_IS_STREAM_KEY};
|
||||
|
||||
pub(crate) fn parse_direct_request_body(
|
||||
|
||||
@@ -18,6 +18,12 @@ pub(crate) fn maybe_build_local_stream_rewriter<'a>(
|
||||
}
|
||||
|
||||
impl LocalStreamRewriter<'_> {
|
||||
pub(crate) fn into_owned(self) -> LocalStreamRewriter<'static> {
|
||||
LocalStreamRewriter {
|
||||
inner: self.inner.into_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn push_chunk(&mut self, chunk: &[u8]) -> Result<Vec<u8>, GatewayError> {
|
||||
self.inner.push_chunk(chunk).map_err(map_surface_error)
|
||||
}
|
||||
|
||||
@@ -2579,6 +2579,8 @@ mod tests {
|
||||
|
||||
let fixed_order_app = AppState::new().expect("state should build");
|
||||
let fixed_order_policy = ResolvedRoutingPolicy {
|
||||
billing_multiplier: 1.0,
|
||||
group_name: None,
|
||||
group_id: Some("routing-group-fixed-order".to_string()),
|
||||
group_version: Some(1),
|
||||
selection_source: "test".to_string(),
|
||||
|
||||
@@ -362,6 +362,8 @@ mod tests {
|
||||
candidate.key_internal_priority = 3;
|
||||
candidate.key_global_priority_for_format = Some(2);
|
||||
let policy = aether_routing_core::ResolvedRoutingPolicy {
|
||||
billing_multiplier: 1.0,
|
||||
group_name: None,
|
||||
group_id: Some("group-1".to_string()),
|
||||
group_version: Some(1),
|
||||
selection_source: "system_default".to_string(),
|
||||
@@ -399,6 +401,8 @@ mod tests {
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(data_state);
|
||||
let policy = aether_routing_core::ResolvedRoutingPolicy {
|
||||
billing_multiplier: 1.0,
|
||||
group_name: None,
|
||||
group_id: Some("group-1".to_string()),
|
||||
group_version: Some(1),
|
||||
selection_source: "system_default".to_string(),
|
||||
@@ -434,6 +438,8 @@ mod tests {
|
||||
candidate.key_internal_priority = 3;
|
||||
candidate.key_global_priority_for_format = Some(2);
|
||||
let policy = aether_routing_core::ResolvedRoutingPolicy {
|
||||
billing_multiplier: 1.0,
|
||||
group_name: None,
|
||||
group_id: Some("group-1".to_string()),
|
||||
group_version: Some(1),
|
||||
selection_source: "system_default".to_string(),
|
||||
|
||||
@@ -93,6 +93,16 @@ impl AiCandidateResolutionPort for GatewayLocalCandidateResolutionPort<'_> {
|
||||
&self,
|
||||
candidate: Self::Candidate,
|
||||
) -> Self::Skipped {
|
||||
warn!(
|
||||
event_name = "local_candidate_skipped",
|
||||
log_type = "event",
|
||||
provider_id = %candidate.provider_id,
|
||||
endpoint_id = %candidate.endpoint_id,
|
||||
key_id = %candidate.key_id,
|
||||
api_format = %candidate.endpoint_api_format,
|
||||
skip_reason = "transport_snapshot_missing",
|
||||
"local execution candidate skipped during planning"
|
||||
);
|
||||
SkippedLocalExecutionCandidate {
|
||||
candidate,
|
||||
skip_reason: "transport_snapshot_missing",
|
||||
@@ -145,6 +155,16 @@ impl AiCandidateResolutionPort for GatewayLocalCandidateResolutionPort<'_> {
|
||||
transport: Self::Transport,
|
||||
skip_reason: &'static str,
|
||||
) -> Self::Skipped {
|
||||
warn!(
|
||||
event_name = "local_candidate_skipped",
|
||||
log_type = "event",
|
||||
provider_id = %candidate.provider_id,
|
||||
endpoint_id = %candidate.endpoint_id,
|
||||
key_id = %candidate.key_id,
|
||||
api_format = %candidate.endpoint_api_format,
|
||||
skip_reason,
|
||||
"local execution candidate skipped during planning"
|
||||
);
|
||||
SkippedLocalExecutionCandidate {
|
||||
candidate,
|
||||
skip_reason,
|
||||
|
||||
@@ -6,7 +6,7 @@ use aether_routing_core::ResolvedRoutingPolicy;
|
||||
use aether_runtime::ConcurrencyPermit;
|
||||
use aether_scheduler_core::{
|
||||
enumerate_minimal_candidate_selection_with_model_directives, normalize_api_format,
|
||||
resolve_requested_global_model_name_with_model_directives_and_request_operation,
|
||||
resolve_requested_global_model_name_with_reserved_global_model,
|
||||
row_supports_requested_model_with_model_directives_and_request_operation,
|
||||
ClientSessionAffinity, EnumerateMinimalCandidateSelectionInput,
|
||||
SchedulerMinimalCandidateSelectionCandidate,
|
||||
@@ -378,6 +378,7 @@ pub(crate) struct LocalCandidatePreselectionPageCursor<'a> {
|
||||
requested_name_offsets: BTreeMap<String, u32>,
|
||||
scanned_rows_by_format: BTreeMap<String, u32>,
|
||||
resolved_global_model_names: BTreeMap<String, String>,
|
||||
reserved_global_model_names: BTreeMap<String, Option<String>>,
|
||||
fallback_offsets: BTreeMap<String, u32>,
|
||||
fallback_scan_epoch: u32,
|
||||
exhausted_api_formats: BTreeSet<String>,
|
||||
@@ -457,6 +458,7 @@ impl<'a> LocalCandidatePreselectionPageCursor<'a> {
|
||||
requested_name_offsets: BTreeMap::new(),
|
||||
scanned_rows_by_format: BTreeMap::new(),
|
||||
resolved_global_model_names: BTreeMap::new(),
|
||||
reserved_global_model_names: BTreeMap::new(),
|
||||
fallback_offsets: BTreeMap::new(),
|
||||
fallback_scan_epoch: 0,
|
||||
exhausted_api_formats: BTreeSet::new(),
|
||||
@@ -555,6 +557,7 @@ impl<'a> LocalCandidatePreselectionPageCursor<'a> {
|
||||
self.requested_name_offsets.clear();
|
||||
self.scanned_rows_by_format.clear();
|
||||
self.resolved_global_model_names.clear();
|
||||
self.reserved_global_model_names.clear();
|
||||
self.fallback_offsets.clear();
|
||||
self.fallback_scan_epoch = self.fallback_scan_epoch.wrapping_add(1);
|
||||
self.exhausted_api_formats.clear();
|
||||
@@ -1185,6 +1188,34 @@ impl<'a> LocalCandidatePreselectionPageCursor<'a> {
|
||||
|| self.exhausted_api_formats.contains(&normalized_api_format)
|
||||
}
|
||||
|
||||
/// Global model names are a reserved routing namespace, so a request that
|
||||
/// names one must not be answered by a provider whose own model merely
|
||||
/// carries that name as an upstream alias. Cached per routing model: the
|
||||
/// answer does not change between pages or API formats.
|
||||
async fn reserved_global_model_name(
|
||||
&mut self,
|
||||
rows: &[StoredMinimalCandidateSelectionRow],
|
||||
routing_model: &str,
|
||||
) -> Result<Option<String>, GatewayError> {
|
||||
if let Some(cached) = self.reserved_global_model_names.get(routing_model) {
|
||||
return Ok(cached.clone());
|
||||
}
|
||||
let state = self.state;
|
||||
let reserved_global_model_name =
|
||||
crate::data::candidate_selection::resolve_reserved_global_model_name(
|
||||
state.app().data.as_ref(),
|
||||
rows,
|
||||
routing_model,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
self.reserved_global_model_names.insert(
|
||||
routing_model.to_string(),
|
||||
reserved_global_model_name.clone(),
|
||||
);
|
||||
Ok(reserved_global_model_name)
|
||||
}
|
||||
|
||||
async fn build_page_outcome_from_rows(
|
||||
&mut self,
|
||||
candidate_api_format: &str,
|
||||
@@ -1216,15 +1247,17 @@ impl<'a> LocalCandidatePreselectionPageCursor<'a> {
|
||||
if let Some(value) = self.resolved_global_model_names.get(normalized_api_format) {
|
||||
value.clone()
|
||||
} else {
|
||||
let Some(value) =
|
||||
resolve_requested_global_model_name_with_model_directives_and_request_operation(
|
||||
&rows,
|
||||
&routing_model,
|
||||
normalized_api_format,
|
||||
false,
|
||||
self.request_operation.as_deref(),
|
||||
)
|
||||
else {
|
||||
let reserved_global_model_name = self
|
||||
.reserved_global_model_name(&rows, &routing_model)
|
||||
.await?;
|
||||
let Some(value) = resolve_requested_global_model_name_with_reserved_global_model(
|
||||
&rows,
|
||||
&routing_model,
|
||||
normalized_api_format,
|
||||
false,
|
||||
self.request_operation.as_deref(),
|
||||
reserved_global_model_name.as_deref(),
|
||||
) else {
|
||||
return Ok(None);
|
||||
};
|
||||
self.resolved_global_model_names
|
||||
@@ -1475,6 +1508,7 @@ mod tests {
|
||||
use crate::AppState;
|
||||
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||
use aether_data::repository::candidate_selection::InMemoryMinimalCandidateSelectionReadRepository;
|
||||
use aether_data::repository::global_models::InMemoryGlobalModelReadRepository;
|
||||
use aether_data::repository::provider_catalog::InMemoryProviderCatalogReadRepository;
|
||||
use aether_data::DataLayerError;
|
||||
use aether_data_contracts::repository::candidate_selection::{
|
||||
@@ -1482,6 +1516,9 @@ mod tests {
|
||||
StoredPoolKeyCandidateRowsByKeyIdsQuery, StoredPoolKeyCandidateRowsQuery,
|
||||
StoredProviderModelMapping, StoredRequestedModelCandidateRowsQuery,
|
||||
};
|
||||
use aether_data_contracts::repository::global_models::{
|
||||
GlobalModelReadRepository, StoredPublicGlobalModel,
|
||||
};
|
||||
use aether_data_contracts::repository::provider_catalog::{
|
||||
StoredProviderCatalogEndpoint, StoredProviderCatalogKey, StoredProviderCatalogProvider,
|
||||
};
|
||||
@@ -1855,6 +1892,191 @@ mod tests {
|
||||
.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn routing_policy_excludes_group_disabled_providers_from_candidate_pages() {
|
||||
let repository: Arc<dyn MinimalCandidateSelectionReadRepository> =
|
||||
Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed([
|
||||
standard_candidate_row("provider-disabled", "openai:chat", 0),
|
||||
standard_candidate_row("provider-enabled", "openai:chat", 1),
|
||||
]));
|
||||
let app = AppState::new()
|
||||
.expect("gateway state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::with_minimal_candidate_selection_reader_for_tests(repository),
|
||||
);
|
||||
let auth_snapshot = unrestricted_auth_snapshot();
|
||||
let model_directive_policy =
|
||||
crate::system_features::ModelDirectivePolicySnapshot::load(&app).await;
|
||||
let config = serde_json::from_value(serde_json::json!({
|
||||
"disabled_providers": ["provider-disabled"],
|
||||
"model_policies": [{
|
||||
"model": "*",
|
||||
"allowed_providers": ["provider-disabled", "provider-enabled"]
|
||||
}]
|
||||
}))
|
||||
.expect("routing config should parse");
|
||||
let routing_policy = aether_routing_core::resolve_routing_policy(
|
||||
&config,
|
||||
aether_routing_core::RoutingPolicyInput {
|
||||
group_id: Some("routing-group-1"),
|
||||
group_version: Some(1),
|
||||
selection_source: "test",
|
||||
requested_model: "gpt-5",
|
||||
resolved_model: "gpt-5",
|
||||
api_format: "openai:chat",
|
||||
user_id: None,
|
||||
api_key_id: None,
|
||||
headers: &serde_json::json!({}),
|
||||
body: &serde_json::json!({}),
|
||||
phase: aether_routing_core::RoutingRulePhase::ClientRequest,
|
||||
},
|
||||
)
|
||||
.expect("routing policy should resolve");
|
||||
let mut cursor = LocalCandidatePreselectionPageCursor::new(
|
||||
PlannerAppState::new(&app),
|
||||
&model_directive_policy,
|
||||
"openai:chat",
|
||||
"gpt-5",
|
||||
None,
|
||||
false,
|
||||
None,
|
||||
&auth_snapshot,
|
||||
Some(&routing_policy),
|
||||
None,
|
||||
None,
|
||||
true,
|
||||
LocalCandidatePreselectionKeyMode::ProviderEndpointKeyModelAndApiFormat,
|
||||
false,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
let page = cursor
|
||||
.next_page()
|
||||
.await
|
||||
.expect("routing candidate scan should succeed")
|
||||
.expect("the enabled provider should remain");
|
||||
assert_eq!(
|
||||
page.candidates
|
||||
.iter()
|
||||
.map(|candidate| candidate.provider_id.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["provider-enabled"]
|
||||
);
|
||||
assert!(cursor
|
||||
.next_page()
|
||||
.await
|
||||
.expect("routing scan should finish")
|
||||
.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn model_provider_enablement_filters_candidate_pages_without_affecting_other_models() {
|
||||
let mut rows = Vec::new();
|
||||
for model in ["model-a", "model-b", "model-c"] {
|
||||
for (provider, priority) in [
|
||||
("provider-legacy-disabled", 0),
|
||||
("provider-model-disabled", 1),
|
||||
("provider-other", 2),
|
||||
("provider-inactive", 3),
|
||||
] {
|
||||
let mut row = standard_candidate_row(provider, "openai:chat", priority);
|
||||
row.global_model_id = format!("global-{model}");
|
||||
row.global_model_name = model.into();
|
||||
row.model_provider_model_name = model.into();
|
||||
row.model_id = format!("{provider}-{model}");
|
||||
row.provider_is_active = provider != "provider-inactive";
|
||||
rows.push(row);
|
||||
}
|
||||
}
|
||||
let repository: Arc<dyn MinimalCandidateSelectionReadRepository> =
|
||||
Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(rows));
|
||||
let app = AppState::new().unwrap().with_data_state_for_tests(
|
||||
GatewayDataState::with_minimal_candidate_selection_reader_for_tests(repository),
|
||||
);
|
||||
let auth = unrestricted_auth_snapshot();
|
||||
let directives = crate::system_features::ModelDirectivePolicySnapshot::load(&app).await;
|
||||
let config = serde_json::from_value(serde_json::json!({
|
||||
"disabled_providers": ["provider-legacy-disabled"],
|
||||
"model_policies": [
|
||||
{ "model": "model-a", "provider_enabled_overrides": {
|
||||
"provider-model-disabled": false, "provider-inactive": true
|
||||
} },
|
||||
{ "model": "model-b", "provider_enabled_overrides": {
|
||||
"provider-legacy-disabled": true, "provider-inactive": true
|
||||
} }
|
||||
],
|
||||
"rules": [{ "id": "legacy-allowlist", "actions": [{
|
||||
"type": "restrict_providers", "provider_ids": [
|
||||
"provider-legacy-disabled", "provider-model-disabled", "provider-other", "provider-inactive"
|
||||
]
|
||||
}] }]
|
||||
})).unwrap();
|
||||
// Revisit A after B to exercise candidate caches shared by the app.
|
||||
for (model, expected) in [
|
||||
("model-a", vec!["provider-other"]),
|
||||
(
|
||||
"model-b",
|
||||
vec![
|
||||
"provider-legacy-disabled",
|
||||
"provider-model-disabled",
|
||||
"provider-other",
|
||||
],
|
||||
),
|
||||
("model-c", vec!["provider-model-disabled", "provider-other"]),
|
||||
("model-a", vec!["provider-other"]),
|
||||
] {
|
||||
let policy = aether_routing_core::resolve_routing_policy(
|
||||
&config,
|
||||
aether_routing_core::RoutingPolicyInput {
|
||||
group_id: Some("group-1"),
|
||||
group_version: Some(1),
|
||||
selection_source: "test",
|
||||
requested_model: model,
|
||||
resolved_model: model,
|
||||
api_format: "openai:chat",
|
||||
user_id: None,
|
||||
api_key_id: None,
|
||||
headers: &serde_json::json!({}),
|
||||
body: &serde_json::json!({}),
|
||||
phase: aether_routing_core::RoutingRulePhase::ClientRequest,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let mut cursor = LocalCandidatePreselectionPageCursor::new(
|
||||
PlannerAppState::new(&app),
|
||||
&directives,
|
||||
"openai:chat",
|
||||
model,
|
||||
None,
|
||||
false,
|
||||
None,
|
||||
&auth,
|
||||
Some(&policy),
|
||||
None,
|
||||
None,
|
||||
true,
|
||||
LocalCandidatePreselectionKeyMode::ProviderEndpointKeyModelAndApiFormat,
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
let mut providers = Vec::new();
|
||||
while let Some(page) = cursor.next_page().await.unwrap() {
|
||||
providers.extend(
|
||||
page.candidates
|
||||
.into_iter()
|
||||
.map(|candidate| candidate.provider_id),
|
||||
);
|
||||
}
|
||||
providers.sort();
|
||||
assert_eq!(
|
||||
providers, expected,
|
||||
"provider enablement must remain isolated for {model}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn routing_policy_collects_candidate_pages_before_final_ranking() {
|
||||
let rows = (0..300)
|
||||
@@ -1877,6 +2099,8 @@ mod tests {
|
||||
let model_directive_policy =
|
||||
crate::system_features::ModelDirectivePolicySnapshot::load(&app).await;
|
||||
let routing_policy = ResolvedRoutingPolicy {
|
||||
billing_multiplier: 1.0,
|
||||
group_name: None,
|
||||
group_id: Some("routing-group-1".to_string()),
|
||||
group_version: Some(1),
|
||||
selection_source: "test".to_string(),
|
||||
@@ -1942,6 +2166,8 @@ mod tests {
|
||||
let model_directive_policy =
|
||||
crate::system_features::ModelDirectivePolicySnapshot::load(&app).await;
|
||||
let routing_policy = ResolvedRoutingPolicy {
|
||||
billing_multiplier: 1.0,
|
||||
group_name: None,
|
||||
group_id: Some("routing-group-fallback".to_string()),
|
||||
group_version: Some(1),
|
||||
selection_source: "test".to_string(),
|
||||
@@ -2090,6 +2316,96 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn public_global_model(name: &str) -> StoredPublicGlobalModel {
|
||||
StoredPublicGlobalModel {
|
||||
id: format!("global-model-{name}"),
|
||||
name: name.to_string(),
|
||||
display_name: None,
|
||||
is_active: true,
|
||||
default_price_per_request: None,
|
||||
default_tiered_pricing: None,
|
||||
supported_capabilities: None,
|
||||
config: None,
|
||||
usage_count: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// The cursor provider reaches its upstream under a name that belongs to another
|
||||
/// global model. A `claude:messages` client asking for `gemini-3.8-flash` has to
|
||||
/// land on the provider bound to that global model — format conversion and all —
|
||||
/// rather than on the one that only borrows the name on the way out, which is the
|
||||
/// one an API-format-ordered scan reaches first.
|
||||
#[tokio::test]
|
||||
async fn paged_preselection_keeps_a_global_model_name_from_a_provider_alias() {
|
||||
let mut aliasing = standard_candidate_row("ursor", "claude:messages", 1);
|
||||
aliasing.global_model_id = "global-model-gemini-3.8-flash-cursor".to_string();
|
||||
aliasing.global_model_name = "gemini-3.8-flash-cursor".to_string();
|
||||
aliasing.model_provider_model_name = "gemini-3.8-flash-cursor".to_string();
|
||||
aliasing.model_provider_model_mappings = Some(vec![StoredProviderModelMapping {
|
||||
name: "gemini-3.8-flash".to_string(),
|
||||
priority: 1,
|
||||
api_formats: None,
|
||||
endpoint_ids: None,
|
||||
operations: None,
|
||||
}]);
|
||||
|
||||
let mut bound = standard_candidate_row("anti", "gemini:generate_content", 2);
|
||||
bound.global_model_id = "global-model-gemini-3.8-flash".to_string();
|
||||
bound.global_model_name = "gemini-3.8-flash".to_string();
|
||||
bound.model_provider_model_name = "gemini-3.8-flash".to_string();
|
||||
|
||||
let repository: Arc<dyn MinimalCandidateSelectionReadRepository> =
|
||||
Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed([
|
||||
aliasing, bound,
|
||||
]));
|
||||
let global_models: Arc<dyn GlobalModelReadRepository> =
|
||||
Arc::new(InMemoryGlobalModelReadRepository::seed([
|
||||
public_global_model("gemini-3.8-flash"),
|
||||
public_global_model("gemini-3.8-flash-cursor"),
|
||||
]));
|
||||
let data_state =
|
||||
GatewayDataState::with_minimal_candidate_selection_reader_for_tests(repository)
|
||||
.with_global_model_reader(global_models);
|
||||
let app = AppState::new()
|
||||
.expect("gateway state should build")
|
||||
.with_data_state_for_tests(data_state);
|
||||
let auth_snapshot = unrestricted_auth_snapshot();
|
||||
let model_directive_policy =
|
||||
crate::system_features::ModelDirectivePolicySnapshot::load(&app).await;
|
||||
let mut cursor = LocalCandidatePreselectionPageCursor::new(
|
||||
PlannerAppState::new(&app),
|
||||
&model_directive_policy,
|
||||
"claude:messages",
|
||||
"gemini-3.8-flash",
|
||||
None,
|
||||
false,
|
||||
None,
|
||||
&auth_snapshot,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
true,
|
||||
LocalCandidatePreselectionKeyMode::ProviderEndpointKeyModelAndApiFormat,
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
let page = cursor
|
||||
.next_page()
|
||||
.await
|
||||
.expect("preselection should succeed")
|
||||
.expect("the bound provider should still be reachable");
|
||||
|
||||
assert_eq!(page.candidates.len(), 1);
|
||||
assert_eq!(page.candidates[0].provider_name, "anti");
|
||||
assert_eq!(page.candidates[0].global_model_name, "gemini-3.8-flash");
|
||||
assert_eq!(
|
||||
page.candidates[0].endpoint_api_format,
|
||||
"gemini:generate_content"
|
||||
);
|
||||
}
|
||||
|
||||
fn standard_candidate_row(
|
||||
provider_id: &str,
|
||||
api_format: &str,
|
||||
@@ -2694,6 +3010,8 @@ mod tests {
|
||||
let model_directive_policy =
|
||||
crate::system_features::ModelDirectivePolicySnapshot::load(&app).await;
|
||||
let routing_policy = ResolvedRoutingPolicy {
|
||||
billing_multiplier: 1.0,
|
||||
group_name: None,
|
||||
group_id: Some("routing-group-codex-first".to_string()),
|
||||
group_version: Some(1),
|
||||
selection_source: "test".to_string(),
|
||||
|
||||
@@ -19,12 +19,12 @@ pub(crate) use crate::ai_serving::{
|
||||
GEMINI_FILES_UPLOAD_PLAN_KIND, GEMINI_VIDEO_CANCEL_SYNC_PLAN_KIND,
|
||||
GEMINI_VIDEO_CREATE_SYNC_PLAN_KIND, OPENAI_CHAT_STREAM_PLAN_KIND, OPENAI_CHAT_SYNC_PLAN_KIND,
|
||||
OPENAI_EMBEDDING_SYNC_PLAN_KIND, OPENAI_IMAGE_STREAM_PLAN_KIND, OPENAI_IMAGE_SYNC_PLAN_KIND,
|
||||
OPENAI_RERANK_SYNC_PLAN_KIND, OPENAI_RESPONSES_COMPACT_STREAM_PLAN_KIND,
|
||||
OPENAI_RESPONSES_COMPACT_SYNC_PLAN_KIND, OPENAI_RESPONSES_STREAM_PLAN_KIND,
|
||||
OPENAI_RESPONSES_SYNC_PLAN_KIND, OPENAI_SEARCH_SYNC_PLAN_KIND,
|
||||
OPENAI_VIDEO_CANCEL_SYNC_PLAN_KIND, OPENAI_VIDEO_CONTENT_PLAN_KIND,
|
||||
OPENAI_VIDEO_CREATE_SYNC_PLAN_KIND, OPENAI_VIDEO_DELETE_SYNC_PLAN_KIND,
|
||||
OPENAI_VIDEO_REMIX_SYNC_PLAN_KIND,
|
||||
OPENAI_MEMORIES_SYNC_PLAN_KIND, OPENAI_RERANK_SYNC_PLAN_KIND,
|
||||
OPENAI_RESPONSES_COMPACT_STREAM_PLAN_KIND, OPENAI_RESPONSES_COMPACT_SYNC_PLAN_KIND,
|
||||
OPENAI_RESPONSES_STREAM_PLAN_KIND, OPENAI_RESPONSES_SYNC_PLAN_KIND,
|
||||
OPENAI_SEARCH_SYNC_PLAN_KIND, OPENAI_VIDEO_CANCEL_SYNC_PLAN_KIND,
|
||||
OPENAI_VIDEO_CONTENT_PLAN_KIND, OPENAI_VIDEO_CREATE_SYNC_PLAN_KIND,
|
||||
OPENAI_VIDEO_DELETE_SYNC_PLAN_KIND, OPENAI_VIDEO_REMIX_SYNC_PLAN_KIND,
|
||||
};
|
||||
|
||||
pub(crate) use aether_ai_serving::AiRequestedModelFamily as RequestedModelFamily;
|
||||
|
||||
@@ -6,12 +6,13 @@ use crate::ai_serving::planner::common::{
|
||||
GEMINI_FILES_DOWNLOAD_PLAN_KIND, GEMINI_FILES_GET_PLAN_KIND, GEMINI_FILES_LIST_PLAN_KIND,
|
||||
GEMINI_VIDEO_CANCEL_SYNC_PLAN_KIND, GEMINI_VIDEO_CREATE_SYNC_PLAN_KIND,
|
||||
OPENAI_CHAT_STREAM_PLAN_KIND, OPENAI_CHAT_SYNC_PLAN_KIND, OPENAI_EMBEDDING_SYNC_PLAN_KIND,
|
||||
OPENAI_IMAGE_STREAM_PLAN_KIND, OPENAI_IMAGE_SYNC_PLAN_KIND, OPENAI_RERANK_SYNC_PLAN_KIND,
|
||||
OPENAI_RESPONSES_COMPACT_STREAM_PLAN_KIND, OPENAI_RESPONSES_COMPACT_SYNC_PLAN_KIND,
|
||||
OPENAI_RESPONSES_STREAM_PLAN_KIND, OPENAI_RESPONSES_SYNC_PLAN_KIND,
|
||||
OPENAI_SEARCH_SYNC_PLAN_KIND, OPENAI_VIDEO_CANCEL_SYNC_PLAN_KIND,
|
||||
OPENAI_VIDEO_CONTENT_PLAN_KIND, OPENAI_VIDEO_CREATE_SYNC_PLAN_KIND,
|
||||
OPENAI_VIDEO_DELETE_SYNC_PLAN_KIND, OPENAI_VIDEO_REMIX_SYNC_PLAN_KIND,
|
||||
OPENAI_IMAGE_STREAM_PLAN_KIND, OPENAI_IMAGE_SYNC_PLAN_KIND, OPENAI_MEMORIES_SYNC_PLAN_KIND,
|
||||
OPENAI_RERANK_SYNC_PLAN_KIND, OPENAI_RESPONSES_COMPACT_STREAM_PLAN_KIND,
|
||||
OPENAI_RESPONSES_COMPACT_SYNC_PLAN_KIND, OPENAI_RESPONSES_STREAM_PLAN_KIND,
|
||||
OPENAI_RESPONSES_SYNC_PLAN_KIND, OPENAI_SEARCH_SYNC_PLAN_KIND,
|
||||
OPENAI_VIDEO_CANCEL_SYNC_PLAN_KIND, OPENAI_VIDEO_CONTENT_PLAN_KIND,
|
||||
OPENAI_VIDEO_CREATE_SYNC_PLAN_KIND, OPENAI_VIDEO_DELETE_SYNC_PLAN_KIND,
|
||||
OPENAI_VIDEO_REMIX_SYNC_PLAN_KIND,
|
||||
};
|
||||
use crate::ai_serving::planner::plan_builders::{
|
||||
build_gemini_stream_plan_from_decision, build_gemini_sync_plan_from_decision,
|
||||
@@ -101,7 +102,9 @@ fn build_sync_plan_payload_from_decision(
|
||||
OPENAI_RESPONSES_SYNC_PLAN_KIND => {
|
||||
build_openai_responses_sync_plan_from_decision(parts, body_json, payload, false)?
|
||||
}
|
||||
OPENAI_IMAGE_SYNC_PLAN_KIND | OPENAI_SEARCH_SYNC_PLAN_KIND => {
|
||||
OPENAI_IMAGE_SYNC_PLAN_KIND
|
||||
| OPENAI_SEARCH_SYNC_PLAN_KIND
|
||||
| OPENAI_MEMORIES_SYNC_PLAN_KIND => {
|
||||
build_passthrough_sync_plan_from_decision(parts, payload)?
|
||||
}
|
||||
OPENAI_RESPONSES_COMPACT_SYNC_PLAN_KIND => {
|
||||
|
||||
@@ -17,7 +17,7 @@ use crate::ai_serving::transport::ProviderOutboundRequestContext;
|
||||
use crate::ai_serving::{
|
||||
ClientSurface, ExecutionRuntimeAuthContext, GatewayAuthApiKeySnapshot,
|
||||
GatewayCredentialCarrier, GatewayProviderTransportSnapshot, PlannerAppState,
|
||||
CODEX_RESPONSES_LITE_HEADER,
|
||||
CODEX_RESPONSES_LITE_HEADER, OPENAI_MEMORIES_SYNC_PLAN_KIND,
|
||||
};
|
||||
use crate::cache::CacheLoadObserver;
|
||||
use crate::client_session_affinity::client_session_affinity_from_api_request;
|
||||
@@ -123,6 +123,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
||||
transport: Option<&GatewayProviderTransportSnapshot>,
|
||||
websocket_continuation: bool,
|
||||
) -> Result<(), GatewayError> {
|
||||
let native_memories = decision.decision_kind.as_deref() == Some(OPENAI_MEMORIES_SYNC_PLAN_KIND);
|
||||
let provider_api_format = decision
|
||||
.provider_api_format
|
||||
.clone()
|
||||
@@ -150,7 +151,12 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
||||
input.requested_model.as_str(),
|
||||
)
|
||||
});
|
||||
crate::ai_serving::apply_codex_openai_responses_lite_header_for_request_body_with_capabilities(
|
||||
if native_memories {
|
||||
decision
|
||||
.provider_request_headers
|
||||
.retain(|name, _| !name.eq_ignore_ascii_case(CODEX_RESPONSES_LITE_HEADER));
|
||||
} else {
|
||||
crate::ai_serving::apply_codex_openai_responses_lite_header_for_request_body_with_capabilities(
|
||||
&mut decision.provider_request_headers,
|
||||
decision.provider_request_body.as_ref(),
|
||||
provider_type.as_str(),
|
||||
@@ -159,6 +165,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
||||
input.requested_model.as_str(),
|
||||
model_capabilities.as_ref(),
|
||||
);
|
||||
}
|
||||
|
||||
let Some(context) = input.routing_context.as_ref() else {
|
||||
// Cache identity headers are projected only at the terminal boundary. Any non-empty
|
||||
@@ -260,7 +267,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
||||
provider_headers.insert(HeaderName::from_static(name), value);
|
||||
}
|
||||
}
|
||||
if original_provider_request_body.is_some() {
|
||||
if original_provider_request_body.is_some() && !native_memories {
|
||||
let provider_model = provider_request_body
|
||||
.get("model")
|
||||
.and_then(Value::as_str)
|
||||
@@ -318,6 +325,12 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
||||
}
|
||||
.map_err(|_| invalid_routing_provider_contract())?;
|
||||
}
|
||||
if native_memories {
|
||||
crate::ai_serving::transport::enforce_same_format_provider_api_operation_body_policy(
|
||||
&mut provider_request_body,
|
||||
Some(crate::ai_serving::ApiOperation::OpenAiMemoriesSummarize),
|
||||
);
|
||||
}
|
||||
let provider_model = provider_request_body
|
||||
.get("model")
|
||||
.and_then(Value::as_str)
|
||||
@@ -339,7 +352,11 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
||||
provider_type.as_str(),
|
||||
provider_api_format.as_str(),
|
||||
);
|
||||
crate::ai_serving::apply_codex_openai_responses_lite_header_for_request_body_with_capabilities(
|
||||
if native_memories {
|
||||
provider_request_headers
|
||||
.retain(|name, _| !name.eq_ignore_ascii_case(CODEX_RESPONSES_LITE_HEADER));
|
||||
} else {
|
||||
crate::ai_serving::apply_codex_openai_responses_lite_header_for_request_body_with_capabilities(
|
||||
&mut provider_request_headers,
|
||||
Some(&provider_request_body),
|
||||
provider_type.as_str(),
|
||||
@@ -348,6 +365,7 @@ pub(crate) fn apply_provider_request_routing_policy_to_decision_with_websocket_m
|
||||
input.requested_model.as_str(),
|
||||
model_capabilities.as_ref(),
|
||||
);
|
||||
}
|
||||
crate::ai_serving::apply_codex_openai_compact_terminal_headers(
|
||||
&mut provider_request_headers,
|
||||
provider_type.as_str(),
|
||||
@@ -382,6 +400,15 @@ fn apply_provider_outbound_request_policies_to_decision(
|
||||
let Some(context) = input.provider_outbound_context.as_ref() else {
|
||||
return;
|
||||
};
|
||||
let native_context;
|
||||
let context = if decision.decision_kind.as_deref() == Some(OPENAI_MEMORIES_SYNC_PLAN_KIND) {
|
||||
native_context = context
|
||||
.clone()
|
||||
.with_api_operation(crate::ai_serving::ApiOperation::OpenAiMemoriesSummarize);
|
||||
&native_context
|
||||
} else {
|
||||
context
|
||||
};
|
||||
let results = crate::ai_serving::transport::apply_provider_outbound_request_policies(
|
||||
transport,
|
||||
provider_api_format,
|
||||
@@ -528,24 +555,42 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
||||
input.provider_outbound_context =
|
||||
Some(crate::ai_serving::codex_context::resolve_codex_fingerprint_context(parts, body_json));
|
||||
let explicit_group = routing_header_value_str(&parts.headers, ROUTING_GROUP_HEADER);
|
||||
let preferred_group = if explicit_group.is_none() && !input.auth_context.api_key_is_standalone {
|
||||
state
|
||||
.read_auth_api_key_feature_settings(
|
||||
&input.auth_context.user_id,
|
||||
&input.auth_context.api_key_id,
|
||||
false,
|
||||
)
|
||||
.await?
|
||||
.as_ref()
|
||||
.and_then(|settings| settings.get("routing_group_id"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(str::to_owned)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let selected_group = match state.routing_group_read_repository() {
|
||||
Some(repository) => {
|
||||
// Explicit non-default groups are authorized against principal
|
||||
// bindings, so both selection and its cache key must retain the
|
||||
// caller context. Only the implicit no-binding system-default
|
||||
// path is global and can skip the membership lookup.
|
||||
let principal_context_required = if explicit_group.is_some() {
|
||||
true
|
||||
} else {
|
||||
repository
|
||||
.has_any_routing_group_binding()
|
||||
.await
|
||||
.map_err(|error| {
|
||||
routing_selection_error(GatewayRoutingSelectionError::Repository(
|
||||
error.to_string(),
|
||||
))
|
||||
})?
|
||||
};
|
||||
let principal_context_required =
|
||||
if explicit_group.is_some() || preferred_group.is_some() {
|
||||
true
|
||||
} else {
|
||||
repository
|
||||
.has_any_routing_group_binding()
|
||||
.await
|
||||
.map_err(|error| {
|
||||
routing_selection_error(GatewayRoutingSelectionError::Repository(
|
||||
error.to_string(),
|
||||
))
|
||||
})?
|
||||
};
|
||||
let user_group_ids = if principal_context_required {
|
||||
let user_groups_lookup_started_at = std::time::Instant::now();
|
||||
let user_groups = state
|
||||
@@ -568,6 +613,7 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
||||
principal_context_required.then(|| input.auth_context.api_key_id.clone());
|
||||
let selection_cache_key = routing_group_selection_cache_key(
|
||||
explicit_group.as_deref(),
|
||||
preferred_group.as_deref(),
|
||||
selection_user_id.as_deref(),
|
||||
selection_api_key_id.as_deref(),
|
||||
&user_group_ids,
|
||||
@@ -585,6 +631,7 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
||||
repository.as_ref(),
|
||||
GatewayRoutingSelectionInput {
|
||||
explicit_group: explicit_group.as_deref(),
|
||||
preferred_group: preferred_group.as_deref(),
|
||||
user_id: selection_user_id.as_deref(),
|
||||
api_key_id: selection_api_key_id.as_deref(),
|
||||
user_group_ids: &user_group_ids,
|
||||
@@ -601,6 +648,7 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
||||
|| {
|
||||
let repository = repository.clone();
|
||||
let explicit_group = explicit_group.clone();
|
||||
let preferred_group = preferred_group.clone();
|
||||
let user_id = selection_user_id.clone();
|
||||
let api_key_id = selection_api_key_id.clone();
|
||||
let user_group_ids = user_group_ids.clone();
|
||||
@@ -610,6 +658,7 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
||||
repository.as_ref(),
|
||||
GatewayRoutingSelectionInput {
|
||||
explicit_group: explicit_group.as_deref(),
|
||||
preferred_group: preferred_group.as_deref(),
|
||||
user_id: user_id.as_deref(),
|
||||
api_key_id: api_key_id.as_deref(),
|
||||
user_group_ids: &user_group_ids,
|
||||
@@ -635,6 +684,7 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
||||
selection.group.map(|group| {
|
||||
(
|
||||
Some(group.id),
|
||||
group.name,
|
||||
Some(group.version),
|
||||
group.config_json,
|
||||
selection.source,
|
||||
@@ -642,13 +692,14 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
||||
})
|
||||
}
|
||||
None => {
|
||||
if explicit_group
|
||||
if let Some(requested_group) = explicit_group
|
||||
.or(preferred_group)
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.is_some_and(|value| !value.is_empty())
|
||||
.filter(|value| !value.is_empty())
|
||||
{
|
||||
return Err(routing_selection_error(
|
||||
GatewayRoutingSelectionError::NotFound(explicit_group.unwrap_or_default()),
|
||||
GatewayRoutingSelectionError::NotFound(requested_group.to_string()),
|
||||
));
|
||||
}
|
||||
return Err(routing_selection_error(
|
||||
@@ -657,7 +708,8 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
||||
}
|
||||
};
|
||||
|
||||
let Some((group_id, group_version, group_config_json, selection_source)) = selected_group
|
||||
let Some((group_id, group_name, group_version, group_config_json, selection_source)) =
|
||||
selected_group
|
||||
else {
|
||||
return Err(routing_selection_error(
|
||||
GatewayRoutingSelectionError::NoDefault,
|
||||
@@ -674,6 +726,12 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
||||
&group_config_json,
|
||||
selection_source.as_str(),
|
||||
)? {
|
||||
if let Some(policy) = input.routing_policy.as_mut() {
|
||||
policy.group_name = Some(group_name.clone());
|
||||
}
|
||||
if let Some(trace) = input.routing_trace_seed.as_mut() {
|
||||
trace.group_name = Some(group_name);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -759,6 +817,7 @@ pub(crate) async fn attach_routing_policy_to_local_requested_model_input(
|
||||
final_policy_resolve_started_at.elapsed().as_millis() as u64,
|
||||
);
|
||||
final_policy.mutation_plan = policy.mutation_plan.clone();
|
||||
final_policy.group_name = Some(group_name);
|
||||
input.routing_trace_seed = Some(build_routing_trace_seed(&final_policy, client_api_format));
|
||||
input.routing_policy = Some(final_policy);
|
||||
input.routing_context = Some(LocalRoutingRequestContext {
|
||||
@@ -939,6 +998,7 @@ fn routing_header_value_str(headers: &http::HeaderMap, key: &str) -> Option<Stri
|
||||
|
||||
fn routing_group_selection_cache_key(
|
||||
explicit_group: Option<&str>,
|
||||
preferred_group: Option<&str>,
|
||||
user_id: Option<&str>,
|
||||
api_key_id: Option<&str>,
|
||||
user_group_ids: &[String],
|
||||
@@ -949,8 +1009,9 @@ fn routing_group_selection_cache_key(
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
format!(
|
||||
"v1|explicit={}|user={}|api_key={}|groups={}",
|
||||
"v2|explicit={}|preferred={}|user={}|api_key={}|groups={}",
|
||||
escape_cache_key_part(explicit_group.unwrap_or_default()),
|
||||
escape_cache_key_part(preferred_group.unwrap_or_default()),
|
||||
escape_cache_key_part(user_id.unwrap_or_default()),
|
||||
escape_cache_key_part(api_key_id.unwrap_or_default()),
|
||||
groups
|
||||
@@ -1148,10 +1209,13 @@ mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::*;
|
||||
use aether_data::repository::auth::{
|
||||
InMemoryAuthApiKeySnapshotRepository, StoredAuthApiKeySnapshot,
|
||||
};
|
||||
use aether_data::repository::routing_profiles::InMemoryRoutingGroupRepository;
|
||||
use aether_data_contracts::repository::routing_profiles::{
|
||||
CreateRoutingGroupBindingRecord, CreateRoutingGroupRecord, RoutingGroupBindingSubject,
|
||||
RoutingGroupWriteRepository,
|
||||
RoutingGroupWriteRepository, UpdateRoutingGroupRecord,
|
||||
};
|
||||
use aether_provider_transport::snapshot::{
|
||||
GatewayProviderTransportEndpoint, GatewayProviderTransportKey,
|
||||
@@ -1162,12 +1226,14 @@ mod tests {
|
||||
fn explicit_routing_selection_cache_key_is_principal_specific() {
|
||||
let first = routing_group_selection_cache_key(
|
||||
Some("private"),
|
||||
None,
|
||||
Some("user-1"),
|
||||
Some("key-1"),
|
||||
&["team-1".to_string()],
|
||||
);
|
||||
let second = routing_group_selection_cache_key(
|
||||
Some("private"),
|
||||
None,
|
||||
Some("user-2"),
|
||||
Some("key-2"),
|
||||
&["team-2".to_string()],
|
||||
@@ -1284,6 +1350,160 @@ mod tests {
|
||||
assert!(matches!(error, GatewayError::Internal(message) if message.contains("identity")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn api_key_routing_selection_applies_at_planner_and_invalidates_after_changes() {
|
||||
let auth_repository = Arc::new(InMemoryAuthApiKeySnapshotRepository::seed(
|
||||
["api-key-1", "api-key-2"].map(|key_id| {
|
||||
(
|
||||
None,
|
||||
StoredAuthApiKeySnapshot::new(
|
||||
"user-1".into(),
|
||||
"alice".into(),
|
||||
None,
|
||||
"user".into(),
|
||||
"local".into(),
|
||||
true,
|
||||
false,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
key_id.into(),
|
||||
Some(key_id.into()),
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
}),
|
||||
));
|
||||
let groups = Arc::new(InMemoryRoutingGroupRepository::default());
|
||||
for (id, visible, is_default, multiplier) in [
|
||||
("default", false, true, 1.0),
|
||||
("discount", true, false, 0.5),
|
||||
("premium", true, false, 2.0),
|
||||
] {
|
||||
groups.create_routing_group(CreateRoutingGroupRecord {
|
||||
id: id.into(), name: format!("{id}-name"), description: None,
|
||||
enabled: true, is_system_default: is_default, sort_order: 0,
|
||||
config_json: json!({ "user_visible": visible, "billing_multiplier": multiplier }),
|
||||
version: 1, created_at: 1, updated_at: 1, published_at: None,
|
||||
}).await.unwrap();
|
||||
}
|
||||
let state = AppState::new().unwrap().with_data_state_for_tests(
|
||||
crate::data::GatewayDataState::with_auth_api_key_repository_for_tests(auth_repository)
|
||||
.with_routing_group_repository_for_tests(groups.clone()),
|
||||
);
|
||||
for (key_id, group_id) in [("api-key-1", "discount"), ("api-key-2", "premium")] {
|
||||
assert!(state
|
||||
.set_user_api_key_feature_settings(
|
||||
"user-1",
|
||||
key_id,
|
||||
Some(json!({ "routing_group_id": group_id }))
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_some());
|
||||
}
|
||||
let (parts, _) = http::Request::builder().body(()).unwrap().into_parts();
|
||||
let (header_parts, _) = http::Request::builder()
|
||||
.header(ROUTING_GROUP_HEADER, "premium")
|
||||
.body(())
|
||||
.unwrap()
|
||||
.into_parts();
|
||||
|
||||
async fn attach(
|
||||
state: &AppState,
|
||||
parts: &http::request::Parts,
|
||||
key_id: &str,
|
||||
) -> Result<LocalRequestedModelDecisionInput, GatewayError> {
|
||||
let mut input = sample_decision_input();
|
||||
input.auth_context.api_key_id = key_id.into();
|
||||
input.auth_snapshot.api_key_id = key_id.into();
|
||||
attach_routing_policy_to_local_requested_model_input(
|
||||
state,
|
||||
parts,
|
||||
&mut input,
|
||||
&json!({ "model": "gpt-5" }),
|
||||
"openai:chat",
|
||||
)
|
||||
.await?;
|
||||
Ok(input)
|
||||
}
|
||||
|
||||
// Revisit the first key after the second to exercise both cached choices.
|
||||
for (key_id, group_id, multiplier) in [
|
||||
("api-key-1", "discount", 0.5),
|
||||
("api-key-2", "premium", 2.0),
|
||||
("api-key-1", "discount", 0.5),
|
||||
] {
|
||||
let input = attach(&state, &parts, key_id).await.unwrap();
|
||||
let policy = input.routing_policy.as_ref().unwrap();
|
||||
assert_eq!(policy.group_id.as_deref(), Some(group_id));
|
||||
assert_eq!(policy.selection_source, "api_key_selection");
|
||||
assert_eq!(policy.billing_multiplier, multiplier);
|
||||
assert_eq!(
|
||||
input
|
||||
.routing_trace_seed
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.billing_multiplier,
|
||||
Some(multiplier)
|
||||
);
|
||||
}
|
||||
let header = attach(&state, &header_parts, "api-key-1").await.unwrap();
|
||||
let policy = header.routing_policy.unwrap();
|
||||
assert_eq!(policy.group_id.as_deref(), Some("premium"));
|
||||
assert_eq!(policy.selection_source, "explicit_header");
|
||||
|
||||
groups
|
||||
.update_routing_group(
|
||||
"discount",
|
||||
UpdateRoutingGroupRecord {
|
||||
config_json: Some(json!({ "user_visible": false, "billing_multiplier": 0.5 })),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
state.invalidate_provider_routing_caches();
|
||||
assert!(matches!(
|
||||
attach(&state, &parts, "api-key-1").await,
|
||||
Err(GatewayError::Client {
|
||||
status: StatusCode::FORBIDDEN,
|
||||
..
|
||||
})
|
||||
));
|
||||
let header = attach(&state, &header_parts, "api-key-1").await.unwrap();
|
||||
assert_eq!(
|
||||
header.routing_policy.unwrap().group_id.as_deref(),
|
||||
Some("premium")
|
||||
);
|
||||
|
||||
assert!(state
|
||||
.set_user_api_key_feature_settings("user-1", "api-key-1", None)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_some());
|
||||
let cleared = attach(&state, &parts, "api-key-1").await.unwrap();
|
||||
let policy = cleared.routing_policy.unwrap();
|
||||
assert_eq!(policy.group_id.as_deref(), Some("default"));
|
||||
assert_eq!(policy.selection_source, "system_default");
|
||||
assert_eq!(policy.billing_multiplier, 1.0);
|
||||
// Clearing one key's preference must not disturb the other key's selection.
|
||||
let other = attach(&state, &parts, "api-key-2").await.unwrap();
|
||||
assert_eq!(
|
||||
other.routing_policy.unwrap().group_id.as_deref(),
|
||||
Some("premium")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn explicit_routing_attachment_authorizes_and_caches_per_principal() {
|
||||
let repository = Arc::new(InMemoryRoutingGroupRepository::default());
|
||||
@@ -1295,7 +1515,7 @@ mod tests {
|
||||
enabled: true,
|
||||
is_system_default: false,
|
||||
sort_order: 0,
|
||||
config_json: json!({}),
|
||||
config_json: json!({"billing_multiplier": 0.5}),
|
||||
version: 1,
|
||||
created_at: 1,
|
||||
updated_at: 1,
|
||||
@@ -1341,6 +1561,11 @@ mod tests {
|
||||
.as_ref()
|
||||
.expect("explicit selection should attach routing policy");
|
||||
assert_eq!(policy.group_id.as_deref(), Some("private-group"));
|
||||
assert_eq!(policy.group_name.as_deref(), Some("private"));
|
||||
assert_eq!(policy.billing_multiplier, 0.5);
|
||||
let trace = allowed.routing_trace_seed.as_ref().unwrap();
|
||||
assert_eq!(trace.group_name.as_deref(), Some("private"));
|
||||
assert_eq!(trace.billing_multiplier, Some(0.5));
|
||||
assert_eq!(policy.selection_source, "explicit_header");
|
||||
|
||||
let mut denied = sample_decision_input();
|
||||
@@ -1708,7 +1933,7 @@ mod tests {
|
||||
assert_eq!(policy.group_version, Some(4));
|
||||
assert_eq!(
|
||||
policy.priority_mode,
|
||||
aether_routing_core::RoutingSetPriorityMode::GlobalKey
|
||||
aether_routing_core::RoutingSetPriorityMode::Provider
|
||||
);
|
||||
assert_eq!(
|
||||
policy.scheduling_mode,
|
||||
|
||||
@@ -7,7 +7,7 @@ use crate::ai_serving::transport::{
|
||||
build_gemini_cli_v1internal_request, build_standard_provider_request_headers,
|
||||
GatewayProviderTransportSnapshot, GeminiCliRequestAuth, GeminiCliRequestAuthSupport,
|
||||
GeminiCliRequestEnvelopeSupport, StandardProviderRequestHeaders,
|
||||
StandardProviderRequestHeadersInput, GEMINI_CLI_USER_AGENT,
|
||||
StandardProviderRequestHeadersInput,
|
||||
};
|
||||
use crate::AppState;
|
||||
|
||||
@@ -64,8 +64,10 @@ pub(crate) async fn build_gemini_cli_v1internal_provider_request(
|
||||
)
|
||||
.ok_or(GeminiCliV1InternalRequestError::UpstreamUrlUnavailable)?;
|
||||
|
||||
let extra_headers =
|
||||
BTreeMap::from([("user-agent".to_string(), GEMINI_CLI_USER_AGENT.to_string())]);
|
||||
let extra_headers = BTreeMap::from([(
|
||||
"user-agent".to_string(),
|
||||
crate::ai_serving::transport::gemini_cli::gemini_cli_client_user_agent(),
|
||||
)]);
|
||||
let headers = build_standard_provider_request_headers(StandardProviderRequestHeadersInput {
|
||||
transport: &payload.transport,
|
||||
provider_api_format: input.provider_api_format,
|
||||
|
||||
@@ -21,10 +21,11 @@ use crate::ai_serving::transport::{
|
||||
build_same_format_provider_headers, resolve_local_gemini_cli_request_auth,
|
||||
GeminiCliRequestAuth, GeminiCliRequestAuthSupport, GeminiCliRequestEnvelopeSupport,
|
||||
GrokHeaderInput, SameFormatProviderCompatibilityEdit,
|
||||
SameFormatProviderCompatibilityEditAction, SameFormatProviderHeadersInput,
|
||||
GEMINI_CLI_USER_AGENT, GROK_CHAT_PATH,
|
||||
SameFormatProviderCompatibilityEditAction, SameFormatProviderHeadersInput, GROK_CHAT_PATH,
|
||||
};
|
||||
use crate::ai_serving::{
|
||||
CandidateFailureDiagnostic, GatewayProviderTransportSnapshot, CODEX_RESPONSES_LITE_HEADER,
|
||||
};
|
||||
use crate::ai_serving::{CandidateFailureDiagnostic, GatewayProviderTransportSnapshot};
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
mod policy;
|
||||
@@ -255,7 +256,9 @@ pub(crate) async fn resolve_local_same_format_provider_candidate_payload_parts(
|
||||
// re-enforce stream-field policy afterward.
|
||||
// Kiro behavior classification already hard-requires upstream streaming,
|
||||
// and the Kiro envelope does not use a top-level body stream field.
|
||||
if prepared.kiro_auth.is_none() {
|
||||
if prepared.kiro_auth.is_none()
|
||||
&& spec.operation != Some(crate::ai_serving::ApiOperation::OpenAiMemoriesSummarize)
|
||||
{
|
||||
enforce_provider_body_stream_policy(
|
||||
&mut base_provider_request_body,
|
||||
prepared.provider_api_format.as_str(),
|
||||
@@ -275,7 +278,8 @@ pub(crate) async fn resolve_local_same_format_provider_candidate_payload_parts(
|
||||
prepared.mapped_model.as_str(),
|
||||
source_model,
|
||||
);
|
||||
if let Err(violation) =
|
||||
if spec.operation != Some(crate::ai_serving::ApiOperation::OpenAiMemoriesSummarize) {
|
||||
if let Err(violation) =
|
||||
crate::ai_serving::finalize_openai_provider_request_with_codex_model_capabilities_and_reasoning_replay_policy(
|
||||
&mut base_provider_request_body,
|
||||
crate::ai_serving::OpenAiProviderRequestFinalization {
|
||||
@@ -313,6 +317,21 @@ pub(crate) async fn resolve_local_same_format_provider_candidate_payload_parts(
|
||||
.await;
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
|
||||
// Same-format requests skip `apply_transport_request_body_semantics`, so the opt-in
|
||||
// Claude Code body mimicry has to be applied here as well.
|
||||
if crate::ai_serving::transport::claude_code::apply_claude_code_body_mimicry_for_transport(
|
||||
&mut base_provider_request_body,
|
||||
&transport,
|
||||
prepared.provider_api_format.as_str(),
|
||||
) {
|
||||
compatibility_edits.push(SameFormatProviderCompatibilityEdit {
|
||||
field: "body".to_string(),
|
||||
action: SameFormatProviderCompatibilityEditAction::ProviderCompatibilityRewrite,
|
||||
detail: "applied Claude Code body mimicry for provider compatibility".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let antigravity_auth = if prepared.is_antigravity {
|
||||
let mut antigravity_support = classify_local_antigravity_request_support(
|
||||
@@ -513,7 +532,10 @@ pub(crate) async fn resolve_local_same_format_provider_candidate_payload_parts(
|
||||
.map(build_antigravity_static_identity_headers)
|
||||
.unwrap_or_default();
|
||||
if prepared.behavior.is_gemini_cli {
|
||||
extra_headers.insert("user-agent".to_string(), GEMINI_CLI_USER_AGENT.to_string());
|
||||
extra_headers.insert(
|
||||
"user-agent".to_string(),
|
||||
crate::ai_serving::transport::gemini_cli::gemini_cli_client_user_agent(),
|
||||
);
|
||||
}
|
||||
let Some(mut provider_request_headers) = (if is_grok {
|
||||
build_grok_browser_headers(GrokHeaderInput {
|
||||
@@ -583,6 +605,16 @@ pub(crate) async fn resolve_local_same_format_provider_candidate_payload_parts(
|
||||
source_model,
|
||||
codex_model_capabilities.as_ref(),
|
||||
);
|
||||
if spec.operation == Some(crate::ai_serving::ApiOperation::OpenAiMemoriesSummarize) {
|
||||
provider_request_headers
|
||||
.retain(|name, _| !name.eq_ignore_ascii_case(CODEX_RESPONSES_LITE_HEADER));
|
||||
provider_request_headers.insert("accept".to_string(), "application/json".to_string());
|
||||
}
|
||||
crate::ai_serving::transport::xai::insert_cli_identity_headers_if_needed(
|
||||
transport.as_ref(),
|
||||
prepared.provider_api_format.as_str(),
|
||||
&mut provider_request_headers,
|
||||
);
|
||||
request_identity_response_encoding_when_redacted(
|
||||
&mut provider_request_headers,
|
||||
redaction.redacted,
|
||||
|
||||
@@ -3,7 +3,7 @@ use serde_json::Value;
|
||||
use super::super::LocalSameFormatProviderSpec;
|
||||
use crate::ai_serving::transport::{
|
||||
build_same_format_provider_request_body as build_same_format_provider_request_body_impl,
|
||||
build_same_format_provider_request_body_with_compatibility_report_and_reasoning_replay_policy as build_same_format_provider_request_body_with_compatibility_report_impl,
|
||||
build_same_format_provider_request_body_for_operation as build_same_format_provider_request_body_with_compatibility_report_impl,
|
||||
SameFormatProviderFamily, SameFormatProviderRequestBodyInput,
|
||||
SameFormatProviderRequestBodyOutput,
|
||||
};
|
||||
@@ -69,6 +69,7 @@ pub(crate) fn build_same_format_provider_request_body_with_compatibility_report(
|
||||
enable_model_directives,
|
||||
},
|
||||
reasoning_replay_policy,
|
||||
spec.operation,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,11 @@ use aether_ai_serving::{
|
||||
provider_stream_event_api_format_for_provider_type as ai_provider_stream_event_api_format_for_provider_type,
|
||||
AiExecutionReportContextParts, AiRequestOrigin, STICKY_KEY_ATTEMPTS_REPORT_FIELD,
|
||||
};
|
||||
use aether_data_contracts::repository::usage::{
|
||||
BillingMultiplierSnapshot, BILLING_MULTIPLIER_SNAPSHOT_METADATA_KEY,
|
||||
ROUTING_GROUP_BILLING_MULTIPLIER_METADATA_KEY, ROUTING_GROUP_ID_METADATA_KEY,
|
||||
ROUTING_GROUP_NAME_METADATA_KEY,
|
||||
};
|
||||
use aether_routing_core::ResolvedRoutingPolicy;
|
||||
use aether_runtime_state::RuntimeLockLease;
|
||||
use aether_scheduler_core::{ClientSessionAffinity, SchedulerRankingOutcome};
|
||||
@@ -87,6 +92,46 @@ pub(crate) fn build_local_execution_report_context(
|
||||
parts.original_request_body_base64,
|
||||
);
|
||||
let mut extra_fields = parts.extra_fields;
|
||||
// Always overwrite caller-supplied extras with the planner's immutable policy snapshot.
|
||||
let billing_multiplier = parts
|
||||
.routing_policy
|
||||
.map(|policy| policy.billing_multiplier)
|
||||
.filter(|value| value.is_finite() && *value >= 0.0)
|
||||
.unwrap_or(1.0);
|
||||
extra_fields.insert(
|
||||
ROUTING_GROUP_BILLING_MULTIPLIER_METADATA_KEY.to_string(),
|
||||
Value::from(billing_multiplier),
|
||||
);
|
||||
extra_fields.insert(
|
||||
BILLING_MULTIPLIER_SNAPSHOT_METADATA_KEY.to_string(),
|
||||
serde_json::to_value(
|
||||
BillingMultiplierSnapshot::from_factors(BTreeMap::from([(
|
||||
"routing_group".to_string(),
|
||||
billing_multiplier,
|
||||
)]))
|
||||
.expect("validated routing multiplier must produce a billing snapshot"),
|
||||
)
|
||||
.expect("validated billing snapshot must serialize"),
|
||||
);
|
||||
for (field, value) in [
|
||||
(
|
||||
ROUTING_GROUP_ID_METADATA_KEY,
|
||||
parts
|
||||
.routing_policy
|
||||
.and_then(|policy| policy.group_id.as_deref()),
|
||||
),
|
||||
(
|
||||
ROUTING_GROUP_NAME_METADATA_KEY,
|
||||
parts
|
||||
.routing_policy
|
||||
.and_then(|policy| policy.group_name.as_deref()),
|
||||
),
|
||||
] {
|
||||
extra_fields.remove(field);
|
||||
if let Some(value) = value {
|
||||
extra_fields.insert(field.to_string(), Value::String(value.to_string()));
|
||||
}
|
||||
}
|
||||
if let Some(value) = parts
|
||||
.client_session_affinity
|
||||
.and_then(client_session_affinity_report_context_value)
|
||||
@@ -341,6 +386,27 @@ mod tests {
|
||||
Some("codex".to_string()),
|
||||
Some("account=account-1;session=session-1".to_string()),
|
||||
);
|
||||
let mut routing_policy = aether_routing_core::resolve_routing_policy(
|
||||
&aether_routing_core::RoutingGroupConfig {
|
||||
billing_multiplier: 0.25,
|
||||
..Default::default()
|
||||
},
|
||||
aether_routing_core::RoutingPolicyInput {
|
||||
group_id: Some("group-1"),
|
||||
group_version: Some(7),
|
||||
selection_source: "system_default",
|
||||
requested_model: "gpt-5",
|
||||
resolved_model: "gpt-5",
|
||||
api_format: "openai:chat",
|
||||
user_id: None,
|
||||
api_key_id: None,
|
||||
headers: &json!({}),
|
||||
body: &json!({}),
|
||||
phase: aether_routing_core::RoutingRulePhase::ClientRequest,
|
||||
},
|
||||
)
|
||||
.expect("routing policy should resolve");
|
||||
routing_policy.group_name = Some("请求时的分组".to_string());
|
||||
|
||||
let report_context =
|
||||
build_local_execution_report_context(LocalExecutionReportContextParts {
|
||||
@@ -379,16 +445,35 @@ mod tests {
|
||||
original_request_body_json: Some(&json!({"model": "gpt-5"})),
|
||||
original_request_body_base64: None,
|
||||
client_session_affinity: Some(&client_session_affinity),
|
||||
routing_policy: None,
|
||||
routing_policy: Some(&routing_policy),
|
||||
scheduler_affinity_epoch: None,
|
||||
sticky_key_attempts: None,
|
||||
client_requested_stream: false,
|
||||
upstream_is_stream: false,
|
||||
has_envelope: false,
|
||||
needs_conversion: false,
|
||||
extra_fields: Map::new(),
|
||||
extra_fields: Map::from_iter([
|
||||
(
|
||||
"billing_multiplier_snapshot".to_string(),
|
||||
json!({
|
||||
"version": 1, "factors": {"routing_group": 99.0}, "multiplier": 99.0
|
||||
}),
|
||||
),
|
||||
("routing_group_billing_multiplier".to_string(), json!(99)),
|
||||
("routing_group_id".to_string(), json!("forged-group")),
|
||||
("routing_group_name".to_string(), json!("forged-name")),
|
||||
]),
|
||||
});
|
||||
|
||||
assert_eq!(report_context["routing_group_billing_multiplier"], 0.25);
|
||||
assert_eq!(
|
||||
report_context["billing_multiplier_snapshot"],
|
||||
json!({
|
||||
"version": 1, "factors": {"routing_group": 0.25}, "multiplier": 0.25
|
||||
})
|
||||
);
|
||||
assert_eq!(report_context["routing_group_id"], "group-1");
|
||||
assert_eq!(report_context["routing_group_name"], "请求时的分组");
|
||||
assert_eq!(
|
||||
report_context["client_ip"],
|
||||
Value::String("203.0.113.8".to_string())
|
||||
|
||||
@@ -17,8 +17,8 @@ use crate::ai_serving::transport::{
|
||||
ProviderOpenAiImageHeadersInput, StandardProviderRequestHeadersInput, GROK_CHAT_PATH,
|
||||
};
|
||||
use crate::ai_serving::{
|
||||
apply_codex_openai_special_headers, build_chatgpt_web_image_request_body,
|
||||
build_codex_openai_image_api_provider_request_body,
|
||||
apply_codex_openai_special_headers, apply_xai_upstream_payload_edits,
|
||||
build_chatgpt_web_image_request_body, build_codex_openai_image_api_provider_request_body,
|
||||
build_gemini_image_request_body_from_openai_image_request,
|
||||
build_openai_image_api_provider_request_body, build_openai_image_provider_request_body,
|
||||
default_model_for_openai_image_operation, normalize_openai_image_request,
|
||||
@@ -211,7 +211,7 @@ pub(super) async fn resolve_local_openai_image_candidate_payload_parts(
|
||||
upstream_is_stream,
|
||||
)
|
||||
};
|
||||
let Some(provider_request_body) = provider_request_body else {
|
||||
let Some(mut provider_request_body) = provider_request_body else {
|
||||
mark_skipped_local_openai_image_candidate_with_failure_diagnostic(
|
||||
state,
|
||||
input,
|
||||
@@ -229,6 +229,11 @@ pub(super) async fn resolve_local_openai_image_candidate_payload_parts(
|
||||
.await;
|
||||
return None;
|
||||
};
|
||||
apply_xai_upstream_payload_edits(
|
||||
&mut provider_request_body,
|
||||
transport.provider.provider_type.as_str(),
|
||||
provider_api_format,
|
||||
);
|
||||
let Some(mut provider_request_headers) = (if is_grok {
|
||||
build_grok_browser_headers(GrokHeaderInput {
|
||||
transport,
|
||||
|
||||
@@ -8,6 +8,7 @@ use crate::ai_serving::planner::{
|
||||
build_ai_execution_decision_response, resolve_transport_request_encoding_policy,
|
||||
AiExecutionDecisionResponseParts,
|
||||
};
|
||||
use crate::ai_serving::transport::xai::video::is_native_video_request;
|
||||
use crate::ai_serving::transport::{
|
||||
resolve_transport_execution_timeouts, resolve_transport_profile,
|
||||
};
|
||||
@@ -33,7 +34,7 @@ pub(super) async fn maybe_build_local_video_create_decision_payload_for_candidat
|
||||
let Some(resolved) = resolve_local_video_create_candidate_payload_parts(
|
||||
state, parts, body_json, trace_id, input, &attempt, spec,
|
||||
)
|
||||
.await
|
||||
.await?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
@@ -52,9 +53,32 @@ pub(super) async fn maybe_build_local_video_create_decision_payload_for_candidat
|
||||
.await;
|
||||
let transport_profile = resolve_transport_profile(&transport);
|
||||
let mut extra_fields = serde_json::Map::new();
|
||||
if is_native_video_request(&transport.provider.provider_type, parts.uri.path()) {
|
||||
extra_fields.insert(
|
||||
"video_client_protocol".to_string(),
|
||||
serde_json::json!("xai"),
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(proxy_value) = build_request_trace_proxy_value(Some(&transport), proxy.as_ref()) {
|
||||
extra_fields.insert("proxy".to_string(), proxy_value);
|
||||
}
|
||||
if transport.provider.provider_type.eq_ignore_ascii_case("xai") {
|
||||
extra_fields.insert("video_provider_xai".into(), serde_json::json!(true));
|
||||
if let Some(duration) = resolved.provider_request_body.get("duration") {
|
||||
extra_fields.insert("video_duration".into(), duration.clone());
|
||||
}
|
||||
if parts.uri.path() == "/openai/v1/videos" {
|
||||
extra_fields.insert(
|
||||
"video_size".into(),
|
||||
body_json
|
||||
.get("size")
|
||||
.filter(|v| v.as_str().is_some_and(|s| !s.trim().is_empty()))
|
||||
.cloned()
|
||||
.unwrap_or_else(|| serde_json::json!("720x1280")),
|
||||
);
|
||||
}
|
||||
}
|
||||
let effective_headers = input.effective_headers(&parts.headers);
|
||||
let report_context = build_local_execution_report_context(LocalExecutionReportContextParts {
|
||||
auth_context: &input.auth_context,
|
||||
|
||||
@@ -3,15 +3,23 @@ use std::sync::Arc;
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::ai_serving::planner::candidate_preparation::resolve_candidate_mapped_model;
|
||||
use crate::ai_serving::planner::candidate_preparation::{
|
||||
prepare_header_authenticated_candidate, resolve_candidate_mapped_model, OauthPreparationContext,
|
||||
};
|
||||
use crate::ai_serving::planner::spec_metadata::local_video_create_spec_metadata;
|
||||
use crate::ai_serving::transport::xai::video::{
|
||||
convert_openai_video_request, is_explicit_native_video_path, is_native_video_request,
|
||||
};
|
||||
use crate::ai_serving::transport::{
|
||||
build_video_create_headers, build_video_create_request_body, build_video_create_upstream_url,
|
||||
resolve_video_create_auth, video_create_transport_unsupported_reason,
|
||||
ProviderVideoCreateFamily, ProviderVideoCreateHeadersInput,
|
||||
};
|
||||
use crate::ai_serving::{CandidateFailureDiagnostic, GatewayProviderTransportSnapshot};
|
||||
use crate::AppState;
|
||||
use crate::ai_serving::{
|
||||
apply_xai_upstream_payload_edits, CandidateFailureDiagnostic, GatewayProviderTransportSnapshot,
|
||||
PlannerAppState,
|
||||
};
|
||||
use crate::{AppState, GatewayError};
|
||||
|
||||
use super::support::{
|
||||
mark_skipped_local_video_candidate, mark_skipped_local_video_candidate_with_failure_diagnostic,
|
||||
@@ -37,11 +45,16 @@ pub(super) async fn resolve_local_video_create_candidate_payload_parts(
|
||||
input: &LocalVideoCreateDecisionInput,
|
||||
attempt: &LocalVideoCreateCandidateAttempt,
|
||||
spec: LocalVideoCreateSpec,
|
||||
) -> Option<LocalVideoCreateCandidatePayloadParts> {
|
||||
) -> Result<Option<LocalVideoCreateCandidatePayloadParts>, GatewayError> {
|
||||
let spec_metadata = local_video_create_spec_metadata(spec);
|
||||
let candidate = &attempt.eligible.candidate;
|
||||
let transport = &attempt.eligible.transport;
|
||||
let effective_headers = input.effective_headers(&parts.headers);
|
||||
if is_explicit_native_video_path(parts.uri.path())
|
||||
&& !transport.provider.provider_type.eq_ignore_ascii_case("xai")
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let provider_family = provider_video_create_family(spec.family);
|
||||
let transport_unsupported_reason = video_create_transport_unsupported_reason(
|
||||
@@ -60,23 +73,39 @@ pub(super) async fn resolve_local_video_create_candidate_payload_parts(
|
||||
skip_reason,
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let auth = resolve_video_create_auth(transport, provider_family);
|
||||
let Some((auth_header, auth_value)) = auth else {
|
||||
mark_skipped_local_video_candidate(
|
||||
state,
|
||||
input,
|
||||
let prepared_candidate = match prepare_header_authenticated_candidate(
|
||||
PlannerAppState::new(state),
|
||||
transport,
|
||||
candidate,
|
||||
resolve_video_create_auth(transport, provider_family),
|
||||
OauthPreparationContext {
|
||||
trace_id,
|
||||
candidate,
|
||||
attempt.candidate_index,
|
||||
&attempt.candidate_id,
|
||||
"transport_auth_unavailable",
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
api_format: spec_metadata.api_format,
|
||||
operation: "video_create_candidate_request",
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(prepared) => prepared,
|
||||
Err(skip_reason) => {
|
||||
mark_skipped_local_video_candidate(
|
||||
state,
|
||||
input,
|
||||
trace_id,
|
||||
candidate,
|
||||
attempt.candidate_index,
|
||||
&attempt.candidate_id,
|
||||
skip_reason,
|
||||
)
|
||||
.await;
|
||||
return Ok(None);
|
||||
}
|
||||
};
|
||||
let auth_header = prepared_candidate.auth_header;
|
||||
let auth_value = prepared_candidate.auth_value;
|
||||
|
||||
let mapped_model = match resolve_candidate_mapped_model(candidate) {
|
||||
Ok(mapped_model) => mapped_model,
|
||||
@@ -91,7 +120,7 @@ pub(super) async fn resolve_local_video_create_candidate_payload_parts(
|
||||
skip_reason,
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
return Ok(None);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -117,10 +146,10 @@ pub(super) async fn resolve_local_video_create_candidate_payload_parts(
|
||||
),
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let Some(provider_request_body) = build_video_create_request_body(
|
||||
let Some(mut provider_request_body) = build_video_create_request_body(
|
||||
body_json,
|
||||
provider_family,
|
||||
&mapped_model,
|
||||
@@ -142,11 +171,28 @@ pub(super) async fn resolve_local_video_create_candidate_payload_parts(
|
||||
),
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
return Ok(None);
|
||||
};
|
||||
if transport.provider.provider_type.eq_ignore_ascii_case("xai")
|
||||
&& !is_native_video_request(&transport.provider.provider_type, parts.uri.path())
|
||||
{
|
||||
provider_request_body =
|
||||
convert_openai_video_request(&provider_request_body).map_err(|message| {
|
||||
GatewayError::Client {
|
||||
status: http::StatusCode::BAD_REQUEST,
|
||||
message: message.to_string(),
|
||||
}
|
||||
})?;
|
||||
}
|
||||
apply_xai_upstream_payload_edits(
|
||||
&mut provider_request_body,
|
||||
transport.provider.provider_type.as_str(),
|
||||
spec_metadata.api_format,
|
||||
);
|
||||
|
||||
let Some(provider_request_headers) =
|
||||
build_video_create_headers(ProviderVideoCreateHeadersInput {
|
||||
transport,
|
||||
headers: effective_headers,
|
||||
auth_header: &auth_header,
|
||||
auth_value: &auth_value,
|
||||
@@ -170,10 +216,10 @@ pub(super) async fn resolve_local_video_create_candidate_payload_parts(
|
||||
),
|
||||
)
|
||||
.await;
|
||||
return None;
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
Some(LocalVideoCreateCandidatePayloadParts {
|
||||
Ok(Some(LocalVideoCreateCandidatePayloadParts {
|
||||
transport: Arc::clone(transport),
|
||||
auth_header,
|
||||
auth_value,
|
||||
@@ -181,7 +227,7 @@ pub(super) async fn resolve_local_video_create_candidate_payload_parts(
|
||||
provider_request_headers,
|
||||
provider_request_body,
|
||||
upstream_url,
|
||||
})
|
||||
}))
|
||||
}
|
||||
|
||||
fn provider_video_create_family(family: LocalVideoCreateFamily) -> ProviderVideoCreateFamily {
|
||||
|
||||
@@ -505,9 +505,12 @@ fn projects_uuid_prompt_cache_identity_into_missing_session_headers() {
|
||||
assert_eq!(headers.get("x-client-request-id"), None);
|
||||
assert_eq!(
|
||||
headers.get("user-agent").map(String::as_str),
|
||||
Some(aether_ai_formats::CODEX_CLIENT_USER_AGENT)
|
||||
Some(aether_ai_formats::codex_client_user_agent().as_str())
|
||||
);
|
||||
assert_eq!(
|
||||
headers.get("originator"),
|
||||
Some(&aether_ai_formats::codex_client_originator())
|
||||
);
|
||||
assert_eq!(headers.get("originator"), Some(&"codex_cli_rs".to_string()));
|
||||
assert!(!headers.contains_key("version"));
|
||||
assert_eq!(headers.get("x-openai-fedramp"), Some(&"true".to_string()));
|
||||
assert_eq!(
|
||||
@@ -615,9 +618,12 @@ fn injects_only_codex_client_headers_for_images_requests() {
|
||||
);
|
||||
assert_eq!(
|
||||
headers.get("user-agent").map(String::as_str),
|
||||
Some(aether_ai_formats::CODEX_CLIENT_USER_AGENT)
|
||||
Some(aether_ai_formats::codex_client_user_agent().as_str())
|
||||
);
|
||||
assert_eq!(
|
||||
headers.get("originator"),
|
||||
Some(&aether_ai_formats::codex_client_originator())
|
||||
);
|
||||
assert_eq!(headers.get("originator"), Some(&"codex_cli_rs".to_string()));
|
||||
assert!(!headers.contains_key("version"));
|
||||
assert_eq!(headers.get("x-openai-fedramp"), Some(&"true".to_string()));
|
||||
for name in ["x-client-request-id", "session-id", "thread-id"] {
|
||||
@@ -699,9 +705,12 @@ fn preserves_client_context_headers_and_enforces_codex_provider_identity() {
|
||||
);
|
||||
assert_eq!(
|
||||
headers.get("user-agent").map(String::as_str),
|
||||
Some(aether_ai_formats::CODEX_CLIENT_USER_AGENT)
|
||||
Some(aether_ai_formats::codex_client_user_agent().as_str())
|
||||
);
|
||||
assert_eq!(
|
||||
headers.get("originator"),
|
||||
Some(&aether_ai_formats::codex_client_originator())
|
||||
);
|
||||
assert_eq!(headers.get("originator"), Some(&"codex_cli_rs".to_string()));
|
||||
assert_eq!(
|
||||
headers
|
||||
.keys()
|
||||
@@ -763,9 +772,12 @@ fn compact_projects_uuid_prompt_cache_identity_into_session_headers() {
|
||||
assert_eq!(headers.get("x-client-request-id"), None);
|
||||
assert_eq!(
|
||||
headers.get("user-agent").map(String::as_str),
|
||||
Some(aether_ai_formats::CODEX_CLIENT_USER_AGENT)
|
||||
Some(aether_ai_formats::codex_client_user_agent().as_str())
|
||||
);
|
||||
assert_eq!(
|
||||
headers.get("originator"),
|
||||
Some(&aether_ai_formats::codex_client_originator())
|
||||
);
|
||||
assert_eq!(headers.get("originator"), Some(&"codex_cli_rs".to_string()));
|
||||
assert!(!headers.contains_key("version"));
|
||||
assert_eq!(headers.get("x-openai-fedramp"), Some(&"true".to_string()));
|
||||
assert_eq!(
|
||||
|
||||
@@ -1,39 +1,21 @@
|
||||
use serde_json::{json, Value};
|
||||
|
||||
pub(crate) fn is_deepseek_provider(provider_type: &str, base_url: &str) -> bool {
|
||||
let provider_type = provider_type.trim().to_ascii_lowercase();
|
||||
if matches!(
|
||||
provider_type.as_str(),
|
||||
"deepseek" | "deepseek_openai" | "deepseek_anthropic" | "deepseek_compatible"
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
pub(crate) fn is_deepseek_provider(_provider_type: &str, base_url: &str) -> bool {
|
||||
let Some(host) = base_url_host(base_url) else {
|
||||
return false;
|
||||
};
|
||||
host == "deepseek.com" || host.ends_with(".deepseek.com")
|
||||
}
|
||||
|
||||
fn is_deepseek_model(provider_model: &str) -> bool {
|
||||
let provider_model = provider_model.trim().to_ascii_lowercase();
|
||||
let leaf = provider_model
|
||||
.rsplit(['/', ':'])
|
||||
.next()
|
||||
.unwrap_or(provider_model.as_str());
|
||||
leaf == "deepseek" || leaf.starts_with("deepseek-") || leaf.starts_with("deepseek_")
|
||||
}
|
||||
|
||||
fn is_deepseek_upstream(provider_type: &str, base_url: &str, provider_model: &str) -> bool {
|
||||
is_deepseek_provider(provider_type, base_url) || is_deepseek_model(provider_model)
|
||||
// 仅官方接口启用专用兼容;供应商类型和模型名称不能代表第三方接口的行为。
|
||||
host == "api.deepseek.com"
|
||||
}
|
||||
|
||||
pub(crate) fn openai_responses_reasoning_replay_policy(
|
||||
provider_type: &str,
|
||||
base_url: &str,
|
||||
provider_model: &str,
|
||||
_provider_model: &str,
|
||||
) -> crate::ai_serving::OpenAiResponsesReasoningReplayPolicy {
|
||||
if is_deepseek_upstream(provider_type, base_url, provider_model) {
|
||||
if provider_type.trim().eq_ignore_ascii_case("xai") {
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::XaiEncrypted
|
||||
} else if is_deepseek_provider(provider_type, base_url) {
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::DeepSeekOpaque
|
||||
} else {
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::OpenAiItemIds
|
||||
@@ -47,11 +29,7 @@ pub(crate) fn apply_deepseek_tool_call_thinking_compat(
|
||||
provider_api_format: &str,
|
||||
original_request_body: Option<&Value>,
|
||||
) {
|
||||
let provider_model = provider_request_body
|
||||
.get("model")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or_default();
|
||||
if !is_deepseek_upstream(provider_type, base_url, provider_model) {
|
||||
if !is_deepseek_provider(provider_type, base_url) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -155,6 +133,9 @@ fn apply_deepseek_openai_chat_thinking_compat(
|
||||
provider_request_body: &mut Value,
|
||||
original_request_body: Option<&Value>,
|
||||
) {
|
||||
// 携带 tools 时,所有历史 reasoning_content 都须完整回传,包括未调用工具的轮次。
|
||||
// 无 tools 时允许回传,且 prefix 续写需要保留输入;因此原样保留 messages,
|
||||
// 不删除思考内容,也不以空字符串冒充缺失内容,由上游校验请求是否完整。
|
||||
let disabled = source_disables_thinking(original_request_body, provider_request_body);
|
||||
set_deepseek_thinking_type(
|
||||
provider_request_body,
|
||||
@@ -170,33 +151,6 @@ fn apply_deepseek_openai_chat_thinking_compat(
|
||||
{
|
||||
object.remove("reasoning_effort");
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
let Some(messages) = object.get_mut("messages").and_then(Value::as_array_mut) else {
|
||||
return;
|
||||
};
|
||||
for message in messages {
|
||||
let Some(message_object) = message.as_object_mut() else {
|
||||
continue;
|
||||
};
|
||||
let is_assistant = message_object
|
||||
.get("role")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|role| role.trim().eq_ignore_ascii_case("assistant"));
|
||||
if !is_assistant {
|
||||
continue;
|
||||
}
|
||||
if message_object
|
||||
.get("reasoning_content")
|
||||
.is_some_and(|value| !value.is_null())
|
||||
{
|
||||
continue;
|
||||
}
|
||||
message_object.insert(
|
||||
"reasoning_content".to_string(),
|
||||
Value::String(String::new()),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -287,8 +241,29 @@ mod tests {
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn detects_deepseek_provider_by_type_or_host() {
|
||||
assert!(is_deepseek_provider(
|
||||
fn xai_reasoning_policy_comes_from_provider_type() {
|
||||
use crate::ai_serving::OpenAiResponsesReasoningReplayPolicy;
|
||||
assert_eq!(
|
||||
openai_responses_reasoning_replay_policy(
|
||||
"xai",
|
||||
"https://custom.example/v1",
|
||||
"grok-4.6"
|
||||
),
|
||||
OpenAiResponsesReasoningReplayPolicy::XaiEncrypted
|
||||
);
|
||||
assert_eq!(
|
||||
openai_responses_reasoning_replay_policy(
|
||||
"openai",
|
||||
"https://custom.example/v1",
|
||||
"grok-4.6"
|
||||
),
|
||||
OpenAiResponsesReasoningReplayPolicy::OpenAiItemIds
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_deepseek_provider_only_by_official_host() {
|
||||
assert!(!is_deepseek_provider(
|
||||
"deepseek",
|
||||
"https://relay.example.com"
|
||||
));
|
||||
@@ -298,6 +273,16 @@ mod tests {
|
||||
));
|
||||
assert!(is_deepseek_provider("custom", "api.deepseek.com/v1"));
|
||||
assert!(is_deepseek_provider("custom", "api.deepseek.com:443/v1"));
|
||||
assert!(!is_deepseek_provider("custom", "https://deepseek.com"));
|
||||
assert!(!is_deepseek_provider("custom", "deepseek.com/v1"));
|
||||
assert!(is_deepseek_provider(
|
||||
"custom",
|
||||
" HTTPS://API.DEEPSEEK.COM:443/beta "
|
||||
));
|
||||
assert!(!is_deepseek_provider(
|
||||
"deepseek",
|
||||
"https://other.deepseek.com/v1"
|
||||
));
|
||||
assert!(!is_deepseek_provider(
|
||||
"custom",
|
||||
"https://example.com/deepseek"
|
||||
@@ -319,6 +304,15 @@ mod tests {
|
||||
"https://evil.example/[email protected]"
|
||||
));
|
||||
assert!(!is_deepseek_provider("custom", "ftp://api.deepseek.com/v1"));
|
||||
assert!(!is_deepseek_provider("deepseek", ""));
|
||||
assert_eq!(
|
||||
openai_responses_reasoning_replay_policy(
|
||||
"deepseek",
|
||||
"https://deepseek.com/v1",
|
||||
"deepseek-chat",
|
||||
),
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::OpenAiItemIds
|
||||
);
|
||||
assert_eq!(
|
||||
openai_responses_reasoning_replay_policy(
|
||||
"custom",
|
||||
@@ -341,7 +335,7 @@ mod tests {
|
||||
"https://api.b.ai/v1",
|
||||
"deepseek-v4-flash",
|
||||
),
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::DeepSeekOpaque
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::OpenAiItemIds
|
||||
);
|
||||
assert_eq!(
|
||||
openai_responses_reasoning_replay_policy(
|
||||
@@ -353,6 +347,33 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn official_deepseek_host_enables_compat_without_type_or_model_hints() {
|
||||
for base_url in [
|
||||
"https://api.deepseek.com/v1",
|
||||
"https://api.deepseek.com/beta",
|
||||
] {
|
||||
let mut body = json!({
|
||||
"model": "mapped-model",
|
||||
"messages": [{"role": "assistant", "content": "answer"}]
|
||||
});
|
||||
|
||||
apply_deepseek_tool_call_thinking_compat(
|
||||
&mut body,
|
||||
"custom",
|
||||
base_url,
|
||||
"openai:chat",
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(body["thinking"]["type"], "enabled");
|
||||
assert_eq!(
|
||||
openai_responses_reasoning_replay_policy("custom", base_url, "mapped-model"),
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::DeepSeekOpaque
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn custom_deepseek_host_preserves_production_shaped_opaque_reasoning_replay() {
|
||||
let reasoning_items = (0..66)
|
||||
@@ -444,7 +465,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_chat_deepseek_adds_thinking_and_empty_reasoning_content() {
|
||||
fn openai_chat_deepseek_enables_thinking_without_fabricating_reasoning() {
|
||||
let mut body = json!({
|
||||
"model": "deepseek-chat",
|
||||
"messages": [
|
||||
@@ -467,11 +488,11 @@ mod tests {
|
||||
);
|
||||
|
||||
assert_eq!(body["thinking"]["type"], "enabled");
|
||||
assert_eq!(body["messages"][1]["reasoning_content"], "");
|
||||
assert!(body["messages"][1].get("reasoning_content").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn custom_relay_deepseek_model_adds_chat_thinking_compat() {
|
||||
fn custom_relay_deepseek_model_preserves_chat_request() {
|
||||
let mut body = json!({
|
||||
"model": "deepseek-v4-flash",
|
||||
"messages": [
|
||||
@@ -484,6 +505,7 @@ mod tests {
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "done"}
|
||||
]
|
||||
});
|
||||
let original = body.clone();
|
||||
|
||||
apply_deepseek_tool_call_thinking_compat(
|
||||
&mut body,
|
||||
@@ -493,8 +515,211 @@ mod tests {
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(body, original);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn third_party_hosts_ignore_deepseek_type_and_model_hints() {
|
||||
for provider_type in [
|
||||
"custom",
|
||||
"deepseek",
|
||||
"deepseek_openai",
|
||||
"deepseek_anthropic",
|
||||
"deepseek_compatible",
|
||||
] {
|
||||
for provider_model in [
|
||||
"other-model",
|
||||
"deepseek-chat",
|
||||
"deepseek-reasoner",
|
||||
"deepseek-v3",
|
||||
"deepseek-v4-flash",
|
||||
"vendor/deepseek-chat",
|
||||
"vendor:deepseek-reasoner",
|
||||
] {
|
||||
let base_url = "https://relay.example.com/v1";
|
||||
assert!(!is_deepseek_provider(provider_type, base_url));
|
||||
assert_eq!(
|
||||
openai_responses_reasoning_replay_policy(
|
||||
provider_type,
|
||||
base_url,
|
||||
provider_model
|
||||
),
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::OpenAiItemIds
|
||||
);
|
||||
|
||||
for api_format in ["openai:chat", "claude:messages"] {
|
||||
let original = json!({
|
||||
"model": provider_model,
|
||||
"messages": [{
|
||||
"role": "assistant",
|
||||
"content": "answer",
|
||||
"reasoning_content": "original plan"
|
||||
}]
|
||||
});
|
||||
let mut body = original.clone();
|
||||
|
||||
apply_deepseek_tool_call_thinking_compat(
|
||||
&mut body,
|
||||
provider_type,
|
||||
base_url,
|
||||
api_format,
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
body, original,
|
||||
"{provider_type} / {provider_model} / {api_format}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_chat_deepseek_preserves_history_without_tools() {
|
||||
let mut body = json!({
|
||||
"model": "deepseek-chat",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Compare 9.11 and 9.8"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "9.8 is greater",
|
||||
"reasoning_content": "Compare the decimal places.\n9.80 > 9.11."
|
||||
},
|
||||
{"role": "user", "content": "Explain again"},
|
||||
{"role": "assistant", "content": "Compare 9.80 with 9.11"}
|
||||
]
|
||||
});
|
||||
let messages = body["messages"].clone();
|
||||
|
||||
apply_deepseek_tool_call_thinking_compat(
|
||||
&mut body,
|
||||
"deepseek",
|
||||
"https://api.deepseek.com/v1",
|
||||
"openai:chat",
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(body["messages"], messages);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_chat_deepseek_preserves_reasoning_across_all_tool_turns() {
|
||||
let mut body = json!({
|
||||
"model": "deepseek-chat",
|
||||
"tools": [{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {"type": "object", "properties": {}}
|
||||
}
|
||||
}],
|
||||
"messages": [
|
||||
{"role": "user", "content": "What is the weather?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": null,
|
||||
"reasoning_content": "Check the weather before answering.\nKeep this full plan.",
|
||||
"tool_calls": [{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "arguments": "{}"}
|
||||
}]
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "Cloudy"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "It is cloudy",
|
||||
"reasoning_content": "The weather result is available; summarize it."
|
||||
},
|
||||
{"role": "user", "content": "Should I take an umbrella?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "An umbrella may be useful",
|
||||
"reasoning_content": "Use the previous weather result without another tool call."
|
||||
},
|
||||
{"role": "user", "content": "Why?"}
|
||||
]
|
||||
});
|
||||
let messages = body["messages"].clone();
|
||||
let tools = body["tools"].clone();
|
||||
|
||||
apply_deepseek_tool_call_thinking_compat(
|
||||
&mut body,
|
||||
"deepseek",
|
||||
"https://api.deepseek.com/v1",
|
||||
"openai:chat",
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(body["messages"], messages);
|
||||
assert_eq!(body["tools"], tools);
|
||||
assert_eq!(body["thinking"]["type"], "enabled");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_chat_deepseek_does_not_fabricate_missing_tool_reasoning() {
|
||||
for tools in [
|
||||
json!([]),
|
||||
json!([{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "lookup",
|
||||
"parameters": {"type": "object", "properties": {}}
|
||||
}
|
||||
}]),
|
||||
] {
|
||||
let mut body = json!({
|
||||
"model": "deepseek-chat",
|
||||
"tools": tools,
|
||||
"messages": [
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "assistant", "content": "missing"},
|
||||
{"role": "assistant", "content": "null", "reasoning_content": null},
|
||||
{"role": "assistant", "content": "empty", "reasoning_content": ""},
|
||||
{"role": "assistant", "content": "answer", "reasoning_content": "original plan"}
|
||||
]
|
||||
});
|
||||
let messages = body["messages"].clone();
|
||||
|
||||
apply_deepseek_tool_call_thinking_compat(
|
||||
&mut body,
|
||||
"deepseek",
|
||||
"https://api.deepseek.com/v1",
|
||||
"openai:chat",
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(body["messages"], messages);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_chat_deepseek_preserves_reasoning_prefix_without_tools() {
|
||||
let mut body = json!({
|
||||
"model": "deepseek-chat",
|
||||
"messages": [
|
||||
{"role": "user", "content": "What is 1 + 1?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"prefix": true,
|
||||
"content": "",
|
||||
"reasoning_content": "Start by adding one to one."
|
||||
}
|
||||
]
|
||||
});
|
||||
let messages = body["messages"].clone();
|
||||
|
||||
apply_deepseek_tool_call_thinking_compat(
|
||||
&mut body,
|
||||
"deepseek",
|
||||
"https://api.deepseek.com/beta",
|
||||
"openai:chat",
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(body["messages"], messages);
|
||||
assert_eq!(body["thinking"]["type"], "enabled");
|
||||
assert_eq!(body["messages"][1]["reasoning_content"], "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -4,7 +4,7 @@ use crate::ai_serving::transport::apply_standard_provider_request_body_rules_wit
|
||||
use crate::ai_serving::{
|
||||
apply_codex_openai_responses_chat_body_edits,
|
||||
apply_openai_responses_compact_special_body_edits,
|
||||
build_cross_format_openai_chat_request_body_with_model_directives as surface_build_cross_format_openai_chat_request_body,
|
||||
build_cross_format_openai_chat_request_body_with_provider_context as surface_build_cross_format_openai_chat_request_body,
|
||||
build_local_openai_chat_request_body_with_model_directives as surface_build_local_openai_chat_request_body,
|
||||
GatewayProviderTransportSnapshot,
|
||||
};
|
||||
@@ -73,9 +73,11 @@ pub(crate) fn build_cross_format_openai_chat_request_body(
|
||||
let provider_request_body = surface_build_cross_format_openai_chat_request_body(
|
||||
body_json,
|
||||
mapped_model,
|
||||
provider_type,
|
||||
provider_api_format,
|
||||
upstream_is_stream,
|
||||
enable_model_directives,
|
||||
user_api_key_id,
|
||||
)?;
|
||||
let mut provider_request_body =
|
||||
apply_standard_provider_request_body_rules_with_request_headers(
|
||||
@@ -125,3 +127,39 @@ pub(crate) fn build_cross_format_openai_chat_upstream_url(
|
||||
parts.uri.query(),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod antigravity_schema_tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn antigravity_chat_route_preserves_tool_schema_and_alternate_responses_shape() {
|
||||
let schema = json!({"type": "object", "properties": {"mode": {"const": "fast"}}});
|
||||
let body = json!({"model": "client", "messages": [{"role": "user", "content": "hi"}],
|
||||
"tools": [{"type": "function", "function": {"name": "probe", "parameters": schema}}]});
|
||||
let responses_body = json!({"model": "client", "input": "hi",
|
||||
"tools": [{"type": "function", "name": "probe", "parameters": schema}]});
|
||||
for input in [body, responses_body] {
|
||||
for provider in ["antigravity", "gemini"] {
|
||||
let output = build_cross_format_openai_chat_request_body(
|
||||
&input,
|
||||
"claude-test",
|
||||
provider,
|
||||
"gemini:generate_content",
|
||||
true,
|
||||
false,
|
||||
None,
|
||||
None,
|
||||
&http::HeaderMap::new(),
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
let parameters = &output["tools"][0]["functionDeclarations"][0]["parameters"];
|
||||
assert_eq!(parameters == &schema, provider == "antigravity");
|
||||
assert!(output.get("stream").is_none());
|
||||
assert_eq!(output["contents"][0]["parts"][0]["text"], "hi");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ use serde_json::Value;
|
||||
use crate::ai_serving::transport::apply_standard_provider_request_body_rules_with_request_headers;
|
||||
use crate::ai_serving::{
|
||||
apply_openai_responses_compact_special_body_edits,
|
||||
build_cross_format_openai_responses_request_body_with_model_directives_and_history_scope as surface_build_cross_format_openai_responses_request_body,
|
||||
build_cross_format_openai_responses_request_body_with_provider_context as surface_build_cross_format_openai_responses_request_body,
|
||||
build_local_openai_responses_request_body_with_model_directives as surface_build_local_openai_responses_request_body,
|
||||
GatewayProviderTransportSnapshot,
|
||||
};
|
||||
@@ -218,6 +218,7 @@ pub(crate) fn build_cross_format_openai_responses_request_body_with_codex_model_
|
||||
body_json,
|
||||
mapped_model,
|
||||
client_api_format,
|
||||
provider_type,
|
||||
provider_api_format,
|
||||
upstream_is_stream,
|
||||
enable_model_directives,
|
||||
@@ -291,3 +292,38 @@ pub(crate) fn build_cross_format_openai_responses_upstream_url(
|
||||
parts.uri.query(),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod antigravity_schema_tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn antigravity_responses_route_preserves_tool_schema_without_changing_public_gemini() {
|
||||
let schema = json!({"type": "object", "properties": {"mode": {"const": "fast"}}});
|
||||
let input = json!({"model": "client", "input": "hi",
|
||||
"tools": [{"type": "function", "name": "probe", "parameters": schema}]});
|
||||
for provider in ["antigravity", "gemini"] {
|
||||
let output =
|
||||
build_cross_format_openai_responses_request_body_with_codex_model_capabilities(
|
||||
&input,
|
||||
"claude-test",
|
||||
"openai:responses",
|
||||
"gemini:generate_content",
|
||||
true,
|
||||
false,
|
||||
provider,
|
||||
None,
|
||||
&http::HeaderMap::new(),
|
||||
Some("antigravity-schema-test"),
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
let parameters = &output["tools"][0]["functionDeclarations"][0]["parameters"];
|
||||
assert_eq!(parameters == &schema, provider == "antigravity");
|
||||
assert!(output.get("stream").is_none());
|
||||
assert_eq!(output["contents"][0]["parts"][0]["text"], "hi");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+13
-3
@@ -140,7 +140,7 @@ fn finalize_openai_chat_provider_request_body(
|
||||
mapped_model,
|
||||
source_model,
|
||||
);
|
||||
crate::ai_serving::finalize_openai_provider_request_with_codex_model_capabilities_and_reasoning_replay_policy(
|
||||
let finalization_failure = crate::ai_serving::finalize_openai_provider_request_with_codex_model_capabilities_and_reasoning_replay_policy(
|
||||
provider_request_body,
|
||||
crate::ai_serving::OpenAiProviderRequestFinalization {
|
||||
source_api_format: "openai:chat",
|
||||
@@ -170,7 +170,17 @@ fn finalize_openai_chat_provider_request_body(
|
||||
provider_api_format,
|
||||
"openai_chat_request_finalization",
|
||||
)
|
||||
})
|
||||
});
|
||||
if finalization_failure.is_none() {
|
||||
// This builder does not go through `apply_transport_request_body_semantics`, so the
|
||||
// Claude Code body mimicry must be applied here for Chat -> claude_code requests.
|
||||
crate::ai_serving::transport::claude_code::apply_claude_code_body_mimicry_for_transport(
|
||||
provider_request_body,
|
||||
transport,
|
||||
provider_api_format,
|
||||
);
|
||||
}
|
||||
finalization_failure
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
@@ -2763,7 +2773,7 @@ mod tests {
|
||||
payload.provider_request_body["userAgent"],
|
||||
"vscode/1.X.X (Antigravity/4.3.0)"
|
||||
);
|
||||
assert_eq!(payload.provider_request_body["requestType"], "agent");
|
||||
assert!(payload.provider_request_body.get("requestType").is_none());
|
||||
assert!(payload.provider_request_body.get("contents").is_none());
|
||||
assert!(payload.provider_request_body["request"]
|
||||
.get("contents")
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use aether_routing_core::RoutingExecutionPolicy;
|
||||
use aether_routing_core::{RoutingExecutionPolicy, RoutingSchedulingMode};
|
||||
use async_trait::async_trait;
|
||||
use std::collections::VecDeque;
|
||||
use tracing::warn;
|
||||
@@ -207,7 +207,12 @@ impl LocalOpenAiChatStreamAttemptSource<'_> {
|
||||
async fn next_raw_attempt_with_target_select(
|
||||
&mut self,
|
||||
) -> Result<Option<LocalOpenAiChatCandidateAttempt>, GatewayError> {
|
||||
let select_window = openai_chat_stream_target_select_window();
|
||||
let select_window = openai_chat_stream_target_select_window_for_mode(
|
||||
self.input
|
||||
.routing_policy
|
||||
.as_ref()
|
||||
.map(|policy| policy.scheduling_mode),
|
||||
);
|
||||
if select_window <= 1 {
|
||||
return self.next_raw_attempt_linear().await;
|
||||
}
|
||||
@@ -365,6 +370,15 @@ fn openai_chat_stream_target_select_window() -> usize {
|
||||
.clamp(1, MAX_OPENAI_CHAT_STREAM_TARGET_SELECT_WINDOW)
|
||||
}
|
||||
|
||||
fn openai_chat_stream_target_select_window_for_mode(
|
||||
scheduling_mode: Option<RoutingSchedulingMode>,
|
||||
) -> usize {
|
||||
if scheduling_mode == Some(RoutingSchedulingMode::FixedOrder) {
|
||||
return 1;
|
||||
}
|
||||
openai_chat_stream_target_select_window()
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct TargetSelectCandidateIdentity<'a> {
|
||||
provider_id: &'a str,
|
||||
@@ -574,4 +588,14 @@ mod tests {
|
||||
|
||||
assert_eq!(select_target_index(19, &choices), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fixed_order_disables_stream_target_selection() {
|
||||
assert_eq!(
|
||||
openai_chat_stream_target_select_window_for_mode(Some(
|
||||
RoutingSchedulingMode::FixedOrder,
|
||||
)),
|
||||
1
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+7
@@ -635,6 +635,13 @@ pub(crate) async fn resolve_local_openai_responses_candidate_payload_parts_with_
|
||||
{
|
||||
log_responses_to_chat_tool_conversion(trace_id, body_json, &base_provider_request_body);
|
||||
}
|
||||
// This builder does not go through `apply_transport_request_body_semantics`, so the
|
||||
// Claude Code body mimicry must be applied here for Responses -> claude_code requests.
|
||||
crate::ai_serving::transport::claude_code::apply_claude_code_body_mimicry_for_transport(
|
||||
&mut base_provider_request_body,
|
||||
&transport,
|
||||
provider_api_format,
|
||||
);
|
||||
let provider_request_body = base_provider_request_body;
|
||||
|
||||
if let Some(kiro_auth) = kiro_auth.as_ref() {
|
||||
|
||||
@@ -488,6 +488,7 @@ impl ResponsesWebSocketBodyNormalization {
|
||||
digest.update([match self.reasoning_replay_policy {
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::OpenAiItemIds => 0,
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::DeepSeekOpaque => 1,
|
||||
crate::ai_serving::OpenAiResponsesReasoningReplayPolicy::XaiEncrypted => 2,
|
||||
}]);
|
||||
update_normalization_optional_json_digest(&mut digest, self.model_directive_patch.as_ref());
|
||||
digest.finalize().into()
|
||||
|
||||
@@ -12,13 +12,16 @@ pub(crate) use aether_ai_formats::api::{
|
||||
apply_codex_openai_responses_websocket_continuation_body_edits_with_source_model_and_capabilities,
|
||||
apply_codex_openai_special_headers, apply_model_directive_mapping_patch,
|
||||
apply_model_directive_overrides_from_model, apply_model_directive_overrides_from_request,
|
||||
apply_openai_responses_compact_special_body_edits, build_chatgpt_web_image_request_body,
|
||||
apply_openai_responses_compact_special_body_edits, apply_xai_upstream_payload_edits,
|
||||
apply_xai_upstream_payload_edits_with_client, build_chatgpt_web_image_request_body,
|
||||
build_codex_model_catalog_metadata, build_codex_openai_image_api_provider_request_body,
|
||||
build_core_error_body_for_client_format, build_cross_format_openai_chat_request_body,
|
||||
build_cross_format_openai_chat_request_body_with_model_directives,
|
||||
build_cross_format_openai_chat_request_body_with_provider_context,
|
||||
build_cross_format_openai_responses_request_body,
|
||||
build_cross_format_openai_responses_request_body_with_model_directives,
|
||||
build_cross_format_openai_responses_request_body_with_model_directives_and_history_scope,
|
||||
build_cross_format_openai_responses_request_body_with_provider_context,
|
||||
build_gemini_image_request_body_from_openai_image_request,
|
||||
build_gemini_image_response_from_openai_image_response,
|
||||
build_gemini_image_response_from_openai_responses_image_response, build_generated_tool_call_id,
|
||||
@@ -160,7 +163,7 @@ pub(crate) use aether_ai_formats::api::{
|
||||
OPENAI_EMBEDDING_SYNC_PLAN_KIND, OPENAI_IMAGE_STREAM_PLAN_KIND,
|
||||
OPENAI_IMAGE_STREAM_SUCCESS_REPORT_KIND, OPENAI_IMAGE_SYNC_FINALIZE_REPORT_KIND,
|
||||
OPENAI_IMAGE_SYNC_PLAN_KIND, OPENAI_IMAGE_SYNC_SUCCESS_REPORT_KIND,
|
||||
OPENAI_REALTIME_STREAM_PLAN_KIND, OPENAI_RERANK_SYNC_PLAN_KIND,
|
||||
OPENAI_MEMORIES_SYNC_PLAN_KIND, OPENAI_REALTIME_STREAM_PLAN_KIND, OPENAI_RERANK_SYNC_PLAN_KIND,
|
||||
OPENAI_RESPONSES_COMPACT_STREAM_PLAN_KIND, OPENAI_RESPONSES_COMPACT_STREAM_SUCCESS_REPORT_KIND,
|
||||
OPENAI_RESPONSES_COMPACT_SYNC_ERROR_REPORT_KIND,
|
||||
OPENAI_RESPONSES_COMPACT_SYNC_FINALIZE_REPORT_KIND, OPENAI_RESPONSES_COMPACT_SYNC_PLAN_KIND,
|
||||
@@ -181,7 +184,7 @@ pub(crate) use aether_ai_formats::{
|
||||
openai_responses_request_operation, openai_responses_synthetic_reasoning_item_id,
|
||||
strip_incompatible_openai_responses_reasoning_items,
|
||||
strip_incompatible_openai_responses_reasoning_items_with_policy, ApiOperation, ClientSurface,
|
||||
CODEX_CLIENT_VERSION, OPENAI_RESPONSES_OPERATION_COMPACT,
|
||||
OPENAI_RESPONSES_OPERATION_COMPACT,
|
||||
};
|
||||
|
||||
pub(crate) fn plan_kind_matches_api_operation(
|
||||
|
||||
@@ -58,6 +58,10 @@ pub(crate) mod windsurf {
|
||||
pub(crate) use aether_provider_transport::windsurf::*;
|
||||
}
|
||||
|
||||
pub(crate) mod xai {
|
||||
pub(crate) use aether_provider_transport::xai::*;
|
||||
}
|
||||
|
||||
pub(crate) use aether_provider_transport::{
|
||||
append_transport_diagnostics_to_value, apply_codex_fingerprint_convergence,
|
||||
apply_codex_fingerprint_convergence_with_context, apply_local_auth_config_header_overrides,
|
||||
@@ -74,7 +78,7 @@ pub(crate) use aether_provider_transport::{
|
||||
build_local_openai_chat_upstream_url, build_local_openai_responses_upstream_url,
|
||||
build_openai_image_headers, build_openai_image_upstream_url, build_passthrough_headers,
|
||||
build_request_trace_proxy_value, build_same_format_provider_headers,
|
||||
build_same_format_provider_request_body,
|
||||
build_same_format_provider_request_body, build_same_format_provider_request_body_for_operation,
|
||||
build_same_format_provider_request_body_with_compatibility_report,
|
||||
build_same_format_provider_request_body_with_compatibility_report_and_reasoning_replay_policy,
|
||||
build_same_format_provider_upstream_url, build_standard_plan_fallback_headers,
|
||||
|
||||
@@ -53,6 +53,8 @@ const AI_ANY_ROUTE_PATTERNS: &[&str] = &[
|
||||
"/v1beta/operations/{*operation_path}",
|
||||
"/v1/videos",
|
||||
"/v1/videos/{*video_path}",
|
||||
"/openai/v1/videos",
|
||||
"/openai/v1/videos/{*video_path}",
|
||||
"/upload/v1beta/files",
|
||||
"/v1beta/files",
|
||||
"/v1beta/files/{*file_path}",
|
||||
|
||||
@@ -174,7 +174,7 @@ async fn authorize_operational_request(
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(error = ?err, "operational admin session authentication failed");
|
||||
warn!(error = %crate::error::redact_error_debug(&err), "operational admin session authentication failed");
|
||||
return operational_error_response(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"operational authentication unavailable",
|
||||
|
||||
@@ -536,6 +536,9 @@ mod tests {
|
||||
|
||||
fn sample_sparse_stored_task() -> StoredVideoTask {
|
||||
let snapshot = LocalVideoTaskSnapshot::OpenAi(OpenAiVideoTaskSeed {
|
||||
local_short_id: None,
|
||||
native_response: None,
|
||||
xai_provider: false,
|
||||
local_task_id: "task-1".to_string(),
|
||||
upstream_task_id: "ext-1".to_string(),
|
||||
created_at_unix_ms: 1,
|
||||
|
||||
@@ -799,6 +799,7 @@ mod tests {
|
||||
use aes_gcm::aead::{Aead, AeadCore, KeyInit, OsRng, Payload};
|
||||
use aes_gcm::Aes256Gcm;
|
||||
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
|
||||
use base64::Engine as _;
|
||||
use bytes::Bytes;
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde_json::json;
|
||||
@@ -1243,8 +1244,18 @@ mod tests {
|
||||
assert_eq!(restored.key_id, None);
|
||||
assert_eq!(restored.export_version.as_deref(), Some("2.3"));
|
||||
|
||||
// 17 个互不相同的合法 base64-32 字节直接密钥:本段只验证“legacy 候选 >16 → TooManyLegacyKeys”,
|
||||
// 不测口令强度、不解密。直接密钥走 decode_direct_fernet_key(生产已支持路径),跳过 PBKDF2,
|
||||
// 避免本用例为计数语义再付 17×10 万次迭代;上半段 DEVELOPMENT_ENCRYPTION_KEY 真实 v1 兼容
|
||||
// 与 wrong-legacy-secret 派生路径保持不变。
|
||||
let too_many: Vec<_> = (0..17)
|
||||
.map(|index| BackupDecryptionKey::historical(format!("legacy-{index}")).unwrap())
|
||||
.map(|index| {
|
||||
let mut material = [0u8; 32];
|
||||
material[0] = index as u8 + 1;
|
||||
material[31] = index as u8 + 1;
|
||||
let secret = base64::engine::general_purpose::STANDARD.encode(material);
|
||||
BackupDecryptionKey::historical(secret).unwrap()
|
||||
})
|
||||
.collect();
|
||||
assert!(matches!(
|
||||
restore_backup_json(
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
#[path = "support/responses_ws_probe.rs"]
|
||||
mod responses_ws_probe;
|
||||
|
||||
use aether_gateway::{CODEX_CLIENT_ORIGINATOR, CODEX_CLIENT_USER_AGENT};
|
||||
use aether_gateway::{codex_client_originator, codex_client_user_agent};
|
||||
use clap::Parser;
|
||||
use http::header::{AUTHORIZATION, USER_AGENT};
|
||||
use http::{HeaderMap, HeaderName, HeaderValue};
|
||||
@@ -78,14 +78,12 @@ fn handshake_headers(access_token: &str, account_id: &str) -> Result<HeaderMap,
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(AUTHORIZATION, bearer_authorization_value(access_token)?);
|
||||
headers.insert(HeaderName::from_static("chatgpt-account-id"), account_id);
|
||||
headers.insert(
|
||||
USER_AGENT,
|
||||
HeaderValue::from_static(CODEX_CLIENT_USER_AGENT),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("originator"),
|
||||
HeaderValue::from_static(CODEX_CLIENT_ORIGINATOR),
|
||||
);
|
||||
let user_agent = HeaderValue::from_str(&codex_client_user_agent())
|
||||
.map_err(|_| ProbeFailure::MissingConfiguration)?;
|
||||
headers.insert(USER_AGENT, user_agent);
|
||||
let originator = HeaderValue::from_str(&codex_client_originator())
|
||||
.map_err(|_| ProbeFailure::MissingConfiguration)?;
|
||||
headers.insert(HeaderName::from_static("originator"), originator);
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
@@ -111,6 +109,18 @@ mod tests {
|
||||
assert!(headers.contains_key("chatgpt-account-id"));
|
||||
assert!(headers.contains_key(USER_AGENT));
|
||||
assert!(headers.contains_key("originator"));
|
||||
assert_eq!(
|
||||
headers
|
||||
.get(USER_AGENT)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some(aether_gateway::codex_client_user_agent().as_str())
|
||||
);
|
||||
assert_eq!(
|
||||
headers
|
||||
.get("originator")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some(aether_gateway::codex_client_originator().as_str())
|
||||
);
|
||||
assert_eq!(
|
||||
CodexResponsesProbeProfile::sent_header_names(),
|
||||
vec![
|
||||
|
||||
Vendored
+2
@@ -4,6 +4,7 @@ mod auth_runtime;
|
||||
mod candidate_page;
|
||||
mod dashboard_response;
|
||||
mod direct_plan_bypass;
|
||||
mod overview_total;
|
||||
mod scheduler_affinity;
|
||||
mod system_config;
|
||||
|
||||
@@ -30,6 +31,7 @@ pub(crate) use candidate_page::{
|
||||
};
|
||||
pub(crate) use dashboard_response::DashboardResponseCache;
|
||||
pub(crate) use direct_plan_bypass::DirectPlanBypassCache;
|
||||
pub(crate) use overview_total::{OverviewTotalCache, OverviewTotalRead};
|
||||
pub(crate) use scheduler_affinity::{
|
||||
SchedulerAffinityCache, SchedulerAffinitySnapshotEntry, SchedulerAffinityTarget,
|
||||
};
|
||||
|
||||
+196
@@ -0,0 +1,196 @@
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use aether_data_contracts::repository::usage::StoredUsageDashboardAnalytics;
|
||||
|
||||
const FRESH_FOR: Duration = Duration::from_secs(5 * 60);
|
||||
const FAILURE_BACKOFF: Duration = Duration::from_secs(10);
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct OverviewTotalCache {
|
||||
state: Mutex<CacheState>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct CacheState {
|
||||
value: Option<(Instant, Arc<StoredUsageDashboardAnalytics>)>,
|
||||
refreshing: bool,
|
||||
retry_after: Option<Instant>,
|
||||
}
|
||||
|
||||
pub(crate) enum OverviewTotalRead {
|
||||
Pending,
|
||||
Failed,
|
||||
Ready {
|
||||
snapshot: Arc<StoredUsageDashboardAnalytics>,
|
||||
stale: bool,
|
||||
},
|
||||
}
|
||||
|
||||
/// Owns the single refresh slot even if the request that launched it disconnects.
|
||||
/// Dropping a cancelled or panicking worker also releases the slot with backoff.
|
||||
pub(crate) struct OverviewTotalRefresh {
|
||||
cache: Arc<OverviewTotalCache>,
|
||||
completed: bool,
|
||||
}
|
||||
|
||||
impl OverviewTotalCache {
|
||||
pub(crate) fn read(
|
||||
self: &Arc<Self>,
|
||||
now: Instant,
|
||||
) -> (OverviewTotalRead, Option<OverviewTotalRefresh>) {
|
||||
let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner());
|
||||
let fresh = state
|
||||
.value
|
||||
.as_ref()
|
||||
.is_some_and(|(at, _)| now.saturating_duration_since(*at) < FRESH_FOR);
|
||||
let retry_allowed = state.retry_after.is_none_or(|after| now >= after);
|
||||
let refresh = if !fresh && !state.refreshing && retry_allowed {
|
||||
state.refreshing = true;
|
||||
Some(OverviewTotalRefresh {
|
||||
cache: Arc::clone(self),
|
||||
completed: false,
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let result = match &state.value {
|
||||
Some((_, snapshot)) => OverviewTotalRead::Ready {
|
||||
snapshot: Arc::clone(snapshot),
|
||||
stale: !fresh,
|
||||
},
|
||||
None if state.refreshing => OverviewTotalRead::Pending,
|
||||
None => OverviewTotalRead::Failed,
|
||||
};
|
||||
(result, refresh)
|
||||
}
|
||||
}
|
||||
|
||||
impl OverviewTotalRefresh {
|
||||
pub(crate) fn finish(mut self, snapshot: Option<StoredUsageDashboardAnalytics>, now: Instant) {
|
||||
let mut state = self
|
||||
.cache
|
||||
.state
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner());
|
||||
state.refreshing = false;
|
||||
if let Some(snapshot) = snapshot {
|
||||
state.value = Some((now, Arc::new(snapshot)));
|
||||
state.retry_after = None;
|
||||
} else {
|
||||
state.retry_after = Some(now + FAILURE_BACKOFF);
|
||||
}
|
||||
self.completed = true;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for OverviewTotalRefresh {
|
||||
fn drop(&mut self) {
|
||||
if !self.completed {
|
||||
let mut state = self
|
||||
.cache
|
||||
.state
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner());
|
||||
state.refreshing = false;
|
||||
state.retry_after = Some(Instant::now() + FAILURE_BACKOFF);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn snapshot() -> StoredUsageDashboardAnalytics {
|
||||
let mut snapshot = StoredUsageDashboardAnalytics::default();
|
||||
snapshot.total.generated_at = "2026-09-18T00:00:00Z".into();
|
||||
snapshot.total.read_revision = "revision-1".into();
|
||||
snapshot.total.summary.request_count = 42;
|
||||
snapshot
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrent_cold_reads_claim_one_refresh() {
|
||||
let cache = Arc::new(OverviewTotalCache::default());
|
||||
let barrier = Arc::new(std::sync::Barrier::new(16));
|
||||
let now = Instant::now();
|
||||
let workers = (0..16)
|
||||
.map(|_| {
|
||||
let cache = Arc::clone(&cache);
|
||||
let barrier = Arc::clone(&barrier);
|
||||
std::thread::spawn(move || {
|
||||
barrier.wait();
|
||||
let (read, refresh) = cache.read(now);
|
||||
assert!(matches!(read, OverviewTotalRead::Pending));
|
||||
refresh
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let mut refreshes = workers
|
||||
.into_iter()
|
||||
.filter_map(|worker| worker.join().unwrap())
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(refreshes.len(), 1);
|
||||
refreshes.pop().unwrap().finish(Some(snapshot()), now);
|
||||
let (read, refresh) = cache.read(now);
|
||||
assert!(matches!(
|
||||
read,
|
||||
OverviewTotalRead::Ready { stale: false, .. }
|
||||
));
|
||||
assert!(refresh.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expiration_returns_original_snapshot_and_failed_refresh_preserves_it() {
|
||||
let cache = Arc::new(OverviewTotalCache::default());
|
||||
let now = Instant::now();
|
||||
cache.read(now).1.unwrap().finish(Some(snapshot()), now);
|
||||
assert!(cache
|
||||
.read(now + FRESH_FOR - Duration::from_secs(1))
|
||||
.1
|
||||
.is_none());
|
||||
let expired = now + FRESH_FOR;
|
||||
let (read, refresh) = cache.read(expired);
|
||||
let OverviewTotalRead::Ready {
|
||||
snapshot: old,
|
||||
stale: true,
|
||||
} = read
|
||||
else {
|
||||
panic!("expired success must remain visible")
|
||||
};
|
||||
assert_eq!(old.total.generated_at, "2026-09-18T00:00:00Z");
|
||||
assert_eq!(old.total.read_revision, "revision-1");
|
||||
assert!(cache.read(expired).1.is_none());
|
||||
refresh.unwrap().finish(None, expired);
|
||||
let (read, retry) = cache.read(expired + FAILURE_BACKOFF - Duration::from_secs(1));
|
||||
let OverviewTotalRead::Ready {
|
||||
snapshot: retained,
|
||||
stale: true,
|
||||
} = read
|
||||
else {
|
||||
panic!("failed refresh must retain stale success")
|
||||
};
|
||||
assert!(Arc::ptr_eq(&old, &retained));
|
||||
assert!(retry.is_none());
|
||||
assert!(cache.read(expired + FAILURE_BACKOFF).1.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cold_failure_and_worker_cancellation_back_off_before_retrying() {
|
||||
let cache = Arc::new(OverviewTotalCache::default());
|
||||
let now = Instant::now();
|
||||
cache.read(now).1.unwrap().finish(None, now);
|
||||
let (read, refresh) = cache.read(now + Duration::from_secs(9));
|
||||
assert!(matches!(read, OverviewTotalRead::Failed));
|
||||
assert!(refresh.is_none());
|
||||
let (read, refresh) = cache.read(now + FAILURE_BACKOFF);
|
||||
assert!(matches!(read, OverviewTotalRead::Pending));
|
||||
drop(refresh);
|
||||
let after_cancel = Instant::now();
|
||||
let (read, refresh) = cache.read(after_cancel);
|
||||
assert!(matches!(read, OverviewTotalRead::Failed));
|
||||
assert!(refresh.is_none());
|
||||
assert!(cache.read(after_cancel + FAILURE_BACKOFF).1.is_some());
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -126,6 +126,7 @@ pub(crate) const RUST_FRONTDOOR_OWNED_ROUTE_PATTERNS: &[&str] = &[
|
||||
"/v1/messages/count_tokens",
|
||||
"/v1/responses",
|
||||
"/v1/responses/compact",
|
||||
"/v1/memories/trace_summarize",
|
||||
"/v1/realtime",
|
||||
"/v1/realtime/calls",
|
||||
"/v1/live",
|
||||
@@ -140,6 +141,8 @@ pub(crate) const RUST_FRONTDOOR_OWNED_ROUTE_PATTERNS: &[&str] = &[
|
||||
"/v1beta/models/{model}/operations/{id}",
|
||||
"/v1beta/operations",
|
||||
"/v1beta/operations/{id}",
|
||||
"/openai/v1/videos",
|
||||
"/openai/v1/videos/{path...}",
|
||||
"/v1/videos",
|
||||
"/v1/videos/{path...}",
|
||||
"/upload/v1beta/files",
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use axum::body::Bytes;
|
||||
use axum::http::Uri;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use super::super::GatewayControlDecision;
|
||||
use super::credentials::{contains_string, extract_requested_model};
|
||||
@@ -230,8 +231,32 @@ pub(crate) async fn estimate_execution_plan_cost_upper_bound_usd(
|
||||
report_context: Option<&serde_json::Value>,
|
||||
) -> Result<Option<f64>, GatewayError> {
|
||||
let started_at = std::time::Instant::now();
|
||||
let result =
|
||||
estimate_execution_plan_cost_upper_bound_usd_inner(state, plan, report_context).await;
|
||||
let result = async {
|
||||
let multiplier_snapshot =
|
||||
aether_data_contracts::repository::usage::billing_multiplier_snapshot(report_context)
|
||||
.map_err(|error| GatewayError::Internal(error.to_string()))?;
|
||||
let estimate = estimate_execution_plan_cost_upper_bound_usd_inner(
|
||||
state,
|
||||
plan,
|
||||
report_context,
|
||||
multiplier_snapshot.is_some(),
|
||||
)
|
||||
.await?;
|
||||
let Some(snapshot) = multiplier_snapshot else {
|
||||
return Ok(estimate);
|
||||
};
|
||||
// Cache the unmultiplied base estimate so different request snapshots
|
||||
// cannot reuse one another's charge. Pricing validation still runs for
|
||||
// a zero multiplier, even when the request has no finite token bound.
|
||||
if snapshot.multiplier() == 0.0 {
|
||||
return Ok(Some(0.0));
|
||||
}
|
||||
estimate
|
||||
.map(|cost| snapshot.cost(cost))
|
||||
.transpose()
|
||||
.map_err(|error| GatewayError::Internal(error.to_string()))
|
||||
}
|
||||
.await;
|
||||
observe_gateway_stage_ms(
|
||||
"auth_capacity_cost_estimate",
|
||||
started_at.elapsed().as_millis() as u64,
|
||||
@@ -243,6 +268,7 @@ async fn estimate_execution_plan_cost_upper_bound_usd_inner(
|
||||
state: &AppState,
|
||||
plan: &aether_contracts::ExecutionPlan,
|
||||
report_context: Option<&serde_json::Value>,
|
||||
use_base_cost: bool,
|
||||
) -> Result<Option<f64>, GatewayError> {
|
||||
let api_format = crate::ai_serving::normalize_api_format_alias(&plan.provider_api_format);
|
||||
let body_json = plan.body.json_body.as_ref();
|
||||
@@ -310,7 +336,7 @@ async fn estimate_execution_plan_cost_upper_bound_usd_inner(
|
||||
if model_id.is_none() && global_model_name.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
let cache_key = execution_plan_cost_upper_bound_cache_key(
|
||||
let mut cache_key = execution_plan_cost_upper_bound_cache_key(
|
||||
plan,
|
||||
model_id,
|
||||
global_model_name,
|
||||
@@ -320,6 +346,11 @@ async fn estimate_execution_plan_cost_upper_bound_usd_inner(
|
||||
requested_processing_tier.as_deref(),
|
||||
cache_ttl_minutes,
|
||||
);
|
||||
if use_base_cost {
|
||||
// Legacy requests cache provider Key cost; new requests cache base cost.
|
||||
// These values must never share a cache entry for the same provider Key.
|
||||
cache_key.insert_str(0, "base\x1f");
|
||||
}
|
||||
let ttl = state.frontdoor_runtime_guards.auth_capacity_cache_ttl;
|
||||
if ttl.is_zero() {
|
||||
let _permit = state.acquire_auth_snapshot_load_gate().await?;
|
||||
@@ -334,6 +365,7 @@ async fn estimate_execution_plan_cost_upper_bound_usd_inner(
|
||||
max_output_tokens,
|
||||
requested_processing_tier.as_deref(),
|
||||
cache_ttl_minutes,
|
||||
use_base_cost,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -352,6 +384,7 @@ async fn estimate_execution_plan_cost_upper_bound_usd_inner(
|
||||
max_output_tokens,
|
||||
requested_processing_tier.as_deref(),
|
||||
cache_ttl_minutes,
|
||||
use_base_cost,
|
||||
)
|
||||
.await
|
||||
})
|
||||
@@ -370,6 +403,7 @@ async fn calculate_execution_plan_cost_upper_bound(
|
||||
max_output_tokens: Option<i64>,
|
||||
requested_processing_tier: Option<&str>,
|
||||
cache_ttl_minutes: Option<i64>,
|
||||
use_base_cost: bool,
|
||||
) -> Result<Option<f64>, GatewayError> {
|
||||
let context =
|
||||
load_execution_plan_billing_context(state, plan, model_id, global_model_name).await?;
|
||||
@@ -382,11 +416,13 @@ async fn calculate_execution_plan_cost_upper_bound(
|
||||
estimate.requested_processing_tier = requested_processing_tier.map(ToOwned::to_owned);
|
||||
estimate.cache_ttl_minutes = cache_ttl_minutes;
|
||||
estimate.max_output_tokens = max_output_tokens;
|
||||
let mut pricing = aether_billing::BillingModelPricingSnapshot::from(context);
|
||||
if use_base_cost {
|
||||
pricing.provider_billing_type = None;
|
||||
pricing.provider_api_key_rate_multipliers = None;
|
||||
}
|
||||
aether_billing::BillingService::new()
|
||||
.estimate_authorization_cost_upper_bound(
|
||||
&aether_billing::BillingModelPricingSnapshot::from(context),
|
||||
&estimate,
|
||||
)
|
||||
.estimate_authorization_cost_upper_bound(&pricing, &estimate)
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))
|
||||
}
|
||||
|
||||
@@ -747,6 +783,11 @@ async fn request_model_resolves_to_allowed_model(
|
||||
return Ok(false);
|
||||
};
|
||||
|
||||
// Global model names are a reserved routing namespace, so authorization has to
|
||||
// resolve a request the same way candidate planning will: a provider whose own
|
||||
// model carries the requested name only as an upstream alias must not make the
|
||||
// request resolve to that provider's global model.
|
||||
let mut reserved_global_model_names: BTreeMap<String, Option<String>> = BTreeMap::new();
|
||||
for api_format in candidate_api_formats_for_model_resolution(&client_api_format) {
|
||||
let resolution = decision
|
||||
.model_directive_policy
|
||||
@@ -762,23 +803,45 @@ async fn request_model_resolves_to_allowed_model(
|
||||
.list_minimal_candidate_selection_rows_for_api_format(&api_format)
|
||||
.await?
|
||||
};
|
||||
let reserved_global_model_name = match reserved_global_model_names.get(routing_model) {
|
||||
Some(cached) => cached.clone(),
|
||||
None => {
|
||||
let reserved_global_model_name =
|
||||
crate::data::candidate_selection::resolve_reserved_global_model_name(
|
||||
state.data.as_ref(),
|
||||
&rows,
|
||||
routing_model,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
reserved_global_model_names.insert(
|
||||
routing_model.to_string(),
|
||||
reserved_global_model_name.clone(),
|
||||
);
|
||||
reserved_global_model_name
|
||||
}
|
||||
};
|
||||
let matching_rows = rows
|
||||
.into_iter()
|
||||
.filter(|row| {
|
||||
aether_scheduler_core::row_supports_requested_model_with_model_directives(
|
||||
aether_scheduler_core::row_supports_requested_model_with_reserved_global_model(
|
||||
row,
|
||||
routing_model,
|
||||
&api_format,
|
||||
false,
|
||||
None,
|
||||
reserved_global_model_name.as_deref(),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let Some(resolved_global_model) =
|
||||
aether_scheduler_core::resolve_requested_global_model_name_with_model_directives(
|
||||
aether_scheduler_core::resolve_requested_global_model_name_with_reserved_global_model(
|
||||
&matching_rows,
|
||||
routing_model,
|
||||
&api_format,
|
||||
false,
|
||||
None,
|
||||
reserved_global_model_name.as_deref(),
|
||||
)
|
||||
else {
|
||||
continue;
|
||||
@@ -832,10 +895,10 @@ mod tests {
|
||||
use serde_json::json;
|
||||
|
||||
use super::{
|
||||
available_balance_capacity_usd, execution_plan_balance_capacity_rejection,
|
||||
execution_plan_cost_upper_bound_cache_key, max_output_tokens_from_request,
|
||||
openai_request_input_is_self_contained, output_choice_count_upper_bound,
|
||||
request_model_local_rejection, GatewayLocalAuthRejection,
|
||||
available_balance_capacity_usd, estimate_execution_plan_cost_upper_bound_usd,
|
||||
execution_plan_balance_capacity_rejection, execution_plan_cost_upper_bound_cache_key,
|
||||
max_output_tokens_from_request, openai_request_input_is_self_contained,
|
||||
output_choice_count_upper_bound, request_model_local_rejection, GatewayLocalAuthRejection,
|
||||
};
|
||||
use crate::control::{GatewayControlAuthContext, GatewayControlDecision};
|
||||
use crate::data::GatewayDataState;
|
||||
@@ -2070,6 +2133,166 @@ mod tests {
|
||||
assert_eq!(estimate, 6.5);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn charge_estimate_and_capacity_use_request_multiplier_without_key_cost_or_cache_leaks() {
|
||||
let context = billing_context_with_pricing(
|
||||
Some(json!({"tiers": [{
|
||||
"up_to": null,
|
||||
"input_price_per_1m": 0.0,
|
||||
"output_price_per_1m": 10.0
|
||||
}]})),
|
||||
None,
|
||||
Some(json!({"openai:chat": 2.0})),
|
||||
None,
|
||||
);
|
||||
let mut state = state_with_quota_and_wallet(quota_availability(15.0, false), context);
|
||||
Arc::make_mut(&mut state.frontdoor_runtime_guards).auth_capacity_cache_ttl =
|
||||
Duration::from_secs(60);
|
||||
let decision = decision_with_allowed_models(vec!["gpt-5".to_string()]);
|
||||
let plan = execution_plan(
|
||||
json!({"model": "gpt-5", "messages": [], "max_tokens": 1_000_000}),
|
||||
"openai:chat",
|
||||
);
|
||||
let legacy = billing_report_context();
|
||||
let mut discounted = legacy.clone();
|
||||
discounted["billing_multiplier_snapshot"] = json!({
|
||||
"version": 1,
|
||||
"factors": {"routing_group": 2.0, "promotion": 0.25},
|
||||
"multiplier": 0.5
|
||||
});
|
||||
let mut marked_up = legacy.clone();
|
||||
marked_up["billing_multiplier_snapshot"] = json!({
|
||||
"version": 1,
|
||||
"factors": {"routing_group": 3.0},
|
||||
"multiplier": 3.0
|
||||
});
|
||||
let mut legacy_group_snapshot = legacy.clone();
|
||||
legacy_group_snapshot["routing_group_billing_multiplier"] = json!(1.0);
|
||||
|
||||
// Reuse the same cache for legacy Key cost, independent request
|
||||
// multipliers, and the old group-only snapshot representation.
|
||||
for (report_context, expected) in [
|
||||
(&legacy, 20.0),
|
||||
(&discounted, 5.0),
|
||||
(&marked_up, 30.0),
|
||||
(&legacy_group_snapshot, 10.0),
|
||||
(&discounted, 5.0),
|
||||
(&legacy, 20.0),
|
||||
] {
|
||||
assert_eq!(
|
||||
estimate_execution_plan_cost_upper_bound_usd(&state, &plan, Some(report_context))
|
||||
.await
|
||||
.expect("charge estimate should resolve"),
|
||||
Some(expected)
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
execution_plan_balance_capacity_rejection(&state, &decision, &plan, Some(&discounted))
|
||||
.await
|
||||
.expect("discounted request capacity should resolve"),
|
||||
None
|
||||
);
|
||||
assert_eq!(
|
||||
execution_plan_balance_capacity_rejection(&state, &decision, &plan, Some(&marked_up))
|
||||
.await
|
||||
.expect("marked-up request capacity should resolve"),
|
||||
Some(GatewayLocalAuthRejection::BalanceDenied {
|
||||
remaining: Some(15.0)
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn charge_estimate_uses_base_price_when_provider_is_free_tier() {
|
||||
let context = billing_context_with_pricing(
|
||||
Some(json!({"tiers": [{
|
||||
"up_to": null,
|
||||
"input_price_per_1m": 0.0,
|
||||
"output_price_per_1m": 10.0
|
||||
}]})),
|
||||
None,
|
||||
Some(json!({"openai:chat": 0.0})),
|
||||
Some("free_tier"),
|
||||
);
|
||||
let state = state_with_quota_and_wallet(quota_availability(15.0, false), context);
|
||||
let plan = execution_plan(
|
||||
json!({"model": "gpt-5", "messages": [], "max_tokens": 1_000_000}),
|
||||
"openai:chat",
|
||||
);
|
||||
let mut report_context = billing_report_context();
|
||||
assert_eq!(
|
||||
estimate_execution_plan_cost_upper_bound_usd(&state, &plan, Some(&report_context))
|
||||
.await
|
||||
.expect("legacy free-tier estimate should resolve"),
|
||||
Some(0.0)
|
||||
);
|
||||
report_context["routing_group_billing_multiplier"] = json!(0.5);
|
||||
assert_eq!(
|
||||
estimate_execution_plan_cost_upper_bound_usd(&state, &plan, Some(&report_context))
|
||||
.await
|
||||
.expect("charge estimate should use the model base price"),
|
||||
Some(5.0)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn zero_charge_multiplier_bounds_unknown_cost_but_still_rejects_invalid_pricing() {
|
||||
let context = billing_context_with_pricing(
|
||||
Some(json!({"tiers": [{
|
||||
"up_to": null,
|
||||
"input_price_per_1m": 0.0,
|
||||
"output_price_per_1m": 10.0
|
||||
}]})),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
);
|
||||
let state = state_with_quota_and_wallet(quota_availability(0.0, false), context);
|
||||
let plan = execution_plan(json!({"model": "gpt-5", "messages": []}), "openai:chat");
|
||||
let mut report_context = billing_report_context();
|
||||
assert_eq!(
|
||||
estimate_execution_plan_cost_upper_bound_usd(&state, &plan, Some(&report_context))
|
||||
.await
|
||||
.expect("an unspecified output limit has no finite estimate"),
|
||||
None
|
||||
);
|
||||
report_context["billing_multiplier_snapshot"] = json!({
|
||||
"version": 1,
|
||||
"factors": {"routing_group": 0.0},
|
||||
"multiplier": 0.0
|
||||
});
|
||||
assert_eq!(
|
||||
estimate_execution_plan_cost_upper_bound_usd(&state, &plan, Some(&report_context))
|
||||
.await
|
||||
.expect("zero multiplier should bound the charge"),
|
||||
Some(0.0)
|
||||
);
|
||||
let invalid_context = billing_context_with_pricing(
|
||||
Some(json!({
|
||||
"tiers": [{"up_to": null, "input_price_per_1m": 1.0}],
|
||||
"processing_tiers": {
|
||||
"priority": {"tiers": [{}], "price_multiplier": -1.0}
|
||||
}
|
||||
})),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
);
|
||||
let invalid_state =
|
||||
state_with_quota_and_wallet(quota_availability(0.0, false), invalid_context);
|
||||
let invalid_plan = execution_plan(
|
||||
json!({"model": "gpt-5", "messages": [], "service_tier": "priority"}),
|
||||
"openai:chat",
|
||||
);
|
||||
assert!(estimate_execution_plan_cost_upper_bound_usd(
|
||||
&invalid_state,
|
||||
&invalid_plan,
|
||||
Some(&report_context)
|
||||
)
|
||||
.await
|
||||
.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn daily_quota_estimate_treats_free_tier_as_zero_cost() {
|
||||
let context = billing_context_with_pricing(
|
||||
|
||||
@@ -7,6 +7,35 @@ pub(super) fn classify_admin_basic_family_route(
|
||||
normalized_path: &str,
|
||||
normalized_path_no_trailing: &str,
|
||||
) -> Option<ClassifiedRoute> {
|
||||
let finance_path = normalized_path_no_trailing;
|
||||
if (method == http::Method::GET
|
||||
&& matches!(
|
||||
finance_path,
|
||||
"/api/admin/billing/provider-accounts" | "/api/admin/billing/provider-expenses"
|
||||
))
|
||||
|| (method == http::Method::POST && finance_path == "/api/admin/billing/provider-expenses")
|
||||
|| (method == http::Method::POST
|
||||
&& finance_path
|
||||
.strip_prefix("/api/admin/billing/provider-expenses/")
|
||||
.and_then(|v| v.strip_suffix("/void"))
|
||||
.is_some_and(|id| !id.is_empty() && !id.contains('/')))
|
||||
{
|
||||
return Some(classified(
|
||||
"admin_proxy",
|
||||
"billing_manage",
|
||||
if finance_path.ends_with("/provider-accounts") {
|
||||
"provider_accounts"
|
||||
} else if method == http::Method::GET {
|
||||
"provider_expenses"
|
||||
} else if finance_path.ends_with("/void") {
|
||||
"void_provider_expense"
|
||||
} else {
|
||||
"create_provider_expense"
|
||||
},
|
||||
"admin:billing",
|
||||
false,
|
||||
));
|
||||
}
|
||||
if method == http::Method::GET
|
||||
&& matches!(
|
||||
normalized_path,
|
||||
|
||||
@@ -6,6 +6,33 @@ pub(super) fn classify_admin_endpoints_family_route(
|
||||
method: &http::Method,
|
||||
normalized_path: &str,
|
||||
) -> Option<ClassifiedRoute> {
|
||||
if normalized_path == "/api/admin/endpoints/health/v2/publication"
|
||||
&& (method == http::Method::GET || method == http::Method::PUT)
|
||||
{
|
||||
return Some(classified(
|
||||
"admin_proxy",
|
||||
"endpoints_health",
|
||||
"health_v2_publication",
|
||||
"admin:endpoints_health",
|
||||
false,
|
||||
));
|
||||
}
|
||||
if method == http::Method::GET
|
||||
&& (matches!(
|
||||
normalized_path,
|
||||
"/api/admin/endpoints/health/v2/summary" | "/api/admin/endpoints/health/v2/objects"
|
||||
) || normalized_path
|
||||
.strip_prefix("/api/admin/endpoints/health/v2/objects/")
|
||||
.is_some_and(|id| !id.is_empty() && !id.contains('/')))
|
||||
{
|
||||
return Some(classified(
|
||||
"admin_proxy",
|
||||
"endpoints_health",
|
||||
"health_v2",
|
||||
"admin:endpoints_health",
|
||||
false,
|
||||
));
|
||||
}
|
||||
if method == http::Method::GET && normalized_path == "/api/admin/endpoints/health/summary" {
|
||||
Some(classified(
|
||||
"admin_proxy",
|
||||
|
||||
@@ -7,6 +7,15 @@ pub(super) fn classify_admin_observability_family_route(
|
||||
normalized_path: &str,
|
||||
normalized_path_no_trailing: &str,
|
||||
) -> Option<ClassifiedRoute> {
|
||||
if let Some(kind) = classify_overview_route(method, normalized_path_no_trailing) {
|
||||
return Some(classified(
|
||||
"admin_proxy",
|
||||
"overview_manage",
|
||||
kind,
|
||||
"admin:stats",
|
||||
false,
|
||||
));
|
||||
}
|
||||
if method == http::Method::POST
|
||||
&& matches!(
|
||||
normalized_path,
|
||||
@@ -596,6 +605,20 @@ pub(super) fn classify_admin_observability_family_route(
|
||||
"admin:stats",
|
||||
false,
|
||||
))
|
||||
} else if method == http::Method::GET
|
||||
&& matches!(
|
||||
normalized_path,
|
||||
"/api/admin/stats/leaderboard/user-groups"
|
||||
| "/api/admin/stats/leaderboard/user-groups/"
|
||||
)
|
||||
{
|
||||
Some(classified(
|
||||
"admin_proxy",
|
||||
"stats_manage",
|
||||
"leaderboard_user_groups",
|
||||
"admin:stats",
|
||||
false,
|
||||
))
|
||||
} else if method == http::Method::GET
|
||||
&& matches!(
|
||||
normalized_path,
|
||||
@@ -713,3 +736,32 @@ pub(super) fn classify_admin_observability_family_route(
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn classify_overview_route(method: &http::Method, path: &str) -> Option<&'static str> {
|
||||
if method != http::Method::GET {
|
||||
return None;
|
||||
}
|
||||
match path.strip_prefix("/api/admin/overview/")? {
|
||||
"dashboard" => Some("dashboard"),
|
||||
"dashboard/summary" => Some("dashboard_summary"),
|
||||
"dashboard/total" => Some("dashboard_total"),
|
||||
"dashboard/charts" => Some("dashboard_charts"),
|
||||
"summary" => Some("summary"),
|
||||
"timeseries" => Some("timeseries"),
|
||||
"breakdown" => Some("breakdown"),
|
||||
"users" => Some("users"),
|
||||
"consumption" => Some("consumption"),
|
||||
"costs" => Some("costs"),
|
||||
"operations/live" => Some("operations_live"),
|
||||
"operations/performance" => Some("operations_performance"),
|
||||
"operations/resources" => Some("operations_resources"),
|
||||
detail
|
||||
if detail
|
||||
.strip_prefix("users/")
|
||||
.is_some_and(|id| !id.is_empty() && !id.contains('/')) =>
|
||||
{
|
||||
Some("user_detail")
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -88,6 +88,11 @@ pub(super) fn classify_ai_public_route(
|
||||
true,
|
||||
))
|
||||
}
|
||||
} else if method == http::Method::POST && normalized_path == "/v1/memories/trace_summarize" {
|
||||
Some(
|
||||
classified("ai_public", "openai", "memories", "openai:responses", true)
|
||||
.with_api_operation(ApiOperation::OpenAiMemoriesSummarize),
|
||||
)
|
||||
} else if method == http::Method::POST && normalized_path == "/v1/alpha/search" {
|
||||
Some(classified(
|
||||
"ai_public",
|
||||
@@ -137,7 +142,11 @@ pub(super) fn classify_ai_public_route(
|
||||
.with_client_surface(detect_claude_client_surface(headers))
|
||||
.with_api_operation(ApiOperation::ClaudeMessagesCreate),
|
||||
)
|
||||
} else if normalized_path.starts_with("/v1/videos") {
|
||||
} else if normalized_path == "/v1/videos"
|
||||
|| normalized_path.starts_with("/v1/videos/")
|
||||
|| normalized_path == "/openai/v1/videos"
|
||||
|| normalized_path.starts_with("/openai/v1/videos/")
|
||||
{
|
||||
Some(classified(
|
||||
"ai_public",
|
||||
"openai",
|
||||
|
||||
@@ -146,6 +146,36 @@ pub(super) fn classify_public_support_route(
|
||||
"public:announcements",
|
||||
false,
|
||||
))
|
||||
} else if method == http::Method::GET
|
||||
&& (matches!(
|
||||
normalized_path,
|
||||
"/api/users/me/health/v2/summary" | "/api/users/me/health/v2/objects"
|
||||
) || normalized_path
|
||||
.strip_prefix("/api/users/me/health/v2/objects/")
|
||||
.is_some_and(|id| !id.is_empty() && !id.contains('/')))
|
||||
{
|
||||
Some(classified(
|
||||
"public_support",
|
||||
"health_user",
|
||||
"health_v2",
|
||||
"user:health",
|
||||
false,
|
||||
))
|
||||
} else if method == http::Method::GET
|
||||
&& (matches!(
|
||||
normalized_path,
|
||||
"/api/public/health/v2/summary" | "/api/public/health/v2/objects"
|
||||
) || normalized_path
|
||||
.strip_prefix("/api/public/health/v2/objects/")
|
||||
.is_some_and(|id| !id.is_empty() && !id.contains('/')))
|
||||
{
|
||||
Some(classified(
|
||||
"public_support",
|
||||
"public_catalog",
|
||||
"health_v2",
|
||||
"public:catalog",
|
||||
false,
|
||||
))
|
||||
} else if method == http::Method::GET
|
||||
&& matches!(
|
||||
normalized_path,
|
||||
@@ -273,6 +303,19 @@ pub(super) fn classify_public_support_route(
|
||||
"user:monitoring",
|
||||
false,
|
||||
))
|
||||
} else if method == http::Method::GET
|
||||
&& matches!(
|
||||
normalized_path,
|
||||
"/api/announcements/users/me" | "/api/announcements/users/me/"
|
||||
)
|
||||
{
|
||||
Some(classified(
|
||||
"public_support",
|
||||
"announcement_user",
|
||||
"list",
|
||||
"user:announcements",
|
||||
false,
|
||||
))
|
||||
} else if method == http::Method::GET
|
||||
&& matches!(
|
||||
normalized_path,
|
||||
@@ -594,6 +637,7 @@ pub(super) fn classify_public_support_route(
|
||||
| "/api/users/me/usage/interval-timeline"
|
||||
| "/api/users/me/usage/heatmap"
|
||||
| "/api/users/me/providers"
|
||||
| "/api/users/me/routing-groups"
|
||||
| "/api/users/me/available-models"
|
||||
| "/api/users/me/client-config"
|
||||
| "/api/users/me/endpoint-status"
|
||||
@@ -611,6 +655,7 @@ pub(super) fn classify_public_support_route(
|
||||
"/api/users/me/usage/interval-timeline" => "usage_interval_timeline",
|
||||
"/api/users/me/usage/heatmap" => "usage_heatmap",
|
||||
"/api/users/me/providers" => "providers",
|
||||
"/api/users/me/routing-groups" => "routing_groups",
|
||||
"/api/users/me/available-models" => "available_models",
|
||||
"/api/users/me/client-config" => "client_config",
|
||||
"/api/users/me/endpoint-status" => "endpoint_status",
|
||||
|
||||
@@ -203,3 +203,50 @@ fn admin_billing_plan_write_routes_buffer_request_body() {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_finance_routes_require_admin_billing_and_buffer_expense_input() {
|
||||
let headers = headers(&[]);
|
||||
for (method, path, kind) in [
|
||||
(
|
||||
http::Method::GET,
|
||||
"/api/admin/billing/provider-accounts",
|
||||
"provider_accounts",
|
||||
),
|
||||
(
|
||||
http::Method::GET,
|
||||
"/api/admin/billing/provider-expenses",
|
||||
"provider_expenses",
|
||||
),
|
||||
(
|
||||
http::Method::POST,
|
||||
"/api/admin/billing/provider-expenses",
|
||||
"create_provider_expense",
|
||||
),
|
||||
(
|
||||
http::Method::POST,
|
||||
"/api/admin/billing/provider-expenses/entry-1/void",
|
||||
"void_provider_expense",
|
||||
),
|
||||
] {
|
||||
let uri: Uri = path.parse().unwrap();
|
||||
let decision = classify_control_route(&method, &uri, &headers).unwrap();
|
||||
assert_eq!(decision.route_family.as_deref(), Some("billing_manage"));
|
||||
assert_eq!(decision.route_kind.as_deref(), Some(kind));
|
||||
assert_eq!(
|
||||
decision.auth_endpoint_signature.as_deref(),
|
||||
Some("admin:billing")
|
||||
);
|
||||
let context = GatewayPublicRequestContext::from_request_parts(
|
||||
"expense-test",
|
||||
&method,
|
||||
&uri,
|
||||
&headers,
|
||||
Some(decision),
|
||||
);
|
||||
assert_eq!(
|
||||
local_proxy_route_requires_buffered_body(&context),
|
||||
kind == "create_provider_expense"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use http::Uri;
|
||||
|
||||
use crate::control::management_token_required_permission;
|
||||
use crate::control::{management_token_required_permission, GatewayPublicRequestContext};
|
||||
use crate::handlers::shared::local_proxy_route_requires_buffered_body;
|
||||
|
||||
use super::{classify_control_route, headers};
|
||||
|
||||
@@ -206,6 +207,10 @@ fn classifies_admin_system_maintenance_write_routes_as_admin_proxy_route() {
|
||||
"/api/admin/system/important-notification/test",
|
||||
"important_notification_test",
|
||||
),
|
||||
(
|
||||
"/api/admin/system/cleanup/usage/manual",
|
||||
"cleanup_usage_manual",
|
||||
),
|
||||
("/api/admin/system/cleanup", "cleanup"),
|
||||
("/api/admin/system/purge/config", "purge_config"),
|
||||
("/api/admin/system/purge/users", "purge_users"),
|
||||
@@ -235,6 +240,28 @@ fn classifies_admin_system_maintenance_write_routes_as_admin_proxy_route() {
|
||||
Some("admin:system")
|
||||
);
|
||||
assert!(!decision.is_execution_runtime_candidate());
|
||||
|
||||
if matches!(
|
||||
expected_kind,
|
||||
"config_import"
|
||||
| "users_import"
|
||||
| "data_import"
|
||||
| "smtp_test"
|
||||
| "important_notification_test"
|
||||
| "cleanup_usage_manual"
|
||||
) {
|
||||
let context = GatewayPublicRequestContext::from_request_parts(
|
||||
"trace-system-maintenance-write",
|
||||
&http::Method::POST,
|
||||
&uri,
|
||||
&headers,
|
||||
Some(decision),
|
||||
);
|
||||
assert!(
|
||||
local_proxy_route_requires_buffered_body(&context),
|
||||
"POST {path} should buffer request body"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -303,6 +330,20 @@ fn classifies_admin_system_update_routes_as_admin_proxy_routes() {
|
||||
Some("admin:system")
|
||||
);
|
||||
assert!(!decision.is_execution_runtime_candidate());
|
||||
|
||||
if matches!(expected_kind, "prepare_update" | "apply_update") {
|
||||
let context = GatewayPublicRequestContext::from_request_parts(
|
||||
"trace-system-update-write",
|
||||
&method,
|
||||
&uri,
|
||||
&headers,
|
||||
Some(decision),
|
||||
);
|
||||
assert!(
|
||||
local_proxy_route_requires_buffered_body(&context),
|
||||
"{method} {path} should buffer request body"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,46 @@ use http::Uri;
|
||||
|
||||
use super::{classify_control_route, headers};
|
||||
|
||||
#[test]
|
||||
fn overview_routes_require_the_admin_stats_principal_and_get_method() {
|
||||
for (suffix, kind) in [
|
||||
("dashboard", "dashboard"),
|
||||
("dashboard/summary", "dashboard_summary"),
|
||||
("dashboard/total", "dashboard_total"),
|
||||
("dashboard/charts", "dashboard_charts"),
|
||||
("summary", "summary"),
|
||||
("timeseries", "timeseries"),
|
||||
("breakdown", "breakdown"),
|
||||
("users", "users"),
|
||||
("users/employee-1", "user_detail"),
|
||||
("consumption", "consumption"),
|
||||
("costs", "costs"),
|
||||
("operations/live", "operations_live"),
|
||||
("operations/performance", "operations_performance"),
|
||||
("operations/resources", "operations_resources"),
|
||||
] {
|
||||
for trailing in ["", "/"] {
|
||||
let uri: Uri = format!("/api/admin/overview/{suffix}{trailing}")
|
||||
.parse()
|
||||
.unwrap();
|
||||
let decision = classify_control_route(&http::Method::GET, &uri, &headers(&[])).unwrap();
|
||||
assert_eq!(decision.route_family.as_deref(), Some("overview_manage"));
|
||||
assert_eq!(decision.route_kind.as_deref(), Some(kind));
|
||||
assert_eq!(
|
||||
decision.auth_endpoint_signature.as_deref(),
|
||||
Some("admin:stats")
|
||||
);
|
||||
assert!(!decision.is_execution_runtime_candidate());
|
||||
let decision = classify_control_route(&http::Method::POST, &uri, &headers(&[]));
|
||||
assert!(
|
||||
decision.is_none_or(
|
||||
|decision| decision.route_family.as_deref() != Some("overview_manage")
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classifies_admin_stats_provider_quota_usage_as_admin_proxy_route() {
|
||||
let headers = headers(&[]);
|
||||
@@ -157,6 +197,28 @@ fn classifies_admin_stats_leaderboard_models_as_admin_proxy_route() {
|
||||
assert!(!decision.is_execution_runtime_candidate());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classifies_admin_stats_leaderboard_user_groups_as_admin_proxy_route() {
|
||||
let headers = headers(&[]);
|
||||
let uri: Uri = "/api/admin/stats/leaderboard/user-groups"
|
||||
.parse()
|
||||
.expect("uri should parse");
|
||||
let decision =
|
||||
classify_control_route(&http::Method::GET, &uri, &headers).expect("route should classify");
|
||||
|
||||
assert_eq!(decision.route_class.as_deref(), Some("admin_proxy"));
|
||||
assert_eq!(decision.route_family.as_deref(), Some("stats_manage"));
|
||||
assert_eq!(
|
||||
decision.route_kind.as_deref(),
|
||||
Some("leaderboard_user_groups")
|
||||
);
|
||||
assert_eq!(
|
||||
decision.auth_endpoint_signature.as_deref(),
|
||||
Some("admin:stats")
|
||||
);
|
||||
assert!(!decision.is_execution_runtime_candidate());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classifies_admin_stats_leaderboard_users_as_admin_proxy_route() {
|
||||
let headers = headers(&[]);
|
||||
|
||||
@@ -261,6 +261,27 @@ fn classifies_wallet_redeem_as_public_support_route() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classifies_personal_announcements_as_authenticated_user_route() {
|
||||
let headers = headers(&[]);
|
||||
for path in [
|
||||
"/api/announcements/users/me?limit=20&offset=0&unread_only=false",
|
||||
"/api/announcements/users/me/",
|
||||
] {
|
||||
let uri: Uri = path.parse().expect("uri should parse");
|
||||
let decision = classify_control_route(&http::Method::GET, &uri, &headers)
|
||||
.expect("route should classify");
|
||||
assert_eq!(decision.route_class.as_deref(), Some("public_support"));
|
||||
assert_eq!(decision.route_family.as_deref(), Some("announcement_user"));
|
||||
assert_eq!(decision.route_kind.as_deref(), Some("list"));
|
||||
assert_eq!(
|
||||
decision.auth_endpoint_signature.as_deref(),
|
||||
Some("user:announcements")
|
||||
);
|
||||
assert!(!decision.is_execution_runtime_candidate());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classifies_announcement_unread_count_as_public_support_route() {
|
||||
let headers = headers(&[]);
|
||||
@@ -440,6 +461,11 @@ fn classifies_users_me_routes_as_public_support_route() {
|
||||
"/api/users/me/available-models",
|
||||
"available_models",
|
||||
),
|
||||
(
|
||||
http::Method::GET,
|
||||
"/api/users/me/routing-groups",
|
||||
"routing_groups",
|
||||
),
|
||||
(
|
||||
http::Method::GET,
|
||||
"/api/users/me/vscodex/devices",
|
||||
|
||||
@@ -6,7 +6,7 @@ use aether_data_contracts::repository::candidate_selection::{
|
||||
use aether_scheduler_core::{
|
||||
auth_constraints_allow_api_format, collect_global_model_names_for_required_capability,
|
||||
enumerate_minimal_candidate_selection_with_model_directives, normalize_api_format,
|
||||
resolve_requested_global_model_name_with_model_directives,
|
||||
resolve_requested_global_model_name_with_reserved_global_model,
|
||||
row_supports_requested_model_with_model_directives, EnumerateMinimalCandidateSelectionInput,
|
||||
SchedulerAuthConstraints, SchedulerMinimalCandidateSelectionCandidate,
|
||||
};
|
||||
@@ -56,6 +56,37 @@ pub(crate) trait MinimalCandidateSelectionRowSource {
|
||||
&self,
|
||||
query: &StoredPoolKeyCandidateRowsQuery,
|
||||
) -> Result<Vec<StoredMinimalCandidateSelectionRow>, DataLayerError>;
|
||||
|
||||
/// Returns the canonical global model name when `model_name` is one, so the
|
||||
/// caller can keep provider-side aliases out of a request that names a
|
||||
/// global model. Sources without a global model reader answer `None`, which
|
||||
/// leaves matching unrestricted.
|
||||
async fn read_reserved_global_model_name(
|
||||
&self,
|
||||
_model_name: &str,
|
||||
) -> Result<Option<String>, DataLayerError> {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolves the reserved global model name for `routing_model`.
|
||||
///
|
||||
/// Rows already in hand answer the question for free whenever one of them is
|
||||
/// bound to a global model of that exact name; only a request that no local row
|
||||
/// claims as a global model needs the lookup, which keeps the extra read off the
|
||||
/// path every ordinary request takes.
|
||||
pub(crate) async fn resolve_reserved_global_model_name(
|
||||
source: &(impl MinimalCandidateSelectionRowSource + Sync),
|
||||
rows: &[StoredMinimalCandidateSelectionRow],
|
||||
routing_model: &str,
|
||||
) -> Result<Option<String>, DataLayerError> {
|
||||
if rows
|
||||
.iter()
|
||||
.any(|row| row.global_model_name == routing_model)
|
||||
{
|
||||
return Ok(Some(routing_model.to_string()));
|
||||
}
|
||||
source.read_reserved_global_model_name(routing_model).await
|
||||
}
|
||||
|
||||
pub(crate) const REQUESTED_MODEL_CANDIDATE_PAGE_SIZE: u32 = 256;
|
||||
@@ -102,12 +133,16 @@ pub(crate) async fn read_requested_model_rows(
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let reserved_global_model_name =
|
||||
resolve_reserved_global_model_name(state, &rows, requested_model_name).await?;
|
||||
let Some(resolved_global_model_name) =
|
||||
resolve_requested_global_model_name_with_model_directives(
|
||||
resolve_requested_global_model_name_with_reserved_global_model(
|
||||
&rows,
|
||||
requested_model_name,
|
||||
api_format,
|
||||
enable_model_directives,
|
||||
None,
|
||||
reserved_global_model_name.as_deref(),
|
||||
)
|
||||
else {
|
||||
return Ok(None);
|
||||
|
||||
@@ -1252,6 +1252,7 @@ impl GatewayDataState {
|
||||
// exists while avoiding an unbounded read during error compensation.
|
||||
let page = repository
|
||||
.list_admin_wallets(&aether_data::repository::wallet::AdminWalletListQuery {
|
||||
user_id: None,
|
||||
status: None,
|
||||
owner_type: Some("api_key".to_string()),
|
||||
limit: 1,
|
||||
|
||||
@@ -562,6 +562,30 @@ impl GatewayDataState {
|
||||
Ok(created)
|
||||
}
|
||||
|
||||
pub(crate) async fn create_provider_catalog_provider_in_routing_group(
|
||||
&self,
|
||||
provider: &StoredProviderCatalogProvider,
|
||||
shift_existing_priorities_from: Option<i32>,
|
||||
routing_group_id: &str,
|
||||
) -> Result<Option<StoredProviderCatalogProvider>, DataLayerError> {
|
||||
let created = match &self.provider_catalog_writer {
|
||||
Some(repository) => repository
|
||||
.create_provider_in_routing_group(
|
||||
provider,
|
||||
shift_existing_priorities_from,
|
||||
routing_group_id,
|
||||
)
|
||||
.await
|
||||
.map(Some),
|
||||
None => Ok(None),
|
||||
}?;
|
||||
if created.is_some() {
|
||||
self.clear_provider_catalog_cache();
|
||||
self.clear_routing_group_cache();
|
||||
}
|
||||
Ok(created)
|
||||
}
|
||||
|
||||
pub(crate) async fn update_provider_catalog_provider(
|
||||
&self,
|
||||
provider: &StoredProviderCatalogProvider,
|
||||
|
||||
@@ -185,6 +185,16 @@ impl MinimalCandidateSelectionRowSource for GatewayDataState {
|
||||
) -> Result<Vec<StoredMinimalCandidateSelectionRow>, DataLayerError> {
|
||||
self.list_pool_key_candidate_rows_for_group(query).await
|
||||
}
|
||||
|
||||
async fn read_reserved_global_model_name(
|
||||
&self,
|
||||
model_name: &str,
|
||||
) -> Result<Option<String>, DataLayerError> {
|
||||
Ok(self
|
||||
.get_public_global_model_by_name(model_name)
|
||||
.await?
|
||||
.map(|global_model| global_model.name))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
|
||||
@@ -62,8 +62,9 @@ pub(crate) use aether_data::repository::users::{
|
||||
StoredUserPreferenceRecord, StoredUserSessionRecord,
|
||||
};
|
||||
use aether_data::repository::wallet::{
|
||||
AdjustWalletBalanceInput, AdminPaymentOrderListQuery, AdminRedeemCodeBatchListQuery,
|
||||
AdminRedeemCodeListQuery, AdminWalletLedgerQuery, AdminWalletListQuery,
|
||||
AdjustWalletBalanceInBatchInput, AdjustWalletBalanceInput, AdminPaymentOrderListQuery,
|
||||
AdminRedeemCodeBatchListQuery, AdminRedeemCodeListQuery,
|
||||
AdminUserWalletBalanceBatchUserOutcome, AdminWalletLedgerQuery, AdminWalletListQuery,
|
||||
AdminWalletRefundRequestListQuery, CompareAndSwapPaymentOrderStripeClientSecretInput,
|
||||
CompleteAdminWalletRefundInput, CreateAdminRedeemCodeBatchInput,
|
||||
CreateAdminRedeemCodeBatchResult, CreateManualWalletRechargeInput,
|
||||
@@ -72,13 +73,14 @@ use aether_data::repository::wallet::{
|
||||
CreateWalletRefundRequestOutcome, CreditAdminPaymentOrderInput,
|
||||
DeleteAdminRedeemCodeBatchInput, DisableAdminRedeemCodeBatchInput, DisableAdminRedeemCodeInput,
|
||||
FailAdminWalletRefundInput, FailWalletRechargeCheckoutInput, InitializeAuthWalletOutcome,
|
||||
PrepareAdminUserWalletBalanceBatchInput, PrepareAdminUserWalletBalanceBatchOutcome,
|
||||
ProcessAdminWalletRefundInput, ProcessPaymentCallbackInput, ProcessPaymentCallbackOutcome,
|
||||
ReclaimWalletRechargeCheckoutInput, RedeemWalletCodeInput, RedeemWalletCodeOutcome,
|
||||
StoredAdminPaymentCallback, StoredAdminPaymentCallbackPage, StoredAdminPaymentOrder,
|
||||
StoredAdminPaymentOrderPage, StoredAdminRedeemCode, StoredAdminRedeemCodeBatch,
|
||||
StoredAdminRedeemCodeBatchPage, StoredAdminRedeemCodePage, StoredAdminWalletLedgerPage,
|
||||
StoredAdminWalletListPage, StoredAdminWalletRefund, StoredAdminWalletRefundPage,
|
||||
StoredAdminWalletRefundRequestPage, StoredAdminWalletTransaction,
|
||||
StoredAdminRedeemCodeBatchPage, StoredAdminRedeemCodePage, StoredAdminUserWalletBalanceBatch,
|
||||
StoredAdminWalletLedgerPage, StoredAdminWalletListPage, StoredAdminWalletRefund,
|
||||
StoredAdminWalletRefundPage, StoredAdminWalletRefundRequestPage, StoredAdminWalletTransaction,
|
||||
StoredAdminWalletTransactionPage, StoredWalletDailyUsageLedger,
|
||||
StoredWalletDailyUsageLedgerPage, StoredWalletSnapshot, UpdateAdminWalletRefundGatewayInput,
|
||||
UpdateWalletRechargeCheckoutInput, WalletLookupKey, WalletMutationOutcome,
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
use super::{
|
||||
read_decision_trace, read_provider_transport_snapshot, read_request_candidate_trace,
|
||||
AdjustWalletBalanceInput, AdminBillingCollectorRecord, AdminBillingCollectorWriteInput,
|
||||
AdminBillingMutationOutcome, AdminBillingPresetApplyResult, AdminBillingRuleRecord,
|
||||
AdminBillingRuleWriteInput, AdminPaymentOrderListQuery, AdminRedeemCodeBatchListQuery,
|
||||
AdminRedeemCodeListQuery, AdminWalletLedgerQuery, AdminWalletListQuery,
|
||||
AdjustWalletBalanceInBatchInput, AdjustWalletBalanceInput, AdminBillingCollectorRecord,
|
||||
AdminBillingCollectorWriteInput, AdminBillingMutationOutcome, AdminBillingPresetApplyResult,
|
||||
AdminBillingRuleRecord, AdminBillingRuleWriteInput, AdminPaymentOrderListQuery,
|
||||
AdminRedeemCodeBatchListQuery, AdminRedeemCodeListQuery,
|
||||
AdminUserWalletBalanceBatchUserOutcome, AdminWalletLedgerQuery, AdminWalletListQuery,
|
||||
AdminWalletRefundRequestListQuery, AnnouncementListQuery, AuditLogListQuery,
|
||||
BackgroundTaskListQuery, BackgroundTaskSummary, BillingModelContextCacheKey,
|
||||
BillingModelContextCacheState, BillingModelContextInflightState, BillingPlanRecord,
|
||||
@@ -17,30 +18,34 @@ use super::{
|
||||
DisableAdminRedeemCodeBatchInput, DisableAdminRedeemCodeInput, FailAdminWalletRefundInput,
|
||||
FailWalletRechargeCheckoutInput, GatewayDataState, GatewayProviderTransportSnapshot,
|
||||
LocalVideoTaskReadResponse, PaymentGatewayConfigCasWriteInput, PaymentGatewayConfigRecord,
|
||||
PaymentGatewayConfigWriteInput, PaymentGatewaySecretCasUpdate, ProcessAdminWalletRefundInput,
|
||||
ProcessPaymentCallbackInput, ProcessPaymentCallbackOutcome, ReclaimWalletRechargeCheckoutInput,
|
||||
ReconcileUsagePolicyCostInput, RedeemWalletCodeInput, RedeemWalletCodeOutcome,
|
||||
ReleaseUsagePolicyRequestAdmissionInput, RequestAuditBundle, RequestCandidateTrace,
|
||||
ReserveUsagePolicyCostInput, ReserveUsagePolicyCostOutcome, ReserveUsagePolicyRequestInput,
|
||||
ReserveUsagePolicyRequestOutcome, StoredAdminAuditLogPage, StoredAdminPaymentCallbackPage,
|
||||
StoredAdminPaymentOrder, StoredAdminPaymentOrderPage, StoredAdminRedeemCodeBatch,
|
||||
StoredAdminRedeemCodeBatchPage, StoredAdminRedeemCodePage, StoredAdminWalletLedgerPage,
|
||||
StoredAdminWalletListPage, StoredAdminWalletRefund, StoredAdminWalletRefundPage,
|
||||
StoredAdminWalletRefundRequestPage, StoredAdminWalletTransaction,
|
||||
StoredAdminWalletTransactionPage, StoredAnnouncement, StoredAnnouncementPage,
|
||||
StoredBackgroundTaskEvent, StoredBackgroundTaskRun, StoredBackgroundTaskRunPage,
|
||||
StoredBillingModelContext, StoredProviderQuotaSnapshot, StoredProviderUsageSummary,
|
||||
StoredRequestUsageAudit, StoredSuspiciousActivity, StoredUsagePolicyCostReservation,
|
||||
StoredUsagePolicyRequestAdmission, StoredUsageSettlement, StoredUserAuditLogPage,
|
||||
StoredUserAuthRecord, StoredUserExportRow, StoredUserSummary, StoredVideoTask,
|
||||
StoredWalletDailyUsageLedger, StoredWalletDailyUsageLedgerPage, StoredWalletSnapshot,
|
||||
UpdateAdminWalletRefundGatewayInput, UpdateAnnouncementRecord,
|
||||
PaymentGatewayConfigWriteInput, PaymentGatewaySecretCasUpdate,
|
||||
PrepareAdminUserWalletBalanceBatchInput, PrepareAdminUserWalletBalanceBatchOutcome,
|
||||
ProcessAdminWalletRefundInput, ProcessPaymentCallbackInput, ProcessPaymentCallbackOutcome,
|
||||
ReclaimWalletRechargeCheckoutInput, ReconcileUsagePolicyCostInput, RedeemWalletCodeInput,
|
||||
RedeemWalletCodeOutcome, ReleaseUsagePolicyRequestAdmissionInput, RequestAuditBundle,
|
||||
RequestCandidateTrace, ReserveUsagePolicyCostInput, ReserveUsagePolicyCostOutcome,
|
||||
ReserveUsagePolicyRequestInput, ReserveUsagePolicyRequestOutcome, StoredAdminAuditLogPage,
|
||||
StoredAdminPaymentCallbackPage, StoredAdminPaymentOrder, StoredAdminPaymentOrderPage,
|
||||
StoredAdminRedeemCodeBatch, StoredAdminRedeemCodeBatchPage, StoredAdminRedeemCodePage,
|
||||
StoredAdminUserWalletBalanceBatch, StoredAdminWalletLedgerPage, StoredAdminWalletListPage,
|
||||
StoredAdminWalletRefund, StoredAdminWalletRefundPage, StoredAdminWalletRefundRequestPage,
|
||||
StoredAdminWalletTransaction, StoredAdminWalletTransactionPage, StoredAnnouncement,
|
||||
StoredAnnouncementPage, StoredBackgroundTaskEvent, StoredBackgroundTaskRun,
|
||||
StoredBackgroundTaskRunPage, StoredBillingModelContext, StoredProviderQuotaSnapshot,
|
||||
StoredProviderUsageSummary, StoredRequestUsageAudit, StoredSuspiciousActivity,
|
||||
StoredUsagePolicyCostReservation, StoredUsagePolicyRequestAdmission, StoredUsageSettlement,
|
||||
StoredUserAuditLogPage, StoredUserAuthRecord, StoredUserExportRow, StoredUserSummary,
|
||||
StoredVideoTask, StoredWalletDailyUsageLedger, StoredWalletDailyUsageLedgerPage,
|
||||
StoredWalletSnapshot, UpdateAdminWalletRefundGatewayInput, UpdateAnnouncementRecord,
|
||||
UpdateWalletRechargeCheckoutInput, UpsertBackgroundTaskEvent, UpsertBackgroundTaskRun,
|
||||
UpsertUsageRecord, UpsertVideoTask, UsageSettlementInput, UserDailyQuotaAvailabilityRecord,
|
||||
UserPlanEntitlementRecord, VideoTaskLookupKey, VideoTaskModelCount, VideoTaskQueryFilter,
|
||||
VideoTaskStatusCount, WalletDailyUsageAggregationInput, WalletDailyUsageAggregationResult,
|
||||
WalletLookupKey, WalletMutationOutcome,
|
||||
};
|
||||
use aether_data_contracts::repository::billing::{
|
||||
ProviderExpenseInput, ProviderExpensePage, ProviderExpenseQuery, ProviderExpenseRecord,
|
||||
};
|
||||
use aether_data_contracts::repository::usage::{
|
||||
PendingUsageCleanupSummary, ProviderApiKeyWindowUsageRequest,
|
||||
StoredProviderApiKeyWindowUsageSummary, StoredUsageDailySummary, UsageAuditListQuery,
|
||||
@@ -364,6 +369,26 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn rebuild_overview_buckets(
|
||||
&self,
|
||||
input: &aether_data::StatsHourlyAggregationInput,
|
||||
) -> Result<usize, DataLayerError> {
|
||||
match &self.backends {
|
||||
Some(backends) => backends.rebuild_overview_buckets(input).await,
|
||||
None => Ok(0),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn drain_overview_dirty_events(
|
||||
&self,
|
||||
now: chrono::DateTime<chrono::Utc>,
|
||||
) -> Result<u64, DataLayerError> {
|
||||
match &self.backends {
|
||||
Some(backends) => backends.drain_overview_dirty_events(now).await,
|
||||
None => Ok(0),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn aggregate_stats_daily(
|
||||
&self,
|
||||
input: &aether_data::StatsDailyAggregationInput,
|
||||
@@ -384,6 +409,18 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn list_user_announcements(
|
||||
&self,
|
||||
user_id: &str,
|
||||
query: &aether_data::repository::announcements::UserAnnouncementListQuery,
|
||||
) -> Result<aether_data::repository::announcements::StoredUserAnnouncementPage, DataLayerError>
|
||||
{
|
||||
match &self.announcement_reader {
|
||||
Some(repository) => repository.list_user_announcements(user_id, query).await,
|
||||
None => Ok(Default::default()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn find_announcement_by_id(
|
||||
&self,
|
||||
announcement_id: &str,
|
||||
@@ -1066,13 +1103,81 @@ impl GatewayDataState {
|
||||
pub(crate) async fn adjust_wallet_balance(
|
||||
&self,
|
||||
input: AdjustWalletBalanceInput,
|
||||
) -> Result<Option<(StoredWalletSnapshot, StoredAdminWalletTransaction)>, DataLayerError> {
|
||||
) -> Result<Option<(StoredWalletSnapshot, Option<StoredAdminWalletTransaction>)>, DataLayerError>
|
||||
{
|
||||
match &self.wallet_writer {
|
||||
Some(repository) => repository.adjust_wallet_balance(input).await,
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_admin_user_wallet_balance_batch(
|
||||
&self,
|
||||
input: PrepareAdminUserWalletBalanceBatchInput,
|
||||
) -> Result<Option<PrepareAdminUserWalletBalanceBatchOutcome>, DataLayerError> {
|
||||
match &self.wallet_writer {
|
||||
Some(repository) => repository
|
||||
.prepare_admin_user_wallet_balance_batch(input)
|
||||
.await
|
||||
.map(Some),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn get_admin_user_wallet_balance_batch(
|
||||
&self,
|
||||
admin_user_id: &str,
|
||||
idempotency_key: &str,
|
||||
request_fingerprint: &str,
|
||||
) -> Result<Option<PrepareAdminUserWalletBalanceBatchOutcome>, DataLayerError> {
|
||||
match &self.wallet_writer {
|
||||
Some(repository) => {
|
||||
repository
|
||||
.get_admin_user_wallet_balance_batch(
|
||||
admin_user_id,
|
||||
idempotency_key,
|
||||
request_fingerprint,
|
||||
)
|
||||
.await
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn adjust_admin_user_wallet_balance_batch_user(
|
||||
&self,
|
||||
input: AdjustWalletBalanceInBatchInput,
|
||||
) -> Result<Option<AdminUserWalletBalanceBatchUserOutcome>, DataLayerError> {
|
||||
match &self.wallet_writer {
|
||||
Some(repository) => repository
|
||||
.adjust_admin_user_wallet_balance_batch_user(input)
|
||||
.await
|
||||
.map(Some),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn record_admin_user_wallet_balance_batch_failure(
|
||||
&self,
|
||||
admin_user_id: &str,
|
||||
idempotency_key: &str,
|
||||
user_id: &str,
|
||||
reason: &str,
|
||||
) -> Result<Option<AdminUserWalletBalanceBatchUserOutcome>, DataLayerError> {
|
||||
match &self.wallet_writer {
|
||||
Some(repository) => repository
|
||||
.record_admin_user_wallet_balance_batch_failure(
|
||||
admin_user_id,
|
||||
idempotency_key,
|
||||
user_id,
|
||||
reason,
|
||||
)
|
||||
.await
|
||||
.map(Some),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn create_manual_wallet_recharge(
|
||||
&self,
|
||||
input: CreateManualWalletRechargeInput,
|
||||
@@ -1659,6 +1764,60 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn query_dashboard_summary(
|
||||
&self,
|
||||
query: &aether_data_contracts::repository::usage::UsageDashboardAnalyticsQuery,
|
||||
) -> Result<aether_data_contracts::repository::usage::StoredDashboardSummary, DataLayerError>
|
||||
{
|
||||
match &self.usage_reader {
|
||||
Some(repository) => repository.query_dashboard_summary(query).await,
|
||||
None => Err(DataLayerError::InvalidInput(
|
||||
"dashboard summary repository is unavailable".into(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn query_dashboard_analytics(
|
||||
&self,
|
||||
query: &aether_data_contracts::repository::usage::UsageDashboardAnalyticsQuery,
|
||||
) -> Result<
|
||||
aether_data_contracts::repository::usage::StoredUsageDashboardAnalytics,
|
||||
DataLayerError,
|
||||
> {
|
||||
match &self.usage_reader {
|
||||
Some(repository) => repository.query_dashboard_analytics(query).await,
|
||||
None => Err(DataLayerError::InvalidInput(
|
||||
"usage analytics repository is unavailable".into(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn query_usage_analytics(
|
||||
&self,
|
||||
query: &aether_data_contracts::repository::usage::UsageAnalyticsQuery,
|
||||
) -> Result<aether_data_contracts::repository::usage::StoredUsageAnalytics, DataLayerError>
|
||||
{
|
||||
match &self.usage_reader {
|
||||
Some(repository) => repository.query_usage_analytics(query).await,
|
||||
None => Err(DataLayerError::InvalidInput(
|
||||
"usage analytics repository is unavailable".into(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn summarize_health_observations(
|
||||
&self,
|
||||
query: &aether_data_contracts::repository::usage::HealthObservationQuery,
|
||||
) -> Result<aether_data_contracts::repository::usage::HealthObservationSummary, DataLayerError>
|
||||
{
|
||||
match &self.usage_reader {
|
||||
Some(repository) => repository.summarize_health_observations(query).await,
|
||||
None => Err(DataLayerError::InvalidInput(
|
||||
"health observations repository is unavailable".into(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn summarize_usage_audits(
|
||||
&self,
|
||||
query: &aether_data_contracts::repository::usage::UsageAuditSummaryQuery,
|
||||
@@ -2747,6 +2906,35 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn list_provider_expenses(
|
||||
&self,
|
||||
query: &ProviderExpenseQuery,
|
||||
) -> Result<Option<ProviderExpensePage>, DataLayerError> {
|
||||
match &self.billing_reader {
|
||||
Some(repo) => repo.list_provider_expenses(query).await,
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
pub(crate) async fn create_provider_expense(
|
||||
&self,
|
||||
input: &ProviderExpenseInput,
|
||||
) -> Result<AdminBillingMutationOutcome<ProviderExpenseRecord>, DataLayerError> {
|
||||
match &self.billing_reader {
|
||||
Some(repo) => repo.create_provider_expense(input).await,
|
||||
None => Ok(AdminBillingMutationOutcome::Unavailable),
|
||||
}
|
||||
}
|
||||
pub(crate) async fn void_provider_expense(
|
||||
&self,
|
||||
id: &str,
|
||||
operator: Option<&str>,
|
||||
) -> Result<AdminBillingMutationOutcome<ProviderExpenseRecord>, DataLayerError> {
|
||||
match &self.billing_reader {
|
||||
Some(repo) => repo.void_provider_expense(id, operator).await,
|
||||
None => Ok(AdminBillingMutationOutcome::Unavailable),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn list_billing_plans(
|
||||
&self,
|
||||
include_disabled: bool,
|
||||
@@ -2819,6 +3007,21 @@ impl GatewayDataState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn list_user_plan_entitlements_with_history(
|
||||
&self,
|
||||
user_id: &str,
|
||||
include_inactive: bool,
|
||||
) -> Result<Option<Vec<UserPlanEntitlementRecord>>, DataLayerError> {
|
||||
match &self.billing_reader {
|
||||
Some(repository) => {
|
||||
repository
|
||||
.list_user_plan_entitlements_with_history(user_id, include_inactive)
|
||||
.await
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn revoke_user_plan_entitlement(
|
||||
&self,
|
||||
user_id: &str,
|
||||
|
||||
@@ -123,6 +123,15 @@ impl GatewayDataState {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn attach_video_task_repository_for_tests<T>(mut self, repository: Arc<T>) -> Self
|
||||
where
|
||||
T: VideoTaskRepository + 'static,
|
||||
{
|
||||
self.video_task_reader = Some(repository.clone());
|
||||
self.video_task_writer = Some(repository);
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn with_video_task_repository_for_tests<T>(repository: Arc<T>) -> Self
|
||||
where
|
||||
T: VideoTaskRepository + 'static,
|
||||
|
||||
@@ -131,8 +131,13 @@ async fn schedule_pool_page_candidates(
|
||||
entry.1.insert(candidate.candidate.key_id.clone());
|
||||
}
|
||||
|
||||
let key_context_by_id =
|
||||
read_pool_catalog_key_contexts_by_id(state, &candidates, provider_model_name).await;
|
||||
let key_context_by_id = read_pool_catalog_key_contexts_by_id(
|
||||
state,
|
||||
&candidates,
|
||||
provider_model_name,
|
||||
effective_pool_config,
|
||||
)
|
||||
.await;
|
||||
|
||||
let mut runtime_by_provider = BTreeMap::new();
|
||||
let mut pool_config_by_provider = BTreeMap::new();
|
||||
@@ -634,7 +639,9 @@ impl<'a> PoolKeyCursor<'a> {
|
||||
|
||||
if !self.score_phase_exhausted {
|
||||
if let Some(score_candidates) = self.next_score_candidates().await {
|
||||
return Some(score_candidates);
|
||||
if !score_candidates.is_empty() {
|
||||
return Some(score_candidates);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1027,6 +1034,26 @@ impl<'a> PoolKeyCursor<'a> {
|
||||
return None;
|
||||
}
|
||||
|
||||
if pool_config.reserve_minimum_quota
|
||||
&& admin_provider_pool_pure::admin_pool_key_minimum_quota_reached(
|
||||
&key,
|
||||
self.group.candidate.provider_type.as_str(),
|
||||
Some(self.group.candidate.selected_provider_model_name.as_str()),
|
||||
)
|
||||
{
|
||||
self.seen_key_ids.insert(key.id.clone());
|
||||
self.record_skip_reason(POOL_ACCOUNT_EXHAUSTED_SKIP_REASON);
|
||||
self.skipped_candidates
|
||||
.push(SkippedLocalExecutionCandidate {
|
||||
candidate: pool_candidate_from_catalog_key(&self.group, key),
|
||||
skip_reason: POOL_ACCOUNT_EXHAUSTED_SKIP_REASON,
|
||||
transport: None,
|
||||
ranking: self.group.ranking.clone(),
|
||||
extra_data: None,
|
||||
});
|
||||
return None;
|
||||
}
|
||||
|
||||
let candidate = pool_candidate_from_catalog_key(&self.group, key);
|
||||
self.build_eligible_candidate(candidate).await
|
||||
}
|
||||
@@ -1427,15 +1454,23 @@ async fn read_pool_catalog_key_contexts_by_id(
|
||||
state: PlannerAppState<'_>,
|
||||
candidates: &[EligibleLocalExecutionCandidate],
|
||||
provider_model_name: Option<&str>,
|
||||
effective_pool_config: Option<&AdminProviderPoolConfig>,
|
||||
) -> BTreeMap<String, PoolCatalogKeyContext> {
|
||||
let mut key_ids = Vec::new();
|
||||
let mut provider_type_by_key_id = BTreeMap::<String, String>::new();
|
||||
let mut reserve_minimum_quota_key_ids = BTreeSet::new();
|
||||
|
||||
for candidate in candidates {
|
||||
if pool_config_for_candidate(candidate).is_none() {
|
||||
let Some(pool_config) = effective_pool_config
|
||||
.cloned()
|
||||
.or_else(|| pool_config_for_candidate(candidate))
|
||||
else {
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let key_id = candidate.candidate.key_id.clone();
|
||||
if pool_config.reserve_minimum_quota {
|
||||
reserve_minimum_quota_key_ids.insert(key_id.clone());
|
||||
}
|
||||
if let Entry::Vacant(entry) = provider_type_by_key_id.entry(key_id.clone()) {
|
||||
entry.insert(candidate.transport.provider.provider_type.clone());
|
||||
key_ids.push(key_id);
|
||||
@@ -1487,16 +1522,20 @@ async fn read_pool_catalog_key_contexts_by_id(
|
||||
.get(&key.id)
|
||||
.map(String::as_str)
|
||||
.unwrap_or_default();
|
||||
(
|
||||
key.id.clone(),
|
||||
build_pool_catalog_key_context(
|
||||
state,
|
||||
&provider_pool_service,
|
||||
let mut context = build_pool_catalog_key_context(
|
||||
state,
|
||||
&provider_pool_service,
|
||||
&key,
|
||||
provider_type,
|
||||
provider_model_name,
|
||||
);
|
||||
context.quota_exhausted |= reserve_minimum_quota_key_ids.contains(&key.id)
|
||||
&& admin_provider_pool_pure::admin_pool_key_minimum_quota_reached(
|
||||
&key,
|
||||
provider_type,
|
||||
provider_model_name,
|
||||
),
|
||||
)
|
||||
);
|
||||
(key.id.clone(), context)
|
||||
})
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
// A key can disappear between the candidate-row and catalog reads. Keep
|
||||
@@ -3962,6 +4001,110 @@ mod tests {
|
||||
}));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pool_key_cursor_reserve_minimum_quota_filters_pages_and_sticky_hits() {
|
||||
for reserve_enabled in [false, true] {
|
||||
for sticky in [false, true] {
|
||||
for used_percent in [99.0, 98.0, 83.0] {
|
||||
let provider_config = Some(json!({
|
||||
"pool_advanced": {
|
||||
"reserve_minimum_quota": reserve_enabled,
|
||||
"skip_exhausted_accounts": false
|
||||
}
|
||||
}));
|
||||
let provider =
|
||||
sample_codex_pool_provider("provider-pool", 0, provider_config.clone());
|
||||
let endpoint = sample_codex_pool_endpoint("provider-pool", "endpoint-1");
|
||||
let mut reserved = sample_codex_pool_key("provider-pool", "key-low");
|
||||
reserved.status_snapshot = Some(json!({
|
||||
"quota": {
|
||||
"provider_type": "codex",
|
||||
"updated_at": 100,
|
||||
"allowed": false,
|
||||
"exhausted": true,
|
||||
"code": "exhausted",
|
||||
"windows": [{
|
||||
"code": "weekly",
|
||||
"scope": "account",
|
||||
"used_ratio": 1.0,
|
||||
"reset_at": 4_102_444_800u64
|
||||
}]
|
||||
}
|
||||
}));
|
||||
reserved.upstream_metadata = Some(json!({
|
||||
"codex": {
|
||||
"updated_at": 200,
|
||||
"primary_used_percent": used_percent,
|
||||
"primary_reset_at": 4_102_444_800u64
|
||||
}
|
||||
}));
|
||||
let ready = sample_codex_pool_key("provider-pool", "key-ready");
|
||||
let rows = vec![
|
||||
sample_codex_pool_row("provider-pool", "endpoint-1", "key-low", 0),
|
||||
sample_codex_pool_row("provider-pool", "endpoint-1", "key-ready", 0),
|
||||
];
|
||||
let data_state = GatewayDataState::with_provider_catalog_and_minimal_candidate_selection_for_tests(
|
||||
Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||
vec![provider], vec![endpoint], vec![reserved, ready],
|
||||
)),
|
||||
Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(rows)),
|
||||
)
|
||||
.with_encryption_key_for_tests(aether_crypto::DEVELOPMENT_ENCRYPTION_KEY);
|
||||
let app = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(data_state);
|
||||
let group =
|
||||
sample_codex_pool_group("provider-pool", "endpoint-1", 0, provider_config);
|
||||
let pool_config =
|
||||
pool_config_for_candidate(&group).expect("pool config should parse");
|
||||
let sticky_token = sticky.then_some("reserve-session");
|
||||
if sticky {
|
||||
record_admin_provider_pool_success(
|
||||
app.runtime_state.as_ref(),
|
||||
"provider-pool",
|
||||
"key-low",
|
||||
&pool_config,
|
||||
sticky_token,
|
||||
0,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let mut cursor = PoolKeyCursor::new(
|
||||
PlannerAppState::new(&app),
|
||||
group,
|
||||
sticky_token,
|
||||
None,
|
||||
None,
|
||||
);
|
||||
cursor.window_size = 1;
|
||||
cursor.page_size = 1;
|
||||
let mut returned = Vec::new();
|
||||
while let Some(candidate) = cursor.next_key().await {
|
||||
returned.push(candidate.candidate.key_id);
|
||||
}
|
||||
let reserve_reached = reserve_enabled && used_percent >= 99.0;
|
||||
assert_eq!(
|
||||
returned.contains(&"key-low".to_string()),
|
||||
!reserve_reached,
|
||||
"reserve={reserve_enabled}, sticky={sticky}, used={used_percent}"
|
||||
);
|
||||
assert!(returned.contains(&"key-ready".to_string()));
|
||||
if reserve_reached {
|
||||
assert_eq!(
|
||||
cursor
|
||||
.skip_reason_counts
|
||||
.get(POOL_ACCOUNT_EXHAUSTED_SKIP_REASON),
|
||||
Some(&1)
|
||||
);
|
||||
} else if sticky {
|
||||
assert_eq!(returned.first().map(String::as_str), Some("key-low"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pool_key_cursor_does_not_spend_effective_scan_budget_on_exhausted_accounts() {
|
||||
let provider_config = Some(json!({
|
||||
@@ -4168,6 +4311,115 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn inactive_pool_key_with_stale_score_does_not_exhaust_pool() {
|
||||
let provider_config = Some(json!({
|
||||
"pool_advanced": {
|
||||
"score_top_n": 128,
|
||||
"scheduling_presets": [
|
||||
{"preset": "single_account", "enabled": true},
|
||||
{"preset": "priority_first", "enabled": true}
|
||||
]
|
||||
}
|
||||
}));
|
||||
let (provider, endpoint, mut keys, mut rows) =
|
||||
large_pool_fixture(2, provider_config.clone());
|
||||
keys[1].is_active = false;
|
||||
rows.retain(|row| row.key_id != "key-00001");
|
||||
let scores = vec![
|
||||
sample_provider_key_pool_score("provider-pool", "key-00000", 5.0),
|
||||
sample_provider_key_pool_score("provider-pool", "key-00001", 20.0),
|
||||
];
|
||||
let data_state =
|
||||
GatewayDataState::with_provider_catalog_and_minimal_candidate_selection_for_tests(
|
||||
Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||
vec![provider],
|
||||
vec![endpoint],
|
||||
keys,
|
||||
)),
|
||||
Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(rows)),
|
||||
)
|
||||
.with_pool_score_repository_for_tests(Arc::new(
|
||||
InMemoryPoolMemberScoreRepository::seed(scores),
|
||||
))
|
||||
.with_encryption_key_for_tests(aether_crypto::DEVELOPMENT_ENCRYPTION_KEY);
|
||||
let app = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(data_state);
|
||||
let group = sample_eligible_candidate(
|
||||
"provider-pool",
|
||||
"endpoint-1",
|
||||
"pool-group",
|
||||
10,
|
||||
provider_config,
|
||||
);
|
||||
let mut cursor = PoolKeyCursor::new(PlannerAppState::new(&app), group, None, None, None);
|
||||
|
||||
let candidate = cursor
|
||||
.next_key()
|
||||
.await
|
||||
.expect("active key must stay schedulable beside a stale inactive score");
|
||||
|
||||
assert_eq!(candidate.candidate.key_id, "key-00000");
|
||||
assert_eq!(
|
||||
cursor.skip_reason_counts.get("pool_score_member_missing"),
|
||||
Some(&1)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_inactive_score_only_does_not_exhaust_pool() {
|
||||
let provider_config = Some(json!({
|
||||
"pool_advanced": {
|
||||
"score_top_n": 128,
|
||||
"scheduling_presets": [
|
||||
{"preset": "single_account", "enabled": true},
|
||||
{"preset": "priority_first", "enabled": true}
|
||||
]
|
||||
}
|
||||
}));
|
||||
let (provider, endpoint, mut keys, mut rows) =
|
||||
large_pool_fixture(2, provider_config.clone());
|
||||
keys[1].is_active = false;
|
||||
rows.retain(|row| row.key_id != "key-00001");
|
||||
let scores = vec![sample_provider_key_pool_score(
|
||||
"provider-pool",
|
||||
"key-00001",
|
||||
20.0,
|
||||
)];
|
||||
let data_state =
|
||||
GatewayDataState::with_provider_catalog_and_minimal_candidate_selection_for_tests(
|
||||
Arc::new(InMemoryProviderCatalogReadRepository::seed(
|
||||
vec![provider],
|
||||
vec![endpoint],
|
||||
keys,
|
||||
)),
|
||||
Arc::new(InMemoryMinimalCandidateSelectionReadRepository::seed(rows)),
|
||||
)
|
||||
.with_pool_score_repository_for_tests(Arc::new(
|
||||
InMemoryPoolMemberScoreRepository::seed(scores),
|
||||
))
|
||||
.with_encryption_key_for_tests(aether_crypto::DEVELOPMENT_ENCRYPTION_KEY);
|
||||
let app = AppState::new()
|
||||
.expect("state should build")
|
||||
.with_data_state_for_tests(data_state);
|
||||
let group = sample_eligible_candidate(
|
||||
"provider-pool",
|
||||
"endpoint-1",
|
||||
"pool-group",
|
||||
10,
|
||||
provider_config,
|
||||
);
|
||||
let mut cursor = PoolKeyCursor::new(PlannerAppState::new(&app), group, None, None, None);
|
||||
|
||||
let candidate = cursor
|
||||
.next_key()
|
||||
.await
|
||||
.expect("catalog rows must remain schedulable when the only score is stale");
|
||||
|
||||
assert_eq!(candidate.candidate.key_id, "key-00000");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn score_candidates_continue_across_pool_windows() {
|
||||
let provider_config = Some(json!({
|
||||
@@ -4913,15 +5165,6 @@ mod tests {
|
||||
))
|
||||
}
|
||||
|
||||
fn provider_catalog_credential_state() -> AppState {
|
||||
AppState::new()
|
||||
.expect("credential state should build")
|
||||
.with_data_state_for_tests(
|
||||
GatewayDataState::disabled()
|
||||
.with_encryption_key_for_tests(aether_crypto::DEVELOPMENT_ENCRYPTION_KEY),
|
||||
)
|
||||
}
|
||||
|
||||
fn large_pool_fixture(
|
||||
key_count: usize,
|
||||
provider_config: Option<serde_json::Value>,
|
||||
@@ -4972,18 +5215,12 @@ mod tests {
|
||||
)
|
||||
.expect("endpoint transport should build");
|
||||
|
||||
let credential_state = provider_catalog_credential_state();
|
||||
// 这些用例只验证池扫描、跳过计数和游标预算,不会发起请求或读取凭据。
|
||||
// 留空凭据可跳过无关的 Fernet 加解密,同时避免复用绑定密文破坏 key_id AAD。
|
||||
let mut keys = Vec::with_capacity(key_count);
|
||||
let mut rows = Vec::with_capacity(key_count);
|
||||
for index in 0..key_count {
|
||||
let key_id = format!("key-{index:05}");
|
||||
let encrypted_api_key = credential_state
|
||||
.seal_provider_catalog_key_api_key(
|
||||
"provider-pool",
|
||||
&key_id,
|
||||
&format!("secret-{index}"),
|
||||
)
|
||||
.expect("api key should encrypt");
|
||||
let mut key = StoredProviderCatalogKey::new(
|
||||
key_id.clone(),
|
||||
"provider-pool".to_string(),
|
||||
@@ -4995,7 +5232,7 @@ mod tests {
|
||||
.expect("key should build")
|
||||
.with_transport_fields(
|
||||
Some(json!(["openai:chat"])),
|
||||
encrypted_api_key,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
@@ -5130,10 +5367,8 @@ mod tests {
|
||||
.expect("endpoint transport should build")
|
||||
}
|
||||
|
||||
/// 这些测试只检查池调度状态,不涉及凭据解密,因此不构造无关的密文。
|
||||
fn sample_codex_pool_key(provider_id: &str, key_id: &str) -> StoredProviderCatalogKey {
|
||||
let encrypted_api_key = provider_catalog_credential_state()
|
||||
.seal_provider_catalog_key_api_key(provider_id, key_id, &format!("secret-{key_id}"))
|
||||
.expect("api key should encrypt");
|
||||
let mut key = StoredProviderCatalogKey::new(
|
||||
key_id.to_string(),
|
||||
provider_id.to_string(),
|
||||
@@ -5145,7 +5380,7 @@ mod tests {
|
||||
.expect("key should build")
|
||||
.with_transport_fields(
|
||||
Some(json!(["openai:responses"])),
|
||||
encrypted_api_key,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
Some(json!({"openai:responses": 1})),
|
||||
@@ -5289,6 +5524,8 @@ mod tests {
|
||||
key_ids: [&str; N],
|
||||
) -> ResolvedRoutingPolicy {
|
||||
ResolvedRoutingPolicy {
|
||||
billing_multiplier: 1.0,
|
||||
group_name: None,
|
||||
group_id: Some("routing-group-1".to_string()),
|
||||
group_version: Some(1),
|
||||
selection_source: "test".to_string(),
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use axum::body::Body;
|
||||
use axum::http::{Response, StatusCode};
|
||||
use axum::response::IntoResponse;
|
||||
use axum::Json;
|
||||
use regex::Regex;
|
||||
use serde_json::json;
|
||||
use sha2::{Digest, Sha256};
|
||||
use tracing::warn;
|
||||
@@ -10,6 +13,134 @@ use crate::ai_serving::AiSurfaceFinalizeError;
|
||||
use crate::constants::*;
|
||||
use crate::insert_header_if_missing;
|
||||
|
||||
/// 开启后记录不截断但仍脱敏的内部错误详情,默认关闭。
|
||||
static GATEWAY_ERROR_DETAIL_LOGGING: LazyLock<bool> = LazyLock::new(|| {
|
||||
parse_gateway_error_detail_logging(
|
||||
std::env::var("AETHER_GATEWAY_ERROR_DETAIL_LOGGING")
|
||||
.ok()
|
||||
.as_deref(),
|
||||
)
|
||||
});
|
||||
|
||||
fn parse_gateway_error_detail_logging(value: Option<&str>) -> bool {
|
||||
// 仅接受精确的小写 true/false;未设置或无效值默认关闭详情日志。
|
||||
value
|
||||
.and_then(|value| value.parse::<bool>().ok())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
// 按 URL authority 的边界匹配 userinfo,避免跨过路径、查询串和片段中的 @。
|
||||
static ERROR_URL_USERINFO: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r#"(?i)([a-z][a-z0-9+.-]*://)[^\s/?\#"<>]*@"#)
|
||||
.expect("error URL userinfo regex should compile")
|
||||
});
|
||||
|
||||
static ERROR_CREDENTIAL_PREFIX: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(
|
||||
r#"(?ix)
|
||||
\b
|
||||
(?:password|passwd|pwd|(?:access|refresh|id|session|auth)[_-]?token|token|
|
||||
(?:client[_-]?)?secret|(?:api|access|secret|private)[_-]?key|
|
||||
(?:proxy[_-])?authorization)
|
||||
(?:\\*["'])?(?:\s|\\+[nrt])*[:=](?:\s|\\+[nrt])*
|
||||
(?:Some\((?:\s|\\+[nrt])*)?
|
||||
(?:(?:Bearer|Basic)(?:\s|\\+[nrt])+)?
|
||||
|(?:\b|\\+[nrt])Bearer(?:\s|\\+[nrt])+"#,
|
||||
)
|
||||
.expect("error credential prefix regex should compile")
|
||||
});
|
||||
|
||||
/// 检查是否启用了内部错误详情日志。
|
||||
pub(crate) fn gateway_error_detail_logging_enabled() -> bool {
|
||||
*GATEWAY_ERROR_DETAIL_LOGGING
|
||||
}
|
||||
|
||||
/// 日志摘要:移除 URL userinfo、常见凭据键值和 Bearer 内容,再限制为 256 字节。
|
||||
/// 这是自由文本的有限规则,不能保证识别任意敏感内容或编码后的字段名。
|
||||
pub(crate) fn redact_error_detail(error: &impl std::fmt::Display) -> String {
|
||||
redact_error_str(&error.to_string())
|
||||
}
|
||||
|
||||
/// 脱敏 Debug 格式的错误详情(用于未实现 Display 的错误类型)。
|
||||
pub(crate) fn redact_error_debug(error: &impl std::fmt::Debug) -> String {
|
||||
redact_error_str(&format!("{error:?}"))
|
||||
}
|
||||
|
||||
pub(crate) fn redact_error_str(message: &str) -> String {
|
||||
const MAX_LEN: usize = 256;
|
||||
// 必须先处理完整凭据,再截断;否则截断位置可能落在密码和 @host 之间。
|
||||
let mut redacted = redact_error_str_unbounded(message);
|
||||
if redacted.len() > MAX_LEN {
|
||||
let mut end = MAX_LEN;
|
||||
while !redacted.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
redacted.truncate(end);
|
||||
redacted.push_str("...");
|
||||
}
|
||||
redacted
|
||||
}
|
||||
|
||||
/// 详情日志仅取消长度限制,继续使用与摘要相同的凭据脱敏规则。
|
||||
fn redact_error_str_unbounded(message: &str) -> String {
|
||||
let urls_redacted = ERROR_URL_USERINFO.replace_all(message, "$1");
|
||||
let mut result = String::with_capacity(urls_redacted.len());
|
||||
let mut cursor = 0;
|
||||
let mut prefixes = ERROR_CREDENTIAL_PREFIX.find_iter(&urls_redacted).peekable();
|
||||
while let Some(prefix) = prefixes.next() {
|
||||
// 带引号的值可能包含 password= 等文本,已遮盖的内容不再重复处理。
|
||||
if prefix.start() < cursor {
|
||||
continue;
|
||||
}
|
||||
let value_start = prefix.end();
|
||||
// 未引用的值最多读到下一个凭据字段,避免吞掉它的开头却留下带空格的值。
|
||||
let unquoted_limit = prefixes
|
||||
.peek()
|
||||
.map_or(urls_redacted.len() - value_start, |next| {
|
||||
urls_redacted[value_start..next.start()]
|
||||
.trim_end_matches([',', ';', '&'])
|
||||
.len()
|
||||
});
|
||||
let value_end =
|
||||
value_start + credential_value_len(&urls_redacted[value_start..], unquoted_limit);
|
||||
result.push_str(&urls_redacted[cursor..value_start]);
|
||||
result.push_str("[REDACTED]");
|
||||
cursor = value_end;
|
||||
}
|
||||
result.push_str(&urls_redacted[cursor..]);
|
||||
result
|
||||
}
|
||||
|
||||
fn credential_value_len(value: &str, unquoted_limit: usize) -> usize {
|
||||
let bytes = value.as_bytes();
|
||||
let opening_slashes = bytes.iter().take_while(|byte| **byte == b'\\').count();
|
||||
if let Some(quote @ (b'"' | b'\'')) = bytes.get(opening_slashes) {
|
||||
// 仅同一转义层的引号可闭合;无法确认边界时多遮盖,避免泄露密码尾部。
|
||||
let mut slashes = 0;
|
||||
for (index, byte) in bytes.iter().enumerate().skip(opening_slashes + 1) {
|
||||
if byte == quote && slashes == opening_slashes {
|
||||
return index + 1;
|
||||
}
|
||||
slashes = if *byte == b'\\' { slashes + 1 } else { 0 };
|
||||
}
|
||||
// 不完整的引号内容整体遮盖,避免保留密码片段。
|
||||
return value.len();
|
||||
}
|
||||
|
||||
let mut escaped = false;
|
||||
// DSN 未引用密码中的标点也可能是凭据,只以未转义的空白为结束边界。
|
||||
for (index, ch) in value[..unquoted_limit].char_indices() {
|
||||
if ch == '\\' {
|
||||
escaped = true;
|
||||
} else if escaped {
|
||||
escaped = false;
|
||||
} else if ch.is_whitespace() {
|
||||
return index;
|
||||
}
|
||||
}
|
||||
unquoted_limit
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) enum GatewayError {
|
||||
UpstreamUnavailable {
|
||||
@@ -211,13 +342,7 @@ impl IntoResponse for GatewayError {
|
||||
)
|
||||
.into_response(),
|
||||
Self::Internal(message) => {
|
||||
let error_fingerprint = gateway_error_fingerprint(&message);
|
||||
tracing::error!(
|
||||
event_name = "gateway_internal_error",
|
||||
error_fingerprint,
|
||||
error_length = message.len(),
|
||||
"internal gateway error hidden from client"
|
||||
);
|
||||
log_gateway_internal_error(&message, gateway_error_detail_logging_enabled());
|
||||
(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
Json(json!({
|
||||
@@ -232,6 +357,26 @@ impl IntoResponse for GatewayError {
|
||||
}
|
||||
}
|
||||
|
||||
fn log_gateway_internal_error(message: &str, detail_logging: bool) {
|
||||
let error_fingerprint = gateway_error_fingerprint(message);
|
||||
if detail_logging {
|
||||
tracing::error!(
|
||||
event_name = "gateway_internal_error",
|
||||
error_fingerprint,
|
||||
error_length = message.len(),
|
||||
error_detail = %redact_error_str_unbounded(message),
|
||||
"internal gateway error hidden from client"
|
||||
);
|
||||
} else {
|
||||
tracing::error!(
|
||||
event_name = "gateway_internal_error",
|
||||
error_fingerprint,
|
||||
error_length = message.len(),
|
||||
"internal gateway error hidden from client"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn gateway_error_fingerprint(message: &str) -> String {
|
||||
let digest = Sha256::digest(message.as_bytes());
|
||||
format!("{:x}", digest)[..16].to_string()
|
||||
@@ -245,13 +390,68 @@ impl From<AiSurfaceFinalizeError> for GatewayError {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use axum::body::to_bytes;
|
||||
use axum::http::{header::RETRY_AFTER, StatusCode};
|
||||
use axum::response::IntoResponse;
|
||||
|
||||
use crate::constants::TRACE_ID_HEADER;
|
||||
|
||||
use super::{gateway_error_fingerprint, GatewayError};
|
||||
use super::{
|
||||
gateway_error_fingerprint, log_gateway_internal_error, parse_gateway_error_detail_logging,
|
||||
redact_error_debug, redact_error_detail, redact_error_str, GatewayError,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn detail_logging_only_accepts_lowercase_true_and_false() {
|
||||
assert!(parse_gateway_error_detail_logging(Some("true")));
|
||||
assert!(!parse_gateway_error_detail_logging(Some("false")));
|
||||
assert!(!parse_gateway_error_detail_logging(None));
|
||||
for value in [
|
||||
"1", "0", "yes", "no", "on", "off", "TRUE", "FALSE", "True", " true ", "true\n", "",
|
||||
"invalid",
|
||||
] {
|
||||
assert!(
|
||||
!parse_gateway_error_detail_logging(Some(value)),
|
||||
"value: {value:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct LogBuffer(Arc<Mutex<Vec<u8>>>);
|
||||
|
||||
impl std::io::Write for LogBuffer {
|
||||
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
|
||||
self.0
|
||||
.lock()
|
||||
.expect("log buffer should lock")
|
||||
.extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn internal_error_log(message: &str, detail_logging: bool) -> serde_json::Value {
|
||||
let buffer = LogBuffer::default();
|
||||
let writer = buffer.clone();
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.json()
|
||||
.without_time()
|
||||
.with_max_level(tracing::Level::ERROR)
|
||||
.with_writer(move || writer.clone())
|
||||
.finish();
|
||||
// 使用线程局部日志捕获和显式开关,不修改进程环境以免干扰并发测试。
|
||||
tracing::subscriber::with_default(subscriber, || {
|
||||
log_gateway_internal_error(message, detail_logging);
|
||||
});
|
||||
let bytes = buffer.0.lock().expect("log buffer should lock");
|
||||
serde_json::from_slice(&bytes).expect("internal error log should be JSON")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn internal_errors_do_not_expose_internal_details() {
|
||||
@@ -308,4 +508,292 @@ mod tests {
|
||||
Some(trace_id.as_str())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_error_str_strips_url_credentials() {
|
||||
let input = "connect failed: postgresql://admin:[email protected]:5432/aether";
|
||||
let redacted = redact_error_str(input);
|
||||
assert!(!redacted.contains("s3cret"));
|
||||
assert!(redacted.contains("postgresql://"));
|
||||
assert!(redacted.contains("db.internal"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_error_str_truncates_long_messages() {
|
||||
let long = "x".repeat(500);
|
||||
let redacted = redact_error_str(&long);
|
||||
assert_eq!(redacted, format!("{}...", "x".repeat(256)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_error_str_preserves_short_messages() {
|
||||
let input = "connection refused";
|
||||
assert_eq!(redact_error_str(input), input);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_error_str_handles_multiple_urls() {
|
||||
for (input, expected) in [
|
||||
(
|
||||
"https://u:first@a/p,https://v:second@b/q",
|
||||
"https://a/p,https://b/q",
|
||||
),
|
||||
(
|
||||
"https://u:first@a,https://v:second@b",
|
||||
"https://a,https://b",
|
||||
),
|
||||
(
|
||||
"failed: https://user:[email protected]/v1 and http://admin:[email protected]",
|
||||
"failed: https://api.example.com/v1 and http://internal.io",
|
||||
),
|
||||
] {
|
||||
assert_eq!(redact_error_str(input), expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_error_str_handles_url_authority_boundaries_and_punctuation() {
|
||||
for (input, expected) in [
|
||||
(
|
||||
r#"url="postgres://u:p@h", retry=2"#,
|
||||
r#"url="postgres://h", retry=2"#,
|
||||
),
|
||||
(
|
||||
"(https://u:p@h?mode=test#detail)",
|
||||
"(https://h?mode=test#detail)",
|
||||
),
|
||||
(
|
||||
"postgres://u:p@ss@[::1]:5432/db",
|
||||
"postgres://[::1]:5432/db",
|
||||
),
|
||||
("https://u:p,a;s's@h/p", "https://h/p"),
|
||||
(
|
||||
"https://h/path@name?email=a@b#ref@c",
|
||||
"https://h/path@name?email=a@b#ref@c",
|
||||
),
|
||||
] {
|
||||
assert_eq!(redact_error_str(input), expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_error_str_redacts_before_truncation() {
|
||||
let prefix = format!("{} ", "x".repeat(239));
|
||||
let input = format!("{prefix}postgres://u:supersecret@db/app");
|
||||
assert_eq!(
|
||||
redact_error_str(&input),
|
||||
format!("{prefix}postgres://db/ap...")
|
||||
);
|
||||
|
||||
let input = format!("password={} host=db", "secret".repeat(100));
|
||||
assert_eq!(redact_error_str(&input), "password=[REDACTED] host=db");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_error_str_preserves_utf8_and_whitespace() {
|
||||
let input = " first \n\tsecond \r\n";
|
||||
assert_eq!(redact_error_str(input), input);
|
||||
assert_eq!(
|
||||
redact_error_str(" failed:\n\tpostgres://u:p@h\r\n retry"),
|
||||
" failed:\n\tpostgres://h\r\n retry"
|
||||
);
|
||||
for length in [254, 255, 256] {
|
||||
let prefix = "x".repeat(length);
|
||||
assert_eq!(
|
||||
redact_error_str(&format!("{prefix}错误")),
|
||||
format!("{prefix}...")
|
||||
);
|
||||
}
|
||||
assert_eq!(redact_error_str(&"x".repeat(256)), "x".repeat(256));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_error_str_masks_common_credentials() {
|
||||
for key in [
|
||||
"password",
|
||||
"PASSWORD",
|
||||
"passwd",
|
||||
"pwd",
|
||||
"token",
|
||||
"access_token",
|
||||
"refresh-token",
|
||||
"idToken",
|
||||
"session_token",
|
||||
"auth-token",
|
||||
"secret",
|
||||
"client_secret",
|
||||
"clientSecret",
|
||||
"api_key",
|
||||
"api-key",
|
||||
"apiKey",
|
||||
"access_key",
|
||||
"secret_key",
|
||||
"private_key",
|
||||
"x-api-key",
|
||||
] {
|
||||
for separator in ["=", ":", " = ", "\t:\t"] {
|
||||
let input = format!("{key}{separator}test-credential retry=2");
|
||||
assert_eq!(
|
||||
redact_error_str(&input),
|
||||
format!("{key}{separator}[REDACTED] retry=2")
|
||||
);
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
redact_error_str("password=one;token=two,secret=three&retry=2"),
|
||||
"password=[REDACTED];token=[REDACTED],secret=[REDACTED]"
|
||||
);
|
||||
assert_eq!(
|
||||
redact_error_str("https://u:p@host/path?token=abc&api_key=xyz#details"),
|
||||
"https://host/path?token=[REDACTED]&api_key=[REDACTED]"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_error_str_does_not_expose_punctuation_in_unquoted_passwords() {
|
||||
for secret in [
|
||||
"one#two", "one?two", "one&two", "one,two", "one;two", "one)two", "one\"two",
|
||||
] {
|
||||
assert_eq!(
|
||||
redact_error_str(&format!("password={secret} host=db")),
|
||||
"password=[REDACTED] host=db"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
redact_error_str("https://host/db?password=one?two&mode=test"),
|
||||
"https://host/db?password=[REDACTED]"
|
||||
);
|
||||
assert_eq!(
|
||||
redact_error_str(r#"password=one,token="two words" retry=2"#),
|
||||
"password=[REDACTED],token=[REDACTED] retry=2"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_error_str_masks_authorization_and_bearer_values() {
|
||||
for (input, expected) in [
|
||||
(
|
||||
"Authorization: Bearer short",
|
||||
"Authorization: Bearer [REDACTED]",
|
||||
),
|
||||
(
|
||||
"authorization=bEaReR\tabc.def",
|
||||
"authorization=bEaReR\t[REDACTED]",
|
||||
),
|
||||
(
|
||||
"Proxy-Authorization: Basic abc==",
|
||||
"Proxy-Authorization: Basic [REDACTED]",
|
||||
),
|
||||
(
|
||||
r#"{"Authorization": "Bearer secret value"}"#,
|
||||
r#"{"Authorization": [REDACTED]}"#,
|
||||
),
|
||||
(
|
||||
"error: BEARER a+/b==, retry=2",
|
||||
"error: BEARER [REDACTED] retry=2",
|
||||
),
|
||||
] {
|
||||
assert_eq!(redact_error_str(input), expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_error_str_masks_quoted_and_escaped_values() {
|
||||
for (input, expected) in [
|
||||
(
|
||||
"password='space secret' host=db",
|
||||
"password=[REDACTED] host=db",
|
||||
),
|
||||
(
|
||||
r#"password="space \"secret" host=db"#,
|
||||
"password=[REDACTED] host=db",
|
||||
),
|
||||
(
|
||||
r"password=space\ secret host=db",
|
||||
"password=[REDACTED] host=db",
|
||||
),
|
||||
(
|
||||
r#"Error { password: "space secret", token: Some("option-secret") }"#,
|
||||
"Error { password: [REDACTED], token: Some([REDACTED]) }",
|
||||
),
|
||||
("password='unterminated secret", "password=[REDACTED]"),
|
||||
(
|
||||
r#"password="another unterminated secret"#,
|
||||
"password=[REDACTED]",
|
||||
),
|
||||
(
|
||||
r#"password="token=inner-secret" retry=2"#,
|
||||
"password=[REDACTED] retry=2",
|
||||
),
|
||||
] {
|
||||
assert_eq!(redact_error_str(input), expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn display_and_debug_error_helpers_redact_real_formatted_values() {
|
||||
let input =
|
||||
r#"{"password": "space \"escaped-secret", "token": "token-secret", "retry": 2}"#;
|
||||
for redacted in [redact_error_detail(&input), redact_error_debug(&input)] {
|
||||
assert!(!redacted.contains("escaped-secret"));
|
||||
assert!(!redacted.contains("token-secret"));
|
||||
assert!(redacted.contains("retry"));
|
||||
assert_eq!(redacted.matches("[REDACTED]").count(), 2);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn debug_error_redaction_handles_escaped_whitespace_and_single_quotes() {
|
||||
for input in [
|
||||
"Authorization:\nBearer test-secret",
|
||||
"Authorization:\tBearer test-secret",
|
||||
"\nBearer test-secret",
|
||||
"Bearer\ntest-secret",
|
||||
r"password=space\ test-secret host=db",
|
||||
r"password='space \'test-secret' host=db",
|
||||
] {
|
||||
let redacted = redact_error_debug(&input);
|
||||
assert!(!redacted.contains("test-secret"), "redacted: {redacted}");
|
||||
assert!(redacted.contains("[REDACTED]"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn internal_error_detail_logging_redacts_without_truncating() {
|
||||
let padding = "x".repeat(300);
|
||||
let message = format!("password=first-secret {padding}\nhttps://u:second-secret@db/path token=third-secret\nretry exhausted");
|
||||
let log = internal_error_log(&message, true);
|
||||
let fields = &log["fields"];
|
||||
assert_eq!(log["level"], "ERROR");
|
||||
assert_eq!(fields["event_name"], "gateway_internal_error");
|
||||
assert_eq!(fields["error_length"], message.len());
|
||||
assert_eq!(
|
||||
fields["error_fingerprint"],
|
||||
gateway_error_fingerprint(&message)
|
||||
);
|
||||
assert_eq!(
|
||||
fields["error_detail"],
|
||||
format!(
|
||||
"password=[REDACTED] {padding}\nhttps://db/path token=[REDACTED]\nretry exhausted"
|
||||
)
|
||||
);
|
||||
for secret in ["first-secret", "second-secret", "third-secret"] {
|
||||
assert!(!log.to_string().contains(secret));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn internal_error_logging_omits_details_when_disabled() {
|
||||
let message = "password=internal-secret";
|
||||
let log = internal_error_log(message, false);
|
||||
let fields = &log["fields"];
|
||||
assert_eq!(fields["event_name"], "gateway_internal_error");
|
||||
assert_eq!(
|
||||
fields["error_fingerprint"],
|
||||
gateway_error_fingerprint(message)
|
||||
);
|
||||
assert_eq!(fields["error_length"], message.len());
|
||||
assert!(fields.get("error_detail").is_none());
|
||||
assert!(!log.to_string().contains("internal-secret"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,538 @@
|
||||
//! Node-local, request-deduplicated activity for provider and requested-model analysis.
|
||||
//!
|
||||
//! RPM counts distinct requests entering upstream execution in the last 60 seconds;
|
||||
//! it is never extrapolated from a shorter observation window. Concurrency follows
|
||||
//! guard lifetimes, including streams, independently of that window. Expiration is
|
||||
//! ordered rather than scanning request history on each lifecycle event.
|
||||
use std::cmp::Reverse;
|
||||
use std::collections::{BinaryHeap, HashMap};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Instant;
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
const WINDOW_US: u64 = 60_000_000;
|
||||
const MAX_REQUESTS: usize = 100_000;
|
||||
const MAX_REQUEST_DIMENSIONS: usize = 200_000;
|
||||
const MAX_LABEL_BYTES: usize = 512;
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct Counts {
|
||||
recent: u64,
|
||||
active: u64,
|
||||
provider_name: Option<Arc<str>>,
|
||||
}
|
||||
|
||||
impl Counts {
|
||||
fn empty(&self) -> bool {
|
||||
self.recent == 0 && self.active == 0
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct ProviderRequest {
|
||||
active: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Request {
|
||||
model: Option<Arc<str>>,
|
||||
active: u64,
|
||||
providers: HashMap<Arc<str>, ProviderRequest>,
|
||||
idle_since: Option<u64>,
|
||||
cleanup_scheduled: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Eq, PartialEq, Ord, PartialOrd)]
|
||||
enum Expiration {
|
||||
Model(Arc<str>),
|
||||
Provider(Arc<str>, Arc<str>),
|
||||
Request(Arc<str>),
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct History {
|
||||
through_us: u64,
|
||||
requests: HashMap<Arc<str>, Request>,
|
||||
providers: HashMap<Arc<str>, Counts>,
|
||||
models: HashMap<Option<Arc<str>>, Counts>,
|
||||
expirations: BinaryHeap<Reverse<(u64, Expiration)>>,
|
||||
request_dimensions: usize,
|
||||
untracked_active: u64,
|
||||
incomplete_until_us: u64,
|
||||
}
|
||||
|
||||
impl History {
|
||||
fn advance(&mut self, now_us: u64) {
|
||||
self.through_us = self.through_us.max(now_us);
|
||||
while self
|
||||
.expirations
|
||||
.peek()
|
||||
.is_some_and(|Reverse((expires_at, _))| *expires_at <= self.through_us)
|
||||
{
|
||||
let Reverse((_, expiration)) = self.expirations.pop().expect("expiration exists");
|
||||
match expiration {
|
||||
Expiration::Model(request_id) => {
|
||||
let Some(request) = self.requests.get_mut(&request_id) else {
|
||||
continue;
|
||||
};
|
||||
if let Some(counts) = self.models.get_mut(&request.model) {
|
||||
counts.recent = counts.recent.saturating_sub(1);
|
||||
if counts.empty() {
|
||||
self.models.remove(&request.model);
|
||||
}
|
||||
}
|
||||
}
|
||||
Expiration::Provider(request_id, provider_id) => {
|
||||
let Some(request) = self.requests.get(&request_id) else {
|
||||
continue;
|
||||
};
|
||||
if !request.providers.contains_key(&provider_id) {
|
||||
continue;
|
||||
}
|
||||
if let Some(counts) = self.providers.get_mut(&provider_id) {
|
||||
counts.recent = counts.recent.saturating_sub(1);
|
||||
if counts.empty() {
|
||||
self.providers.remove(&provider_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
Expiration::Request(request_id) => {
|
||||
let Some(request) = self.requests.get_mut(&request_id) else {
|
||||
continue;
|
||||
};
|
||||
request.cleanup_scheduled = false;
|
||||
if let Some(idle_since) = request.idle_since {
|
||||
let expires_at = idle_since.saturating_add(WINDOW_US);
|
||||
if expires_at <= self.through_us {
|
||||
self.request_dimensions -= request.providers.len() + 1;
|
||||
self.requests.remove(&request_id);
|
||||
} else {
|
||||
// A retry reused the record while its first cleanup was
|
||||
// pending. Keep at most one cleanup entry per request.
|
||||
request.cleanup_scheduled = true;
|
||||
self.expirations
|
||||
.push(Reverse((expires_at, Expiration::Request(request_id))));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn begin(
|
||||
&mut self,
|
||||
now_us: u64,
|
||||
request_id: &str,
|
||||
provider_id: &str,
|
||||
provider_name: Option<&str>,
|
||||
requested_model: Option<&str>,
|
||||
) -> GuardIdentity {
|
||||
self.advance(now_us);
|
||||
let existing = self.requests.get(request_id);
|
||||
let new_request = existing.is_none();
|
||||
let new_provider = existing.is_none_or(|r| !r.providers.contains_key(provider_id));
|
||||
let new_dimensions = usize::from(new_request) + usize::from(new_provider);
|
||||
let valid_labels = !request_id.is_empty()
|
||||
&& !provider_id.is_empty()
|
||||
&& [
|
||||
Some(request_id),
|
||||
Some(provider_id),
|
||||
provider_name,
|
||||
requested_model,
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.all(|label| label.len() <= MAX_LABEL_BYTES);
|
||||
if !valid_labels
|
||||
|| (new_request && self.requests.len() >= MAX_REQUESTS)
|
||||
|| self.request_dimensions.saturating_add(new_dimensions) > MAX_REQUEST_DIMENSIONS
|
||||
{
|
||||
// Telemetry must not affect admission. Explicitly mark incomplete
|
||||
// coverage instead of silently returning plausible but partial counts.
|
||||
self.untracked_active += 1;
|
||||
self.incomplete_until_us = self.through_us.saturating_add(WINDOW_US);
|
||||
return GuardIdentity::Untracked;
|
||||
}
|
||||
|
||||
let request_id: Arc<str> = self
|
||||
.requests
|
||||
.get_key_value(request_id)
|
||||
.map(|(key, _)| Arc::clone(key))
|
||||
.unwrap_or_else(|| Arc::from(request_id));
|
||||
let request = self
|
||||
.requests
|
||||
.entry(Arc::clone(&request_id))
|
||||
.or_insert_with(|| Request {
|
||||
model: requested_model
|
||||
.filter(|model| !model.is_empty())
|
||||
.map(Arc::from),
|
||||
active: 0,
|
||||
providers: HashMap::new(),
|
||||
idle_since: None,
|
||||
cleanup_scheduled: false,
|
||||
});
|
||||
let model_counts = self.models.entry(request.model.clone()).or_default();
|
||||
if new_request {
|
||||
model_counts.recent += 1;
|
||||
self.expirations.push(Reverse((
|
||||
self.through_us.saturating_add(WINDOW_US),
|
||||
Expiration::Model(Arc::clone(&request_id)),
|
||||
)));
|
||||
}
|
||||
if request.active == 0 {
|
||||
model_counts.active += 1;
|
||||
}
|
||||
request.active += 1;
|
||||
request.idle_since = None;
|
||||
|
||||
let provider_id: Arc<str> = request
|
||||
.providers
|
||||
.get_key_value(provider_id)
|
||||
.map(|(key, _)| Arc::clone(key))
|
||||
.unwrap_or_else(|| Arc::from(provider_id));
|
||||
let provider = request
|
||||
.providers
|
||||
.entry(Arc::clone(&provider_id))
|
||||
.or_insert(ProviderRequest { active: 0 });
|
||||
let provider_counts = self.providers.entry(Arc::clone(&provider_id)).or_default();
|
||||
if let Some(name) = provider_name.filter(|name| !name.is_empty()) {
|
||||
provider_counts.provider_name = Some(Arc::from(name));
|
||||
}
|
||||
if new_provider {
|
||||
provider_counts.recent += 1;
|
||||
self.expirations.push(Reverse((
|
||||
self.through_us.saturating_add(WINDOW_US),
|
||||
Expiration::Provider(Arc::clone(&request_id), Arc::clone(&provider_id)),
|
||||
)));
|
||||
}
|
||||
if provider.active == 0 {
|
||||
provider_counts.active += 1;
|
||||
}
|
||||
provider.active += 1;
|
||||
self.request_dimensions += new_dimensions;
|
||||
GuardIdentity::Tracked {
|
||||
request_id,
|
||||
provider_id,
|
||||
}
|
||||
}
|
||||
|
||||
fn release(&mut self, now_us: u64, identity: GuardIdentity) {
|
||||
self.advance(now_us);
|
||||
let GuardIdentity::Tracked {
|
||||
request_id,
|
||||
provider_id,
|
||||
} = identity
|
||||
else {
|
||||
self.untracked_active = self.untracked_active.saturating_sub(1);
|
||||
return;
|
||||
};
|
||||
let Some(request) = self.requests.get_mut(&request_id) else {
|
||||
return;
|
||||
};
|
||||
let Some(provider) = request.providers.get_mut(&provider_id) else {
|
||||
return;
|
||||
};
|
||||
provider.active = provider.active.saturating_sub(1);
|
||||
if provider.active == 0 {
|
||||
if let Some(counts) = self.providers.get_mut(&provider_id) {
|
||||
counts.active = counts.active.saturating_sub(1);
|
||||
if counts.empty() {
|
||||
self.providers.remove(&provider_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
request.active = request.active.saturating_sub(1);
|
||||
if request.active == 0 {
|
||||
if let Some(counts) = self.models.get_mut(&request.model) {
|
||||
counts.active = counts.active.saturating_sub(1);
|
||||
if counts.empty() {
|
||||
self.models.remove(&request.model);
|
||||
}
|
||||
}
|
||||
// Retain deduplication briefly after completion as failover may begin
|
||||
// after the old guard drops, including after a >60-second attempt.
|
||||
request.idle_since = Some(self.through_us);
|
||||
if !request.cleanup_scheduled {
|
||||
request.cleanup_scheduled = true;
|
||||
self.expirations.push(Reverse((
|
||||
self.through_us.saturating_add(WINDOW_US),
|
||||
Expiration::Request(request_id),
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn snapshot(&mut self, now_us: u64, started_at_us: i64) -> Value {
|
||||
self.advance(now_us);
|
||||
let mut providers: Vec<_> = self.providers.iter().collect();
|
||||
providers.sort_unstable_by_key(|(provider, _)| *provider);
|
||||
let mut models: Vec<_> = self.models.iter().collect();
|
||||
models.sort_unstable_by_key(|(model, _)| *model);
|
||||
json!({
|
||||
"observed_at": DateTime::from_timestamp_micros(started_at_us.saturating_add(self.through_us.min(i64::MAX as u64) as i64)),
|
||||
"observed_from": DateTime::from_timestamp_micros(started_at_us),
|
||||
"window_seconds": 60,
|
||||
"observed_window_seconds": (self.through_us as f64 / 1_000_000.0).min(60.0),
|
||||
"scope": {"kind": "node"},
|
||||
"measurement": "http_and_responses_websocket_requests",
|
||||
"coverage": if self.untracked_active > 0 || self.through_us < self.incomplete_until_us { "partial" } else { "complete" },
|
||||
"providers": providers.into_iter().map(|(id, counts)| json!({
|
||||
"provider_id": id.as_ref(),
|
||||
"provider": counts.provider_name.as_deref().unwrap_or(id.as_ref()),
|
||||
"requests_per_minute": counts.recent,
|
||||
"current_concurrency": counts.active,
|
||||
})).collect::<Vec<_>>(),
|
||||
"models": models.into_iter().map(|(model, counts)| json!({
|
||||
"model": model.as_deref(),
|
||||
"requests_per_minute": counts.recent,
|
||||
"current_concurrency": counts.active,
|
||||
})).collect::<Vec<_>>(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct ExecutionActivity {
|
||||
started_at: Instant,
|
||||
started_at_us: i64,
|
||||
history: Mutex<History>,
|
||||
}
|
||||
|
||||
impl Default for ExecutionActivity {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
started_at: Instant::now(),
|
||||
started_at_us: Utc::now().timestamp_micros(),
|
||||
history: Mutex::new(History::default()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ExecutionActivity {
|
||||
fn elapsed_us(&self) -> u64 {
|
||||
self.started_at.elapsed().as_micros().min(u64::MAX as u128) as u64
|
||||
}
|
||||
|
||||
pub(crate) fn begin(
|
||||
self: &Arc<Self>,
|
||||
request_id: &str,
|
||||
provider_id: &str,
|
||||
provider_name: Option<&str>,
|
||||
requested_model: Option<&str>,
|
||||
) -> ExecutionActivityGuard {
|
||||
let identity = self
|
||||
.history
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.begin(
|
||||
self.elapsed_us(),
|
||||
request_id,
|
||||
provider_id,
|
||||
provider_name,
|
||||
requested_model,
|
||||
);
|
||||
ExecutionActivityGuard {
|
||||
activity: Arc::clone(self),
|
||||
identity: Some(identity),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn snapshot(&self) -> Value {
|
||||
self.history
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.snapshot(self.elapsed_us(), self.started_at_us)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
enum GuardIdentity {
|
||||
Tracked {
|
||||
request_id: Arc<str>,
|
||||
provider_id: Arc<str>,
|
||||
},
|
||||
Untracked,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct ExecutionActivityGuard {
|
||||
activity: Arc<ExecutionActivity>,
|
||||
identity: Option<GuardIdentity>,
|
||||
}
|
||||
|
||||
impl Drop for ExecutionActivityGuard {
|
||||
fn drop(&mut self) {
|
||||
if let Some(identity) = self.identity.take() {
|
||||
self.activity
|
||||
.history
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.release(self.activity.elapsed_us(), identity);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn begin(history: &mut History, at_us: u64, id: &str, provider: &str) -> GuardIdentity {
|
||||
history.begin(at_us, id, provider, Some(provider), Some("requested-model"))
|
||||
}
|
||||
|
||||
fn value(history: &mut History, at_us: u64) -> Value {
|
||||
history.snapshot(at_us, 0)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rpm_has_an_exact_rolling_window_and_never_extrapolates_startup() {
|
||||
let mut history = History::default();
|
||||
let a = begin(&mut history, 0, "a", "provider");
|
||||
history.release(1, a);
|
||||
let b = begin(&mut history, 30_000_000, "b", "provider");
|
||||
history.release(30_000_001, b);
|
||||
let early = value(&mut history, 30_000_001);
|
||||
assert_eq!(early["providers"][0]["requests_per_minute"], 2);
|
||||
assert_eq!(early["coverage"], "complete");
|
||||
assert!(early["observed_window_seconds"].as_f64().unwrap() < 60.0);
|
||||
assert_eq!(
|
||||
value(&mut history, WINDOW_US - 1)["providers"][0]["requests_per_minute"],
|
||||
2
|
||||
);
|
||||
assert_eq!(
|
||||
value(&mut history, WINDOW_US)["providers"][0]["requests_per_minute"],
|
||||
1
|
||||
);
|
||||
assert!(value(&mut history, 90_000_000)["providers"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.is_empty());
|
||||
assert_eq!(
|
||||
value(&mut history, 90_000_000)["observed_window_seconds"],
|
||||
60.0
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn overlapping_guards_and_sequential_retries_count_one_request() {
|
||||
let mut history = History::default();
|
||||
let a = begin(&mut history, 0, "request", "provider");
|
||||
let b = begin(&mut history, 1, "request", "provider");
|
||||
let c = begin(&mut history, 2, "other-request", "provider");
|
||||
assert_eq!(
|
||||
value(&mut history, 2)["providers"][0]["current_concurrency"],
|
||||
2
|
||||
);
|
||||
assert_eq!(
|
||||
value(&mut history, 2)["providers"][0]["requests_per_minute"],
|
||||
2
|
||||
);
|
||||
history.release(3, a);
|
||||
assert_eq!(
|
||||
value(&mut history, 3)["models"][0]["current_concurrency"],
|
||||
2
|
||||
);
|
||||
history.release(4, b);
|
||||
history.release(5, c);
|
||||
let retry = begin(&mut history, 6, "request", "provider");
|
||||
let result = value(&mut history, 6);
|
||||
assert_eq!(result["providers"][0]["requests_per_minute"], 2);
|
||||
assert_eq!(result["models"][0]["current_concurrency"], 1);
|
||||
history.release(7, retry);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failover_counts_each_provider_but_deduplicates_the_requested_model() {
|
||||
let mut history = History::default();
|
||||
let first = begin(&mut history, 0, "request", "first");
|
||||
let second = begin(&mut history, 1, "request", "second");
|
||||
let result = value(&mut history, 2);
|
||||
assert_eq!(result["providers"].as_array().unwrap().len(), 2);
|
||||
assert_eq!(result["providers"][0]["requests_per_minute"], 1);
|
||||
assert_eq!(result["providers"][1]["current_concurrency"], 1);
|
||||
assert_eq!(result["models"][0]["requests_per_minute"], 1);
|
||||
assert_eq!(result["models"][0]["current_concurrency"], 1);
|
||||
history.release(3, first);
|
||||
history.release(4, second);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn long_stream_retains_concurrency_and_retry_does_not_restart_model_rpm() {
|
||||
let mut history = History::default();
|
||||
let stream = begin(&mut history, 0, "request", "provider");
|
||||
let result = value(&mut history, 2 * WINDOW_US);
|
||||
assert_eq!(result["providers"][0]["requests_per_minute"], 0);
|
||||
assert_eq!(result["models"][0]["current_concurrency"], 1);
|
||||
history.release(2 * WINDOW_US + 1, stream);
|
||||
let retry = begin(&mut history, 2 * WINDOW_US + 2, "request", "provider");
|
||||
let result = value(&mut history, 2 * WINDOW_US + 2);
|
||||
assert_eq!(result["providers"][0]["requests_per_minute"], 0);
|
||||
assert_eq!(result["models"][0]["requests_per_minute"], 0);
|
||||
assert_eq!(result["models"][0]["current_concurrency"], 1);
|
||||
history.release(2 * WINDOW_US + 3, retry);
|
||||
assert!(value(&mut history, 3 * WINDOW_US + 3)["models"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.is_empty());
|
||||
assert!(history.requests.is_empty());
|
||||
assert!(history.expirations.is_empty());
|
||||
assert_eq!(history.request_dimensions, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancellation_drop_releases_concurrency_but_keeps_rpm() {
|
||||
let activity = Arc::new(ExecutionActivity::default());
|
||||
let guard = activity.begin("request", "provider", Some("Provider name"), None);
|
||||
assert_eq!(
|
||||
activity.snapshot()["providers"][0]["current_concurrency"],
|
||||
1
|
||||
);
|
||||
drop(guard);
|
||||
let result = activity.snapshot();
|
||||
assert_eq!(result["providers"][0]["current_concurrency"], 0);
|
||||
assert_eq!(result["providers"][0]["requests_per_minute"], 1);
|
||||
assert_eq!(result["providers"][0]["provider"], "Provider name");
|
||||
assert!(result["models"][0]["model"].is_null());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_cleanup_entries_stay_bounded_and_idle_memory_is_released() {
|
||||
let mut history = History::default();
|
||||
for n in 0..1_000 {
|
||||
let guard = begin(&mut history, n, "request", "provider");
|
||||
history.release(n, guard);
|
||||
}
|
||||
assert_eq!(history.expirations.len(), 3);
|
||||
value(&mut history, WINDOW_US);
|
||||
assert_eq!(history.expirations.len(), 1);
|
||||
assert_eq!(history.requests.len(), 1);
|
||||
value(&mut history, WINDOW_US + 1_000);
|
||||
assert!(history.requests.is_empty());
|
||||
assert!(history.providers.is_empty());
|
||||
assert!(history.models.is_empty());
|
||||
assert!(history.expirations.is_empty());
|
||||
assert_eq!(history.request_dimensions, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sampling_limits_report_incomplete_coverage_until_unobserved_work_expires() {
|
||||
let mut history = History::default();
|
||||
history.request_dimensions = MAX_REQUEST_DIMENSIONS;
|
||||
let untracked = begin(&mut history, 0, "request", "provider");
|
||||
assert_eq!(value(&mut history, 1)["coverage"], "partial");
|
||||
assert_eq!(value(&mut history, 2 * WINDOW_US)["coverage"], "partial");
|
||||
history.release(2 * WINDOW_US, untracked);
|
||||
assert_eq!(value(&mut history, 2 * WINDOW_US)["coverage"], "complete");
|
||||
history.request_dimensions = 0;
|
||||
let long_id = "x".repeat(MAX_LABEL_BYTES + 1);
|
||||
let untracked = begin(&mut history, 3 * WINDOW_US, &long_id, "provider");
|
||||
history.release(3 * WINDOW_US, untracked);
|
||||
assert_eq!(
|
||||
value(&mut history, 4 * WINDOW_US - 1)["coverage"],
|
||||
"partial"
|
||||
);
|
||||
assert_eq!(value(&mut history, 4 * WINDOW_US)["coverage"], "complete");
|
||||
}
|
||||
}
|
||||
@@ -181,6 +181,12 @@ async fn settle_cancelled_attempt(
|
||||
usage_data.request_metadata.take(),
|
||||
request_diagnostics.as_ref(),
|
||||
);
|
||||
usage_data.request_metadata = crate::usage::reporting::failure::with_analytics_failure(
|
||||
usage_data.request_metadata.as_ref(),
|
||||
"unknown",
|
||||
"finalize",
|
||||
"request_task_cancelled",
|
||||
);
|
||||
usage_data.status_code = Some(CLIENT_CANCELLED_STATUS_CODE);
|
||||
usage_data.error_message = Some(error_message.to_string());
|
||||
usage_data.error_category = Some("cancelled".to_string());
|
||||
|
||||
@@ -668,8 +668,22 @@ impl ExecutionAttemptLifecycle {
|
||||
});
|
||||
|
||||
// 1. usage terminal
|
||||
let analytics_context = if facts.provider.cancelled_by_provider() {
|
||||
crate::usage::reporting::failure::with_analytics_failure(
|
||||
payload.report_context.as_ref(),
|
||||
"upstream",
|
||||
"stream_read",
|
||||
"provider_cancelled",
|
||||
)
|
||||
} else {
|
||||
crate::usage::reporting::failure::stream_analytics_context(
|
||||
payload.report_context.as_ref(),
|
||||
&payload,
|
||||
facts.delivery.is_aborted() && !facts.provider.is_terminal(),
|
||||
)
|
||||
};
|
||||
let context_seed =
|
||||
build_terminal_usage_context_seed(&self.plan, payload.report_context.as_ref());
|
||||
build_terminal_usage_context_seed(&self.plan, analytics_context.as_ref());
|
||||
let payload_seed = build_stream_terminal_usage_payload_seed(&payload);
|
||||
let billing_void = settlement.billing.is_void();
|
||||
let usage_runtime = Arc::clone(&state.usage_runtime);
|
||||
|
||||
@@ -43,12 +43,11 @@ use crate::AppState;
|
||||
|
||||
const CHATGPT_WEB_INTERNAL_HEADER: &str = "x-aether-chatgpt-web-image";
|
||||
const CHATGPT_WEB_DEFAULT_BASE_URL: &str = "https://chatgpt.com";
|
||||
const CHATGPT_WEB_USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36 Edg/143.0.0.0";
|
||||
const CHATGPT_WEB_CLIENT_VERSION: &str = "prod-be885abbfcfe7b1f511e88b3003d9ee44757fbad";
|
||||
const CHATGPT_WEB_BUILD_NUMBER: &str = "5955942";
|
||||
const CHATGPT_WEB_SEC_CH_UA: &str =
|
||||
r#""Microsoft Edge";v="143", "Chromium";v="143", "Not A(Brand";v="24""#;
|
||||
const CHATGPT_WEB_BROWSER_PROFILE: &str = "chrome143";
|
||||
use aether_provider_transport::client_identity::CHATGPT_WEB_BROWSER_PROFILE;
|
||||
use aether_provider_transport::client_identity::{
|
||||
CHATGPT_WEB_BUILD_NUMBER, CHATGPT_WEB_CLIENT_VERSION, CHATGPT_WEB_SEC_CH_UA,
|
||||
CHATGPT_WEB_USER_AGENT,
|
||||
};
|
||||
const CHATGPT_WEB_QUOTA_REFRESH_TIMEOUT_MS: u64 = 30_000;
|
||||
const CHATGPT_WEB_QUOTA_REFRESH_PROXY_TIMEOUT_MS: u64 = 60_000;
|
||||
const RUNTIME_METADATA_CAS_MAX_ATTEMPTS: usize = 16;
|
||||
|
||||
@@ -191,6 +191,7 @@ pub(crate) fn resolve_core_sync_error_finalize_report_kind(
|
||||
let report_kind = match plan_kind {
|
||||
"openai_chat_sync" => "openai_chat_sync_finalize",
|
||||
"openai_responses_sync" => "openai_responses_sync_finalize",
|
||||
"openai_memories_sync" => "openai_memories_sync_finalize",
|
||||
"openai_responses_compact_sync" => "openai_responses_compact_sync_finalize",
|
||||
"claude_chat_sync" => "claude_chat_sync_finalize",
|
||||
"gemini_chat_sync" => "gemini_chat_sync_finalize",
|
||||
@@ -576,6 +577,14 @@ mod tests {
|
||||
error: None,
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
resolve_core_sync_error_finalize_report_kind(
|
||||
"openai_memories_sync",
|
||||
&result,
|
||||
Some(&serde_json::json!({"error":{"message":"synthetic"}}))
|
||||
),
|
||||
Some("openai_memories_sync_finalize".to_string())
|
||||
);
|
||||
for body_json in [
|
||||
serde_json::json!({"status": "failed", "error": null}),
|
||||
serde_json::json!({"type": "error"}),
|
||||
|
||||
@@ -1114,6 +1114,7 @@ fn grok_canonical_usage(usage: GrokUsageEstimate) -> StreamingCanonicalUsage {
|
||||
|
||||
fn grok_standardized_usage(usage: GrokUsageEstimate) -> StandardizedUsage {
|
||||
let mut standardized = StandardizedUsage::new();
|
||||
standardized.token_source = Some(aether_contracts::UsageTokenSource::Estimated);
|
||||
standardized.input_tokens = i64::try_from(usage.input_tokens).unwrap_or(i64::MAX);
|
||||
standardized.output_tokens = i64::try_from(usage.output_tokens).unwrap_or(i64::MAX);
|
||||
standardized.reasoning_tokens = i64::try_from(usage.reasoning_tokens).unwrap_or(i64::MAX);
|
||||
@@ -3198,13 +3199,15 @@ fn openai_responses_body(
|
||||
let response_id = format!("resp_{}", Uuid::new_v4());
|
||||
let mut output = Vec::new();
|
||||
if !collected.thinking.trim().is_empty() {
|
||||
let thinking = collected.thinking.trim();
|
||||
output.push(json!({
|
||||
"id": openai_responses_synthetic_reasoning_item_id(&response_id, 0),
|
||||
"type": "reasoning",
|
||||
"status": "completed",
|
||||
"summary": [{
|
||||
"type": "summary_text",
|
||||
"text": collected.thinking.trim(),
|
||||
"summary": [],
|
||||
"content": [{
|
||||
"type": "reasoning_text",
|
||||
"text": thinking,
|
||||
}],
|
||||
}));
|
||||
}
|
||||
@@ -4574,6 +4577,101 @@ mod tests {
|
||||
assert!(adapter.text.contains("[[1]](https://example.com/source"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn grok_usage_reports_preserve_estimated_provenance_after_wire_roundtrip() {
|
||||
use aether_usage_runtime::{
|
||||
build_stream_terminal_usage_event, build_sync_terminal_usage_event,
|
||||
GatewayStreamReportRequest, GatewaySyncReportRequest, UsageEventType,
|
||||
};
|
||||
|
||||
for (format, report_prefix) in [
|
||||
("openai:chat", "openai_chat"),
|
||||
("openai:responses", "openai_responses"),
|
||||
] {
|
||||
let mut plan = sample_plan(
|
||||
serde_json::json!({
|
||||
"messages": [{"role": "user", "content": "hello"}]
|
||||
}),
|
||||
format,
|
||||
);
|
||||
plan.stream = false;
|
||||
plan.provider_api_format = format.to_string();
|
||||
// The trusted planner binds this hint to the Grok runtime adapter.
|
||||
// Exercise its transport through the same serialized report as usage.
|
||||
let context = serde_json::json!({
|
||||
"provider_type": "grok",
|
||||
"provider_api_format": format,
|
||||
"client_api_format": format,
|
||||
"usage_token_source": "estimated"
|
||||
});
|
||||
let collected = GrokCollected {
|
||||
status_code: 200,
|
||||
text: "hello back".to_string(),
|
||||
thinking: "short reasoning".to_string(),
|
||||
..GrokCollected::default()
|
||||
};
|
||||
let expected = grok_usage_estimate(&plan, &collected);
|
||||
let result = grok_execution_result(&plan, collected, Some(&context));
|
||||
let sync_report = GatewaySyncReportRequest {
|
||||
trace_id: plan.request_id.clone(),
|
||||
report_kind: format!("{report_prefix}_sync_success"),
|
||||
report_context: Some(context.clone()),
|
||||
status_code: result.status_code,
|
||||
headers: result.headers,
|
||||
body_json: result.body.and_then(|body| body.json_body),
|
||||
client_body_json: None,
|
||||
body_base64: None,
|
||||
telemetry: result.telemetry,
|
||||
};
|
||||
let sync_report: GatewaySyncReportRequest =
|
||||
serde_json::from_slice(&serde_json::to_vec(&sync_report).unwrap()).unwrap();
|
||||
let sync_event = build_sync_terminal_usage_event(
|
||||
&plan,
|
||||
sync_report.report_context.as_ref(),
|
||||
&sync_report,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
plan.stream = true;
|
||||
let stream_report = GatewayStreamReportRequest {
|
||||
trace_id: plan.request_id.clone(),
|
||||
report_kind: format!("{report_prefix}_stream_success"),
|
||||
report_context: Some(context),
|
||||
status_code: 200,
|
||||
headers: BTreeMap::new(),
|
||||
provider_body_base64: None,
|
||||
provider_body_state: None,
|
||||
client_body_base64: None,
|
||||
client_body_state: None,
|
||||
terminal_summary: Some(super::grok_stream_terminal_summary(&plan, expected)),
|
||||
telemetry: None,
|
||||
};
|
||||
let stream_report: GatewayStreamReportRequest =
|
||||
serde_json::from_slice(&serde_json::to_vec(&stream_report).unwrap()).unwrap();
|
||||
let stream_event = build_stream_terminal_usage_event(
|
||||
&plan,
|
||||
stream_report.report_context.as_ref(),
|
||||
&stream_report,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Sync honors the response's explicit total. The existing stream
|
||||
// summary has no explicit total, so its fallback also adds reasoning.
|
||||
let sync_total = expected.input_tokens + expected.output_tokens;
|
||||
let stream_total = sync_total + expected.reasoning_tokens;
|
||||
for (event, expected_total) in [(sync_event, sync_total), (stream_event, stream_total)]
|
||||
{
|
||||
assert_eq!(event.event_type, UsageEventType::Completed, "{format}");
|
||||
assert_eq!(event.data.input_tokens, Some(expected.input_tokens));
|
||||
assert_eq!(event.data.output_tokens, Some(expected.output_tokens));
|
||||
assert_eq!(event.data.total_tokens, Some(expected_total));
|
||||
let metadata = event.data.request_metadata.unwrap();
|
||||
assert_eq!(metadata["analytics_measurement"]["source"], "estimated");
|
||||
assert!(metadata.get("usage_token_source").is_none());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_chat_body_includes_estimated_usage() {
|
||||
let plan = sample_plan(
|
||||
@@ -4627,6 +4725,15 @@ mod tests {
|
||||
serde_json::json!(usage.reasoning_tokens)
|
||||
);
|
||||
assert_eq!(body["output"][0]["type"], serde_json::json!("reasoning"));
|
||||
assert_eq!(
|
||||
body["output"][0]["content"][0]["type"],
|
||||
serde_json::json!("reasoning_text")
|
||||
);
|
||||
assert_eq!(
|
||||
body["output"][0]["content"][0]["text"],
|
||||
serde_json::json!("short reasoning")
|
||||
);
|
||||
assert_eq!(body["output"][0]["summary"], serde_json::json!([]));
|
||||
assert_eq!(body["output"][1]["type"], serde_json::json!("message"));
|
||||
assert!(body["output"][1]["id"]
|
||||
.as_str()
|
||||
@@ -4810,7 +4917,12 @@ mod tests {
|
||||
|
||||
assert!(body.contains("event: response.created"));
|
||||
assert!(body.contains("event: response.in_progress"));
|
||||
assert!(body.contains("event: response.reasoning_summary_part.added"));
|
||||
// Thinking must stay off the summary channel or clients that render
|
||||
// both (Codex) print the raw chain-of-thought twice.
|
||||
assert!(!body.contains("event: response.reasoning_summary_part.added"));
|
||||
assert!(!body.contains("event: response.reasoning_summary_text.delta"));
|
||||
assert!(!body.contains("event: response.reasoning_summary_text.done"));
|
||||
assert!(body.contains("\"type\":\"reasoning_text\""));
|
||||
assert!(body.contains("event: response.content_part.added"));
|
||||
assert!(body.contains("event: response.output_text.done"));
|
||||
assert!(body.contains("event: response.completed"));
|
||||
|
||||
@@ -164,7 +164,7 @@ pub(crate) async fn refresh_oauth_plan_auth_for_retry(
|
||||
key_id = %plan.key_id,
|
||||
status_code,
|
||||
refresh_status_code,
|
||||
error = ?err,
|
||||
error = %crate::error::redact_error_debug(&err),
|
||||
"gateway failed to persist oauth retry refresh failure"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ use std::time::{Duration, Instant};
|
||||
use aether_ai_serving::{AiAttemptExecutionOutcome, AiAttemptRetryScope};
|
||||
use aether_contracts::{
|
||||
ExecutionPlan, ExecutionResponseObservation, ExecutionStreamTerminalSummary,
|
||||
ExecutionTelemetry, StandardizedUsage, StreamFrame, StreamFramePayload,
|
||||
ExecutionTelemetry, StandardizedUsage, StreamFrame, StreamFramePayload, UsageTokenSource,
|
||||
};
|
||||
use aether_data_contracts::repository::candidates::{
|
||||
RequestCandidateStatus, UpsertRequestCandidateRecord,
|
||||
@@ -445,11 +445,15 @@ fn build_sync_terminal_usage_seeds(
|
||||
report_context: Option<&serde_json::Value>,
|
||||
payload: &GatewaySyncReportRequest,
|
||||
) -> (TerminalUsageContextSeed, SyncTerminalUsagePayloadSeed) {
|
||||
let analytics_context =
|
||||
crate::usage::reporting::failure::sync_analytics_context(report_context, payload);
|
||||
let report_context_with_diagnostics =
|
||||
attach_current_request_diagnostics_to_report_context(report_context);
|
||||
attach_current_request_diagnostics_to_report_context(analytics_context.as_ref());
|
||||
let context_seed = build_terminal_usage_context_seed(
|
||||
plan,
|
||||
report_context_with_diagnostics.as_ref().or(report_context),
|
||||
report_context_with_diagnostics
|
||||
.as_ref()
|
||||
.or(analytics_context.as_ref()),
|
||||
);
|
||||
let payload_seed = build_sync_terminal_usage_payload_seed(payload);
|
||||
(context_seed, payload_seed)
|
||||
@@ -586,7 +590,12 @@ async fn record_stream_terminal_usage(
|
||||
cancelled: bool,
|
||||
) {
|
||||
crate::execution_runtime::mark_stream_candidate_watchdog_terminal_started();
|
||||
let context_seed = build_terminal_usage_context_seed(plan, report_context);
|
||||
let analytics_context = crate::usage::reporting::failure::stream_analytics_context(
|
||||
report_context,
|
||||
payload,
|
||||
cancelled,
|
||||
);
|
||||
let context_seed = build_terminal_usage_context_seed(plan, analytics_context.as_ref());
|
||||
let payload_seed = build_stream_terminal_usage_payload_seed(payload);
|
||||
state
|
||||
.usage_runtime
|
||||
@@ -976,6 +985,9 @@ async fn maybe_apply_kiro_prompt_cache_usage_to_stream_summary(
|
||||
usage.cache_read_tokens = 0;
|
||||
if usage.input_tokens <= 0 {
|
||||
usage.input_tokens = estimated_input_tokens as i64;
|
||||
if usage.input_tokens > 0 {
|
||||
mark_kiro_stream_estimated_usage(usage, report_context, false);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
@@ -984,6 +996,10 @@ async fn maybe_apply_kiro_prompt_cache_usage_to_stream_summary(
|
||||
usage.input_tokens = kiro_billed_input_tokens(estimated_input_tokens, cache_usage) as i64;
|
||||
usage.cache_creation_tokens = cache_usage.cache_creation_input_tokens as i64;
|
||||
usage.cache_read_tokens = cache_usage.cache_read_input_tokens as i64;
|
||||
if usage.input_tokens > 0 || usage.cache_creation_tokens > 0 || usage.cache_read_tokens > 0
|
||||
{
|
||||
mark_kiro_stream_estimated_usage(usage, report_context, false);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -996,12 +1012,18 @@ async fn maybe_apply_kiro_prompt_cache_usage_to_stream_summary(
|
||||
cache_read_input_tokens: usage.cache_read_tokens.max(0) as u64,
|
||||
},
|
||||
) as i64;
|
||||
if usage.input_tokens > 0 {
|
||||
mark_kiro_stream_estimated_usage(usage, report_context, true);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if usage.input_tokens <= 0 {
|
||||
usage.input_tokens = estimated_input_tokens as i64;
|
||||
if usage.input_tokens > 0 {
|
||||
mark_kiro_stream_estimated_usage(usage, report_context, true);
|
||||
}
|
||||
}
|
||||
|
||||
let Some(profile) =
|
||||
@@ -1024,6 +1046,35 @@ async fn maybe_apply_kiro_prompt_cache_usage_to_stream_summary(
|
||||
usage.input_tokens = billed_input_tokens as i64;
|
||||
usage.cache_creation_tokens = cache_usage.cache_creation_input_tokens as i64;
|
||||
usage.cache_read_tokens = cache_usage.cache_read_input_tokens as i64;
|
||||
mark_kiro_stream_estimated_usage(usage, report_context, false);
|
||||
}
|
||||
|
||||
fn mark_kiro_stream_estimated_usage(
|
||||
usage: &mut StandardizedUsage,
|
||||
report_context: &Value,
|
||||
retains_cache: bool,
|
||||
) {
|
||||
let retained_source = usage.token_source.unwrap_or_else(|| {
|
||||
match report_context
|
||||
.get("usage_token_source")
|
||||
.and_then(Value::as_str)
|
||||
{
|
||||
Some("estimated") => UsageTokenSource::Estimated,
|
||||
Some("mixed") => UsageTokenSource::Mixed,
|
||||
_ => UsageTokenSource::Reported,
|
||||
}
|
||||
});
|
||||
let retains_reported_tokens = retained_source != UsageTokenSource::Estimated
|
||||
&& (usage.output_tokens > 0
|
||||
|| usage.reasoning_tokens > 0
|
||||
|| usage.cache_creation_ephemeral_5m_tokens > 0
|
||||
|| usage.cache_creation_ephemeral_1h_tokens > 0
|
||||
|| (retains_cache && (usage.cache_creation_tokens > 0 || usage.cache_read_tokens > 0)));
|
||||
usage.token_source = Some(if retains_reported_tokens {
|
||||
UsageTokenSource::Mixed
|
||||
} else {
|
||||
UsageTokenSource::Estimated
|
||||
});
|
||||
}
|
||||
|
||||
fn append_stream_capture_bytes(
|
||||
@@ -3963,7 +4014,7 @@ async fn execute_execution_runtime_stream_inner(
|
||||
let candidate_started_unix_secs = current_request_candidate_unix_ms();
|
||||
let provider_in_flight_started_at = Instant::now();
|
||||
let mut provider_pool_in_flight_guard =
|
||||
match acquire_provider_pool_execution_guard(state, &plan).await? {
|
||||
match acquire_provider_pool_execution_guard(state, &plan, report_context.as_ref()).await? {
|
||||
ProviderPoolInFlightAdmission::Acquired(guard) => guard,
|
||||
ProviderPoolInFlightAdmission::Saturated { limit } => {
|
||||
record_local_runtime_candidate_skip_reason(
|
||||
@@ -6462,6 +6513,21 @@ async fn execute_stream_from_frame_stream_with_retry_scope(
|
||||
|
||||
let normalized_stream_report_context =
|
||||
normalize_provider_private_report_context(report_context.as_ref());
|
||||
// Observers follow the live protocol stream across prefetch and transfer.
|
||||
// Diagnostic capture limits must never determine parser state.
|
||||
let stream_usage_report_context = normalized_stream_report_context.clone().or_else(|| {
|
||||
Some(json!({
|
||||
"provider_api_format": plan.provider_api_format.as_str(),
|
||||
"client_api_format": plan.client_api_format.as_str(),
|
||||
}))
|
||||
});
|
||||
let mut stream_usage_observer = stream_usage_report_context
|
||||
.as_ref()
|
||||
.map(|_| StreamingStandardTerminalObserver::default());
|
||||
let mut stream_usage_observer_buffered =
|
||||
StreamUsageObservationBuffer::new(max_stream_body_buffer_bytes);
|
||||
let mut provider_error_inspection = ProviderStreamErrorInspection::default();
|
||||
let mut prefetched_provider_error = None;
|
||||
let upstream_headers = headers.clone();
|
||||
let mut private_stream_normalizer =
|
||||
maybe_build_provider_private_stream_normalizer(report_context.as_ref());
|
||||
@@ -6562,7 +6628,8 @@ async fn execute_stream_from_frame_stream_with_retry_scope(
|
||||
stream_commit_gate.commit();
|
||||
}
|
||||
let mut prefetched_chunks: Vec<Bytes> = Vec::new();
|
||||
let mut provider_prefetched_body = Vec::new();
|
||||
let mut provider_prefetched_body = StreamBodyCapture::default();
|
||||
let mut provider_prefetched_bytes = 0_u64;
|
||||
let mut provider_prefetched_body_truncated = false;
|
||||
let mut prefetched_body = Vec::new();
|
||||
let mut prefetched_inspection_body = Vec::new();
|
||||
@@ -6815,10 +6882,12 @@ async fn execute_stream_from_frame_stream_with_retry_scope(
|
||||
}
|
||||
}
|
||||
|
||||
append_stream_capture_bytes(
|
||||
provider_prefetched_bytes =
|
||||
provider_prefetched_bytes.saturating_add(chunk.len() as u64);
|
||||
append_budgeted_stream_capture_bytes(
|
||||
&mut provider_prefetched_body,
|
||||
&chunk,
|
||||
MAX_STREAM_PREFETCH_BYTES,
|
||||
max_stream_body_buffer_bytes,
|
||||
&mut provider_prefetched_body_truncated,
|
||||
);
|
||||
append_stream_capture_bytes(
|
||||
@@ -7063,6 +7132,22 @@ async fn execute_stream_from_frame_stream_with_retry_scope(
|
||||
} else {
|
||||
chunk
|
||||
};
|
||||
if let Some(error) = provider_error_inspection
|
||||
.observe(stream_usage_report_context.as_ref(), &normalized_chunk)
|
||||
{
|
||||
prefetched_provider_error.get_or_insert(error);
|
||||
}
|
||||
if let (Some(observer), Some(context)) = (
|
||||
stream_usage_observer.as_mut(),
|
||||
stream_usage_report_context.as_ref(),
|
||||
) {
|
||||
observe_stream_usage_bytes(
|
||||
observer,
|
||||
context,
|
||||
&mut stream_usage_observer_buffered,
|
||||
&normalized_chunk,
|
||||
);
|
||||
}
|
||||
let rewritten_chunk = if let Some(rewriter) = local_stream_rewriter.as_mut() {
|
||||
match rewriter.push_chunk(&normalized_chunk) {
|
||||
Ok(rewritten_chunk) => rewritten_chunk,
|
||||
@@ -7193,17 +7278,21 @@ async fn execute_stream_from_frame_stream_with_retry_scope(
|
||||
if stream_commit_gate.is_uncommitted() {
|
||||
stream_commit_gate.commit();
|
||||
}
|
||||
let prefetched_response_history_persisted = if let Some(record) = local_stream_rewriter
|
||||
if let Some(record) = local_stream_rewriter
|
||||
.as_mut()
|
||||
.and_then(|rewriter| rewriter.take_response_history_record())
|
||||
{
|
||||
crate::ai_serving::persist_response_history_record(state, record).await;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
};
|
||||
drop(private_stream_normalizer);
|
||||
drop(local_stream_rewriter);
|
||||
}
|
||||
// Keep partial records and conversion state; replaying the bounded
|
||||
// inspection/capture prefix loses any bytes consumed beyond that prefix.
|
||||
let mut private_stream_normalizer = private_stream_normalizer.map(|parser| parser.into_owned());
|
||||
let mut local_stream_rewriter = local_stream_rewriter.map(|parser| parser.into_owned());
|
||||
if sync_json_stream_bridge_active {
|
||||
private_stream_normalizer = None;
|
||||
local_stream_rewriter = None;
|
||||
stream_usage_observer = None;
|
||||
}
|
||||
|
||||
let initial_usage_telemetry = prefetched_usage_telemetry.clone().or_else(|| {
|
||||
prefetched_telemetry
|
||||
@@ -7246,7 +7335,6 @@ async fn execute_stream_from_frame_stream_with_retry_scope(
|
||||
let headers_for_report = headers.clone();
|
||||
let report_kind_owned = report_kind;
|
||||
let report_context_owned = report_context;
|
||||
let normalized_stream_report_context_owned = normalized_stream_report_context;
|
||||
let lifecycle_seed_for_report = lifecycle_seed;
|
||||
let provider_prefetched_body_for_report = provider_prefetched_body;
|
||||
let prefetched_body_for_report = prefetched_body;
|
||||
@@ -7288,40 +7376,10 @@ async fn execute_stream_from_frame_stream_with_retry_scope(
|
||||
let _stream_total_guard =
|
||||
StageElapsedGuard::from_started_at("stream_total", stream_started_at_for_report);
|
||||
let _provider_pool_in_flight_guard = provider_pool_in_flight_guard_for_report;
|
||||
let mut provider_buffered_body = StreamBodyCapture::default();
|
||||
let mut provider_buffered_body = provider_prefetched_body_for_report;
|
||||
let mut buffered_body = StreamBodyCapture::default();
|
||||
let mut provider_body_truncated = false;
|
||||
let mut provider_body_truncated = provider_prefetched_body_truncated;
|
||||
let mut client_body_truncated = false;
|
||||
let mut private_stream_normalizer = if sync_json_stream_bridge_active_for_report {
|
||||
None
|
||||
} else {
|
||||
maybe_build_provider_private_stream_normalizer(report_context_owned.as_ref())
|
||||
};
|
||||
let mut local_stream_rewriter = if sync_json_stream_bridge_active_for_report {
|
||||
None
|
||||
} else {
|
||||
maybe_build_stream_response_rewriter(normalized_stream_report_context_owned.as_ref())
|
||||
};
|
||||
let stream_usage_report_context =
|
||||
normalized_stream_report_context_owned.clone().or_else(|| {
|
||||
Some(serde_json::json!({
|
||||
"provider_api_format": plan_for_report.provider_api_format.as_str(),
|
||||
"client_api_format": plan_for_report.client_api_format.as_str(),
|
||||
}))
|
||||
});
|
||||
let mut stream_usage_observer = stream_usage_report_context
|
||||
.as_ref()
|
||||
.filter(|_| !sync_json_stream_bridge_active_for_report)
|
||||
.map(|_| StreamingStandardTerminalObserver::default());
|
||||
let mut stream_usage_observer_buffered =
|
||||
StreamUsageObservationBuffer::new(max_stream_body_buffer_bytes);
|
||||
let mut provider_error_inspection = ProviderStreamErrorInspection::default();
|
||||
append_budgeted_stream_capture_bytes(
|
||||
&mut provider_buffered_body,
|
||||
&provider_prefetched_body_for_report,
|
||||
max_stream_body_buffer_bytes,
|
||||
&mut provider_body_truncated,
|
||||
);
|
||||
append_budgeted_stream_capture_bytes(
|
||||
&mut buffered_body,
|
||||
&prefetched_body_for_report,
|
||||
@@ -7365,9 +7423,7 @@ async fn execute_stream_from_frame_stream_with_retry_scope(
|
||||
} else {
|
||||
initial_elapsed_ms
|
||||
}));
|
||||
let provider_stream_bytes = Arc::new(AtomicU64::new(
|
||||
u64::try_from(provider_prefetched_body_for_report.len()).unwrap_or(u64::MAX),
|
||||
));
|
||||
let provider_stream_bytes = Arc::new(AtomicU64::new(provider_prefetched_bytes));
|
||||
let client_stream_bytes = Arc::new(AtomicU64::new(
|
||||
u64::try_from(prefetched_body_for_report.len()).unwrap_or(u64::MAX),
|
||||
));
|
||||
@@ -7463,96 +7519,20 @@ async fn execute_stream_from_frame_stream_with_retry_scope(
|
||||
}
|
||||
})
|
||||
};
|
||||
if !provider_prefetched_body_for_report.is_empty() {
|
||||
let normalized_prefetched_chunk = if let Some(normalizer) =
|
||||
private_stream_normalizer.as_mut()
|
||||
{
|
||||
match normalizer.push_chunk(&provider_prefetched_body_for_report) {
|
||||
Ok(normalized_chunk) => Some(normalized_chunk),
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event_name = "stream_execution_prefetch_normalize_restore_failed",
|
||||
log_type = "ops",
|
||||
trace_id = %trace_id_owned,
|
||||
request_id = %request_id_for_report_log,
|
||||
candidate_id = ?candidate_id_for_report.as_deref(),
|
||||
error_category = "stream_normalization_restore_failed",
|
||||
"gateway failed to restore private stream normalization state after prefetch"
|
||||
);
|
||||
terminal_failure = Some(build_stream_failure_report(
|
||||
"execution_runtime_stream_rewrite_error",
|
||||
format!(
|
||||
"failed to restore private stream normalization state after prefetch: {err:?}"
|
||||
),
|
||||
502,
|
||||
));
|
||||
None
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let replay_chunk = normalized_prefetched_chunk
|
||||
.as_deref()
|
||||
.unwrap_or(provider_prefetched_body_for_report.as_slice());
|
||||
if let Some(error_body_json) = provider_error_inspection
|
||||
.observe(stream_usage_report_context.as_ref(), replay_chunk)
|
||||
{
|
||||
provider_error_forwarded_to_client = !prefetched_body_for_report.is_empty();
|
||||
let error_status_code = resolve_provider_stream_error_status_code(
|
||||
plan_for_report.provider_api_format.as_str(),
|
||||
status_code,
|
||||
&error_body_json,
|
||||
);
|
||||
terminal_failure = Some(build_stream_failure_from_provider_error_body(
|
||||
error_status_code,
|
||||
&error_body_json,
|
||||
));
|
||||
}
|
||||
if let (Some(observer), Some(report_context)) = (
|
||||
stream_usage_observer.as_mut(),
|
||||
stream_usage_report_context.as_ref(),
|
||||
) {
|
||||
observe_stream_usage_bytes(
|
||||
observer,
|
||||
report_context,
|
||||
&mut stream_usage_observer_buffered,
|
||||
replay_chunk,
|
||||
);
|
||||
}
|
||||
if terminal_failure.is_none() {
|
||||
if let Some(rewriter) = local_stream_rewriter.as_mut() {
|
||||
if let Err(err) = rewriter.push_chunk(replay_chunk) {
|
||||
warn!(
|
||||
event_name = "stream_execution_prefetch_rewrite_restore_failed",
|
||||
log_type = "ops",
|
||||
trace_id = %trace_id_owned,
|
||||
request_id = %request_id_for_report_log,
|
||||
candidate_id = ?candidate_id_for_report.as_deref(),
|
||||
error_category = "stream_rewrite_restore_failed",
|
||||
"gateway failed to restore local stream rewrite state after prefetch"
|
||||
);
|
||||
terminal_failure = Some(build_stream_failure_report(
|
||||
"execution_runtime_stream_rewrite_error",
|
||||
format!(
|
||||
"failed to restore local stream rewrite state after prefetch: {err:?}"
|
||||
),
|
||||
502,
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
if prefetched_response_history_persisted {
|
||||
if let Some(rewriter) = local_stream_rewriter.as_mut() {
|
||||
let _ = rewriter.take_response_history_record();
|
||||
}
|
||||
}
|
||||
if let Some(error_body_json) = prefetched_provider_error {
|
||||
provider_error_forwarded_to_client = !prefetched_body_for_report.is_empty();
|
||||
let error_status_code = resolve_provider_stream_error_status_code(
|
||||
plan_for_report.provider_api_format.as_str(),
|
||||
status_code,
|
||||
&error_body_json,
|
||||
);
|
||||
terminal_failure = Some(build_stream_failure_from_provider_error_body(
|
||||
error_status_code,
|
||||
&error_body_json,
|
||||
));
|
||||
}
|
||||
|
||||
// These buffers restore parser/rewriter state above. Audit capture owns
|
||||
// its budgeted copies; retaining semantic prefetch duplicates for the
|
||||
// rest of the stream would bypass the capture memory limit.
|
||||
drop(provider_prefetched_body_for_report);
|
||||
// Parser state is already current and capture owns its budgeted bytes.
|
||||
// This output prefix is needed only to initialize client-side trackers.
|
||||
drop(prefetched_body_for_report);
|
||||
|
||||
if terminal_failure.is_none() && !reached_eof {
|
||||
@@ -9312,6 +9292,188 @@ mod tests {
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prefetch_handoff_preserves_large_responses_setup_event() {
|
||||
let event = format!(
|
||||
"event: response.created\ndata: {}\n\n",
|
||||
json!({"type":"response.created", "response": {
|
||||
"id":"resp-large-setup", "status":"in_progress", "output":[],
|
||||
"tools":[{"name":"write", "description":"x".repeat(64 * 1024)}]
|
||||
}})
|
||||
);
|
||||
let done = "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp-large-setup\",\"status\":\"completed\",\"output\":[],\"usage\":{\"input_tokens\":7,\"output_tokens\":2}}}\n\n";
|
||||
// Include the two observed transport boundaries, exact/near budget
|
||||
// boundaries, and multiple prefetch chunks crossing the budget.
|
||||
for cuts in [
|
||||
vec![16_383],
|
||||
vec![16_384],
|
||||
vec![17_735],
|
||||
vec![17_741],
|
||||
vec![8_192, 17_735],
|
||||
] {
|
||||
let mut chunks = Vec::new();
|
||||
let mut start = 0;
|
||||
for end in cuts {
|
||||
chunks.push(&event[start..end]);
|
||||
start = end;
|
||||
}
|
||||
chunks.push(&event[start..]);
|
||||
chunks.push(done);
|
||||
let response = execute_generic_sse_precommit(chunks, json!({}), None, false)
|
||||
.await
|
||||
.expect("large setup should commit at the bounded prefetch limit");
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
|
||||
let body = String::from_utf8(body.to_vec()).unwrap();
|
||||
assert!(
|
||||
body.starts_with(&event),
|
||||
"setup bytes lost or duplicated at split {start}"
|
||||
);
|
||||
let events: Vec<Value> = body
|
||||
.lines()
|
||||
.filter_map(|line| line.strip_prefix("data: "))
|
||||
.filter(|payload| *payload != "[DONE]")
|
||||
.map(|payload| {
|
||||
serde_json::from_str(payload).expect("every SSE payload must be valid JSON")
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(events.len(), 2, "events must be forwarded exactly once");
|
||||
assert_eq!(events[1]["type"], "response.completed");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prefetch_handoff_keeps_audit_usage_and_private_conversion() {
|
||||
for private in [false, true] {
|
||||
let request_id = format!("handoff-audit-{}", uuid::Uuid::new_v4());
|
||||
let mut plan = if private {
|
||||
antigravity_gemini_stream_plan(&request_id)
|
||||
} else {
|
||||
native_anthropic_stream_plan(&request_id)
|
||||
};
|
||||
if !private {
|
||||
plan.provider_api_format = "openai:responses".into();
|
||||
plan.client_api_format = "openai:responses".into();
|
||||
}
|
||||
let context = json!({
|
||||
"request_id": request_id, "candidate_id": plan.candidate_id,
|
||||
"candidate_index":0, "retry_index":0,
|
||||
"provider_api_format": plan.provider_api_format,
|
||||
"client_api_format": plan.client_api_format,
|
||||
"needs_conversion": private, "has_envelope": private,
|
||||
"envelope_name": if private { "antigravity:v1internal" } else { "" },
|
||||
});
|
||||
let repository = Arc::new(InMemoryUsageReadRepository::default());
|
||||
let catalog = provider_catalog_for_plan(&plan, None);
|
||||
let state = AppState::new()
|
||||
.unwrap()
|
||||
.with_data_state_for_tests(
|
||||
crate::data::GatewayDataState::with_usage_repository_for_tests(Arc::clone(
|
||||
&repository,
|
||||
))
|
||||
.with_provider_catalog_reader(Arc::new(catalog))
|
||||
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY)
|
||||
.with_system_config_values_for_tests([(
|
||||
"request_record_level".into(),
|
||||
json!("full"),
|
||||
)]),
|
||||
)
|
||||
.with_usage_runtime_for_tests(UsageRuntimeConfig {
|
||||
enabled: true,
|
||||
..Default::default()
|
||||
});
|
||||
let text = "hello".repeat(12_000);
|
||||
let payload = if private {
|
||||
json!({"response":{"candidates":[{"content":{"role":"model","parts":[{"text":text}]},
|
||||
"finishReason":"STOP"}],"usageMetadata":{"promptTokenCount":1234,"candidatesTokenCount":567},
|
||||
"modelVersion":"gemini-3.7-flash-tiered"}})
|
||||
} else {
|
||||
json!({"type":"response.completed","response":{"id":"resp-handoff-usage","status":"completed",
|
||||
"output":[{"type":"message","id":"msg-handoff","role":"assistant","status":"completed",
|
||||
"content":[{"type":"output_text","text":text,"annotations":[]}]}],
|
||||
"usage":{"input_tokens":1234,"output_tokens":567,"total_tokens":1801}}})
|
||||
};
|
||||
let input = format!("data: {payload}\n\n");
|
||||
// One complete large chunk exercises an already-emitted prefetch
|
||||
// result; the private path exercises incomplete normalization too.
|
||||
let chunks = if private {
|
||||
vec![input[..17_735].to_string(), input[17_735..].to_string()]
|
||||
} else {
|
||||
vec![input.clone()]
|
||||
};
|
||||
let frames = stream! {
|
||||
yield Ok::<Bytes, std::io::Error>(ndjson_frame(StreamFrame {
|
||||
frame_type:StreamFrameType::Headers,
|
||||
payload:StreamFramePayload::Headers { status_code:200,
|
||||
headers:BTreeMap::from([("content-type".into(),"text/event-stream".into())]),
|
||||
response_observation:None },
|
||||
}));
|
||||
for chunk in chunks {
|
||||
yield Ok(ndjson_frame(StreamFrame { frame_type:StreamFrameType::Data,
|
||||
payload:StreamFramePayload::Data { text:Some(chunk),chunk_b64:None } }));
|
||||
}
|
||||
yield Ok(ndjson_frame(StreamFrame::eof()));
|
||||
}.boxed();
|
||||
let response = execute_stream_from_frame_stream(
|
||||
&state,
|
||||
plan,
|
||||
"trace-handoff-audit",
|
||||
&test_decision(),
|
||||
OPENAI_RESPONSES_STREAM_PLAN_KIND,
|
||||
Some("openai_responses_stream_success".into()),
|
||||
Some(context),
|
||||
crate::clock::current_unix_ms(),
|
||||
Instant::now(),
|
||||
RequestStageTrace::from_env(),
|
||||
false,
|
||||
frames,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
|
||||
let body = String::from_utf8(body.to_vec()).unwrap();
|
||||
let events: Vec<Value> = body
|
||||
.lines()
|
||||
.filter_map(|l| l.strip_prefix("data: "))
|
||||
.filter(|p| *p != "[DONE]")
|
||||
.map(|p| serde_json::from_str(p).unwrap())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
events
|
||||
.iter()
|
||||
.filter(|e| e["type"] == "response.completed")
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
assert!(body.contains(&text));
|
||||
let usage = tokio::time::timeout(Duration::from_secs(3), async {
|
||||
loop {
|
||||
if let Some(u) = repository
|
||||
.find_by_request_id(&request_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.filter(|u| u.status == "completed" || u.status == "failed")
|
||||
{
|
||||
break u;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("usage should finalize");
|
||||
assert_eq!(usage.status, "completed", "{:?}", usage.error_message);
|
||||
assert_eq!(usage.input_tokens, 1234);
|
||||
assert_eq!(usage.output_tokens, 567);
|
||||
let captured = usage.response_body.as_ref().expect("provider capture");
|
||||
assert!(
|
||||
captured["metadata"].get("dropped_chunks").is_none(),
|
||||
"{captured}"
|
||||
);
|
||||
assert_eq!(captured["chunks"].as_array().unwrap(), &vec![payload]);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn generic_stream_success_regex_matches_fragmented_plain_body() {
|
||||
for chunks in [
|
||||
@@ -9850,7 +10012,7 @@ mod tests {
|
||||
let mut buffer = super::StreamUsageObservationBuffer::new(32 * 1024);
|
||||
let mut rewriter = super::maybe_build_stream_response_rewriter(Some(&context)).unwrap();
|
||||
let mut delivered = Vec::new();
|
||||
for chunk in chunks {
|
||||
for (index, chunk) in chunks.into_iter().enumerate() {
|
||||
provider.append(chunk, 32 * 1024, &mut provider_truncated);
|
||||
super::observe_stream_usage_bytes(
|
||||
observer.as_mut().unwrap(),
|
||||
@@ -9861,6 +10023,10 @@ mod tests {
|
||||
let output = rewriter.push_chunk(chunk).unwrap();
|
||||
client.append(&output, 32 * 1024, &mut client_truncated);
|
||||
delivered.extend(output);
|
||||
if index == 0 {
|
||||
// Task handoff must also work when audit admits no bytes.
|
||||
rewriter = rewriter.into_owned();
|
||||
}
|
||||
}
|
||||
let tail = rewriter.finish().unwrap();
|
||||
client.append(&tail, 32 * 1024, &mut client_truncated);
|
||||
@@ -11880,7 +12046,11 @@ mod tests {
|
||||
.expect("response body should read");
|
||||
let body = String::from_utf8(body.to_vec()).expect("response body should be utf8");
|
||||
assert!(
|
||||
body.contains("event: response.reasoning_summary_text.delta\n"),
|
||||
body.contains("event: response.reasoning_text.delta\n"),
|
||||
"{body}"
|
||||
);
|
||||
assert!(
|
||||
!body.contains("event: response.reasoning_summary_text.delta\n"),
|
||||
"{body}"
|
||||
);
|
||||
assert!(
|
||||
@@ -12293,6 +12463,10 @@ mod tests {
|
||||
.expect("first usage should exist");
|
||||
assert!(first_usage.cache_creation_tokens > 0);
|
||||
assert_eq!(first_usage.cache_read_tokens, 0);
|
||||
assert_eq!(
|
||||
first_usage.token_source,
|
||||
Some(aether_contracts::UsageTokenSource::Mixed)
|
||||
);
|
||||
|
||||
let mut second_summary = Some(ExecutionStreamTerminalSummary {
|
||||
standardized_usage: Some(StandardizedUsage {
|
||||
@@ -12317,6 +12491,10 @@ mod tests {
|
||||
assert_eq!(second_usage.cache_creation_tokens, 0);
|
||||
assert!(second_usage.input_tokens < 6_000);
|
||||
assert_eq!(second_usage.output_tokens, 19);
|
||||
assert_eq!(
|
||||
second_usage.token_source,
|
||||
Some(aether_contracts::UsageTokenSource::Mixed)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -12512,6 +12690,49 @@ mod tests {
|
||||
assert_eq!(usage.cache_creation_tokens, 0);
|
||||
assert_eq!(usage.cache_read_tokens, 0);
|
||||
assert_eq!(usage.output_tokens, 13);
|
||||
assert_eq!(
|
||||
usage.token_source,
|
||||
Some(aether_contracts::UsageTokenSource::Mixed)
|
||||
);
|
||||
|
||||
use aether_contracts::UsageTokenSource::{Estimated, Mixed};
|
||||
for (hint, source, input, output, cache, expected) in [
|
||||
(Some("estimated"), None, 0, 13, 0, Some(Estimated)),
|
||||
(None, Some(Estimated), 0, 13, 0, Some(Estimated)),
|
||||
(None, None, 0, 0, 200, Some(Mixed)),
|
||||
(None, None, 0, 0, 0, Some(Estimated)),
|
||||
(None, None, 50, 13, 0, None),
|
||||
] {
|
||||
let mut context = report_context.clone();
|
||||
if let Some(hint) = hint {
|
||||
context["usage_token_source"] = json!(hint);
|
||||
}
|
||||
let mut summary = Some(ExecutionStreamTerminalSummary {
|
||||
standardized_usage: Some(StandardizedUsage {
|
||||
token_source: source,
|
||||
input_tokens: input,
|
||||
output_tokens: output,
|
||||
cache_read_tokens: cache,
|
||||
..StandardizedUsage::new()
|
||||
}),
|
||||
..Default::default()
|
||||
});
|
||||
maybe_apply_kiro_prompt_cache_usage_to_stream_summary(
|
||||
&state,
|
||||
&plan,
|
||||
Some(&context),
|
||||
&mut summary,
|
||||
)
|
||||
.await;
|
||||
let usage = summary.unwrap().standardized_usage.unwrap();
|
||||
assert!(usage.input_tokens > 0);
|
||||
assert_eq!(usage.output_tokens, output);
|
||||
assert_eq!(usage.cache_read_tokens, cache);
|
||||
assert_eq!(
|
||||
usage.token_source, expected,
|
||||
"hint={hint:?}, source={source:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -12751,6 +12972,10 @@ mod tests {
|
||||
assert_eq!(usage.cache_creation_tokens, 175);
|
||||
assert_eq!(usage.cache_read_tokens, 24_463);
|
||||
assert_eq!(usage.output_tokens, 167);
|
||||
assert_eq!(
|
||||
usage.token_source,
|
||||
Some(aether_contracts::UsageTokenSource::Mixed)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -45,6 +45,7 @@ pub(super) struct StreamFailureReport {
|
||||
honor_http_failover: bool,
|
||||
extra_error_fields: Map<String, Value>,
|
||||
provider_body_json: Option<Value>,
|
||||
analytics_failure: Option<Value>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
@@ -133,6 +134,7 @@ impl StreamFailureReport {
|
||||
honor_http_failover: _,
|
||||
mut extra_error_fields,
|
||||
provider_body_json,
|
||||
analytics_failure: _,
|
||||
} = self;
|
||||
extra_error_fields.insert("type".to_string(), Value::String(error_type));
|
||||
extra_error_fields.insert("message".to_string(), Value::String(error_message));
|
||||
@@ -178,6 +180,7 @@ pub(super) fn build_stream_failure_report(
|
||||
honor_http_failover: false,
|
||||
extra_error_fields: Map::new(),
|
||||
provider_body_json: None,
|
||||
analytics_failure: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -196,6 +199,7 @@ pub(super) fn build_stream_transport_failure_report(
|
||||
honor_http_failover: false,
|
||||
extra_error_fields: Map::new(),
|
||||
provider_body_json: None,
|
||||
analytics_failure: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -241,6 +245,10 @@ pub(super) fn build_stream_failure_from_execution_error(
|
||||
honor_http_failover: error.upstream_status.is_some(),
|
||||
extra_error_fields: error_object,
|
||||
provider_body_json: None,
|
||||
analytics_failure: crate::usage::reporting::failure::execution_error_analytics_context(
|
||||
None, error,
|
||||
)
|
||||
.and_then(|context| context.get("analytics_failure").cloned()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -271,6 +279,7 @@ pub(super) fn build_stream_failure_from_provider_error_body(
|
||||
honor_http_failover: true,
|
||||
extra_error_fields: Map::new(),
|
||||
provider_body_json: Some(body_json.clone()),
|
||||
analytics_failure: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -334,6 +343,7 @@ fn build_stream_failure_sync_payload(
|
||||
let status_code = failure.status_code;
|
||||
let upstream_status_code = failure.upstream_status_code;
|
||||
let transport_error = failure.transport_error;
|
||||
let analytics_failure = failure.analytics_failure.clone();
|
||||
let (body, client_body) = failure.into_body_jsons();
|
||||
headers.retain(|name, _| {
|
||||
!name.eq_ignore_ascii_case("content-encoding")
|
||||
@@ -355,6 +365,9 @@ fn build_stream_failure_sync_payload(
|
||||
.or(report_context);
|
||||
let report_context = report_context.map(|mut context| {
|
||||
if let Some(object) = context.as_object_mut() {
|
||||
if let Some(failure) = analytics_failure {
|
||||
object.insert("analytics_failure".into(), failure);
|
||||
}
|
||||
let response_headers = serde_json::to_value(&headers).unwrap_or(Value::Null);
|
||||
if upstream_status_code.is_some() {
|
||||
object.insert(
|
||||
@@ -499,9 +512,11 @@ async fn record_stream_sync_failure(
|
||||
);
|
||||
if !matches!(handling, StreamFailureHandling::HonorLocalFailover) || !retrying_next_candidate {
|
||||
crate::execution_runtime::mark_stream_candidate_watchdog_terminal_started();
|
||||
let analytics_context =
|
||||
crate::usage::reporting::failure::sync_analytics_context(report_context, payload);
|
||||
let report_context_with_diagnostics =
|
||||
attach_current_request_diagnostics_and_candidate_timing_to_report_context(
|
||||
report_context,
|
||||
analytics_context.as_ref(),
|
||||
payload
|
||||
.telemetry
|
||||
.as_ref()
|
||||
@@ -513,7 +528,9 @@ async fn record_stream_sync_failure(
|
||||
);
|
||||
let context_seed = build_terminal_usage_context_seed(
|
||||
plan,
|
||||
report_context_with_diagnostics.as_ref().or(report_context),
|
||||
report_context_with_diagnostics
|
||||
.as_ref()
|
||||
.or(analytics_context.as_ref()),
|
||||
);
|
||||
let payload_seed = build_sync_terminal_usage_payload_seed(payload);
|
||||
state
|
||||
@@ -779,9 +796,13 @@ async fn handle_prefetch_transport_stream_failure(
|
||||
&& matches!(analysis.decision, LocalFailoverDecision::RetryNextCandidate);
|
||||
if !retrying_next_candidate {
|
||||
crate::execution_runtime::mark_stream_candidate_watchdog_terminal_started();
|
||||
let analytics_context = crate::usage::reporting::failure::sync_analytics_context(
|
||||
payload.report_context.as_ref(),
|
||||
&payload,
|
||||
);
|
||||
let report_context_with_diagnostics =
|
||||
attach_current_request_diagnostics_and_candidate_timing_to_report_context(
|
||||
payload.report_context.as_ref(),
|
||||
analytics_context.as_ref(),
|
||||
payload
|
||||
.telemetry
|
||||
.as_ref()
|
||||
@@ -796,7 +817,7 @@ async fn handle_prefetch_transport_stream_failure(
|
||||
plan,
|
||||
report_context_with_diagnostics
|
||||
.as_ref()
|
||||
.or(payload.report_context.as_ref()),
|
||||
.or(analytics_context.as_ref()),
|
||||
);
|
||||
let payload_seed = build_sync_terminal_usage_payload_seed(&payload);
|
||||
state
|
||||
|
||||
@@ -196,6 +196,78 @@ fn maybe_build_invalid_provider_success_finalize_response(
|
||||
)?))
|
||||
}
|
||||
|
||||
fn local_sync_needs_conversion(payload: &GatewaySyncReportRequest) -> bool {
|
||||
payload
|
||||
.report_context
|
||||
.as_ref()
|
||||
.and_then(|value| value.get("needs_conversion"))
|
||||
.and_then(|value| value.as_bool())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// A successful upstream response that needed conversion but could not be
|
||||
/// converted must not reach the client in the provider's own format.
|
||||
fn maybe_build_unconverted_cross_format_success_response(
|
||||
trace_id: &str,
|
||||
decision: &GatewayControlDecision,
|
||||
payload: &GatewaySyncReportRequest,
|
||||
) -> Result<Option<Response<Body>>, GatewayError> {
|
||||
if payload.status_code >= 400
|
||||
|| !local_sync_needs_conversion(payload)
|
||||
|| !is_core_error_finalize_kind(payload.report_kind.as_str())
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let client_api_format = resolve_local_sync_client_api_format(payload);
|
||||
let provider_api_format = resolve_local_sync_provider_api_format(payload);
|
||||
warn!(
|
||||
event_name = "local_core_finalize_cross_format_success_unconverted",
|
||||
log_type = "event",
|
||||
trace_id = %trace_id,
|
||||
report_kind = %payload.report_kind,
|
||||
status_code = payload.status_code,
|
||||
client_api_format = %client_api_format,
|
||||
provider_api_format = %provider_api_format,
|
||||
"gateway could not convert a successful provider response to the client format"
|
||||
);
|
||||
let message = format!(
|
||||
"Provider returned HTTP {} but its {provider_api_format} response could not be converted to {client_api_format}.",
|
||||
payload.status_code
|
||||
);
|
||||
let body_json = build_core_error_body_for_client_format(
|
||||
&client_api_format,
|
||||
&message,
|
||||
Some("response_conversion_failed"),
|
||||
LocalCoreSyncErrorKind::ServerError,
|
||||
)
|
||||
.unwrap_or_else(|| {
|
||||
serde_json::json!({
|
||||
"error": {
|
||||
"message": message,
|
||||
"type": "server_error",
|
||||
"code": "response_conversion_failed"
|
||||
}
|
||||
})
|
||||
});
|
||||
|
||||
let mut response_headers = payload.headers.clone();
|
||||
response_headers.remove("content-encoding");
|
||||
response_headers.remove("content-length");
|
||||
response_headers.insert("content-type".to_string(), "application/json".to_string());
|
||||
let body_bytes =
|
||||
serde_json::to_vec(&body_json).map_err(|err| GatewayError::Internal(err.to_string()))?;
|
||||
response_headers.insert("content-length".to_string(), body_bytes.len().to_string());
|
||||
|
||||
Ok(Some(build_client_response_from_parts(
|
||||
StatusCode::BAD_GATEWAY.as_u16(),
|
||||
&response_headers,
|
||||
Body::from(body_bytes),
|
||||
trace_id,
|
||||
Some(decision),
|
||||
)?))
|
||||
}
|
||||
|
||||
fn local_core_sync_finalize_has_invalid_provider_success(
|
||||
payload: &GatewaySyncReportRequest,
|
||||
) -> Result<bool, GatewayError> {
|
||||
@@ -274,6 +346,12 @@ pub(crate) fn resolve_local_core_error_response_body_json(
|
||||
return Ok(Some(body_json));
|
||||
}
|
||||
|
||||
// A 2xx cross-format body that is not JSON (e.g. an aggregated SSE capture)
|
||||
// carries no upstream error; wrapping it as one would ship raw provider
|
||||
// bytes to the client under the success status.
|
||||
if payload.status_code < 400 && local_sync_needs_conversion(payload) {
|
||||
return Ok(None);
|
||||
}
|
||||
let Some(body_text) = decode_local_sync_body_text(payload)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
@@ -626,6 +704,10 @@ pub(crate) async fn submit_local_core_error_or_sync_finalize(
|
||||
maybe_build_local_core_error_response(trace_id, decision, &payload)?
|
||||
{
|
||||
response
|
||||
} else if let Some(response) =
|
||||
maybe_build_unconverted_cross_format_success_response(trace_id, decision, &payload)?
|
||||
{
|
||||
response
|
||||
} else {
|
||||
warn!(
|
||||
event_name = "local_core_finalize_fallback_raw_response_body",
|
||||
@@ -937,6 +1019,128 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_core_sync_finalize_converts_forced_responses_stream_for_gemini_client() {
|
||||
use base64::Engine as _;
|
||||
|
||||
// Forced-stream xAI shape: the terminal response echoes request
|
||||
// metadata and encrypted reasoning next to the real answer.
|
||||
let raw_sse = concat!(
|
||||
"event: response.created\n",
|
||||
"data: {\"type\":\"response.created\",\"sequence_number\":0,\"response\":{\"id\":\"resp_xai_123\",\"object\":\"response\",\"status\":\"in_progress\",\"model\":\"grok-4.7-build\",\"output\":[],\"parallel_tool_calls\":true,\"tools\":[]}}\n\n",
|
||||
"event: response.output_item.done\n",
|
||||
"data: {\"type\":\"response.output_item.done\",\"sequence_number\":1,\"output_index\":0,\"item\":{\"id\":\"rs_xai_123\",\"type\":\"reasoning\",\"status\":\"completed\",\"summary\":[],\"encrypted_content\":\"opaque-xai-reasoning\"}}\n\n",
|
||||
"event: response.output_text.delta\n",
|
||||
"data: {\"type\":\"response.output_text.delta\",\"sequence_number\":2,\"item_id\":\"msg_xai_123\",\"output_index\":1,\"content_index\":0,\"delta\":\"Hi there, friend\"}\n\n",
|
||||
"event: response.output_item.done\n",
|
||||
"data: {\"type\":\"response.output_item.done\",\"sequence_number\":3,\"output_index\":1,\"item\":{\"id\":\"msg_xai_123\",\"type\":\"message\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"Hi there, friend\",\"annotations\":[]}]}}\n\n",
|
||||
"event: response.completed\n",
|
||||
"data: {\"type\":\"response.completed\",\"sequence_number\":4,\"response\":{\"id\":\"resp_xai_123\",\"object\":\"response\",\"status\":\"completed\",\"model\":\"grok-4.7-build\",\"output\":[],\"parallel_tool_calls\":true,\"tool_choice\":\"auto\",\"tools\":[],\"text\":{\"format\":{\"type\":\"text\"}},\"temperature\":0.7,\"store\":false,\"usage\":{\"input_tokens\":1249,\"output_tokens\":12,\"total_tokens\":1261}}}\n\n",
|
||||
);
|
||||
let mut payload = core_finalize_payload(
|
||||
"gemini_chat_sync_finalize",
|
||||
"gemini:generate_content",
|
||||
"openai:responses",
|
||||
200,
|
||||
json!(null),
|
||||
);
|
||||
payload.body_json = None;
|
||||
payload.body_base64 = Some(base64::engine::general_purpose::STANDARD.encode(raw_sse));
|
||||
payload.report_context = Some(json!({
|
||||
"client_api_format": "gemini:generate_content",
|
||||
"provider_api_format": "openai:responses",
|
||||
"provider_stream_event_api_format": "openai:responses",
|
||||
"model": "grok-4.7",
|
||||
"mapped_model": "grok-4.7",
|
||||
"needs_conversion": true,
|
||||
}));
|
||||
|
||||
let state = AppState::new().expect("state should build");
|
||||
let response = submit_local_core_error_or_sync_finalize(
|
||||
&state,
|
||||
"trace-forced-responses-gemini",
|
||||
&test_decision(),
|
||||
payload,
|
||||
)
|
||||
.await
|
||||
.expect("finalize should build a response");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::OK);
|
||||
let body_bytes = to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read");
|
||||
let body =
|
||||
serde_json::from_slice::<serde_json::Value>(&body_bytes).expect("body should decode");
|
||||
assert!(body.get("error").is_none(), "unexpected error body: {body}");
|
||||
let parts = body["candidates"][0]["content"]["parts"]
|
||||
.as_array()
|
||||
.expect("gemini parts");
|
||||
assert!(parts.iter().any(|part| part["text"] == "Hi there, friend"));
|
||||
let text = String::from_utf8_lossy(&body_bytes);
|
||||
assert!(!text.contains("opaque-xai-reasoning") && !text.contains("response.created"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_core_sync_finalize_never_wraps_unconvertible_success_sse_as_client_error() {
|
||||
use base64::Engine as _;
|
||||
|
||||
// A complete stream whose output the Gemini client cannot represent.
|
||||
let raw_sse = concat!(
|
||||
"event: response.output_item.done\n",
|
||||
"data: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"future_item_123\",\"type\":\"future_output\",\"payload\":\"must-not-drop\"}}\n\n",
|
||||
"event: response.completed\n",
|
||||
"data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_raw_123\",\"object\":\"response\",\"status\":\"completed\",\"model\":\"grok-4.7\",\"output\":[]}}\n\n",
|
||||
);
|
||||
let mut payload = core_finalize_payload(
|
||||
"gemini_chat_sync_finalize",
|
||||
"gemini:generate_content",
|
||||
"openai:responses",
|
||||
200,
|
||||
json!(null),
|
||||
);
|
||||
payload.body_json = None;
|
||||
payload.body_base64 = Some(base64::engine::general_purpose::STANDARD.encode(raw_sse));
|
||||
payload.report_context = Some(json!({
|
||||
"client_api_format": "gemini:generate_content",
|
||||
"provider_api_format": "openai:responses",
|
||||
"provider_stream_event_api_format": "openai:responses",
|
||||
"needs_conversion": true,
|
||||
}));
|
||||
|
||||
assert!(maybe_build_local_core_error_response(
|
||||
"trace-raw-success-sse",
|
||||
&test_decision(),
|
||||
&payload,
|
||||
)
|
||||
.expect("response build should not error")
|
||||
.is_none());
|
||||
|
||||
let state = AppState::new().expect("state should build");
|
||||
let response = submit_local_core_error_or_sync_finalize(
|
||||
&state,
|
||||
"trace-raw-success-sse",
|
||||
&test_decision(),
|
||||
payload,
|
||||
)
|
||||
.await
|
||||
.expect("finalize should build a response");
|
||||
|
||||
assert_eq!(response.status(), http::StatusCode::BAD_GATEWAY);
|
||||
let body = serde_json::from_slice::<serde_json::Value>(
|
||||
&to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.expect("body should read"),
|
||||
)
|
||||
.expect("body should decode");
|
||||
let message = body["error"]["message"]
|
||||
.as_str()
|
||||
.expect("error message should exist");
|
||||
assert!(
|
||||
message.contains("could not be converted") && !message.contains("must-not-drop"),
|
||||
"unexpected message: {message}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn submit_local_core_finalize_keeps_http_200_for_success_image_body() {
|
||||
let payload = core_finalize_payload(
|
||||
|
||||
@@ -243,7 +243,10 @@ impl SyncAttemptTerminalGuard {
|
||||
record_sync_attempt_forced_terminal_state(
|
||||
self.state.clone(),
|
||||
self.plan.clone(),
|
||||
self.report_context.clone(),
|
||||
crate::usage::reporting::failure::gateway_error_analytics_context(
|
||||
self.report_context.as_ref(),
|
||||
error,
|
||||
),
|
||||
self.request_diagnostics.clone(),
|
||||
self.candidate_started_unix_ms,
|
||||
self.candidate_started_at,
|
||||
@@ -317,6 +320,16 @@ async fn record_sync_attempt_forced_terminal_state(
|
||||
let error_message = error_message.into();
|
||||
let report_context =
|
||||
attach_request_diagnostics_to_report_context(report_context, request_diagnostics.as_ref());
|
||||
let report_context = if matches!(usage_event_type, UsageEventType::Cancelled) {
|
||||
crate::usage::reporting::failure::with_analytics_failure(
|
||||
report_context.as_ref(),
|
||||
"unknown",
|
||||
"finalize",
|
||||
"request_task_cancelled",
|
||||
)
|
||||
} else {
|
||||
report_context
|
||||
};
|
||||
let terminal_unix_ms = current_request_candidate_unix_ms();
|
||||
let latency_ms = elapsed_ms_since(candidate_started_at);
|
||||
record_local_request_candidate_status(
|
||||
@@ -614,15 +627,19 @@ async fn record_sync_terminal_usage(
|
||||
candidate_started_at: Instant,
|
||||
candidate_first_byte_elapsed_ms: Option<u64>,
|
||||
) {
|
||||
let analytics_context =
|
||||
crate::usage::reporting::failure::sync_analytics_context(report_context, payload);
|
||||
let report_context_with_diagnostics =
|
||||
attach_current_request_diagnostics_and_candidate_start_timing_to_report_context(
|
||||
report_context,
|
||||
analytics_context.as_ref(),
|
||||
candidate_started_at,
|
||||
candidate_first_byte_elapsed_ms,
|
||||
);
|
||||
let context_seed = build_terminal_usage_context_seed(
|
||||
plan,
|
||||
report_context_with_diagnostics.as_ref().or(report_context),
|
||||
report_context_with_diagnostics
|
||||
.as_ref()
|
||||
.or(analytics_context.as_ref()),
|
||||
);
|
||||
let payload_seed = build_sync_terminal_usage_payload_seed(payload);
|
||||
state
|
||||
@@ -2074,37 +2091,38 @@ async fn execute_execution_runtime_sync_impl(
|
||||
.unwrap_or_else(|| "-".to_string());
|
||||
let candidate_started_at = Instant::now();
|
||||
let candidate_started_unix_secs = current_request_candidate_unix_ms();
|
||||
let _provider_pool_in_flight_guard = match acquire_provider_pool_execution_guard(state, &plan)
|
||||
.await?
|
||||
{
|
||||
ProviderPoolInFlightAdmission::Acquired(guard) => guard,
|
||||
ProviderPoolInFlightAdmission::Saturated { limit } => {
|
||||
record_local_runtime_candidate_skip_reason(
|
||||
state,
|
||||
trace_id,
|
||||
"provider_key_concurrency_limit_reached",
|
||||
);
|
||||
if let Some(retry_scope) = retry_scope_out.as_deref_mut() {
|
||||
*retry_scope = AiAttemptRetryScope::Candidate;
|
||||
let _provider_pool_in_flight_guard =
|
||||
match acquire_provider_pool_execution_guard(state, &plan, report_context.as_ref()).await? {
|
||||
ProviderPoolInFlightAdmission::Acquired(guard) => guard,
|
||||
ProviderPoolInFlightAdmission::Saturated { limit } => {
|
||||
record_local_runtime_candidate_skip_reason(
|
||||
state,
|
||||
trace_id,
|
||||
"provider_key_concurrency_limit_reached",
|
||||
);
|
||||
if let Some(retry_scope) = retry_scope_out.as_deref_mut() {
|
||||
*retry_scope = AiAttemptRetryScope::Candidate;
|
||||
}
|
||||
record_local_request_candidate_status(
|
||||
state,
|
||||
&plan,
|
||||
report_context.as_ref(),
|
||||
SchedulerRequestCandidateStatusUpdate {
|
||||
status: RequestCandidateStatus::Skipped,
|
||||
status_code: Some(StatusCode::TOO_MANY_REQUESTS.as_u16()),
|
||||
error_type: Some("provider_key_concurrency_limit_reached".to_string()),
|
||||
error_message: Some(format!(
|
||||
"provider key concurrency limit reached: {limit}"
|
||||
)),
|
||||
latency_ms: Some(0),
|
||||
started_at_unix_ms: Some(candidate_started_unix_secs),
|
||||
finished_at_unix_ms: Some(candidate_started_unix_secs),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
return Ok(None);
|
||||
}
|
||||
record_local_request_candidate_status(
|
||||
state,
|
||||
&plan,
|
||||
report_context.as_ref(),
|
||||
SchedulerRequestCandidateStatusUpdate {
|
||||
status: RequestCandidateStatus::Skipped,
|
||||
status_code: Some(StatusCode::TOO_MANY_REQUESTS.as_u16()),
|
||||
error_type: Some("provider_key_concurrency_limit_reached".to_string()),
|
||||
error_message: Some(format!("provider key concurrency limit reached: {limit}")),
|
||||
latency_ms: Some(0),
|
||||
started_at_unix_ms: Some(candidate_started_unix_secs),
|
||||
finished_at_unix_ms: Some(candidate_started_unix_secs),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
return Ok(None);
|
||||
}
|
||||
};
|
||||
};
|
||||
let lifecycle_seed = build_lifecycle_usage_seed(&plan, report_context.as_ref());
|
||||
let usage_data = state.usage_lifecycle_data_state().as_ref().clone();
|
||||
state
|
||||
@@ -2804,6 +2822,9 @@ async fn execute_execution_runtime_sync_impl(
|
||||
provider_response_observation.response_headers_observed_at_unix_ms,
|
||||
&provider_response_observation.request_order_id,
|
||||
);
|
||||
if let Some(error) = result.error.as_ref() {
|
||||
report_context = crate::usage::reporting::failure::execution_error_analytics_context(report_context.as_ref(), error);
|
||||
}
|
||||
if result.status_code >= 400 {
|
||||
apply_local_execution_effect(
|
||||
state,
|
||||
|
||||
@@ -9737,7 +9737,10 @@ mod tests {
|
||||
headers: BTreeMap::from([("content-type".into(), "application/json".into())]),
|
||||
content_type: Some("application/json".into()),
|
||||
content_encoding: Some(encoding.into()),
|
||||
body: RequestBody::from_json(json!({"model": "gpt-4.1"})),
|
||||
body: RequestBody::from_json(json!({
|
||||
"model": "gpt-4.1",
|
||||
"service_tier": "ultrafast"
|
||||
})),
|
||||
stream: false,
|
||||
client_api_format: "openai:chat".into(),
|
||||
provider_api_format: "openai:chat".into(),
|
||||
@@ -9758,7 +9761,7 @@ mod tests {
|
||||
result.body.and_then(|body| body.json_body),
|
||||
Some(json!({
|
||||
"content_encoding": encoding,
|
||||
"body": {"model": "gpt-4.1"},
|
||||
"body": {"model": "gpt-4.1", "service_tier": "ultrafast"},
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
@@ -130,6 +130,9 @@ pub(crate) async fn build_transport_error_stop_response(
|
||||
None => serde_json::Map::new(),
|
||||
};
|
||||
request_metadata.insert("transport_error".to_string(), Value::Bool(true));
|
||||
request_metadata.insert("analytics_failure".into(), json!({
|
||||
"origin": "transport", "stage": "connect", "reason": "upstream_transport_error", "schema_version": 1,
|
||||
}));
|
||||
request_metadata.insert(
|
||||
"transport_error_type".to_string(),
|
||||
Value::String(error_type.to_string()),
|
||||
|
||||
@@ -1464,6 +1464,22 @@ pub(crate) async fn maybe_execute_sync_via_local_video_decision(
|
||||
.await
|
||||
}
|
||||
|
||||
fn supports_local_video_get(
|
||||
parts: &http::request::Parts,
|
||||
decision: &GatewayControlDecision,
|
||||
) -> bool {
|
||||
parts.method == http::Method::GET
|
||||
&& decision.route_kind.as_deref() == Some("video")
|
||||
&& (crate::video_tasks::resolve_video_task_read_lookup_key(
|
||||
decision.route_family.as_deref(),
|
||||
parts.uri.path(),
|
||||
)
|
||||
.is_some()
|
||||
|| (decision.route_family.as_deref() == Some("openai")
|
||||
&& crate::video_tasks::extract_openai_task_id_from_content_path(parts.uri.path())
|
||||
.is_some()))
|
||||
}
|
||||
|
||||
pub(crate) fn maybe_execute_sync_request<'a>(
|
||||
state: &'a AppState,
|
||||
parts: &'a http::request::Parts,
|
||||
@@ -1477,7 +1493,7 @@ pub(crate) fn maybe_execute_sync_request<'a>(
|
||||
};
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
if parts.method != http::Method::POST {
|
||||
if parts.method != http::Method::POST && !supports_local_video_get(parts, decision) {
|
||||
return Ok(LocalExecutionRequestOutcome::NoPath);
|
||||
}
|
||||
return maybe_execute_sync_local_path(state, parts, body_bytes, trace_id, decision)
|
||||
@@ -1490,6 +1506,7 @@ pub(crate) fn maybe_execute_sync_request<'a>(
|
||||
.unwrap_or_default()
|
||||
.is_empty()
|
||||
&& parts.method != http::Method::POST
|
||||
&& !supports_local_video_get(parts, decision)
|
||||
{
|
||||
return Ok(LocalExecutionRequestOutcome::NoPath);
|
||||
}
|
||||
@@ -1511,7 +1528,7 @@ pub(crate) fn maybe_execute_stream_request<'a>(
|
||||
};
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
if parts.method != http::Method::POST {
|
||||
if parts.method != http::Method::POST && !supports_local_video_get(parts, decision) {
|
||||
return Ok(LocalExecutionRequestOutcome::NoPath);
|
||||
}
|
||||
return maybe_execute_stream_local_path(state, parts, body_bytes, trace_id, decision)
|
||||
@@ -1524,6 +1541,7 @@ pub(crate) fn maybe_execute_stream_request<'a>(
|
||||
.unwrap_or_default()
|
||||
.is_empty()
|
||||
&& parts.method != http::Method::POST
|
||||
&& !supports_local_video_get(parts, decision)
|
||||
{
|
||||
return Ok(LocalExecutionRequestOutcome::NoPath);
|
||||
}
|
||||
|
||||
@@ -111,6 +111,12 @@ pub(crate) fn record_failed_usage_for_deferred_response<'a>(
|
||||
return;
|
||||
};
|
||||
let mut data = build_usage_event_data_seed(&context.plan, context.report_context.as_ref());
|
||||
data.request_metadata = crate::usage::reporting::failure::with_analytics_failure(
|
||||
data.request_metadata.as_ref(),
|
||||
"upstream",
|
||||
"response",
|
||||
"candidates_exhausted",
|
||||
);
|
||||
data.status_code = Some(status_code);
|
||||
data.error_message =
|
||||
Some("all local candidates failed; returning preserved upstream error".to_string());
|
||||
@@ -390,6 +396,15 @@ pub(crate) async fn record_failed_usage_for_exhausted_request(
|
||||
None => Map::new(),
|
||||
};
|
||||
request_metadata.insert("trace_id".to_string(), Value::String(request_id.clone()));
|
||||
if !request_metadata.contains_key("analytics_failure") {
|
||||
request_metadata.insert(
|
||||
"analytics_failure".into(),
|
||||
json!({
|
||||
"origin": if upstream_status_code.is_some() { "upstream" } else { "gateway" },
|
||||
"stage": "routing", "reason": "candidates_exhausted", "schema_version": 1,
|
||||
}),
|
||||
);
|
||||
}
|
||||
apply_runtime_miss_usage_routing(
|
||||
&mut data,
|
||||
&mut request_metadata,
|
||||
@@ -471,6 +486,9 @@ pub(crate) async fn record_failed_usage_for_runtime_miss_request(
|
||||
}
|
||||
|
||||
let mut request_metadata = Map::new();
|
||||
request_metadata.insert("analytics_failure".into(), json!({
|
||||
"origin": "gateway", "stage": "routing", "reason": "execution_route_unavailable", "schema_version": 1,
|
||||
}));
|
||||
request_metadata.insert(
|
||||
"trace_id".to_string(),
|
||||
Value::String(request_id.to_string()),
|
||||
|
||||
@@ -32,6 +32,10 @@ fn request_has_execution_runtime_via_guard(headers: &HeaderMap) -> bool {
|
||||
}
|
||||
|
||||
pub(crate) fn frontdoor_self_loop_public_ai_path(path: &str) -> bool {
|
||||
let path = path
|
||||
.strip_prefix("/openai")
|
||||
.filter(|p| *p == "/v1/videos" || p.starts_with("/v1/videos/"))
|
||||
.unwrap_or(path);
|
||||
matches!(
|
||||
path,
|
||||
"/v1/messages"
|
||||
@@ -41,6 +45,7 @@ pub(crate) fn frontdoor_self_loop_public_ai_path(path: &str) -> bool {
|
||||
| "/v1/rerank"
|
||||
| "/v1/responses"
|
||||
| "/v1/responses/compact"
|
||||
| "/v1/memories/trace_summarize"
|
||||
| "/v1/realtime"
|
||||
| "/v1/realtime/calls"
|
||||
| "/v1/live"
|
||||
|
||||
@@ -253,7 +253,7 @@ pub(super) async fn build_admin_create_api_key_response(
|
||||
{
|
||||
tracing::error!(
|
||||
api_key_id = %created.api_key_id,
|
||||
error = ?error,
|
||||
error = %crate::error::redact_error_debug(&error),
|
||||
"standalone API key wallet provisioning cleanup failed"
|
||||
);
|
||||
return Err(error);
|
||||
@@ -266,7 +266,7 @@ pub(super) async fn build_admin_create_api_key_response(
|
||||
{
|
||||
tracing::error!(
|
||||
api_key_id = %created.api_key_id,
|
||||
error = ?cleanup_error,
|
||||
error = %crate::error::redact_error_debug(&cleanup_error),
|
||||
"standalone API key wallet provisioning cleanup failed"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -16,6 +16,8 @@ mod collectors;
|
||||
mod payments;
|
||||
mod plans;
|
||||
mod presets;
|
||||
mod provider_accounts;
|
||||
mod provider_expenses;
|
||||
mod routes;
|
||||
mod rules;
|
||||
mod wallets;
|
||||
@@ -207,6 +209,15 @@ pub(crate) async fn maybe_build_local_admin_billing_response(
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
if let Some(response) = provider_accounts::response(state, request_context).await? {
|
||||
return Ok(Some(response));
|
||||
}
|
||||
if let Some(response) =
|
||||
provider_expenses::response(state, request_context, request_body).await?
|
||||
{
|
||||
return Ok(Some(response));
|
||||
}
|
||||
|
||||
let path = request_context.path();
|
||||
let is_billing_route = (request_context.method() == http::Method::GET
|
||||
&& matches!(
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
//! Current provider finance snapshots. This endpoint never calls upstream services.
|
||||
use super::build_admin_billing_data_unavailable_response;
|
||||
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
|
||||
use crate::GatewayError;
|
||||
use axum::{
|
||||
body::Body,
|
||||
http,
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
fn finite(value: Option<&Value>) -> Option<f64> {
|
||||
value
|
||||
.and_then(|v| {
|
||||
v.as_f64()
|
||||
.or_else(|| v.as_str().and_then(|v| v.parse::<f64>().ok()))
|
||||
})
|
||||
.filter(|v| v.is_finite())
|
||||
}
|
||||
fn text(value: Option<&Value>) -> Option<&str> {
|
||||
value
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|v| !v.is_empty() && v.len() <= 256 && !v.chars().any(char::is_control))
|
||||
}
|
||||
fn timestamp(value: Option<&Value>) -> Option<String> {
|
||||
let value = value?;
|
||||
if let Some(raw) = value.as_str() {
|
||||
if let Ok(date) = chrono::DateTime::parse_from_rfc3339(raw) {
|
||||
return Some(date.to_rfc3339_opts(chrono::SecondsFormat::Millis, true));
|
||||
}
|
||||
}
|
||||
let secs = finite(Some(value))?;
|
||||
if !(0.0..=253_402_300_799.0).contains(&secs) {
|
||||
return None;
|
||||
}
|
||||
chrono::DateTime::from_timestamp(secs as i64, 0)
|
||||
.map(|v| v.to_rfc3339_opts(chrono::SecondsFormat::Millis, true))
|
||||
}
|
||||
fn subscription(value: &Value) -> Value {
|
||||
json!({
|
||||
"group_name": text(value.get("group_name")),
|
||||
"status": text(value.get("status")),
|
||||
"daily_used_usd": finite(value.get("daily_used_usd")),
|
||||
"daily_limit_usd": finite(value.get("daily_limit_usd")),
|
||||
"weekly_used_usd": finite(value.get("weekly_used_usd")),
|
||||
"weekly_limit_usd": finite(value.get("weekly_limit_usd")),
|
||||
"monthly_used_usd": finite(value.get("monthly_used_usd")),
|
||||
"monthly_limit_usd": finite(value.get("monthly_limit_usd")),
|
||||
"expires_at": timestamp(value.get("expires_at")),
|
||||
})
|
||||
}
|
||||
fn balance(value: &Value) -> Option<Value> {
|
||||
if value.get("action_type").and_then(Value::as_str) != Some("query_balance") {
|
||||
return None;
|
||||
}
|
||||
let status = text(value.get("status"))?;
|
||||
if !matches!(status, "success" | "auth_expired" | "auth_failed") {
|
||||
return None;
|
||||
}
|
||||
let data = value
|
||||
.get("data")
|
||||
.filter(|_| matches!(status, "success" | "auth_expired"));
|
||||
let extra = data.and_then(|d| d.get("extra"));
|
||||
let subscriptions = extra
|
||||
.and_then(|e| e.get("subscriptions"))
|
||||
.and_then(Value::as_array)
|
||||
.map(|items| {
|
||||
items
|
||||
.iter()
|
||||
.filter(|v| v.is_object())
|
||||
.take(128)
|
||||
.map(subscription)
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
Some(json!({
|
||||
"status": status,
|
||||
"observed_at": timestamp(value.get("executed_at")),
|
||||
"currency": data.and_then(|d| text(d.get("currency"))),
|
||||
"available": data.and_then(|d| finite(d.get("total_available"))),
|
||||
"used": data.and_then(|d| finite(d.get("total_used"))),
|
||||
"granted": data.and_then(|d| finite(d.get("total_granted"))),
|
||||
"plan_name": extra.and_then(|e| text(e.get("plan_name"))),
|
||||
"subscriptions": subscriptions,
|
||||
}))
|
||||
}
|
||||
pub(super) async fn response(
|
||||
state: &AdminAppState<'_>,
|
||||
context: &AdminRequestContext<'_>,
|
||||
) -> Result<Option<Response<Body>>, GatewayError> {
|
||||
if context.method() != http::Method::GET
|
||||
|| context.path().trim_end_matches('/') != "/api/admin/billing/provider-accounts"
|
||||
|| context.route_family() != Some("billing_manage")
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
if !state.has_provider_catalog_data_reader() {
|
||||
return Ok(Some(build_admin_billing_data_unavailable_response()));
|
||||
}
|
||||
let mut providers = state.list_provider_catalog_providers(false).await?;
|
||||
providers.sort_by(|a, b| a.name.cmp(&b.name).then_with(|| a.id.cmp(&b.id)));
|
||||
let keys = providers
|
||||
.iter()
|
||||
.map(|p| format!("provider_ops:balance:{}", p.id))
|
||||
.collect::<Vec<_>>();
|
||||
let (cached, unavailable) = if keys.is_empty() {
|
||||
(Vec::new(), false)
|
||||
} else {
|
||||
match state.runtime_state().kv_get_many(&keys).await {
|
||||
Ok(v) => (v, false),
|
||||
Err(_) => (vec![None; keys.len()], true),
|
||||
}
|
||||
};
|
||||
let items = providers.iter().enumerate().map(|(index, p)| {
|
||||
let limit = p.monthly_quota_usd.filter(|v| v.is_finite() && *v >= 0.0);
|
||||
let used = p.monthly_used_usd.filter(|v| v.is_finite() && *v >= 0.0);
|
||||
let quota = if p.billing_type.as_deref() == Some("monthly_quota") || limit.is_some() {
|
||||
json!({
|
||||
"limit": limit, "used": used,
|
||||
"remaining": limit.zip(used).map(|(l,u)| (l-u).max(0.0)),
|
||||
"currency": "USD",
|
||||
"period_start": p.quota_last_reset_at_unix_secs.and_then(|v| timestamp(Some(&json!(v)))),
|
||||
"expires_at": p.quota_expires_at_unix_secs.and_then(|v| timestamp(Some(&json!(v)))),
|
||||
})
|
||||
} else { Value::Null };
|
||||
let balance = cached.get(index).and_then(|v| v.as_deref())
|
||||
.and_then(|v| serde_json::from_str::<Value>(v).ok()).and_then(|v| balance(&v));
|
||||
json!({
|
||||
"provider_id": p.id, "provider_name": p.name, "is_active": p.is_active,
|
||||
"billing_type": p.billing_type, "quota": quota, "balance": balance,
|
||||
})
|
||||
}).collect::<Vec<_>>();
|
||||
Ok(Some((
|
||||
[(http::header::CACHE_CONTROL, "private, no-store")],
|
||||
Json(json!({
|
||||
"observed_at": chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis,true),
|
||||
"items": items, "balance_snapshot_unavailable": unavailable,
|
||||
})),
|
||||
).into_response()))
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn provider_accounts_only_expose_finance_allowlist_and_preserve_unknown() {
|
||||
let snapshot=balance(&json!({"status":"success","action_type":"query_balance","executed_at":"2026-09-20T00:00:00Z","data":{"currency":"USD","total_available":null,"extra":{"access_token":"secret","plan_name":"Pro","subscriptions":[{"group_name":"Team","monthly_used_usd":"12.25","expires_at":1800000000,"private_token":"secret"}]}}})).unwrap();
|
||||
assert!(snapshot["available"].is_null());
|
||||
assert_eq!(
|
||||
snapshot["subscriptions"][0]["monthly_used_usd"],
|
||||
json!(12.25)
|
||||
);
|
||||
assert!(!snapshot.to_string().contains("secret"));
|
||||
assert!(!snapshot.to_string().contains("access_token"));
|
||||
let failed=balance(&json!({"status":"auth_failed","action_type":"query_balance","data":{"total_available":999}})).unwrap();
|
||||
assert!(failed["available"].is_null());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,334 @@
|
||||
use super::{
|
||||
build_admin_billing_bad_request_response as bad_request,
|
||||
build_admin_billing_conflict_response as conflict,
|
||||
build_admin_billing_data_unavailable_response as unavailable,
|
||||
build_admin_billing_not_found_response as not_found,
|
||||
};
|
||||
use crate::handlers::admin::{
|
||||
request::{AdminAppState, AdminRequestContext},
|
||||
shared::{attach_admin_audit_response, query_param_value},
|
||||
};
|
||||
use crate::handlers::shared::normalize_payment_currency;
|
||||
use crate::GatewayError;
|
||||
use aether_data_contracts::repository::billing::*;
|
||||
use axum::{
|
||||
body::{Body, Bytes},
|
||||
http::{self, StatusCode},
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{json, Value};
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct ExpenseRequest {
|
||||
client_request_id: String,
|
||||
provider_id: String,
|
||||
kind: String,
|
||||
amount: String,
|
||||
currency: String,
|
||||
paid_at: String,
|
||||
period_start: Option<String>,
|
||||
period_end: Option<String>,
|
||||
note: Option<String>,
|
||||
external_reference: Option<String>,
|
||||
}
|
||||
fn datetime(value: u64) -> String {
|
||||
chrono::DateTime::from_timestamp_millis(value as i64)
|
||||
.expect("valid stored timestamp")
|
||||
.to_rfc3339_opts(chrono::SecondsFormat::Millis, true)
|
||||
}
|
||||
fn parse_date(value: &str) -> Result<u64, String> {
|
||||
chrono::DateTime::parse_from_rfc3339(value)
|
||||
.ok()
|
||||
.and_then(|v| u64::try_from(v.timestamp_millis()).ok())
|
||||
.filter(|v| *v <= 253_402_300_799_000)
|
||||
.ok_or_else(|| "timestamps must be RFC3339 dates on or after 1970".into())
|
||||
}
|
||||
fn optional_text(value: Option<String>) -> Option<String> {
|
||||
value.map(|v| v.trim().to_owned()).filter(|v| !v.is_empty())
|
||||
}
|
||||
fn expense_json(record: &ProviderExpenseRecord) -> Value {
|
||||
let e = &record.entry;
|
||||
json!({
|
||||
"id": record.id, "client_request_id": e.client_request_id,
|
||||
"provider_id": e.provider_id, "provider_name": e.provider_name,
|
||||
"kind": e.kind, "amount": e.amount, "currency": e.currency,
|
||||
"paid_at": datetime(e.paid_at_unix_ms),
|
||||
"period_start": e.period_start_unix_ms.map(datetime),
|
||||
"period_end": e.period_end_unix_ms.map(datetime),
|
||||
"note": e.note, "external_reference": e.external_reference,
|
||||
"created_by": e.created_by, "created_at": datetime(record.created_at_unix_ms),
|
||||
"status": if record.voided_at_unix_ms.is_some() { "void" } else { "recorded" },
|
||||
"voided_at": record.voided_at_unix_ms.map(datetime), "voided_by": record.voided_by,
|
||||
})
|
||||
}
|
||||
fn csv_cell(value: &str) -> String {
|
||||
let value = if value.trim_start().starts_with(['=', '+', '-', '@'])
|
||||
|| value.starts_with(['\t', '\r', '\n'])
|
||||
{
|
||||
format!("'{value}")
|
||||
} else {
|
||||
value.to_string()
|
||||
};
|
||||
format!("\"{}\"", value.replace('"', "\"\""))
|
||||
}
|
||||
fn csv_report(items: &[ProviderExpenseRecord]) -> String {
|
||||
let mut result=String::from("\u{feff}id,provider_id,provider_name,kind,amount,currency,paid_at,period_start,period_end,note,external_reference,created_by,created_at\r\n");
|
||||
for r in items {
|
||||
let e = &r.entry;
|
||||
let fields = [
|
||||
r.id.clone(),
|
||||
e.provider_id.clone(),
|
||||
e.provider_name.clone(),
|
||||
e.kind.clone(),
|
||||
e.amount.clone(),
|
||||
e.currency.clone(),
|
||||
datetime(e.paid_at_unix_ms),
|
||||
e.period_start_unix_ms.map(datetime).unwrap_or_default(),
|
||||
e.period_end_unix_ms.map(datetime).unwrap_or_default(),
|
||||
e.note.clone().unwrap_or_default(),
|
||||
e.external_reference.clone().unwrap_or_default(),
|
||||
e.created_by.clone().unwrap_or_default(),
|
||||
datetime(r.created_at_unix_ms),
|
||||
];
|
||||
result.push_str(
|
||||
&fields
|
||||
.iter()
|
||||
.map(|s| csv_cell(s))
|
||||
.collect::<Vec<_>>()
|
||||
.join(","),
|
||||
);
|
||||
result.push_str("\r\n");
|
||||
}
|
||||
result
|
||||
}
|
||||
fn query(context: &AdminRequestContext<'_>, csv: bool) -> Result<ProviderExpenseQuery, String> {
|
||||
let q = context.query_string();
|
||||
let now = chrono::Utc::now().timestamp_millis().max(0) as u64;
|
||||
let from = query_param_value(q, "from")
|
||||
.map(|v| parse_date(&v))
|
||||
.transpose()?
|
||||
.unwrap_or(now.saturating_sub(30 * 86_400_000));
|
||||
let to = query_param_value(q, "to")
|
||||
.map(|v| parse_date(&v))
|
||||
.transpose()?
|
||||
.unwrap_or(now);
|
||||
let limit = if csv {
|
||||
10_001
|
||||
} else {
|
||||
query_param_value(q, "limit")
|
||||
.map(|v| v.parse::<u32>().map_err(|_| "invalid limit".to_string()))
|
||||
.transpose()?
|
||||
.unwrap_or(25)
|
||||
};
|
||||
let offset = if csv {
|
||||
0
|
||||
} else {
|
||||
query_param_value(q, "offset")
|
||||
.map(|v| v.parse::<u64>().map_err(|_| "invalid offset".to_string()))
|
||||
.transpose()?
|
||||
.unwrap_or(0)
|
||||
};
|
||||
if !csv && limit > 200 {
|
||||
return Err("limit must be at most 200".into());
|
||||
}
|
||||
let q = ProviderExpenseQuery {
|
||||
from_unix_ms: from,
|
||||
to_unix_ms: to,
|
||||
limit,
|
||||
offset,
|
||||
};
|
||||
q.validate().map_err(|e| e.to_string())?;
|
||||
Ok(q)
|
||||
}
|
||||
pub(super) async fn response(
|
||||
state: &AdminAppState<'_>,
|
||||
context: &AdminRequestContext<'_>,
|
||||
body: Option<&Bytes>,
|
||||
) -> Result<Option<Response<Body>>, GatewayError> {
|
||||
let path = context.path().trim_end_matches('/');
|
||||
if context.route_family() != Some("billing_manage")
|
||||
|| !path.starts_with("/api/admin/billing/provider-expenses")
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let operator = context
|
||||
.decision()
|
||||
.and_then(|d| d.admin_principal.as_ref())
|
||||
.map(|p| p.user_id.clone());
|
||||
if path == "/api/admin/billing/provider-expenses" && context.method() == http::Method::GET {
|
||||
let csv = query_param_value(context.query_string(), "format").as_deref() == Some("csv");
|
||||
let q = match query(context, csv) {
|
||||
Ok(v) => v,
|
||||
Err(e) => return Ok(Some(bad_request(e))),
|
||||
};
|
||||
let Some(page) = state
|
||||
.app()
|
||||
.data
|
||||
.list_provider_expenses(&q)
|
||||
.await
|
||||
.map_err(|e| GatewayError::Internal(e.to_string()))?
|
||||
else {
|
||||
return Ok(Some(unavailable()));
|
||||
};
|
||||
if csv {
|
||||
if page.total > 10_000 {
|
||||
return Ok(Some(
|
||||
(
|
||||
StatusCode::UNPROCESSABLE_ENTITY,
|
||||
Json(json!({"detail":"导出超过 10000 条,请缩小时间范围"})),
|
||||
)
|
||||
.into_response(),
|
||||
));
|
||||
}
|
||||
return Ok(Some(
|
||||
(
|
||||
[
|
||||
(http::header::CONTENT_TYPE, "text/csv; charset=utf-8"),
|
||||
(
|
||||
http::header::CONTENT_DISPOSITION,
|
||||
"attachment; filename=provider-expenses.csv",
|
||||
),
|
||||
(http::header::CACHE_CONTROL, "private, no-store"),
|
||||
],
|
||||
csv_report(&page.items),
|
||||
)
|
||||
.into_response(),
|
||||
));
|
||||
}
|
||||
return Ok(Some(
|
||||
(
|
||||
[(http::header::CACHE_CONTROL, "private, no-store")],
|
||||
Json(json!({
|
||||
"items": page.items.iter().map(expense_json).collect::<Vec<_>>(),
|
||||
"total": page.total, "totals": page.totals, "providers": page.providers,
|
||||
"limit": q.limit, "offset": q.offset,
|
||||
"from": datetime(q.from_unix_ms), "to": datetime(q.to_unix_ms),
|
||||
"time_basis": "paid_at", "source": "manual_ledger",
|
||||
})),
|
||||
)
|
||||
.into_response(),
|
||||
));
|
||||
}
|
||||
if path == "/api/admin/billing/provider-expenses" && context.method() == http::Method::POST {
|
||||
let Some(body) = body else {
|
||||
return Ok(Some(bad_request("缺少请求体")));
|
||||
};
|
||||
let payload = match serde_json::from_slice::<ExpenseRequest>(body) {
|
||||
Ok(v) => v,
|
||||
Err(_) => return Ok(Some(bad_request("输入验证失败"))),
|
||||
};
|
||||
let input = (|| -> Result<ProviderExpenseInput, String> {
|
||||
let units = provider_expense_amount_units(&payload.amount)
|
||||
.ok_or("amount must be a positive decimal string with at most 8 decimal places")?;
|
||||
let input = ProviderExpenseInput {
|
||||
client_request_id: uuid::Uuid::parse_str(&payload.client_request_id)
|
||||
.map_err(|_| "client_request_id must be a UUID")?
|
||||
.to_string(),
|
||||
provider_id: payload.provider_id.trim().into(),
|
||||
provider_name: "pending".into(),
|
||||
kind: payload.kind,
|
||||
amount: format_provider_expense_amount(units),
|
||||
currency: normalize_payment_currency(&payload.currency, "currency")?,
|
||||
paid_at_unix_ms: parse_date(&payload.paid_at)?,
|
||||
period_start_unix_ms: payload
|
||||
.period_start
|
||||
.as_deref()
|
||||
.map(parse_date)
|
||||
.transpose()?,
|
||||
period_end_unix_ms: payload.period_end.as_deref().map(parse_date).transpose()?,
|
||||
note: optional_text(payload.note),
|
||||
external_reference: optional_text(payload.external_reference),
|
||||
created_by: operator.clone(),
|
||||
};
|
||||
input.validate()?;
|
||||
Ok(input)
|
||||
})();
|
||||
let mut input = match input {
|
||||
Ok(v) => v,
|
||||
Err(e) => return Ok(Some(bad_request(e))),
|
||||
};
|
||||
let providers = state
|
||||
.read_provider_catalog_providers_by_ids(&[input.provider_id.clone()])
|
||||
.await?;
|
||||
let Some(provider) = providers.first() else {
|
||||
return Ok(Some(not_found("Provider not found")));
|
||||
};
|
||||
input.provider_name = provider.name.clone();
|
||||
let result = state
|
||||
.app()
|
||||
.data
|
||||
.create_provider_expense(&input)
|
||||
.await
|
||||
.map_err(|e| GatewayError::Internal(e.to_string()))?;
|
||||
return Ok(Some(mutation_response(
|
||||
result,
|
||||
"admin_provider_expense_recorded",
|
||||
"record_provider_expense",
|
||||
)));
|
||||
}
|
||||
if context.method() == http::Method::POST {
|
||||
if let Some(id) = path
|
||||
.strip_prefix("/api/admin/billing/provider-expenses/")
|
||||
.and_then(|v| v.strip_suffix("/void"))
|
||||
.filter(|v| !v.is_empty() && !v.contains('/'))
|
||||
{
|
||||
if uuid::Uuid::parse_str(id).is_err() {
|
||||
return Ok(Some(bad_request("invalid expense id")));
|
||||
}
|
||||
let result = state
|
||||
.app()
|
||||
.data
|
||||
.void_provider_expense(id, operator.as_deref())
|
||||
.await
|
||||
.map_err(|e| GatewayError::Internal(e.to_string()))?;
|
||||
return Ok(Some(mutation_response(
|
||||
result,
|
||||
"admin_provider_expense_voided",
|
||||
"void_provider_expense",
|
||||
)));
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
fn mutation_response(
|
||||
outcome: AdminBillingMutationOutcome<ProviderExpenseRecord>,
|
||||
event: &'static str,
|
||||
action: &'static str,
|
||||
) -> Response<Body> {
|
||||
match outcome {
|
||||
AdminBillingMutationOutcome::Applied(record) => attach_admin_audit_response(
|
||||
Json(json!({"item":expense_json(&record)})).into_response(),
|
||||
event,
|
||||
action,
|
||||
"provider_expense",
|
||||
&record.id,
|
||||
),
|
||||
AdminBillingMutationOutcome::Invalid(e) => conflict(e),
|
||||
AdminBillingMutationOutcome::NotFound => not_found("Provider expense not found"),
|
||||
AdminBillingMutationOutcome::Unavailable => unavailable(),
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn provider_expense_csv_neutralizes_formulas_and_quotes_fields() {
|
||||
assert_eq!(csv_cell("=cmd()"), "\"'=cmd()\"");
|
||||
assert_eq!(csv_cell(" @cmd"), "\"' @cmd\"");
|
||||
assert_eq!(csv_cell("\tcmd"), "\"'\tcmd\"");
|
||||
assert_eq!(csv_cell("a,\"b\"\nc"), "\"a,\"\"b\"\"\nc\"");
|
||||
assert_eq!(csv_cell("12.34"), "\"12.34\"");
|
||||
}
|
||||
#[test]
|
||||
fn provider_expense_dates_require_explicit_timezone_and_nonnegative_epoch() {
|
||||
assert_eq!(
|
||||
parse_date("2026-09-20T08:00:00+08:00"),
|
||||
parse_date("2026-09-20T00:00:00Z")
|
||||
);
|
||||
assert!(parse_date("2026-09-20").is_err());
|
||||
assert!(parse_date("1969-01-01T00:00:00Z").is_err());
|
||||
}
|
||||
}
|
||||
@@ -63,13 +63,14 @@ pub(in super::super) async fn build_admin_wallet_adjust_response(
|
||||
}
|
||||
let operator_id = admin_wallet_operator_id(request_context);
|
||||
let has_wallet_writer = state.has_wallet_data_writer();
|
||||
let Some((wallet, transaction)) = state
|
||||
let Some((wallet, Some(transaction))) = state
|
||||
.admin_adjust_wallet_balance(
|
||||
&wallet_id,
|
||||
amount_usd,
|
||||
&balance_type,
|
||||
operator_id.as_deref(),
|
||||
description.as_deref(),
|
||||
false,
|
||||
)
|
||||
.await?
|
||||
else {
|
||||
|
||||
@@ -27,11 +27,18 @@ pub(in super::super) async fn build_admin_wallet_list_response(
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(build_admin_wallets_bad_request_response(detail)),
|
||||
};
|
||||
let user_id = query_param_value(query, "user_id");
|
||||
let status = query_param_value(query, "status");
|
||||
let owner_type = parse_admin_wallets_owner_type_filter(query);
|
||||
|
||||
let (wallets, total) = state
|
||||
.list_admin_wallets(status.as_deref(), owner_type.as_deref(), limit, offset)
|
||||
.list_admin_wallets(
|
||||
user_id.as_deref(),
|
||||
status.as_deref(),
|
||||
owner_type.as_deref(),
|
||||
limit,
|
||||
offset,
|
||||
)
|
||||
.await?;
|
||||
let mut items = Vec::with_capacity(wallets.len());
|
||||
for wallet in wallets {
|
||||
|
||||
@@ -35,11 +35,39 @@ fn build_admin_endpoint_health_bad_request_response(detail: &str) -> Response<Bo
|
||||
pub(super) async fn maybe_build_local_admin_endpoints_health_response(
|
||||
state: &AdminAppState<'_>,
|
||||
request_context: &AdminRequestContext<'_>,
|
||||
request_body: Option<&axum::body::Bytes>,
|
||||
) -> Result<Option<Response<Body>>, GatewayError> {
|
||||
let Some(decision) = request_context.decision() else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
if decision.route_family.as_deref() == Some("endpoints_health") {
|
||||
if decision.route_kind.as_deref() == Some("health_v2") {
|
||||
return Ok(Some(
|
||||
crate::handlers::shared::health_monitor::build_health_v2_response(
|
||||
state.app(),
|
||||
request_context.path(),
|
||||
request_context.query_string(),
|
||||
crate::handlers::shared::health_monitor::HealthAudience::Admin,
|
||||
)
|
||||
.await,
|
||||
));
|
||||
}
|
||||
if decision.route_kind.as_deref() == Some("health_v2_publication") {
|
||||
return Ok(Some(
|
||||
crate::handlers::shared::health_monitor::build_publication_response(
|
||||
state.app(),
|
||||
if request_context.method() == http::Method::PUT {
|
||||
Some(request_body.map_or(&[][..], |body| body.as_ref()))
|
||||
} else {
|
||||
None
|
||||
},
|
||||
)
|
||||
.await,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
if decision.route_family.as_deref() == Some("endpoints_health")
|
||||
&& decision.route_kind.as_deref() == Some("health_summary")
|
||||
&& request_context.path() == "/api/admin/endpoints/health/summary"
|
||||
|
||||
@@ -8,6 +8,7 @@ pub(crate) async fn maybe_build_local_admin_endpoints_response(
|
||||
if let Some(response) = health::maybe_build_local_admin_endpoints_health_response(
|
||||
&request.state(),
|
||||
&request.request_context(),
|
||||
request.request_body(),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
mod monitoring;
|
||||
mod overview;
|
||||
mod routes;
|
||||
mod stats;
|
||||
mod usage;
|
||||
@@ -11,3 +12,138 @@ pub(crate) use self::stats::{
|
||||
};
|
||||
pub(crate) use self::stats::{AdminStatsTimeRange, AdminStatsUsageFilter};
|
||||
pub(crate) use self::usage::maybe_build_local_admin_usage_response;
|
||||
|
||||
pub(crate) async fn resolve_usage_user_group_scope(
|
||||
state: &crate::handlers::admin::request::AdminAppState<'_>,
|
||||
query: Option<&str>,
|
||||
include_inactive: bool,
|
||||
exclude_admin: bool,
|
||||
) -> Result<Result<Option<Vec<String>>, String>, crate::GatewayError> {
|
||||
let group_id = crate::handlers::admin::shared::query_param_value(query, "user_group_id");
|
||||
let Some(group_id) = group_id else {
|
||||
return Ok(Ok(None));
|
||||
};
|
||||
if crate::handlers::admin::shared::query_param_value(query, "user_id").is_some() {
|
||||
return Ok(Err(
|
||||
"user_id and user_group_id cannot be used together".to_string()
|
||||
));
|
||||
}
|
||||
if !state.has_user_data_reader() {
|
||||
return Ok(Err("user group data is unavailable".to_string()));
|
||||
}
|
||||
|
||||
if group_id == UNGROUPED_USAGE_ID {
|
||||
let ids = ungrouped_usage_users(state)
|
||||
.await?
|
||||
.into_iter()
|
||||
.filter(|user| include_inactive || user.is_active)
|
||||
.filter(|user| !exclude_admin || !user.role.eq_ignore_ascii_case("admin"))
|
||||
.map(|user| user.id)
|
||||
.collect();
|
||||
return Ok(Ok(Some(ids)));
|
||||
}
|
||||
match state
|
||||
.resolve_usage_user_group_member_ids(&group_id, include_inactive, exclude_admin)
|
||||
.await?
|
||||
{
|
||||
Some(user_ids) => Ok(Ok(Some(user_ids))),
|
||||
None => Ok(Err("user_group_id does not exist".to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Reserved statistics-only scope; never a permission group.
|
||||
pub(crate) const UNGROUPED_USAGE_ID: &str = "__ungrouped__";
|
||||
|
||||
pub(crate) async fn ungrouped_usage_users(
|
||||
state: &crate::handlers::admin::request::AdminAppState<'_>,
|
||||
) -> Result<Vec<aether_data::repository::users::StoredUserSummary>, crate::GatewayError> {
|
||||
use aether_data::repository::users::UserExportListQuery;
|
||||
let mut users = Vec::new();
|
||||
let mut skip = 0;
|
||||
loop {
|
||||
let page = state
|
||||
.list_export_users_page(&UserExportListQuery {
|
||||
skip,
|
||||
limit: 500,
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
let count = page.len();
|
||||
if count == 0 {
|
||||
break;
|
||||
}
|
||||
let ids = page.into_iter().map(|user| user.id).collect::<Vec<_>>();
|
||||
let grouped = state
|
||||
.list_user_group_memberships_by_user_ids(&ids)
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|membership| membership.user_id)
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
let ids = ids
|
||||
.into_iter()
|
||||
.filter(|id| !grouped.contains(id))
|
||||
.collect::<Vec<_>>();
|
||||
users.extend(
|
||||
state
|
||||
.list_users_by_ids(&ids)
|
||||
.await?
|
||||
.into_iter()
|
||||
.filter(|user| !user.is_deleted),
|
||||
);
|
||||
skip += count;
|
||||
if count < 500 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(users)
|
||||
}
|
||||
|
||||
/// Current group provider policy, resolved to the provider-name dimension used by usage rollups.
|
||||
/// None is unrestricted; Some(empty) deliberately matches no usage.
|
||||
pub(crate) async fn usage_group_provider_names(
|
||||
state: &crate::handlers::admin::request::AdminAppState<'_>,
|
||||
group: &aether_data::repository::users::StoredUserGroup,
|
||||
) -> Result<Option<Vec<String>>, crate::GatewayError> {
|
||||
if matches!(
|
||||
group.allowed_providers_mode.as_str(),
|
||||
"unrestricted" | "inherit"
|
||||
) {
|
||||
return Ok(None);
|
||||
}
|
||||
if group.allowed_providers_mode != "specific" {
|
||||
return Ok(Some(Vec::new()));
|
||||
}
|
||||
let allowed = group.allowed_providers.as_deref().unwrap_or_default();
|
||||
let providers = state.list_provider_catalog_providers(false).await?;
|
||||
let mut names = providers
|
||||
.into_iter()
|
||||
.filter(|provider| {
|
||||
allowed.iter().any(|value| {
|
||||
let value = value.trim();
|
||||
value.eq_ignore_ascii_case(&provider.id)
|
||||
|| value.eq_ignore_ascii_case(&provider.name)
|
||||
|| value.eq_ignore_ascii_case(&provider.provider_type)
|
||||
})
|
||||
})
|
||||
.map(|provider| provider.name)
|
||||
.collect::<Vec<_>>();
|
||||
names.sort();
|
||||
names.dedup();
|
||||
Ok(Some(names))
|
||||
}
|
||||
|
||||
pub(crate) async fn resolve_usage_group_provider_names(
|
||||
state: &crate::handlers::admin::request::AdminAppState<'_>,
|
||||
query: Option<&str>,
|
||||
) -> Result<Option<Vec<String>>, crate::GatewayError> {
|
||||
let Some(id) = crate::handlers::admin::shared::query_param_value(query, "user_group_id") else {
|
||||
return Ok(None);
|
||||
};
|
||||
if id == UNGROUPED_USAGE_ID {
|
||||
return Ok(None);
|
||||
}
|
||||
let Some(group) = state.find_user_group_by_id(&id).await? else {
|
||||
return Ok(Some(Vec::new()));
|
||||
};
|
||||
usage_group_provider_names(state, &group).await
|
||||
}
|
||||
|
||||
@@ -169,9 +169,11 @@ pub(super) async fn build_admin_monitoring_system_status_response(
|
||||
|
||||
let today_usage = state
|
||||
.summarize_usage_audits(&UsageAuditSummaryQuery {
|
||||
provider_names: None,
|
||||
created_from_unix_secs: today_start.timestamp().max(0) as u64,
|
||||
created_until_unix_secs: now_unix_secs.saturating_add(1),
|
||||
user_id: None,
|
||||
user_ids: None,
|
||||
provider_name: None,
|
||||
model: None,
|
||||
})
|
||||
|
||||
@@ -22,6 +22,8 @@ pub(crate) mod test_support;
|
||||
mod trace;
|
||||
mod usage_helpers;
|
||||
|
||||
pub(super) use resilience::overview_resilience_payload;
|
||||
|
||||
pub(crate) async fn maybe_build_local_admin_monitoring_response(
|
||||
state: &AdminAppState<'_>,
|
||||
request_context: &AdminRequestContext<'_>,
|
||||
|
||||
@@ -6,3 +6,23 @@ mod status;
|
||||
pub(super) use history::build_admin_monitoring_resilience_circuit_history_response;
|
||||
pub(super) use reset::build_admin_monitoring_reset_error_stats_response;
|
||||
pub(super) use status::build_admin_monitoring_resilience_status_response;
|
||||
|
||||
pub(in super::super) async fn overview_resilience_payload(
|
||||
state: &crate::handlers::admin::request::AdminAppState<'_>,
|
||||
) -> Result<serde_json::Value, crate::GatewayError> {
|
||||
let snapshot = snapshot::build_admin_monitoring_resilience_snapshot(state).await?;
|
||||
let from = (snapshot.timestamp - chrono::Duration::hours(24))
|
||||
.timestamp()
|
||||
.max(
|
||||
state
|
||||
.admin_monitoring_error_stats_reset_at()
|
||||
.unwrap_or_default() as i64,
|
||||
);
|
||||
Ok(serde_json::json!({
|
||||
"scope": {"kind": "installation"},
|
||||
"error_range": {"from": chrono::DateTime::from_timestamp(from, 0), "to": snapshot.timestamp},
|
||||
"timestamp": snapshot.timestamp, "health_score": snapshot.health_score,
|
||||
"status": snapshot.status, "error_statistics": snapshot.error_statistics,
|
||||
"recent_errors": snapshot.recent_errors, "recommendations": snapshot.recommendations,
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
|
||||
use crate::GatewayError;
|
||||
use aether_admin::observability::analytics::{dashboard_value, parse_dashboard_query};
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{self, StatusCode},
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
|
||||
pub(super) async fn response(
|
||||
state: &AdminAppState<'_>,
|
||||
context: &AdminRequestContext<'_>,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let query = match parse_dashboard_query(context.query_string()) {
|
||||
Ok(query) => query,
|
||||
Err(detail) => return Ok(super::error(StatusCode::BAD_REQUEST, &detail)),
|
||||
};
|
||||
if !state.as_ref().has_usage_data_reader() {
|
||||
return Ok(super::error(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"usage analytics is unavailable",
|
||||
));
|
||||
}
|
||||
let snapshot = match tokio::time::timeout(
|
||||
std::time::Duration::from_secs(15),
|
||||
state.as_ref().query_dashboard_analytics(&query),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => result?,
|
||||
Err(_) => {
|
||||
return Ok(super::error(
|
||||
StatusCode::GATEWAY_TIMEOUT,
|
||||
"dashboard query exceeded its time budget",
|
||||
))
|
||||
}
|
||||
};
|
||||
let data = dashboard_value(&query, &snapshot).map_err(GatewayError::Internal)?;
|
||||
Ok((
|
||||
[(http::header::CACHE_CONTROL, "private, no-store")],
|
||||
Json(data),
|
||||
)
|
||||
.into_response())
|
||||
}
|
||||
|
||||
pub(super) async fn total_response(
|
||||
state: &AdminAppState<'_>,
|
||||
context: &AdminRequestContext<'_>,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
use crate::cache::OverviewTotalRead;
|
||||
use aether_data_contracts::repository::usage::UsageDashboardAnalyticsQuery;
|
||||
use serde_json::json;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
let query = match parse_dashboard_query(context.query_string()) {
|
||||
Ok(query) => query,
|
||||
Err(detail) => return Ok(super::error(StatusCode::BAD_REQUEST, &detail)),
|
||||
};
|
||||
if !state.as_ref().has_usage_data_reader() {
|
||||
return Ok(super::error(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"usage analytics is unavailable",
|
||||
));
|
||||
}
|
||||
let (cached, refresh) = state.as_ref().overview_total_cache.read(Instant::now());
|
||||
if let Some(refresh) = refresh {
|
||||
let app = state.as_ref();
|
||||
let data = if app.background_data.has_usage_reader() {
|
||||
Arc::clone(&app.background_data)
|
||||
} else {
|
||||
Arc::clone(&app.data)
|
||||
};
|
||||
// Lifetime boundaries do not depend on the viewer's timezone. Every
|
||||
// administrator shares one refresh, including after a page reload.
|
||||
tokio::spawn(async move {
|
||||
let query = UsageDashboardAnalyticsQuery {
|
||||
timezone: "UTC".into(),
|
||||
};
|
||||
let result = tokio::time::timeout(
|
||||
Duration::from_secs(185),
|
||||
data.query_dashboard_analytics(&query),
|
||||
)
|
||||
.await;
|
||||
let snapshot = match result {
|
||||
Ok(Ok(snapshot)) => Some(snapshot),
|
||||
Ok(Err(error)) => {
|
||||
tracing::warn!(%error, "dashboard lifetime refresh failed");
|
||||
None
|
||||
}
|
||||
Err(_) => {
|
||||
tracing::warn!("dashboard lifetime refresh exceeded its time budget");
|
||||
None
|
||||
}
|
||||
};
|
||||
refresh.finish(snapshot, Instant::now());
|
||||
});
|
||||
}
|
||||
let (status, body, retry_after) = match cached {
|
||||
OverviewTotalRead::Pending => {
|
||||
(StatusCode::ACCEPTED, json!({"status":"pending"}), Some("3"))
|
||||
}
|
||||
OverviewTotalRead::Failed => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
json!({"status":"failed", "detail":"cumulative dashboard totals are temporarily unavailable; retry shortly"}),
|
||||
Some("10"),
|
||||
),
|
||||
OverviewTotalRead::Ready { snapshot, stale } => {
|
||||
let mut value = dashboard_value(&query, &snapshot).map_err(GatewayError::Internal)?;
|
||||
(
|
||||
StatusCode::OK,
|
||||
json!({
|
||||
"status":"ready", "total": value["total"].take(),
|
||||
"history_complete": snapshot.history_complete, "stale": stale,
|
||||
}),
|
||||
None,
|
||||
)
|
||||
}
|
||||
};
|
||||
let mut response = (
|
||||
status,
|
||||
[(http::header::CACHE_CONTROL, "private, no-store")],
|
||||
Json(body),
|
||||
)
|
||||
.into_response();
|
||||
if let Some(retry_after) = retry_after {
|
||||
response.headers_mut().insert(
|
||||
http::header::RETRY_AFTER,
|
||||
http::HeaderValue::from_static(retry_after),
|
||||
);
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
|
||||
use crate::GatewayError;
|
||||
use aether_admin::observability::analytics::{dashboard_summary_value, parse_dashboard_query};
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{header, StatusCode},
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
|
||||
pub(super) async fn response(
|
||||
state: &AdminAppState<'_>,
|
||||
context: &AdminRequestContext<'_>,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
let query = match parse_dashboard_query(context.query_string()) {
|
||||
Ok(query) => query,
|
||||
Err(detail) => return Ok(super::error(StatusCode::BAD_REQUEST, &detail)),
|
||||
};
|
||||
if !state.as_ref().has_usage_data_reader() {
|
||||
return Ok(super::error(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"dashboard statistics are unavailable",
|
||||
));
|
||||
}
|
||||
let snapshot = match tokio::time::timeout(
|
||||
std::time::Duration::from_secs(5),
|
||||
state.as_ref().data.query_dashboard_summary(&query),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(snapshot)) => snapshot,
|
||||
Ok(Err(error)) => return Err(GatewayError::Internal(error.to_string())),
|
||||
Err(_) => {
|
||||
return Ok(super::error(
|
||||
StatusCode::GATEWAY_TIMEOUT,
|
||||
"dashboard statistics exceeded their time budget",
|
||||
))
|
||||
}
|
||||
};
|
||||
let mut value = dashboard_summary_value(&snapshot);
|
||||
value["concurrency"] = state
|
||||
.as_ref()
|
||||
.today_concurrency(&query.timezone)
|
||||
.map_err(GatewayError::Internal)?;
|
||||
Ok(([(header::CACHE_CONTROL, "private, no-store")], Json(value)).into_response())
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
use super::error;
|
||||
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
|
||||
use crate::GatewayError;
|
||||
use aether_admin::observability::analytics::{envelope, metrics_value, OverviewRequest};
|
||||
use aether_data_contracts::repository::usage::{UsageAnalyticsQuery, USAGE_ANALYTICS_VERSION};
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{header, HeaderValue, StatusCode},
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
pub(super) async fn response(
|
||||
state: &AdminAppState<'_>,
|
||||
context: &AdminRequestContext<'_>,
|
||||
) -> Result<Response<Body>, GatewayError> {
|
||||
if context
|
||||
.query_string()
|
||||
.is_some_and(|query| !query.is_empty())
|
||||
{
|
||||
return Ok(error(
|
||||
StatusCode::BAD_REQUEST,
|
||||
"live diagnostics do not accept historical filters",
|
||||
));
|
||||
}
|
||||
let app = state.as_ref();
|
||||
let _ = app.metric_samples().await;
|
||||
let snapshot = app.metric_snapshot.read().await.clone();
|
||||
let captured = snapshot.as_ref().map(|(captured, _)| *captured);
|
||||
let now = chrono::Utc::now();
|
||||
let observed_at = captured
|
||||
.and_then(|captured| chrono::Duration::from_std(captured.elapsed()).ok())
|
||||
.map(|age| now - age);
|
||||
let mut unavailable = Vec::new();
|
||||
let (resilience_result, recent_result) = tokio::join!(
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_secs(3),
|
||||
super::super::monitoring::overview_resilience_payload(state)
|
||||
),
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_secs(3),
|
||||
recent_activity(state, now)
|
||||
),
|
||||
);
|
||||
let resilience = match resilience_result {
|
||||
Ok(Ok(value)) => Some(value),
|
||||
_ => {
|
||||
tracing::warn!("overview resilience snapshot unavailable");
|
||||
unavailable.push("resilience");
|
||||
None
|
||||
}
|
||||
};
|
||||
let recent_activity = match recent_result {
|
||||
Ok(Ok(value)) => Some(value),
|
||||
_ => {
|
||||
unavailable.push("recent_activity");
|
||||
None
|
||||
}
|
||||
};
|
||||
if captured.is_none() {
|
||||
unavailable.push("metrics");
|
||||
}
|
||||
let mut response = Json(json!({
|
||||
"meta": {
|
||||
"schema_version": 1, "metric_version": USAGE_ANALYTICS_VERSION, "scope": {"kind": "node"},
|
||||
"generated_at": now, "data_through": observed_at, "read_revision": observed_at.map(|value| value.timestamp_millis().to_string()),
|
||||
"coverage": {"status": if unavailable.is_empty() {"complete"} else {"partial"}},
|
||||
},
|
||||
"data": {
|
||||
"observed_at": observed_at, "window_seconds": null, "node_id": null,
|
||||
"scope": {"kind": "node", "node_ids": []},
|
||||
"metrics_text": snapshot.map(|(_, samples)| aether_runtime::metrics::render_prometheus_text(&samples)),
|
||||
"resilience": resilience, "recent_activity": recent_activity,
|
||||
"execution_activity": app.execution_activity.snapshot(),
|
||||
"unavailable_sections": unavailable,
|
||||
},
|
||||
})).into_response();
|
||||
response.headers_mut().insert(
|
||||
header::CACHE_CONTROL,
|
||||
HeaderValue::from_static("private, no-store"),
|
||||
);
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
async fn recent_activity(
|
||||
state: &AdminAppState<'_>,
|
||||
now: chrono::DateTime<chrono::Utc>,
|
||||
) -> Result<serde_json::Value, GatewayError> {
|
||||
let to = now.timestamp_millis().max(60_000) as u64;
|
||||
let request = OverviewRequest {
|
||||
query: UsageAnalyticsQuery {
|
||||
from_unix_ms: to - 60_000,
|
||||
to_unix_ms: to,
|
||||
timezone: "UTC".into(),
|
||||
limit: 1,
|
||||
..Default::default()
|
||||
},
|
||||
amount_basis: "billable".into(),
|
||||
csv: false,
|
||||
};
|
||||
let snapshot = state.as_ref().query_usage_analytics(&request.query).await?;
|
||||
let data = recent_activity_data(&snapshot);
|
||||
Ok(envelope(&request, &snapshot, data))
|
||||
}
|
||||
|
||||
fn recent_activity_data(
|
||||
snapshot: &aether_data_contracts::repository::usage::StoredUsageAnalytics,
|
||||
) -> serde_json::Value {
|
||||
let mut data = metrics_value(&snapshot.summary);
|
||||
data["requests_per_second"] = json!(snapshot.summary.request_count as f64 / 60.0);
|
||||
data["requests_per_minute"] = json!(snapshot.summary.request_count);
|
||||
data["tokens_per_minute"] = data["total_tokens"].clone();
|
||||
data["window_seconds"] = json!(60);
|
||||
data
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use aether_data_contracts::repository::usage::{StoredUsageAnalytics, UsageAnalyticsMetrics};
|
||||
|
||||
#[test]
|
||||
fn recent_activity_reports_one_minute_rates_without_inventing_missing_tokens() {
|
||||
let mut snapshot = StoredUsageAnalytics {
|
||||
summary: UsageAnalyticsMetrics {
|
||||
request_count: 120,
|
||||
usage_available_count: 120,
|
||||
total_tokens: 4200,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
let value = recent_activity_data(&snapshot);
|
||||
assert_eq!(value["window_seconds"], 60);
|
||||
assert_eq!(value["requests_per_second"], 2.0);
|
||||
assert_eq!(value["requests_per_minute"], 120);
|
||||
assert_eq!(value["tokens_per_minute"], 4200);
|
||||
snapshot.summary.usage_available_count = 0;
|
||||
assert!(recent_activity_data(&snapshot)["tokens_per_minute"].is_null());
|
||||
snapshot.summary = UsageAnalyticsMetrics::default();
|
||||
assert_eq!(recent_activity_data(&snapshot)["tokens_per_minute"], 0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
mod dashboard;
|
||||
mod dashboard_summary;
|
||||
mod live;
|
||||
|
||||
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
|
||||
use crate::GatewayError;
|
||||
use aether_admin::observability::analytics::{
|
||||
costs_value, dashboard_charts_value, envelope, export_csv, metrics_value, page_value,
|
||||
parse_dashboard_charts_query, parse_overview_query, performance_value, user_finance_value,
|
||||
user_payments_value,
|
||||
};
|
||||
use aether_data_contracts::repository::usage::{UsageAnalyticsGranularity, UsageAnalyticsView};
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{self, StatusCode},
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
pub(crate) async fn maybe_build_overview_response(
|
||||
state: &AdminAppState<'_>,
|
||||
context: &AdminRequestContext<'_>,
|
||||
) -> Result<Option<Response<Body>>, GatewayError> {
|
||||
if context.route_family() != Some("overview_manage") || context.method() != http::Method::GET {
|
||||
return Ok(None);
|
||||
}
|
||||
let kind = context.route_kind().unwrap_or_default();
|
||||
if kind == "dashboard_summary" {
|
||||
return dashboard_summary::response(state, context).await.map(Some);
|
||||
}
|
||||
if kind == "dashboard_total" {
|
||||
return dashboard::total_response(state, context).await.map(Some);
|
||||
}
|
||||
if kind == "dashboard" {
|
||||
return dashboard::response(state, context).await.map(Some);
|
||||
}
|
||||
if matches!(kind, "operations_live" | "operations_resources") {
|
||||
return live::response(state, context).await.map(Some);
|
||||
}
|
||||
let view = match kind {
|
||||
"dashboard_charts" => UsageAnalyticsView::DashboardCharts,
|
||||
"summary" => UsageAnalyticsView::Summary,
|
||||
"timeseries" | "costs" => UsageAnalyticsView::Timeseries,
|
||||
"operations_performance" => UsageAnalyticsView::Performance,
|
||||
"users" | "user_detail" => UsageAnalyticsView::Users,
|
||||
"breakdown" => UsageAnalyticsView::Breakdown,
|
||||
"consumption" => UsageAnalyticsView::Consumption,
|
||||
_ => return Ok(None),
|
||||
};
|
||||
let parsed = if kind == "dashboard_charts" {
|
||||
parse_dashboard_charts_query(context.query_string())
|
||||
} else {
|
||||
parse_overview_query(context.query_string(), view)
|
||||
};
|
||||
let mut request = match parsed {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(error(StatusCode::BAD_REQUEST, &detail))),
|
||||
};
|
||||
if kind == "user_detail" {
|
||||
let encoded = context
|
||||
.path()
|
||||
.trim_end_matches('/')
|
||||
.rsplit('/')
|
||||
.next()
|
||||
.unwrap_or_default();
|
||||
let Ok(id) = percent_encoding::percent_decode_str(encoded).decode_utf8() else {
|
||||
return Ok(Some(error(
|
||||
StatusCode::BAD_REQUEST,
|
||||
"invalid user identifier",
|
||||
)));
|
||||
};
|
||||
let id = id.as_ref();
|
||||
if id.is_empty() || id.len() > 512 || id.contains('/') || id.chars().any(char::is_control) {
|
||||
return Ok(Some(error(
|
||||
StatusCode::BAD_REQUEST,
|
||||
"invalid user identifier",
|
||||
)));
|
||||
}
|
||||
if request
|
||||
.query
|
||||
.actor_user_id
|
||||
.as_deref()
|
||||
.is_some_and(|value| value != id)
|
||||
|| request
|
||||
.query
|
||||
.credential_owner_id
|
||||
.as_deref()
|
||||
.is_some_and(|value| value != id)
|
||||
{
|
||||
return Ok(Some(error(
|
||||
StatusCode::BAD_REQUEST,
|
||||
"user filter conflicts with the requested employee",
|
||||
)));
|
||||
}
|
||||
if request.query.attribution_kind.as_deref() == Some("employee") {
|
||||
request.query.actor_user_id = Some(id.into());
|
||||
} else {
|
||||
request.query.credential_owner_id = Some(id.into());
|
||||
}
|
||||
request.query.limit = 1;
|
||||
request.query.offset = 0;
|
||||
}
|
||||
if matches!(
|
||||
view,
|
||||
UsageAnalyticsView::Timeseries | UsageAnalyticsView::Performance
|
||||
) {
|
||||
request.query.limit = 10_000;
|
||||
request.query.offset = 0;
|
||||
}
|
||||
if kind == "costs" {
|
||||
request.query.granularity = UsageAnalyticsGranularity::Day;
|
||||
}
|
||||
if !state.as_ref().has_usage_data_reader() {
|
||||
return Ok(Some(error(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"usage analytics is unavailable",
|
||||
)));
|
||||
}
|
||||
let snapshot = match tokio::time::timeout(
|
||||
std::time::Duration::from_secs(if request.csv { 30 } else { 15 }),
|
||||
state.as_ref().query_usage_analytics(&request.query),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => result?,
|
||||
Err(_) => {
|
||||
return Ok(Some(error(
|
||||
StatusCode::GATEWAY_TIMEOUT,
|
||||
"report query exceeded its time budget; narrow the range or filters",
|
||||
)))
|
||||
}
|
||||
};
|
||||
if request.csv {
|
||||
return Ok(Some(match export_csv(&request, &snapshot) {
|
||||
Ok(csv) => (
|
||||
[
|
||||
(http::header::CONTENT_TYPE, "text/csv; charset=utf-8"),
|
||||
(
|
||||
http::header::CONTENT_DISPOSITION,
|
||||
"attachment; filename=overview.csv",
|
||||
),
|
||||
(http::header::CACHE_CONTROL, "private, no-store"),
|
||||
],
|
||||
csv,
|
||||
)
|
||||
.into_response(),
|
||||
Err(detail) => error(StatusCode::UNPROCESSABLE_ENTITY, &detail),
|
||||
}));
|
||||
}
|
||||
let data = match kind {
|
||||
"dashboard_charts" => dashboard_charts_value(&snapshot),
|
||||
"summary" => metrics_value(&snapshot.summary),
|
||||
"user_detail" => {
|
||||
let Some(user) = snapshot.users.first() else {
|
||||
return Ok(Some(error(StatusCode::NOT_FOUND, "employee not found")));
|
||||
};
|
||||
json!({
|
||||
"user": { "id": user.user_id, "username": user.username, "email": user.email, "is_active": user.is_active },
|
||||
"summary": metrics_value(&user.metrics),
|
||||
"finance": user_finance_value(user.finance.as_ref()),
|
||||
"payments": user_payments_value(snapshot.user_payments.as_ref()),
|
||||
})
|
||||
}
|
||||
"costs" => costs_value(&request, &snapshot),
|
||||
"timeseries" => {
|
||||
let mut page = page_value(&request, &snapshot);
|
||||
page["granularity"] = json!(request.query.granularity);
|
||||
page
|
||||
}
|
||||
"operations_performance" => performance_value(&request, &snapshot),
|
||||
_ => page_value(&request, &snapshot),
|
||||
};
|
||||
let mut response = Json(envelope(&request, &snapshot, data)).into_response();
|
||||
response.headers_mut().insert(
|
||||
http::header::CACHE_CONTROL,
|
||||
http::HeaderValue::from_static("private, no-store"),
|
||||
);
|
||||
Ok(Some(response))
|
||||
}
|
||||
|
||||
fn error(status: StatusCode, detail: &str) -> Response<Body> {
|
||||
(status, Json(json!({"detail": detail}))).into_response()
|
||||
}
|
||||
@@ -1,9 +1,15 @@
|
||||
use super::{monitoring, stats, usage};
|
||||
use super::{monitoring, overview, stats, usage};
|
||||
use crate::handlers::admin::request::{AdminRouteRequest, AdminRouteResult};
|
||||
|
||||
pub(crate) async fn maybe_build_local_admin_observability_response(
|
||||
request: AdminRouteRequest<'_>,
|
||||
) -> AdminRouteResult {
|
||||
if let Some(response) =
|
||||
overview::maybe_build_overview_response(&request.state(), &request.request_context())
|
||||
.await?
|
||||
{
|
||||
return Ok(Some(response));
|
||||
}
|
||||
if let Some(response) =
|
||||
stats::maybe_build_local_admin_stats_response(&request.state(), &request.request_context())
|
||||
.await?
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
use super::range::{build_comparison_range, parse_bounded_u32};
|
||||
use super::super::resolve_usage_user_group_scope;
|
||||
use super::range::{
|
||||
build_comparison_range, parse_bounded_u32, precise_admin_stats_time_range,
|
||||
resolve_precise_time_bounds,
|
||||
};
|
||||
use super::resolve_admin_usage_time_range;
|
||||
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
|
||||
use crate::handlers::admin::shared::{
|
||||
@@ -109,6 +113,7 @@ pub(super) async fn maybe_build_local_admin_stats_analytics_response(
|
||||
};
|
||||
let current_summary = state
|
||||
.summarize_usage_audits(&UsageAuditSummaryQuery {
|
||||
provider_names: None,
|
||||
created_from_unix_secs: current_from_unix_secs,
|
||||
created_until_unix_secs: current_until_unix_secs,
|
||||
..Default::default()
|
||||
@@ -116,6 +121,7 @@ pub(super) async fn maybe_build_local_admin_stats_analytics_response(
|
||||
.await?;
|
||||
let comparison_summary = state
|
||||
.summarize_usage_audits(&UsageAuditSummaryQuery {
|
||||
provider_names: None,
|
||||
created_from_unix_secs: comparison_from_unix_secs,
|
||||
created_until_unix_secs: comparison_until_unix_secs,
|
||||
..Default::default()
|
||||
@@ -282,41 +288,83 @@ pub(super) async fn maybe_build_local_admin_stats_analytics_response(
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
let time_range = match resolve_admin_usage_time_range(request_context.query_string()) {
|
||||
let legacy_time_range = match resolve_admin_usage_time_range(request_context.query_string())
|
||||
{
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
if let Err(detail) = time_range.validate_for_time_series(granularity) {
|
||||
return Ok(Some(admin_stats_bad_request_response(detail)));
|
||||
let precise_bounds = match resolve_precise_time_bounds(request_context.query_string()) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
let precise_time_range = match precise_bounds {
|
||||
Some((from, to)) => {
|
||||
match precise_admin_stats_time_range(request_context.query_string(), from, to) {
|
||||
Ok(value) => Some(value),
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
}
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
let time_range = precise_time_range.as_ref().unwrap_or(&legacy_time_range);
|
||||
if precise_bounds.is_none() {
|
||||
if let Err(detail) = time_range.validate_for_time_series(granularity) {
|
||||
return Ok(Some(admin_stats_bad_request_response(detail)));
|
||||
}
|
||||
} else if precise_bounds
|
||||
.and_then(|(from, to)| to.checked_sub(from))
|
||||
.is_some_and(|seconds| seconds > 90 * 86_400)
|
||||
{
|
||||
return Ok(Some(admin_stats_bad_request_response(
|
||||
"Query range cannot exceed 90 days".to_string(),
|
||||
)));
|
||||
}
|
||||
if !state.has_usage_data_reader() {
|
||||
return Ok(Some(admin_stats_time_series_empty_response()));
|
||||
}
|
||||
|
||||
let filters = AdminStatsUsageFilter::from_query(request_context.query_string());
|
||||
let user_ids = match resolve_usage_user_group_scope(
|
||||
state,
|
||||
request_context.query_string(),
|
||||
false,
|
||||
false,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
let query_granularity = match granularity {
|
||||
AdminStatsGranularity::Hour => UsageTimeSeriesGranularity::Hour,
|
||||
AdminStatsGranularity::Day
|
||||
| AdminStatsGranularity::Week
|
||||
| AdminStatsGranularity::Month => UsageTimeSeriesGranularity::Day,
|
||||
};
|
||||
let Some((created_from_unix_secs, created_until_unix_secs)) = time_range.to_unix_bounds()
|
||||
let Some((created_from_unix_secs, created_until_unix_secs)) =
|
||||
precise_bounds.or_else(|| time_range.to_unix_bounds())
|
||||
else {
|
||||
return Ok(Some(admin_stats_time_series_empty_response()));
|
||||
};
|
||||
let buckets = state
|
||||
.summarize_usage_time_series(&UsageTimeSeriesQuery {
|
||||
provider_names: super::super::resolve_usage_group_provider_names(
|
||||
state,
|
||||
request_context.query_string(),
|
||||
)
|
||||
.await?,
|
||||
created_from_unix_secs,
|
||||
created_until_unix_secs,
|
||||
granularity: query_granularity,
|
||||
tz_offset_minutes: time_range.tz_offset_minutes,
|
||||
user_id: filters.user_id,
|
||||
user_ids,
|
||||
provider_name: filters.provider_name,
|
||||
model: filters.model,
|
||||
})
|
||||
.await?;
|
||||
return Ok(Some(build_admin_stats_time_series_response_from_summaries(
|
||||
&time_range,
|
||||
time_range,
|
||||
granularity,
|
||||
&buckets,
|
||||
)));
|
||||
|
||||
@@ -72,11 +72,13 @@ pub(super) async fn maybe_build_local_admin_stats_cost_response(
|
||||
};
|
||||
let buckets = state
|
||||
.summarize_usage_time_series(&UsageTimeSeriesQuery {
|
||||
provider_names: None,
|
||||
created_from_unix_secs,
|
||||
created_until_unix_secs,
|
||||
granularity: UsageTimeSeriesGranularity::Day,
|
||||
tz_offset_minutes: time_range.tz_offset_minutes,
|
||||
user_id: None,
|
||||
user_ids: None,
|
||||
provider_name: None,
|
||||
model: None,
|
||||
})
|
||||
|
||||
@@ -3,12 +3,13 @@ use crate::GatewayError;
|
||||
use aether_data_contracts::repository::usage::StoredRequestUsageAudit;
|
||||
|
||||
pub(super) use aether_admin::observability::stats::{
|
||||
build_admin_stats_leaderboard_response, build_api_key_leaderboard_items,
|
||||
build_api_key_leaderboard_items_from_summaries, build_model_leaderboard_items,
|
||||
build_model_leaderboard_items_from_summaries, build_user_leaderboard_items,
|
||||
build_user_leaderboard_items_from_summaries, compare_leaderboard_items, compute_dense_rank,
|
||||
AdminStatsLeaderboardItem, AdminStatsLeaderboardMetric, AdminStatsLeaderboardNameMode,
|
||||
AdminStatsSortOrder, AdminStatsUserMetadata,
|
||||
build_admin_stats_leaderboard_response, build_admin_stats_user_group_leaderboard_response,
|
||||
build_api_key_leaderboard_items, build_api_key_leaderboard_items_from_summaries,
|
||||
build_model_leaderboard_items, build_model_leaderboard_items_from_summaries,
|
||||
build_user_leaderboard_items, build_user_leaderboard_items_from_summaries,
|
||||
compare_leaderboard_items, compute_dense_rank, AdminStatsLeaderboardItem,
|
||||
AdminStatsLeaderboardMetric, AdminStatsLeaderboardNameMode, AdminStatsSortOrder,
|
||||
AdminStatsUserMetadata,
|
||||
};
|
||||
|
||||
pub(super) async fn load_user_leaderboard_metadata(
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
use super::super::resolve_usage_user_group_scope;
|
||||
use super::leaderboard::{
|
||||
build_admin_stats_leaderboard_response, build_api_key_leaderboard_items_from_summaries,
|
||||
build_model_leaderboard_items_from_summaries, build_user_leaderboard_items_from_summaries,
|
||||
compare_leaderboard_items, load_user_leaderboard_metadata, AdminStatsLeaderboardNameMode,
|
||||
build_admin_stats_leaderboard_response, build_admin_stats_user_group_leaderboard_response,
|
||||
build_api_key_leaderboard_items_from_summaries, build_model_leaderboard_items_from_summaries,
|
||||
build_user_leaderboard_items_from_summaries, compare_leaderboard_items,
|
||||
load_user_leaderboard_metadata, AdminStatsLeaderboardItem, AdminStatsLeaderboardNameMode,
|
||||
};
|
||||
use super::range::{parse_bounded_u32, parse_nonnegative_usize};
|
||||
use super::range::{parse_bounded_u32, parse_nonnegative_usize, resolve_precise_time_bounds};
|
||||
use super::resolve_admin_usage_time_range;
|
||||
use crate::handlers::admin::request::{AdminAppState, AdminRequestContext};
|
||||
use crate::handlers::admin::shared::{query_param_bool, query_param_value};
|
||||
@@ -14,6 +16,7 @@ use aether_admin::observability::stats::{
|
||||
};
|
||||
use aether_data_contracts::repository::usage::{UsageLeaderboardGroupBy, UsageLeaderboardQuery};
|
||||
use axum::{body::Body, http, response::Response};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
|
||||
state: &AdminAppState<'_>,
|
||||
@@ -75,10 +78,12 @@ pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
|
||||
};
|
||||
let summaries = state
|
||||
.summarize_usage_leaderboard(&UsageLeaderboardQuery {
|
||||
provider_names: None,
|
||||
created_from_unix_secs,
|
||||
created_until_unix_secs,
|
||||
group_by: UsageLeaderboardGroupBy::Model,
|
||||
user_id: filters.user_id,
|
||||
user_ids: None,
|
||||
provider_name: filters.provider_name,
|
||||
model: filters.model,
|
||||
})
|
||||
@@ -152,10 +157,12 @@ pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
|
||||
};
|
||||
let summaries = state
|
||||
.summarize_usage_leaderboard(&UsageLeaderboardQuery {
|
||||
provider_names: None,
|
||||
created_from_unix_secs,
|
||||
created_until_unix_secs,
|
||||
group_by: UsageLeaderboardGroupBy::ApiKey,
|
||||
user_id: filters.user_id,
|
||||
user_ids: None,
|
||||
provider_name: filters.provider_name,
|
||||
model: filters.model,
|
||||
})
|
||||
@@ -206,6 +213,186 @@ pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
|
||||
)));
|
||||
}
|
||||
|
||||
if request_context
|
||||
.decision()
|
||||
.and_then(|decision| decision.route_kind.as_deref())
|
||||
== Some("leaderboard_user_groups")
|
||||
&& request_context.method() == http::Method::GET
|
||||
&& matches!(
|
||||
request_context.path(),
|
||||
"/api/admin/stats/leaderboard/user-groups"
|
||||
| "/api/admin/stats/leaderboard/user-groups/"
|
||||
)
|
||||
{
|
||||
let time_range = match resolve_admin_usage_time_range(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
let precise_bounds = match resolve_precise_time_bounds(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
let metric = match AdminStatsLeaderboardMetric::parse(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
let order = match AdminStatsSortOrder::parse(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
let limit = match query_param_value(query, "limit")
|
||||
.map(|value| parse_bounded_u32("limit", &value, 1, 100))
|
||||
.transpose()
|
||||
{
|
||||
Ok(Some(value)) => value as usize,
|
||||
Ok(None) => 10,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
let offset = match query_param_value(query, "offset")
|
||||
.map(|value| parse_nonnegative_usize("offset", &value))
|
||||
.transpose()
|
||||
{
|
||||
Ok(Some(value)) => value,
|
||||
Ok(None) => 0,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
let empty_counts = BTreeMap::new();
|
||||
if !state.has_usage_data_reader() || !state.has_user_data_reader() {
|
||||
return Ok(Some(build_admin_stats_user_group_leaderboard_response(
|
||||
metric,
|
||||
Some(&time_range),
|
||||
&[],
|
||||
&empty_counts,
|
||||
&empty_counts,
|
||||
offset,
|
||||
limit,
|
||||
)));
|
||||
}
|
||||
let include_inactive = query_param_bool(query, "include_inactive", false);
|
||||
let exclude_admin = query_param_bool(query, "exclude_admin", false);
|
||||
let filters = AdminStatsUsageFilter::from_query(query);
|
||||
if filters.user_id.is_some() {
|
||||
return Ok(Some(admin_stats_bad_request_response(
|
||||
"user_id is not supported for the user group leaderboard".to_string(),
|
||||
)));
|
||||
}
|
||||
let Some((created_from_unix_secs, created_until_unix_secs)) =
|
||||
precise_bounds.or_else(|| time_range.to_unix_bounds())
|
||||
else {
|
||||
return Ok(Some(build_admin_stats_user_group_leaderboard_response(
|
||||
metric,
|
||||
Some(&time_range),
|
||||
&[],
|
||||
&empty_counts,
|
||||
&empty_counts,
|
||||
offset,
|
||||
limit,
|
||||
)));
|
||||
};
|
||||
|
||||
let mut leaderboard = Vec::new();
|
||||
let mut member_counts = BTreeMap::new();
|
||||
let mut active_member_counts = BTreeMap::new();
|
||||
for group in state.list_user_groups().await? {
|
||||
let members = state.list_user_group_members(&group.id).await?;
|
||||
let member_count = members.iter().filter(|member| !member.is_deleted).count();
|
||||
let active_member_count = members
|
||||
.iter()
|
||||
.filter(|member| !member.is_deleted && member.is_active)
|
||||
.count();
|
||||
let user_ids = members
|
||||
.iter()
|
||||
.filter(|member| !member.is_deleted)
|
||||
.filter(|member| include_inactive || member.is_active)
|
||||
.filter(|member| !exclude_admin || !member.role.eq_ignore_ascii_case("admin"))
|
||||
.map(|member| member.user_id.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let summaries = state
|
||||
.summarize_usage_leaderboard(&UsageLeaderboardQuery {
|
||||
created_from_unix_secs,
|
||||
created_until_unix_secs,
|
||||
group_by: UsageLeaderboardGroupBy::User,
|
||||
user_id: None,
|
||||
user_ids: Some(user_ids),
|
||||
provider_names: super::super::usage_group_provider_names(state, &group).await?,
|
||||
provider_name: filters.provider_name.clone(),
|
||||
model: filters.model.clone(),
|
||||
})
|
||||
.await?;
|
||||
let user_ids = summaries
|
||||
.iter()
|
||||
.map(|row| row.group_key.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let metadata = load_user_leaderboard_metadata(state, &user_ids).await?;
|
||||
let users = build_user_leaderboard_items_from_summaries(
|
||||
&summaries,
|
||||
&metadata,
|
||||
state.has_auth_user_data_reader(),
|
||||
state.has_user_data_reader(),
|
||||
include_inactive,
|
||||
exclude_admin,
|
||||
);
|
||||
let mut item = AdminStatsLeaderboardItem {
|
||||
id: group.id.clone(),
|
||||
name: group.name,
|
||||
requests: 0,
|
||||
tokens: 0,
|
||||
cost: 0.0,
|
||||
};
|
||||
for user in users {
|
||||
item.requests = item.requests.saturating_add(user.requests);
|
||||
item.tokens = item.tokens.saturating_add(user.tokens);
|
||||
item.cost += user.cost;
|
||||
}
|
||||
member_counts.insert(group.id.clone(), member_count);
|
||||
active_member_counts.insert(group.id, active_member_count);
|
||||
leaderboard.push(item);
|
||||
}
|
||||
let ungrouped = super::super::ungrouped_usage_users(state).await?;
|
||||
let id = super::super::UNGROUPED_USAGE_ID.to_string();
|
||||
member_counts.insert(id.clone(), ungrouped.len());
|
||||
active_member_counts.insert(
|
||||
id.clone(),
|
||||
ungrouped.iter().filter(|user| user.is_active).count(),
|
||||
);
|
||||
let user_ids = ungrouped
|
||||
.into_iter()
|
||||
.filter(|user| include_inactive || user.is_active)
|
||||
.filter(|user| !exclude_admin || !user.role.eq_ignore_ascii_case("admin"))
|
||||
.map(|user| user.id)
|
||||
.collect();
|
||||
let rows = state
|
||||
.summarize_usage_leaderboard(&UsageLeaderboardQuery {
|
||||
created_from_unix_secs,
|
||||
created_until_unix_secs,
|
||||
group_by: UsageLeaderboardGroupBy::User,
|
||||
user_id: None,
|
||||
user_ids: Some(user_ids),
|
||||
provider_names: None,
|
||||
provider_name: filters.provider_name.clone(),
|
||||
model: filters.model.clone(),
|
||||
})
|
||||
.await?;
|
||||
leaderboard.push(AdminStatsLeaderboardItem {
|
||||
id,
|
||||
name: "Ungrouped".to_string(),
|
||||
requests: rows.iter().map(|row| row.request_count).sum(),
|
||||
tokens: rows.iter().map(|row| row.total_tokens).sum(),
|
||||
cost: rows.iter().map(|row| row.total_cost_usd).sum(),
|
||||
});
|
||||
leaderboard.sort_by(|left, right| compare_leaderboard_items(metric, order, left, right));
|
||||
|
||||
return Ok(Some(build_admin_stats_user_group_leaderboard_response(
|
||||
metric,
|
||||
Some(&time_range),
|
||||
&leaderboard,
|
||||
&member_counts,
|
||||
&active_member_counts,
|
||||
offset,
|
||||
limit,
|
||||
)));
|
||||
}
|
||||
|
||||
if request_context
|
||||
.decision()
|
||||
.and_then(|decision| decision.route_kind.as_deref())
|
||||
@@ -220,6 +407,10 @@ pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
let precise_bounds = match resolve_precise_time_bounds(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
let metric = match AdminStatsLeaderboardMetric::parse(query) {
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
@@ -253,7 +444,15 @@ pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
|
||||
let include_inactive = query_param_bool(query, "include_inactive", false);
|
||||
let exclude_admin = query_param_bool(query, "exclude_admin", false);
|
||||
let filters = AdminStatsUsageFilter::from_query(query);
|
||||
let Some((created_from_unix_secs, created_until_unix_secs)) = time_range.to_unix_bounds()
|
||||
let scoped_user_ids =
|
||||
match resolve_usage_user_group_scope(state, query, include_inactive, exclude_admin)
|
||||
.await?
|
||||
{
|
||||
Ok(value) => value,
|
||||
Err(detail) => return Ok(Some(admin_stats_bad_request_response(detail))),
|
||||
};
|
||||
let Some((created_from_unix_secs, created_until_unix_secs)) =
|
||||
precise_bounds.or_else(|| time_range.to_unix_bounds())
|
||||
else {
|
||||
return Ok(Some(admin_stats_leaderboard_empty_response(
|
||||
metric,
|
||||
@@ -262,10 +461,13 @@ pub(super) async fn maybe_build_local_admin_stats_leaderboard_response(
|
||||
};
|
||||
let summaries = state
|
||||
.summarize_usage_leaderboard(&UsageLeaderboardQuery {
|
||||
provider_names: super::super::resolve_usage_group_provider_names(state, query)
|
||||
.await?,
|
||||
created_from_unix_secs,
|
||||
created_until_unix_secs,
|
||||
group_by: UsageLeaderboardGroupBy::User,
|
||||
user_id: filters.user_id,
|
||||
user_ids: scoped_user_ids,
|
||||
provider_name: filters.provider_name,
|
||||
model: filters.model,
|
||||
})
|
||||
|
||||
@@ -8,7 +8,10 @@ mod leaderboard;
|
||||
mod leaderboard_routes;
|
||||
mod provider_quota_routes;
|
||||
mod range;
|
||||
pub(crate) use self::range::{parse_bounded_u32, resolve_admin_usage_time_range};
|
||||
pub(crate) use self::range::{
|
||||
parse_bounded_u32, precise_admin_stats_time_range, resolve_admin_usage_time_range,
|
||||
resolve_precise_time_bounds, resolve_usage_time_bounds,
|
||||
};
|
||||
pub(crate) use aether_admin::observability::stats::{
|
||||
admin_stats_bad_request_response, aggregate_usage_stats, round_to, AdminStatsTimeRange,
|
||||
AdminStatsUsageFilter,
|
||||
|
||||
@@ -4,10 +4,14 @@ pub(super) use aether_admin::observability::stats::{
|
||||
admin_usage_default_days, build_comparison_range, build_time_range_from_days, parse_naive_date,
|
||||
parse_nonnegative_usize, parse_tz_offset_minutes, resolve_preset_dates, user_today,
|
||||
};
|
||||
use chrono::{DateTime, Offset, TimeZone, Utc};
|
||||
|
||||
pub(crate) fn resolve_admin_usage_time_range(
|
||||
query: Option<&str>,
|
||||
) -> Result<AdminStatsTimeRange, String> {
|
||||
if let Some((from, to)) = resolve_precise_time_bounds(query)? {
|
||||
return precise_admin_stats_time_range(query, from, to);
|
||||
}
|
||||
match AdminStatsTimeRange::resolve_optional(query)? {
|
||||
Some(time_range) => Ok(time_range),
|
||||
None => {
|
||||
@@ -20,3 +24,153 @@ pub(crate) fn resolve_admin_usage_time_range(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve an exact UTC range supplied by the shared admin range picker.
|
||||
///
|
||||
/// The older stats handlers use `start_date`/`end_date` and fixed offsets. Keep
|
||||
/// that parser intact and only opt into this path when both RFC 3339 endpoints
|
||||
/// are present, so existing callers retain their behavior.
|
||||
pub(crate) fn resolve_precise_time_bounds(
|
||||
query: Option<&str>,
|
||||
) -> Result<Option<(u64, u64)>, String> {
|
||||
let entries =
|
||||
url::form_urlencoded::parse(query.unwrap_or_default().as_bytes()).collect::<Vec<_>>();
|
||||
let from = entries
|
||||
.iter()
|
||||
.filter(|(key, _)| key == "from")
|
||||
.collect::<Vec<_>>();
|
||||
let to = entries
|
||||
.iter()
|
||||
.filter(|(key, _)| key == "to")
|
||||
.collect::<Vec<_>>();
|
||||
if from.is_empty() && to.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
if from.len() != 1 || to.len() != 1 {
|
||||
return Err("from and to must each be provided once".into());
|
||||
}
|
||||
if entries
|
||||
.iter()
|
||||
.any(|(key, _)| matches!(key.as_ref(), "start_date" | "end_date" | "preset" | "days"))
|
||||
{
|
||||
return Err("precise from/to cannot be combined with date presets".into());
|
||||
}
|
||||
if let Some(zone) = query_param_value(query, "timezone") {
|
||||
zone.parse::<chrono_tz::Tz>()
|
||||
.map_err(|_| "invalid timezone".to_string())?;
|
||||
}
|
||||
let parse = |value: &str| -> Result<u64, String> {
|
||||
let value = DateTime::parse_from_rfc3339(value)
|
||||
.map_err(|_| "from/to must be RFC 3339 timestamps".to_string())?;
|
||||
if value.timestamp_subsec_nanos() != 0 {
|
||||
return Err("request records support second-aligned ranges".into());
|
||||
}
|
||||
u64::try_from(value.timestamp()).map_err(|_| "from/to must not precede Unix epoch".into())
|
||||
};
|
||||
let bounds = (parse(&from[0].1)?, parse(&to[0].1)?);
|
||||
if bounds.0 >= bounds.1 || bounds.1 - bounds.0 > 366 * 86_400 {
|
||||
return Err("from/to must define a nonempty range of at most 366 days".into());
|
||||
}
|
||||
Ok(Some(bounds))
|
||||
}
|
||||
|
||||
/// Return the exact range when present, otherwise preserve the legacy stats
|
||||
/// date/preset behavior.
|
||||
pub(crate) fn resolve_usage_time_bounds(query: Option<&str>) -> Result<Option<(u64, u64)>, String> {
|
||||
if let Some(bounds) = resolve_precise_time_bounds(query)? {
|
||||
return Ok(Some(bounds));
|
||||
}
|
||||
Ok(resolve_admin_usage_time_range(query)?.to_unix_bounds())
|
||||
}
|
||||
|
||||
/// Build the date metadata used by the existing stats response builders for an
|
||||
/// exact range. The data query still uses the exact UTC bounds; this metadata
|
||||
/// only supplies the local date labels and offset expected by old clients.
|
||||
pub(crate) fn precise_admin_stats_time_range(
|
||||
query: Option<&str>,
|
||||
from: u64,
|
||||
to: u64,
|
||||
) -> Result<AdminStatsTimeRange, String> {
|
||||
let timezone_name = query_param_value(query, "timezone");
|
||||
let (start_date, end_date, tz_offset_minutes) = if let Some(name) = timezone_name {
|
||||
let timezone = name
|
||||
.parse::<chrono_tz::Tz>()
|
||||
.map_err(|_| "invalid timezone".to_string())?;
|
||||
let start = Utc
|
||||
.timestamp_opt(
|
||||
i64::try_from(from).map_err(|_| "invalid from timestamp")?,
|
||||
0,
|
||||
)
|
||||
.single()
|
||||
.ok_or_else(|| "invalid from timestamp".to_string())?
|
||||
.with_timezone(&timezone);
|
||||
let end = Utc
|
||||
.timestamp_opt(
|
||||
i64::try_from(to.saturating_sub(1)).map_err(|_| "invalid to timestamp")?,
|
||||
0,
|
||||
)
|
||||
.single()
|
||||
.ok_or_else(|| "invalid to timestamp".to_string())?
|
||||
.with_timezone(&timezone);
|
||||
(
|
||||
start.date_naive(),
|
||||
end.date_naive(),
|
||||
start.offset().fix().local_minus_utc() / 60,
|
||||
)
|
||||
} else {
|
||||
let offset = parse_tz_offset_minutes(query)?;
|
||||
let fixed = chrono::FixedOffset::east_opt(offset * 60)
|
||||
.ok_or_else(|| "invalid timezone offset".to_string())?;
|
||||
let start = Utc
|
||||
.timestamp_opt(
|
||||
i64::try_from(from).map_err(|_| "invalid from timestamp")?,
|
||||
0,
|
||||
)
|
||||
.single()
|
||||
.ok_or_else(|| "invalid from timestamp".to_string())?
|
||||
.with_timezone(&fixed);
|
||||
let end = Utc
|
||||
.timestamp_opt(
|
||||
i64::try_from(to.saturating_sub(1)).map_err(|_| "invalid to timestamp")?,
|
||||
0,
|
||||
)
|
||||
.single()
|
||||
.ok_or_else(|| "invalid to timestamp".to_string())?
|
||||
.with_timezone(&fixed);
|
||||
(start.date_naive(), end.date_naive(), offset)
|
||||
};
|
||||
|
||||
Ok(AdminStatsTimeRange {
|
||||
start_date,
|
||||
end_date,
|
||||
tz_offset_minutes,
|
||||
})
|
||||
}
|
||||
|
||||
fn query_param_value(query: Option<&str>, key: &str) -> Option<String> {
|
||||
url::form_urlencoded::parse(query.unwrap_or_default().as_bytes())
|
||||
.find(|(name, _)| name == key)
|
||||
.map(|(_, value)| value.into_owned())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{precise_admin_stats_time_range, resolve_precise_time_bounds};
|
||||
|
||||
#[test]
|
||||
fn precise_stats_range_preserves_subday_bounds_and_timezone_labels() {
|
||||
let query = "from=2026-09-01T23:45:00Z&to=2026-09-02T00:15:00Z&timezone=Asia%2FShanghai";
|
||||
let (from, to) = resolve_precise_time_bounds(Some(query)).unwrap().unwrap();
|
||||
assert_eq!(to - from, 30 * 60);
|
||||
let range = precise_admin_stats_time_range(Some(query), from, to).unwrap();
|
||||
assert_eq!(range.start_date.to_string(), "2026-09-02");
|
||||
assert_eq!(range.end_date.to_string(), "2026-09-02");
|
||||
assert_eq!(range.tz_offset_minutes, 480);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn precise_stats_range_rejects_mixed_legacy_presets() {
|
||||
let query = "from=2026-09-01T00:00:00Z&to=2026-09-02T00:00:00Z&preset=today";
|
||||
assert!(resolve_precise_time_bounds(Some(query)).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user